20, 'redirection' => 5, 'limit_response_size' => 6 * MB_IN_BYTES, 'user-agent' => 'Mozilla/5.0 (compatible; Domopage/' . DOMO_VERSION . '; +' . DOMO_SITE_URL . ')', ]); if (is_wp_error($response)) { /* translators: placeholders are filled in at runtime (numbers, names or messages). */ return new WP_Error('domo_fetch', sprintf(__('The page could not be downloaded: %s', 'domopage'), $response->get_error_message())); } $code = wp_remote_retrieve_response_code($response); if ($code < 200 || $code >= 300) { /* translators: placeholders are filled in at runtime (numbers, names or messages). */ return new WP_Error('domo_fetch', sprintf(__('The page returned HTTP %d. It may block automated access.', 'domopage'), $code)); } $data = domo_extract(wp_remote_retrieve_body($response), $url); if (!$data['name'] || !$data['description']) { return new WP_Error('domo_extract', __('No usable title and description were found. The site may need JavaScript to show its listings. Try another listing URL, or add the details by hand after import.', 'domopage')); } return $data; } /** Pure extraction from HTML (no WordPress calls), so it can be unit tested. */ function domo_extract(string $html, string $url): array { $d = ['source' => $url, 'name' => '', 'description' => '', 'location' => '', 'price' => '', 'facts' => [], 'images' => []]; if (trim($html) === '') { return $d; } $dom = new DOMDocument(); $prev = libxml_use_internal_errors(true); $dom->loadHTML('' . $html); libxml_clear_errors(); libxml_use_internal_errors($prev); $xp = new DOMXPath($dom); $meta = function (string $key) use ($xp): string { return domo_clean($xp->evaluate('string(//meta[@property="' . $key . '" or @name="' . $key . '"]/@content)')); }; // 1. Structured data (JSON-LD). $nodes = []; $walk = function ($v) use (&$walk, &$nodes) { if (!is_array($v)) { return; } if (isset($v['@type'])) { $nodes[] = $v; } foreach ($v as $child) { if (is_array($child)) { $walk($child); } } }; foreach ($xp->query('//script[@type="application/ld+json"]') as $s) { $walk(json_decode($s->textContent, true)); } $byId = []; foreach ($nodes as $n) { if (isset($n['@id']) && is_string($n['@id'])) { $byId[$n['@id']] = $n; } } $resolve = function ($v) use ($byId) { if (is_array($v) && isset($v['@id']) && is_string($v['@id']) && isset($byId[$v['@id']])) { return array_replace($byId[$v['@id']], $v); } return $v; }; $wanted = ['RealEstateListing', 'Accommodation', 'Apartment', 'House', 'SingleFamilyResidence', 'Residence', 'Product']; $h1 = domo_clean($xp->evaluate('string(//h1[1])')); $chosen = null; $best = -1; foreach ($nodes as $n) { $types = (array) $n['@type']; if (!array_intersect($types, $wanted)) { continue; } $score = 0; if ($h1 !== '' && domo_clean($n['name'] ?? '') === $h1) { $score += 100; } if (strtok((string) ($n['url'] ?? ''), '#') === $url) { $score += 200; } if (in_array('RealEstateListing', $types, true)) { $score += 20; } if (in_array('Product', $types, true)) { $score -= 30; } if (!empty($n['description'])) { $score += 10; } if ($score > $best) { $best = $score; $chosen = $n; } } if ($chosen) { $d['name'] = domo_clean($chosen['name'] ?? ''); $d['description'] = trim((string) ($chosen['description'] ?? '')); $entities = [$chosen]; foreach (['mainEntity', 'about', 'itemOffered'] as $k) { if (!empty($chosen[$k])) { $entities[] = $resolve($chosen[$k]); } } $offer = $resolve($chosen['offers'] ?? []); if (is_array($offer) && isset($offer[0])) { $offer = $resolve($offer[0]); } if (is_array($offer)) { $entities[] = $offer; if (!empty($offer['itemOffered'])) { $entities[] = $resolve($offer['itemOffered']); } } foreach ($entities as $e) { if (!is_array($e)) { continue; } $imgs = $e['image'] ?? []; if (is_string($imgs) || isset($imgs['url'])) { $imgs = [$imgs]; } foreach ((array) $imgs as $im) { $src = is_array($im) ? ($im['url'] ?? $im['contentUrl'] ?? '') : $im; if (is_string($src)) { $d['images'][] = domo_abs_url($src, $url); } } $amount = $e['price'] ?? $e['lowPrice'] ?? null; if (!$d['price'] && is_numeric($amount) && $amount > 0) { $d['price'] = domo_format_price((float) $amount, (string) ($e['priceCurrency'] ?? 'EUR')); } $addr = $resolve($e['address'] ?? null); if (!$d['location'] && is_array($addr)) { $d['location'] = domo_clean(implode(', ', array_filter([$addr['addressLocality'] ?? '', $addr['addressRegion'] ?? '']))); } $facts = [ 'Bedrooms' => $e['numberOfBedrooms'] ?? $e['numberOfRooms'] ?? null, 'Bathrooms' => $e['numberOfBathroomsTotal'] ?? $e['numberOfFullBathrooms'] ?? null, ]; foreach ($facts as $label => $value) { if (is_numeric($value)) { $d['facts'][$label] = $label . ': ' . (0 + $value); } } $size = $e['floorSize'] ?? null; if (is_array($size) && is_numeric($size['value'] ?? null)) { $unit = strtoupper((string) ($size['unitCode'] ?? $size['unitText'] ?? 'MTK')); $d['facts']['Size'] = 'Size: ' . (0 + $size['value']) . (in_array($unit, ['FTK', 'SQFT', 'FT2'], true) ? ' sq ft' : ' m²'); } } } // 2. Visible page content. if (!$d['name']) { $d['name'] = $h1 ?: $meta('og:title'); } $desc = domo_description_block($xp); if ($desc && mb_strlen($desc) > mb_strlen(domo_clean($d['description']))) { $d['description'] = $desc; } if (!$d['description']) { $d['description'] = $meta('og:description') ?: $meta('description'); } $d['description'] = domo_tidy_paragraphs($d['description']); if (!$d['price']) { foreach ($xp->query('//*[@itemprop="price"]') as $el) { $v = domo_clean($el->getAttribute('content') ?: $el->textContent); if ($v !== '' && mb_strlen($v) < 60) { $d['price'] = is_numeric($v) ? domo_format_price((float) $v, 'EUR') : $v; break; } } } if (!$d['price']) { $top = ''; foreach ($xp->query('//h1[1]/following::*[position() <= 12]') as $el) { $t = domo_clean($el->textContent); if (mb_strlen($t) < 300) { $top .= ' ' . $t; } } if (preg_match('/(?:€|EUR|£|GBP)\s?\d[\d.,\s]{3,}\d|\d[\d.,\s]{3,}\d\s?(?:€|EUR|£)/u', $top, $m)) { $d['price'] = domo_clean($m[0]); } } if (!$d['location']) { foreach ($xp->query('//*[@itemprop="addressLocality"]') as $el) { $d['location'] = domo_clean($el->getAttribute('content') ?: $el->textContent); break; } } // 3. Images. Strong signals first (og:image, gallery links, gallery elements), then a wider sweep if the gallery was not found. $og = $meta('og:image'); if ($og) { array_unshift($d['images'], domo_abs_url($og, $url)); } $isPhoto = '~\.(jpe?g|png|webp|avif)(\?[^"\s]*)?$~i'; $skip = '~logo|icon|flag|sprite|avatar|placeholder|blank\.|loader|spinner|banner-ad|/agents?/|/team/|badge|social|whatsapp|facebook|instagram~i'; // Image servers often have no file extension (e.g. Resales Online's ShowFeedImage.asp?...): these are accepted too. $isEndpoint = '~/[\w-]*(image|photo|picture|img)[\w-]*\.(asp|aspx|php|ashx|jsp)\?|[?&](img|image|photo|pic)_?(id|url)?=~i'; $add = function (string $src, bool $trusted = false) use (&$d, $url, $isPhoto, $isEndpoint, $skip) { $src = trim(html_entity_decode($src, ENT_QUOTES | ENT_HTML5, 'UTF-8')); if ($src === '' || str_starts_with($src, 'data:') || (!$trusted && !preg_match($isPhoto, $src) && !preg_match($isEndpoint, $src)) || preg_match($skip, $src)) { return; } $d['images'][] = domo_abs_url($src, $url); }; // Largest candidate in a srcset ("a.jpg 400w, b.jpg 1600w"). $best_src = function (string $srcset): string { $best = ''; $max = -1; foreach (explode(',', $srcset) as $part) { $bits = preg_split('/\s+/', trim($part)); $w = isset($bits[1]) ? (float) $bits[1] : 0; if ($bits[0] !== '' && $w >= $max) { $max = $w; $best = $bits[0]; } } return $best; }; $gallery = '/gallery|slide|swiper|carousel|lightbox|photo|property|fancybox|glide|splide|slick|owl|media|image/i'; // Images inside "similar properties" cards belong to other listings: skip any element whose nearby // container links to a different page in the same section of the site (e.g. /property/abc vs /property/xyz). $self_path = rtrim((string) wp_parse_url($url, PHP_URL_PATH), '/'); $section = explode('/', trim($self_path, '/'))[0] ?? ''; $other_listing = function (DOMElement $el) use ($xp, $self_path, $section): bool { if (preg_match('/related|similar|recommend|you-may|more-propert|other-propert/i', $el->getAttribute('class'))) { return true; } for ($n = $el->parentNode, $i = 0; $n instanceof DOMElement && $i < 5; $n = $n->parentNode, $i++) { if (preg_match('/related|similar|recommend|you-may|more-propert|other-propert/i', $n->getAttribute('class') . ' ' . $n->getAttribute('id'))) { return true; } $links = $n->tagName === 'a' ? [$n] : iterator_to_array($xp->query('.//a[@href]', $n)); foreach ($links as $a) { $path = rtrim((string) wp_parse_url($a->getAttribute('href'), PHP_URL_PATH), '/'); if ($path !== '' && $path !== $self_path && $section !== '' && str_starts_with(trim($path, '/'), $section . '/') && !preg_match('~\.(jpe?g|png|webp)$~i', $path)) { return true; } } } return false; }; foreach ($xp->query('//a[@href]') as $a) { if (!$other_listing($a)) { $add($a->getAttribute('href')); } } $sweep = function (bool $everything) use ($xp, $add, $best_src, $gallery, $d, $other_listing) { foreach ($xp->query('//img | //source | //*[@data-src or @data-large or @data-full or @data-lazy or @data-srcset or @data-bg or @data-bg-image or @data-background or @style]') as $el) { if ($other_listing($el)) { continue; } if (!$everything) { $context = $el->getAttribute('class') . ' ' . $el->getAttribute('id'); for ($n = $el->parentNode, $i = 0; $n instanceof DOMElement && $i < 3; $n = $n->parentNode, $i++) { $context .= ' ' . $n->getAttribute('class') . ' ' . $n->getAttribute('id'); } $big = (int) $el->getAttribute('width') >= 600; $named = $d['name'] !== '' && stripos($el->getAttribute('alt'), $d['name']) !== false; if (!$big && !$named && !preg_match($gallery, $context)) { continue; } } elseif ((int) $el->getAttribute('width') > 0 && (int) $el->getAttribute('width') < 300) { continue; } foreach (['data-srcset', 'srcset'] as $attr) { if ($el->getAttribute($attr) !== '') { $add($best_src($el->getAttribute($attr))); } } foreach (['data-full', 'data-large', 'data-lazy', 'data-src', 'data-bg', 'data-bg-image', 'data-background', 'src'] as $attr) { if ($el->getAttribute($attr) !== '') { // An inside a gallery is a photo even when its address has no file extension. $add($el->getAttribute($attr), !$everything && $el->tagName === 'img' && $attr === 'src'); } } if (preg_match_all('~url\(\s*[\'"]?([^\'")]+)~i', $el->getAttribute('style'), $bg)) { foreach ($bg[1] as $src) { $add($src); } } } }; $sweep(false); // Galleries built by JavaScript: photo URLs inside inline scripts (JSON or arrays) on the same site. $host = (string) wp_parse_url($url, PHP_URL_HOST); foreach ($xp->query('//script[not(@src)]') as $script) { $text = str_replace('\\/', '/', $script->textContent); if (preg_match_all('~(?:https?:)?//[^\s"\'<>()]+?\.(?:jpe?g|png|webp)(?:\?[^\s"\'<>()]*)?|/[\w./%-]+?\.(?:jpe?g|png|webp)~i', $text, $m)) { foreach ($m[0] as $src) { $abs = domo_abs_url($src, $url); $img_host = (string) wp_parse_url($abs, PHP_URL_HOST); if ($img_host === $host || str_ends_with($img_host, '.' . preg_replace('/^www\./', '', $host)) || preg_match('/cdn|img|images|media|cloudfront|imgix/i', $img_host)) { $add($abs); } } } } // Still almost nothing found: take every reasonably sized image on the page (the user unticks extras on review). if (count(array_unique($d['images'])) < 3) { $sweep(true); } // Upgrade WordPress-style thumbnails ("photo-300x200.jpg") to the full image and drop duplicates of the same photo. $seen = []; $clean = []; foreach ($d['images'] as $src) { if (!is_string($src) || !preg_match('~^https?://~i', $src) || preg_match($skip, (string) wp_parse_url($src, PHP_URL_PATH))) { continue; } $full = preg_replace('~-\d{2,4}x\d{2,4}(?=\.(?:jpe?g|png|webp)(?:\?|$))~i', '', $src); $path = (string) wp_parse_url($full, PHP_URL_PATH); // Same file with a different size or cache query is one photo; an image server address (no file extension) is told apart by its query. $key = preg_match('~\.(jpe?g|png|webp|avif|gif)$~i', $path) ? strtolower(basename($path)) : strtolower($full); if (!isset($seen[$key])) { $seen[$key] = true; $clean[] = $full; } } $d['images'] = array_slice($clean, 0, DOMO_MAX_IMAGES); $d['facts'] = array_values($d['facts']); return $d; } /** Text under a "Description" heading, stopping at the next heading. */ function domo_description_block(DOMXPath $xp): string { $labels = '/^(property\s+)?(description|details|about this property|overview|descripción|beschreibung|omschrijving|description du bien|descrizione)$/iu'; foreach ($xp->query('//h2|//h3|//h4') as $heading) { if (!preg_match($labels, domo_clean($heading->textContent))) { continue; } $texts = []; for ($s = $heading->nextSibling; $s; $s = $s->nextSibling) { if (!$s instanceof DOMElement) { continue; } if (preg_match('/^h[1-6]$/i', $s->tagName)) { break; } $ps = $s->tagName === 'p' ? [$s] : iterator_to_array($s->getElementsByTagName('p')); if (!$ps && mb_strlen(domo_clean($s->textContent)) > 60) { $ps = [$s]; } foreach ($ps as $p) { $t = domo_clean($p->textContent); if (mb_strlen($t) > 40) { $texts[] = $t; } } if (count($texts) > 14) { break; } } if ($texts) { return implode("\n\n", array_unique($texts)); } } return ''; } function domo_tidy_paragraphs(string $text): string { $text = str_replace(["\r\n", "\r"], "\n", $text); $text = preg_replace('~~i', "\n", $text); $parts = preg_split('/\n\s*\n|\n/', $text); $parts = array_values(array_filter(array_map('domo_clean', $parts), fn($p) => $p !== '')); return implode("\n\n", $parts); } function domo_format_price(float $amount, string $currency): string { $symbols = ['EUR' => '€', 'GBP' => '£', 'USD' => '$', 'CHF' => 'CHF ', 'SEK' => 'SEK ', 'NOK' => 'NOK ', 'DKK' => 'DKK ']; $currency = strtoupper($currency ?: 'EUR'); $number = number_format($amount, 0, ',', '.'); if ($currency === 'EUR') { return $number . ' €'; } return ($symbols[$currency] ?? $currency . ' ') . number_format($amount, 0, '.', ','); }