20,
'redirection' => 5,
'limit_response_size' => 6 * MB_IN_BYTES,
'user-agent' => 'Mozilla/5.0 (compatible; Domopage/' . DOMO_VERSION . '; +' . DOMO_SITE_URL . ')',
]);
if (is_wp_error($response)) {
/* translators: placeholders are filled in at runtime (numbers, names or messages). */
return new WP_Error('domo_fetch', sprintf(__('The page could not be downloaded: %s', 'domopage'), $response->get_error_message()));
}
$code = wp_remote_retrieve_response_code($response);
if ($code < 200 || $code >= 300) {
/* translators: placeholders are filled in at runtime (numbers, names or messages). */
return new WP_Error('domo_fetch', sprintf(__('The page returned HTTP %d. It may block automated access.', 'domopage'), $code));
}
$data = domo_extract(wp_remote_retrieve_body($response), $url);
if (!$data['name'] || !$data['description']) {
return new WP_Error('domo_extract', __('No usable title and description were found. The site may need JavaScript to show its listings. Try another listing URL, or add the details by hand after import.', 'domopage'));
}
return $data;
}
/** Pure extraction from HTML (no WordPress calls), so it can be unit tested. */
function domo_extract(string $html, string $url): array
{
$d = ['source' => $url, 'name' => '', 'description' => '', 'location' => '', 'price' => '', 'facts' => [], 'images' => []];
if (trim($html) === '') {
return $d;
}
$dom = new DOMDocument();
$prev = libxml_use_internal_errors(true);
$dom->loadHTML('' . $html);
libxml_clear_errors();
libxml_use_internal_errors($prev);
$xp = new DOMXPath($dom);
$meta = function (string $key) use ($xp): string {
return domo_clean($xp->evaluate('string(//meta[@property="' . $key . '" or @name="' . $key . '"]/@content)'));
};
// 1. Structured data (JSON-LD).
$nodes = [];
$walk = function ($v) use (&$walk, &$nodes) {
if (!is_array($v)) {
return;
}
if (isset($v['@type'])) {
$nodes[] = $v;
}
foreach ($v as $child) {
if (is_array($child)) {
$walk($child);
}
}
};
foreach ($xp->query('//script[@type="application/ld+json"]') as $s) {
$walk(json_decode($s->textContent, true));
}
$byId = [];
foreach ($nodes as $n) {
if (isset($n['@id']) && is_string($n['@id'])) {
$byId[$n['@id']] = $n;
}
}
$resolve = function ($v) use ($byId) {
if (is_array($v) && isset($v['@id']) && is_string($v['@id']) && isset($byId[$v['@id']])) {
return array_replace($byId[$v['@id']], $v);
}
return $v;
};
$wanted = ['RealEstateListing', 'Accommodation', 'Apartment', 'House', 'SingleFamilyResidence', 'Residence', 'Product'];
$h1 = domo_clean($xp->evaluate('string(//h1[1])'));
$chosen = null;
$best = -1;
foreach ($nodes as $n) {
$types = (array) $n['@type'];
if (!array_intersect($types, $wanted)) {
continue;
}
$score = 0;
if ($h1 !== '' && domo_clean($n['name'] ?? '') === $h1) {
$score += 100;
}
if (strtok((string) ($n['url'] ?? ''), '#') === $url) {
$score += 200;
}
if (in_array('RealEstateListing', $types, true)) {
$score += 20;
}
if (in_array('Product', $types, true)) {
$score -= 30;
}
if (!empty($n['description'])) {
$score += 10;
}
if ($score > $best) {
$best = $score;
$chosen = $n;
}
}
if ($chosen) {
$d['name'] = domo_clean($chosen['name'] ?? '');
$d['description'] = trim((string) ($chosen['description'] ?? ''));
$entities = [$chosen];
foreach (['mainEntity', 'about', 'itemOffered'] as $k) {
if (!empty($chosen[$k])) {
$entities[] = $resolve($chosen[$k]);
}
}
$offer = $resolve($chosen['offers'] ?? []);
if (is_array($offer) && isset($offer[0])) {
$offer = $resolve($offer[0]);
}
if (is_array($offer)) {
$entities[] = $offer;
if (!empty($offer['itemOffered'])) {
$entities[] = $resolve($offer['itemOffered']);
}
}
foreach ($entities as $e) {
if (!is_array($e)) {
continue;
}
$imgs = $e['image'] ?? [];
if (is_string($imgs) || isset($imgs['url'])) {
$imgs = [$imgs];
}
foreach ((array) $imgs as $im) {
$src = is_array($im) ? ($im['url'] ?? $im['contentUrl'] ?? '') : $im;
if (is_string($src)) {
$d['images'][] = domo_abs_url($src, $url);
}
}
$amount = $e['price'] ?? $e['lowPrice'] ?? null;
if (!$d['price'] && is_numeric($amount) && $amount > 0) {
$d['price'] = domo_format_price((float) $amount, (string) ($e['priceCurrency'] ?? 'EUR'));
}
$addr = $resolve($e['address'] ?? null);
if (!$d['location'] && is_array($addr)) {
$d['location'] = domo_clean(implode(', ', array_filter([$addr['addressLocality'] ?? '', $addr['addressRegion'] ?? ''])));
}
$facts = [
'Bedrooms' => $e['numberOfBedrooms'] ?? $e['numberOfRooms'] ?? null,
'Bathrooms' => $e['numberOfBathroomsTotal'] ?? $e['numberOfFullBathrooms'] ?? null,
];
foreach ($facts as $label => $value) {
if (is_numeric($value)) {
$d['facts'][$label] = $label . ': ' . (0 + $value);
}
}
$size = $e['floorSize'] ?? null;
if (is_array($size) && is_numeric($size['value'] ?? null)) {
$unit = strtoupper((string) ($size['unitCode'] ?? $size['unitText'] ?? 'MTK'));
$d['facts']['Size'] = 'Size: ' . (0 + $size['value']) . (in_array($unit, ['FTK', 'SQFT', 'FT2'], true) ? ' sq ft' : ' m²');
}
}
}
// 2. Visible page content.
if (!$d['name']) {
$d['name'] = $h1 ?: $meta('og:title');
}
$desc = domo_description_block($xp);
if ($desc && mb_strlen($desc) > mb_strlen(domo_clean($d['description']))) {
$d['description'] = $desc;
}
if (!$d['description']) {
$d['description'] = $meta('og:description') ?: $meta('description');
}
$d['description'] = domo_tidy_paragraphs($d['description']);
if (!$d['price']) {
foreach ($xp->query('//*[@itemprop="price"]') as $el) {
$v = domo_clean($el->getAttribute('content') ?: $el->textContent);
if ($v !== '' && mb_strlen($v) < 60) {
$d['price'] = is_numeric($v) ? domo_format_price((float) $v, 'EUR') : $v;
break;
}
}
}
if (!$d['price']) {
$top = '';
foreach ($xp->query('//h1[1]/following::*[position() <= 12]') as $el) {
$t = domo_clean($el->textContent);
if (mb_strlen($t) < 300) {
$top .= ' ' . $t;
}
}
if (preg_match('/(?:€|EUR|£|GBP)\s?\d[\d.,\s]{3,}\d|\d[\d.,\s]{3,}\d\s?(?:€|EUR|£)/u', $top, $m)) {
$d['price'] = domo_clean($m[0]);
}
}
if (!$d['location']) {
foreach ($xp->query('//*[@itemprop="addressLocality"]') as $el) {
$d['location'] = domo_clean($el->getAttribute('content') ?: $el->textContent);
break;
}
}
// 3. Images. Strong signals first (og:image, gallery links, gallery elements), then a wider sweep if the gallery was not found.
$og = $meta('og:image');
if ($og) {
array_unshift($d['images'], domo_abs_url($og, $url));
}
$isPhoto = '~\.(jpe?g|png|webp|avif)(\?[^"\s]*)?$~i';
$skip = '~logo|icon|flag|sprite|avatar|placeholder|blank\.|loader|spinner|banner-ad|/agents?/|/team/|badge|social|whatsapp|facebook|instagram~i';
// Image servers often have no file extension (e.g. Resales Online's ShowFeedImage.asp?...): these are accepted too.
$isEndpoint = '~/[\w-]*(image|photo|picture|img)[\w-]*\.(asp|aspx|php|ashx|jsp)\?|[?&](img|image|photo|pic)_?(id|url)?=~i';
$add = function (string $src, bool $trusted = false) use (&$d, $url, $isPhoto, $isEndpoint, $skip) {
$src = trim(html_entity_decode($src, ENT_QUOTES | ENT_HTML5, 'UTF-8'));
if ($src === '' || str_starts_with($src, 'data:') || (!$trusted && !preg_match($isPhoto, $src) && !preg_match($isEndpoint, $src)) || preg_match($skip, $src)) {
return;
}
$d['images'][] = domo_abs_url($src, $url);
};
// Largest candidate in a srcset ("a.jpg 400w, b.jpg 1600w").
$best_src = function (string $srcset): string {
$best = '';
$max = -1;
foreach (explode(',', $srcset) as $part) {
$bits = preg_split('/\s+/', trim($part));
$w = isset($bits[1]) ? (float) $bits[1] : 0;
if ($bits[0] !== '' && $w >= $max) {
$max = $w;
$best = $bits[0];
}
}
return $best;
};
$gallery = '/gallery|slide|swiper|carousel|lightbox|photo|property|fancybox|glide|splide|slick|owl|media|image/i';
// Images inside "similar properties" cards belong to other listings: skip any element whose nearby
// container links to a different page in the same section of the site (e.g. /property/abc vs /property/xyz).
$self_path = rtrim((string) wp_parse_url($url, PHP_URL_PATH), '/');
$section = explode('/', trim($self_path, '/'))[0] ?? '';
$other_listing = function (DOMElement $el) use ($xp, $self_path, $section): bool {
if (preg_match('/related|similar|recommend|you-may|more-propert|other-propert/i', $el->getAttribute('class'))) {
return true;
}
for ($n = $el->parentNode, $i = 0; $n instanceof DOMElement && $i < 5; $n = $n->parentNode, $i++) {
if (preg_match('/related|similar|recommend|you-may|more-propert|other-propert/i', $n->getAttribute('class') . ' ' . $n->getAttribute('id'))) {
return true;
}
$links = $n->tagName === 'a' ? [$n] : iterator_to_array($xp->query('.//a[@href]', $n));
foreach ($links as $a) {
$path = rtrim((string) wp_parse_url($a->getAttribute('href'), PHP_URL_PATH), '/');
if ($path !== '' && $path !== $self_path && $section !== '' && str_starts_with(trim($path, '/'), $section . '/') && !preg_match('~\.(jpe?g|png|webp)$~i', $path)) {
return true;
}
}
}
return false;
};
foreach ($xp->query('//a[@href]') as $a) {
if (!$other_listing($a)) {
$add($a->getAttribute('href'));
}
}
$sweep = function (bool $everything) use ($xp, $add, $best_src, $gallery, $d, $other_listing) {
foreach ($xp->query('//img | //source | //*[@data-src or @data-large or @data-full or @data-lazy or @data-srcset or @data-bg or @data-bg-image or @data-background or @style]') as $el) {
if ($other_listing($el)) {
continue;
}
if (!$everything) {
$context = $el->getAttribute('class') . ' ' . $el->getAttribute('id');
for ($n = $el->parentNode, $i = 0; $n instanceof DOMElement && $i < 3; $n = $n->parentNode, $i++) {
$context .= ' ' . $n->getAttribute('class') . ' ' . $n->getAttribute('id');
}
$big = (int) $el->getAttribute('width') >= 600;
$named = $d['name'] !== '' && stripos($el->getAttribute('alt'), $d['name']) !== false;
if (!$big && !$named && !preg_match($gallery, $context)) {
continue;
}
} elseif ((int) $el->getAttribute('width') > 0 && (int) $el->getAttribute('width') < 300) {
continue;
}
foreach (['data-srcset', 'srcset'] as $attr) {
if ($el->getAttribute($attr) !== '') {
$add($best_src($el->getAttribute($attr)));
}
}
foreach (['data-full', 'data-large', 'data-lazy', 'data-src', 'data-bg', 'data-bg-image', 'data-background', 'src'] as $attr) {
if ($el->getAttribute($attr) !== '') {
// An
inside a gallery is a photo even when its address has no file extension.
$add($el->getAttribute($attr), !$everything && $el->tagName === 'img' && $attr === 'src');
}
}
if (preg_match_all('~url\(\s*[\'"]?([^\'")]+)~i', $el->getAttribute('style'), $bg)) {
foreach ($bg[1] as $src) {
$add($src);
}
}
}
};
$sweep(false);
// Galleries built by JavaScript: photo URLs inside inline scripts (JSON or arrays) on the same site.
$host = (string) wp_parse_url($url, PHP_URL_HOST);
foreach ($xp->query('//script[not(@src)]') as $script) {
$text = str_replace('\\/', '/', $script->textContent);
if (preg_match_all('~(?:https?:)?//[^\s"\'<>()]+?\.(?:jpe?g|png|webp)(?:\?[^\s"\'<>()]*)?|/[\w./%-]+?\.(?:jpe?g|png|webp)~i', $text, $m)) {
foreach ($m[0] as $src) {
$abs = domo_abs_url($src, $url);
$img_host = (string) wp_parse_url($abs, PHP_URL_HOST);
if ($img_host === $host || str_ends_with($img_host, '.' . preg_replace('/^www\./', '', $host)) || preg_match('/cdn|img|images|media|cloudfront|imgix/i', $img_host)) {
$add($abs);
}
}
}
}
// Still almost nothing found: take every reasonably sized image on the page (the user unticks extras on review).
if (count(array_unique($d['images'])) < 3) {
$sweep(true);
}
// Upgrade WordPress-style thumbnails ("photo-300x200.jpg") to the full image and drop duplicates of the same photo.
$seen = [];
$clean = [];
foreach ($d['images'] as $src) {
if (!is_string($src) || !preg_match('~^https?://~i', $src) || preg_match($skip, (string) wp_parse_url($src, PHP_URL_PATH))) {
continue;
}
$full = preg_replace('~-\d{2,4}x\d{2,4}(?=\.(?:jpe?g|png|webp)(?:\?|$))~i', '', $src);
$path = (string) wp_parse_url($full, PHP_URL_PATH);
// Same file with a different size or cache query is one photo; an image server address (no file extension) is told apart by its query.
$key = preg_match('~\.(jpe?g|png|webp|avif|gif)$~i', $path) ? strtolower(basename($path)) : strtolower($full);
if (!isset($seen[$key])) {
$seen[$key] = true;
$clean[] = $full;
}
}
$d['images'] = array_slice($clean, 0, DOMO_MAX_IMAGES);
$d['facts'] = array_values($d['facts']);
return $d;
}
/** Text under a "Description" heading, stopping at the next heading. */
function domo_description_block(DOMXPath $xp): string
{
$labels = '/^(property\s+)?(description|details|about this property|overview|descripción|beschreibung|omschrijving|description du bien|descrizione)$/iu';
foreach ($xp->query('//h2|//h3|//h4') as $heading) {
if (!preg_match($labels, domo_clean($heading->textContent))) {
continue;
}
$texts = [];
for ($s = $heading->nextSibling; $s; $s = $s->nextSibling) {
if (!$s instanceof DOMElement) {
continue;
}
if (preg_match('/^h[1-6]$/i', $s->tagName)) {
break;
}
$ps = $s->tagName === 'p' ? [$s] : iterator_to_array($s->getElementsByTagName('p'));
if (!$ps && mb_strlen(domo_clean($s->textContent)) > 60) {
$ps = [$s];
}
foreach ($ps as $p) {
$t = domo_clean($p->textContent);
if (mb_strlen($t) > 40) {
$texts[] = $t;
}
}
if (count($texts) > 14) {
break;
}
}
if ($texts) {
return implode("\n\n", array_unique($texts));
}
}
return '';
}
function domo_tidy_paragraphs(string $text): string
{
$text = str_replace(["\r\n", "\r"], "\n", $text);
$text = preg_replace('~
~i', "\n", $text);
$parts = preg_split('/\n\s*\n|\n/', $text);
$parts = array_values(array_filter(array_map('domo_clean', $parts), fn($p) => $p !== ''));
return implode("\n\n", $parts);
}
function domo_format_price(float $amount, string $currency): string
{
$symbols = ['EUR' => '€', 'GBP' => '£', 'USD' => '$', 'CHF' => 'CHF ', 'SEK' => 'SEK ', 'NOK' => 'NOK ', 'DKK' => 'DKK '];
$currency = strtoupper($currency ?: 'EUR');
$number = number_format($amount, 0, ',', '.');
if ($currency === 'EUR') {
return $number . ' €';
}
return ($symbols[$currency] ?? $currency . ' ') . number_format($amount, 0, '.', ',');
}