, an llms.txt, and * the AI search crawlers robots.txt should let in. Every part is re-checked here before it is used — * the same rules Promvia applies before it publishes. */ if ( ! defined( 'ABSPATH' ) ) { exit; } const PROMVIA_FEED_PATH_PATTERN = '#^/api/public/fixes/site_[a-z0-9]{12}/[A-Za-z0-9_-]{24}$#'; const PROMVIA_JSONLD_PATTERN = '#^\s*(?:\s*)*$#'; const PROMVIA_CRAWLER_PATTERN = '/^[A-Za-z0-9._-]{1,40}$/'; const PROMVIA_FEED_TTL = 600; /** A feed address on the Promvia endpoint this site uses, or null. */ function promvia_valid_feed_url( $url, $endpoint ) { $url = trim( (string) $url ); $u = wp_parse_url( $url ); $e = wp_parse_url( (string) $endpoint ); if ( ! is_array( $u ) || ! is_array( $e ) || empty( $u['host'] ) || empty( $u['path'] ) ) { return null; } if ( isset( $u['query'] ) || isset( $u['fragment'] ) || isset( $u['user'] ) ) { return null; } if ( ( $u['port'] ?? null ) !== ( $e['port'] ?? null ) ) { return null; } if ( ( $u['scheme'] ?? '' ) !== ( $e['scheme'] ?? '' ) || strtolower( $u['host'] ) !== strtolower( $e['host'] ?? '' ) ) { return null; } if ( ! preg_match( PROMVIA_FEED_PATH_PATTERN, $u['path'] ) ) { return null; } return $url; } /** * The feed body as the three parts this plugin serves, or null when it is not a Promvia feed. The * JSON-LD arrives as script blocks; only their JSON is kept, and WordPress prints it (wp_print_inline_script_tag). */ function promvia_parse_feed( $body ) { $f = json_decode( (string) $body, true ); if ( ! is_array( $f ) || ( $f['v'] ?? null ) !== 1 ) { return null; } $jsonld = array(); if ( isset( $f['head'] ) && is_string( $f['head'] ) && preg_match( PROMVIA_JSONLD_PATTERN, $f['head'] ) && preg_match_all( '##', $f['head'], $m ) ) { foreach ( $m[1] as $json ) { if ( is_array( json_decode( $json, true ) ) ) { $jsonld[] = trim( $json ); } } } $llms = isset( $f['llmsTxt'] ) && is_string( $f['llmsTxt'] ) && '' !== trim( $f['llmsTxt'] ) ? $f['llmsTxt'] : null; $robots = array(); foreach ( ( isset( $f['robotsAllow'] ) && is_array( $f['robotsAllow'] ) ) ? $f['robotsAllow'] : array() as $t ) { if ( is_string( $t ) && preg_match( PROMVIA_CRAWLER_PATTERN, $t ) ) { $robots[] = $t; } } return array( 'jsonld' => $jsonld, 'llms' => $llms, 'robots' => array_values( array_unique( $robots ) ) ); } /** * What to serve after a refresh. A 404 means the address was replaced or publishing was stopped: * serve nothing. Any other failure keeps the last good feed, so a Promvia outage never removes what * the owner approved. */ function promvia_next_feed( $status, $body, $last ) { if ( 200 === (int) $status ) { $feed = promvia_parse_feed( $body ); return null === $feed ? $last : $feed; } if ( 404 === (int) $status ) { return null; } return $last; } function promvia_robots_rule_matches_root( $val ) { return '/' === $val || '/*' === $val || '*' === $val; } /** * The smallest robots.txt edit that lets the named crawlers in — the same rules as Promvia's GitHub * fix and its Cloudflare worker: a group naming only those crawlers loses its whole-site Disallow; a * group shared with another crawler loses only that crawler's User-agent line; each crawler then gets * its own group that keeps the path rules it was under, plus Allow: /. Everything else is kept. */ function promvia_merge_robots_allow( $existing, array $tokens ) { $existing = (string) $existing; if ( empty( $tokens ) ) { return $existing; } $eol = false !== strpos( $existing, "\r\n" ) ? "\r\n" : "\n"; $lines = preg_split( '/\r?\n/', $existing ); $groups = array(); $cur = -1; foreach ( $lines as $i => $raw ) { $line = trim( preg_replace( '/#.*$/', '', $raw ) ); if ( '' === $line ) { continue; } $at = strpos( $line, ':' ); if ( false === $at ) { continue; } $key = strtolower( trim( substr( $line, 0, $at ) ) ); $val = trim( substr( $line, $at + 1 ) ); if ( 'user-agent' === $key ) { if ( $cur < 0 || $groups[ $cur ]['saw_rule'] ) { $groups[] = array( 'agents' => array(), 'ua_lines' => array(), 'rules' => array(), 'saw_rule' => false ); $cur = count( $groups ) - 1; } $groups[ $cur ]['agents'][] = strtolower( $val ); $groups[ $cur ]['ua_lines'][] = $i; } elseif ( ( 'disallow' === $key || 'allow' === $key ) && $cur >= 0 ) { $groups[ $cur ]['saw_rule'] = true; $groups[ $cur ]['rules'][] = array( 'line' => $i, 'allow' => 'allow' === $key, 'val' => $val ); } } $targets = array_map( 'strtolower', $tokens ); $drop = array(); $carried = array(); $carry = function ( $token, $g ) use ( &$carried ) { $list = isset( $carried[ $token ] ) ? $carried[ $token ] : array(); foreach ( $g['rules'] as $r ) { if ( '' === $r['val'] || promvia_robots_rule_matches_root( $r['val'] ) ) { continue; } $rule = ( $r['allow'] ? 'Allow: ' : 'Disallow: ' ) . $r['val']; if ( ! in_array( $rule, $list, true ) ) { $list[] = $rule; } } $carried[ $token ] = $list; }; foreach ( $tokens as $token ) { $t = strtolower( $token ); $own = array_filter( $groups, function ( $g ) use ( $t ) { return in_array( $t, $g['agents'], true ); } ); if ( empty( $own ) ) { foreach ( $groups as $g ) { if ( in_array( '*', $g['agents'], true ) ) { $carry( $token, $g ); } } if ( ! isset( $carried[ $token ] ) ) { $carried[ $token ] = array(); } continue; } foreach ( $own as $g ) { if ( ! array_diff( $g['agents'], $targets ) ) { foreach ( $g['rules'] as $r ) { if ( ! $r['allow'] && promvia_robots_rule_matches_root( $r['val'] ) ) { $drop[ $r['line'] ] = true; } } } else { foreach ( $g['ua_lines'] as $j => $idx ) { if ( $g['agents'][ $j ] === $t ) { $drop[ $idx ] = true; } } $carry( $token, $g ); } } } $kept = array(); foreach ( $lines as $i => $l ) { if ( ! isset( $drop[ $i ] ) ) { $kept[] = $l; } } $out = implode( $eol, $kept ); if ( $carried ) { $out = preg_replace( '/(\r?\n)*$/', '', $out ) . ( '' !== trim( $out ) ? $eol . $eol : '' ); $blocks = array(); foreach ( $carried as $token => $rules ) { $blocks[] = implode( $eol, array_merge( array( '# ' . $token . ' (AI search) allowed — Promvia fix', 'User-agent: ' . $token ), $rules, array( 'Allow: /' ) ) ); } $out .= implode( $eol . $eol, $blocks ) . $eol; } return $out; } /** /llms.txt is answered only when WordPress lives at the domain root (llms.txt belongs there). */ function promvia_is_llms_request( $request_uri, $home_url ) { $home_path = (string) wp_parse_url( (string) $home_url, PHP_URL_PATH ); if ( '' !== trim( $home_path, '/' ) ) { return false; } return '/llms.txt' === (string) wp_parse_url( (string) $request_uri, PHP_URL_PATH ); }