array{v:'ok'|'fake',bot:string,t:int}. Not autoloaded. */ const OPTION_VERDICTS = 'csdt_crawler_verdicts'; /** Running totals since the counter started, for the Block guard and the panel. */ const OPTION_STATS = 'csdt_crawler_verify_stats'; /** Rule id under which a fake claim is recorded in the shared firewall log. */ const RULE_ID = 'fake_crawler'; /** How long a proven crawler stays proven. */ const TTL_OK = 7 * DAY_IN_SECONDS; /** How long a failed claim stays failed. Shorter: addresses are reassigned. */ const TTL_FAKE = DAY_IN_SECONDS; /** Verdict cache ceiling. The oldest verdicts are dropped first. */ const MAX_VERDICTS = 2000; /** New addresses verified per minute, site-wide. */ const MAX_LOOKUPS_PER_MINUTE = 20; /** Failed claims, with zero proven ones, at which Block is refused as a misconfiguration. */ const GUARD_MIN_FAKES = 5; /** * Crawlers this check knows how to prove, keyed by a stable name. * * Only operators that publish a reverse-DNS domain for their crawlers. A crawler that * publishes only an address list (DuckDuckBot, GPTBot) is not listed, because without * a domain there is nothing to confirm, and an unprovable claim must not read as fake. * * @return array}> */ public static function bots(): array { return array( 'google' => array( 'label' => 'Googlebot', 'ua' => '/\b(?:googlebot|google-inspectiontool|googleother|adsbot-google|mediapartners-google|storebot-google|feedfetcher-google)\b/i', 'domains' => array( 'googlebot.com', 'google.com' ), ), 'bing' => array( 'label' => 'Bingbot', 'ua' => '/\b(?:bingbot|bingpreview|msnbot|adidxbot)\b/i', 'domains' => array( 'search.msn.com' ), ), 'apple' => array( 'label' => 'Applebot', 'ua' => '/\bapplebot\b/i', 'domains' => array( 'applebot.apple.com' ), ), 'yandex' => array( 'label' => 'YandexBot', 'ua' => '/\byandex(?:bot|images|mobilebot|video|news)?\b/i', 'domains' => array( 'yandex.ru', 'yandex.net', 'yandex.com' ), ), 'baidu' => array( 'label' => 'Baiduspider', 'ua' => '/\bbaiduspider\b/i', 'domains' => array( 'baidu.com', 'baidu.jp' ), ), 'yahoo' => array( 'label' => 'Yahoo Slurp', 'ua' => '/\bslurp\b/i', 'domains' => array( 'crawl.yahoo.net' ), ), ); } /** * Resolver override for the tests: array{ptr:callable,a:callable}. Production leaves it * null and the real resolver is used. * * @var array|null */ public static $resolver = null; /** The address and bot queued for verification after this response, if any. */ private static $pending = null; // ── Mode ───────────────────────────────────────────────────────────── /** The three modes, in the order they are offered. */ public static function mode_labels(): array { return array( self::MODE_OFF => __( 'Off: do not check crawler claims', 'cloudscale-cyber-devtools' ), self::MODE_LOG => __( 'Log only: record fake crawlers, refuse nothing', 'cloudscale-cyber-devtools' ), self::MODE_BLOCK => __( 'Block: refuse fake crawlers', 'cloudscale-cyber-devtools' ), ); } /** Current mode, validated against the constants (this runs before init). */ public static function mode(): string { $mode = (string) get_option( self::OPTION_MODE, self::DEFAULT_MODE ); return in_array( $mode, array( self::MODE_OFF, self::MODE_LOG, self::MODE_BLOCK ), true ) ? $mode : self::MODE_OFF; } // ── Pure helpers ───────────────────────────────────────────────────── /** * Which crawler a user agent claims to be, or '' for none. * * @param string $ua User agent. * @return string Key into bots(). */ public static function claimed_bot( string $ua ): string { // One cheap scan on the common path; the per-bot patterns run only after it passes. if ( '' === $ua || ! preg_match( '/bot|google|yandex|spider|slurp|bing/i', $ua ) ) { return ''; } foreach ( self::bots() as $key => $bot ) { if ( preg_match( $bot['ua'], $ua ) ) { return $key; } } return ''; } /** * Whether a host belongs to one of the operator's domains: the domain itself, or a * name under it. "evilgooglebot.com" and "googlebot.com.evil.net" are not. * * @param string $host Host from the PTR record. * @param array $domains Operator domains. */ public static function host_matches( string $host, array $domains ): bool { $host = strtolower( rtrim( $host, '.' ) ); foreach ( $domains as $d ) { if ( $host === $d || substr( $host, -strlen( '.' . $d ) ) === '.' . $d ) { return true; } } return false; } /** * Forward-confirmed reverse DNS for one address. * * @param string $ip Visiting address. * @param array $domains Operator domains. * @return string 'ok', 'fake', or '' when the resolver could not answer. */ public static function verify( string $ip, array $domains ): string { if ( false === filter_var( $ip, FILTER_VALIDATE_IP ) ) { return ''; } $ip = self::canonical( $ip ); // A private, loopback or reserved source address is internal traffic: a container gateway, // a health check, a local test harness, never a real external crawler. It has no public // PTR and so would always read as 'fake', the same own-host false positive that has bitten // the auto-blocker. There is nothing to forward-confirm, so nothing is concluded. In // production verify() is fed the resolved visitor IP (get_client_ip()), so a genuine // Googlebot still arrives as its public address and is checked as before. if ( false === filter_var( $ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE ) ) { return ''; } $host = self::ptr( $ip ); if ( '' === $host ) { // No PTR. Every documented crawler has one, so this is a fake, unless the // resolver itself is down, in which case nothing can be concluded. return self::resolver_works() ? 'fake' : ''; } if ( ! self::host_matches( $host, $domains ) ) { return 'fake'; } $back = array_map( array( __CLASS__, 'canonical' ), self::forward( $host ) ); return in_array( $ip, $back, true ) ? 'ok' : 'fake'; } /** PTR name for an address, '' when there is none. */ private static function ptr( string $ip ): string { if ( isset( self::$resolver['ptr'] ) ) { return (string) call_user_func( self::$resolver['ptr'], $ip ); } $host = gethostbyaddr( $ip ); // gethostbyaddr() returns the address itself when there is no PTR record. return ( is_string( $host ) && $host !== $ip ) ? $host : ''; } /** * Addresses a host resolves to, IPv4 and IPv6. * * @return array */ private static function forward( string $host ): array { if ( isset( self::$resolver['a'] ) ) { return (array) call_user_func( self::$resolver['a'], $host ); } $out = (array) gethostbynamel( $host ); if ( function_exists( 'dns_get_record' ) ) { $aaaa = @dns_get_record( $host, DNS_AAAA ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- a failed lookup is an expected outcome here and is handled by the empty result; the warning it raises would otherwise land in the error log on every fake claim foreach ( is_array( $aaaa ) ? $aaaa : array() as $rec ) { if ( ! empty( $rec['ipv6'] ) ) { $out[] = (string) $rec['ipv6']; } } } return array_values( array_filter( array_map( 'strval', $out ) ) ); } /** An address in canonical form, so 2001:4860:0:0::1 and 2001:4860::1 compare equal. */ public static function canonical( string $ip ): string { $bin = @inet_pton( $ip ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- an unparseable address is handled by the fallback below $out = false === $bin ? false : inet_ntop( $bin ); return false === $out ? $ip : (string) $out; } /** Whether the resolver answers at all, so an outage is not read as a fake crawler. */ private static function resolver_works(): bool { if ( isset( self::$resolver['a'] ) ) { return array() !== (array) call_user_func( self::$resolver['a'], 'www.google.com' ); } return false !== gethostbynamel( 'www.google.com' ); } // ── Verdict cache ──────────────────────────────────────────────────── /** * The cached, unexpired verdict for an address: 'ok', 'fake' or ''. * * @param string $ip Address. * @param int|null $now Unix time, for tests. */ public static function verdict( string $ip, ?int $now = null ): string { $all = get_option( self::OPTION_VERDICTS, array() ); $v = is_array( $all ) ? ( $all[ $ip ] ?? null ) : null; if ( ! is_array( $v ) ) { return ''; } $ttl = 'ok' === ( $v['v'] ?? '' ) ? self::TTL_OK : self::TTL_FAKE; if ( ( null === $now ? time() : $now ) - (int) ( $v['t'] ?? 0 ) > $ttl ) { return ''; } return in_array( $v['v'] ?? '', array( 'ok', 'fake' ), true ) ? (string) $v['v'] : ''; } /** Whether this address has been proven to be the crawler it claims. */ public static function is_verified( string $ip ): bool { return 'ok' === self::verdict( $ip ); } /** Stores one verdict and counts it, keeping the cache under MAX_VERDICTS. */ private static function store( string $ip, string $bot, string $verdict, int $now ): void { $all = get_option( self::OPTION_VERDICTS, array() ); $all = is_array( $all ) ? $all : array(); unset( $all[ $ip ] ); // Re-inserted at the end, so the array stays oldest-first. $all[ $ip ] = array( 'v' => $verdict, 'bot' => $bot, 't' => $now, ); if ( count( $all ) > self::MAX_VERDICTS ) { $all = array_slice( $all, -self::MAX_VERDICTS, null, true ); } update_option( self::OPTION_VERDICTS, $all, false ); $stats = self::stats(); ++$stats[ $verdict ]; if ( 0 === (int) $stats['since'] ) { $stats['since'] = $now; } update_option( self::OPTION_STATS, $stats, false ); } /** @return array{ok:int,fake:int,since:int} */ public static function stats(): array { $s = get_option( self::OPTION_STATS, array() ); $s = is_array( $s ) ? $s : array(); return array( 'ok' => (int) ( $s['ok'] ?? 0 ), 'fake' => (int) ( $s['fake'] ?? 0 ), 'since' => (int) ( $s['since'] ?? 0 ), ); } // ── Enforcement ────────────────────────────────────────────────────── /** * Hooked to `plugins_loaded` priority 3, after the blocklist and country rules and * beside the request firewall. Costs one autoloaded read and one substring scan of * the user agent on every request that does not claim to be a crawler. */ public static function enforce(): void { try { if ( self::MODE_OFF === self::mode() ) { return; } $bot = self::claimed_bot( CSDT_Request::server( 'HTTP_USER_AGENT', 512, false ) ); if ( '' === $bot ) { return; } // The firewall's own gate: never cron, CLI, the site calling itself, or a // signed-in editor. if ( class_exists( 'CSDT_Firewall' ) && CSDT_Firewall::is_exempt() ) { return; } $ip = class_exists( 'CSDT_DevTools' ) ? (string) CSDT_DevTools::get_client_ip() : ''; if ( '' === $ip ) { return; } $verdict = self::verdict( $ip ); if ( '' === $verdict ) { self::$pending = array( $ip, $bot ); // Last on shutdown, after core has flushed the page, so verify_pending() can // release the visitor before it waits on DNS. // accepted_args 0: do_action( 'shutdown' ) passes '' to a one-argument callback, // which a typed $now refuses with a TypeError before the try below can catch it. add_action( 'shutdown', array( __CLASS__, 'verify_pending' ), PHP_INT_MAX, 0 ); return; } if ( 'fake' !== $verdict ) { return; } $blocking = self::MODE_BLOCK === self::mode(); self::record( $ip, $blocking ? 'blocked' : 'logged' ); if ( $blocking ) { status_header( 403 ); wp_die( esc_html__( 'Access denied.', 'cloudscale-cyber-devtools' ), '', array( 'response' => 403 ) ); } } catch ( \Throwable $e ) { // Fails OPEN, like every control on this path. if ( class_exists( 'CSDT_Monitor' ) ) { CSDT_Monitor::log_exception( $e, __METHOD__ ); } } } /** * Verifies the address queued by enforce(), after the response has been built. * * A fake is recorded here as well as on its next request, so Log only shows the first * fake claim from an address rather than only the repeats. * * @param mixed $now Unix time, for tests. Untyped and coerced, because a hook that * passes '' must never be able to fatal the end of a request. */ public static function verify_pending( $now = null ): void { try { if ( null === self::$pending ) { return; } list( $ip, $bot ) = self::$pending; self::$pending = null; $now = is_int( $now ) ? $now : time(); // Under PHP-FPM, hand the finished response to the visitor first, so a slow PTR // zone costs this worker a moment and the crawler nothing. Core's wp-cron.php // does the same before running scheduled tasks. if ( function_exists( 'fastcgi_finish_request' ) && ! self::$resolver ) { fastcgi_finish_request(); } // Site-wide budget, so rotating addresses cannot drive unbounded DNS traffic. $bucket = 'csdt_crawler_lookups_' . (int) floor( $now / 60 ); $used = (int) get_transient( $bucket ); if ( $used >= self::MAX_LOOKUPS_PER_MINUTE ) { return; } set_transient( $bucket, $used + 1, 2 * MINUTE_IN_SECONDS ); $bots = self::bots(); $verdict = self::verify( $ip, $bots[ $bot ]['domains'] ?? array() ); if ( '' === $verdict ) { return; // Resolver down: no verdict, try again on a later request. } self::store( $ip, $bot, $verdict, $now ); if ( 'fake' === $verdict ) { self::record( $ip, 'logged' ); } } catch ( \Throwable $e ) { if ( class_exists( 'CSDT_Monitor' ) ) { CSDT_Monitor::log_exception( $e, __METHOD__ ); } } } /** Records a fake claim in the firewall log, in its own try/catch so a log fault never costs the block. */ private static function record( string $ip, string $action ): void { try { if ( class_exists( 'CSDT_Storage' ) ) { CSDT_Storage::record_match( $ip, self::RULE_ID, $action ); } } catch ( \Throwable $e ) { if ( class_exists( 'CSDT_Monitor' ) ) { CSDT_Monitor::log_exception( $e, __METHOD__ ); } } } // ── Save / panel ───────────────────────────────────────────────────── /** * Why a mode may not be saved, or '' when it may. * * @param string $mode Requested mode. * @param array $stats From stats(). */ public static function validate_mode( string $mode, array $stats ): string { if ( ! in_array( $mode, array( self::MODE_OFF, self::MODE_LOG, self::MODE_BLOCK ), true ) ) { return __( 'Nothing was saved. Unknown mode.', 'cloudscale-cyber-devtools' ); } if ( self::MODE_BLOCK === $mode && 0 === (int) ( $stats['ok'] ?? 0 ) && (int) ( $stats['fake'] ?? 0 ) >= self::GUARD_MIN_FAKES ) { return __( 'Nothing was saved. Every crawler checked so far has failed, including any real ones, which means this site is probably reading a proxy\'s address instead of the visitor\'s. Check Visitor IP Source below before blocking, or Block would refuse Google itself.', 'cloudscale-cyber-devtools' ); } return ''; } /** Saves the mode. Same nonce and capability shape as the firewall's own save. */ public static function ajax_save(): void { check_ajax_referer( CSDT_DevTools::SECURITY_NONCE, 'nonce' ); if ( ! current_user_can( 'manage_options' ) ) { wp_send_json_error( 'Unauthorized', 403 ); } $mode = sanitize_text_field( wp_unslash( $_POST['mode'] ?? '' ) ); $refusal = self::validate_mode( $mode, self::stats() ); if ( '' !== $refusal ) { wp_send_json_error( $refusal ); return; } update_option( self::OPTION_MODE, $mode, true ); wp_send_json_success( array( 'msg' => __( 'Saved.', 'cloudscale-cyber-devtools' ) ) ); } /** Renders the Fake Crawler Check panel on the Firewall tab. */ public static function render_panel(): void { $mode = self::mode(); $stats = self::stats(); $counts = class_exists( 'CSDT_Storage' ) ? CSDT_Storage::match_counts() : null; $stat = is_array( $counts ) ? ( $counts[ self::RULE_ID ] ?? array() ) : array(); ?>
🕷️ 'Why fake crawlers matter', 'rec' => 'Read this first', 'html' => 'Any program can say it is Googlebot. Scrapers and vulnerability scanners do it on purpose, because sites tend to wave search engines through. This check asks each one to prove it.', ), array( 'name' => 'How a claim is proven', 'rec' => 'How it works', 'html' => 'The method Google and Microsoft document for exactly this: look up the name behind the visiting address, check it belongs to the search engine (for example googlebot.com or search.msn.com), then look that name up again and confirm it points back to the same address. Only the search engine can make both directions agree. Covers Google, Bing, Apple, Yandex, Baidu and Yahoo. These are ordinary DNS lookups by your own server; no outside service is contacted.', ), array( 'name' => 'No delay for real crawlers', 'rec' => 'Performance', 'html' => 'The lookup runs after the page is finished, never before it, and each address is checked once and remembered (a week for a real crawler, a day for a fake). The first request from a new address is always let through; the verdict applies from the next one. At most 20 new addresses are checked a minute. If DNS itself is down, nothing is concluded.', ), array( 'name' => 'The guard on Block', 'rec' => 'Honest limits', 'html' => 'If every crawler checked so far has failed, Block is refused. Real crawlers do not all disappear; that pattern means the site is reading a proxy\'s address rather than the visitor\'s, and blocking then would refuse Google itself. Fix Visitor IP Source first. Matches appear in Recent Matches as fake_crawler.', ), ) ); ?>

' . esc_html( number_format_i18n( $stats['ok'] ) ) . '', '' . esc_html( number_format_i18n( $stats['fake'] ) ) . '', '' . esc_html( null === $counts ? '-' : number_format_i18n( (int) ( $stat['hits'] ?? 0 ) ) ) . '' ); ?>