kind. Checked longest-first, case-insensitive. */ public const TOKENS = array( 'GPTBot' => 'training', 'OAI-SearchBot' => 'search', 'ChatGPT-User' => 'user', 'OAI-AdsBot' => 'other', 'ClaudeBot' => 'training', 'Claude-SearchBot' => 'search', 'Claude-User' => 'user', 'PerplexityBot' => 'search', 'Perplexity-User' => 'user', 'Google-Extended' => 'training', 'GoogleOther' => 'other', 'Googlebot' => 'search', 'Google-Agent' => 'user', 'bingbot' => 'search', 'Applebot-Extended' => 'training', 'Applebot' => 'search', 'Meta-ExternalAgent' => 'training', 'Bytespider' => 'training', 'Amazonbot' => 'search', 'DuckAssistBot' => 'user', 'MistralAI-User' => 'user', 'cohere-ai' => 'training', ); /** * Classic high-volume search crawlers. They are logged, but we do NOT force them past * the page cache (that would add server load without PoC value). */ public const CACHE_BYPASS_EXCLUDE = array( 'Googlebot', 'bingbot', 'Applebot' ); public const RAW_RETENTION_DAYS = 30; public const DAILY_RETENTION_DAYS = 400; private static ?array $match = null; /** Tokens used for page-cache exclusion. */ public static function cache_bypass_tokens(): array { return array_values( array_diff( array_keys( self::TOKENS ), self::CACHE_BYPASS_EXCLUDE ) ); } /** * Detect the bot for a UA string. * * @return array{bot:string,kind:string}|null */ public static function detect( string $ua ): ?array { if ( '' === $ua ) { return null; } $tokens = array_keys( self::TOKENS ); usort( $tokens, static fn( $a, $b ) => strlen( $b ) <=> strlen( $a ) ); foreach ( $tokens as $t ) { if ( false !== stripos( $ua, $t ) ) { return array( 'bot' => $t, 'kind' => self::TOKENS[ $t ], ); } } return null; } public static function current(): ?array { if ( null === self::$match ) { $ua = Util::server_text( 'HTTP_USER_AGENT' ); self::$match = self::detect( $ua ) ?? array(); } return self::$match ? self::$match : null; } /** Runs on plugins_loaded (early). */ public static function on_request(): void { $m = self::current(); if ( ! $m ) { return; } // Never store an AI-bot response in a page cache. if ( ! in_array( $m['bot'], self::CACHE_BYPASS_EXCLUDE, true ) && ! defined( 'DONOTCACHEPAGE' ) ) { define( 'DONOTCACHEPAGE', true ); } if ( ( defined( 'DOING_CRON' ) && DOING_CRON ) || ( defined( 'WP_CLI' ) && WP_CLI ) ) { return; } if ( ! Settings::feature( 'bot_logging' ) ) { return; } // Detect now; write at shutdown so we know the status code and product. register_shutdown_function( array( __CLASS__, 'write_hit' ) ); } public static function write_hit(): void { try { global $wpdb; $m = self::current(); if ( ! $m || ! $wpdb ) { return; } // Each value is sanitized and validated in Util: path without query string, single-line UA, real IPs only. $uri = Util::request_path(); $ua = Util::server_text( 'HTTP_USER_AGENT' ); $ip = Util::server_ip( 'REMOTE_ADDR' ); $fwd = Util::server_ip( 'HTTP_CF_CONNECTING_IP' ); if ( '' === $fwd ) { $fwd = Util::server_ip( 'HTTP_X_FORWARDED_FOR' ); } $product_id = null; if ( function_exists( 'is_singular' ) && did_action( 'wp' ) && is_singular( 'product' ) ) { $product_id = (int) get_queried_object_id(); } $status = http_response_code(); $wpdb->suppress_errors( true ); $wpdb->insert( // phpcs:ignore WordPress.DB Install::table( 'bot_hits' ), array( 'at' => gmdate( 'Y-m-d H:i:s' ), 'bot' => $m['bot'], 'kind' => $m['kind'], 'ua' => $ua, 'ip' => $ip, 'fwd_ip' => $fwd, 'path' => $uri, 'product_id' => $product_id ? $product_id : null, 'status' => is_int( $status ) ? $status : 0, ) ); } catch ( \Throwable $e ) { Util::log( 'bot log: ' . $e->getMessage() ); } } /** * Recompute daily rows from raw rows for the raw-retention window (idempotent). */ public static function rollup(): void { global $wpdb; $raw = Install::table( 'bot_hits' ); $daily = Install::table( 'bot_daily' ); $from = gmdate( 'Y-m-d', time() - self::RAW_RETENTION_DAYS * DAY_IN_SECONDS ); // phpcs:disable WordPress.DB $rows = $wpdb->get_results( $wpdb->prepare( "SELECT DATE(at) AS day, bot, kind, path, product_id, COUNT(*) AS hits FROM %i WHERE at >= %s GROUP BY DATE(at), bot, kind, path, product_id", $raw, $from . ' 00:00:00' ), ARRAY_A ); $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE day >= %s', $daily, $from ) ); foreach ( (array) $rows as $r ) { $wpdb->insert( $daily, array( 'day' => substr( (string) $r['day'], 0, 10 ), 'bot' => $r['bot'], 'kind' => $r['kind'], 'path' => $r['path'], 'product_id' => $r['product_id'] ? (int) $r['product_id'] : null, 'hits' => (int) $r['hits'], ) ); } // phpcs:enable } /** Raw IPs are kept at most this long, then truncated (IPv4 /24, IPv6 /48). Israeli PPL Amendment 13 treats IPs as personal data. */ public const RAW_IP_DAYS = 7; /** Truncate an IP: 203.0.113.77 -> 203.0.113.0, 2001:db8:abcd:12::1 -> 2001:db8:abcd::. Non-IPs -> ''. */ public static function truncate_ip( string $ip ): string { if ( '' === $ip ) { return ''; } if ( filter_var( $ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 ) ) { $p = explode( '.', $ip ); $p[3] = '0'; return implode( '.', $p ); } if ( filter_var( $ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6 ) ) { $bin = inet_pton( $ip ); if ( false === $bin ) { return ''; } $out = inet_ntop( substr( $bin, 0, 6 ) . str_repeat( "\0", 10 ) ); return false === $out ? '' : $out; } return ''; } /** Replace raw IPs older than RAW_IP_DAYS with their truncated form (idempotent, batched). */ public static function anonymize_old_ips(): int { global $wpdb; $t = Install::table( 'bot_hits' ); $cutoff = gmdate( 'Y-m-d H:i:s', time() - self::RAW_IP_DAYS * DAY_IN_SECONDS ); $done = 0; // phpcs:disable WordPress.DB for ( $round = 0; $round < 20; $round++ ) { $rows = $wpdb->get_results( $wpdb->prepare( "SELECT id, ip, fwd_ip FROM %i WHERE at < %s AND ( ( ip <> '' AND ip NOT LIKE %s AND ip NOT LIKE %s ) OR ( fwd_ip <> '' AND fwd_ip NOT LIKE %s AND fwd_ip NOT LIKE %s ) ) LIMIT 1000", $t, $cutoff, '%.0', '%::', '%.0', '%::' ), ARRAY_A ); if ( ! $rows ) { break; } foreach ( $rows as $r ) { $wpdb->update( $t, array( 'ip' => self::truncate_ip( (string) $r['ip'] ), 'fwd_ip' => self::truncate_ip( (string) $r['fwd_ip'] ), ), array( 'id' => (int) $r['id'] ) ); ++$done; } } // phpcs:enable return $done; } public static function purge(): void { global $wpdb; // phpcs:disable WordPress.DB $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE at < %s', Install::table( 'bot_hits' ), gmdate( 'Y-m-d H:i:s', time() - self::RAW_RETENTION_DAYS * DAY_IN_SECONDS ) ) ); $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE day < %s', Install::table( 'bot_daily' ), gmdate( 'Y-m-d', time() - self::DAILY_RETENTION_DAYS * DAY_IN_SECONDS ) ) ); // phpcs:enable } public static function daily_cron(): void { Util::safe( array( __CLASS__, 'rollup' ) ); Util::safe( array( __CLASS__, 'anonymize_old_ips' ) ); Util::safe( array( __CLASS__, 'purge' ) ); } /** Data for GET /bot-log. */ public static function report( string $since ): array { global $wpdb; self::rollup(); $ip_cutoff = time() - self::RAW_IP_DAYS * DAY_IN_SECONDS; // phpcs:disable WordPress.DB $daily = $wpdb->get_results( $wpdb->prepare( 'SELECT day, bot, kind, path, product_id, hits FROM %i WHERE day >= %s ORDER BY day ASC, hits DESC', Install::table( 'bot_daily' ), $since ), ARRAY_A ); $recent = $wpdb->get_results( $wpdb->prepare( 'SELECT at, bot, kind, ua, ip, fwd_ip, path, product_id, status FROM %i WHERE at >= %s ORDER BY id DESC LIMIT 200', Install::table( 'bot_hits' ), $since . ' 00:00:00' ), ARRAY_A ); // phpcs:enable return array( 'daily' => array_map( static fn( $r ) => array( 'date' => substr( (string) $r['day'], 0, 10 ), 'bot' => $r['bot'], 'kind' => $r['kind'], 'path' => $r['path'], 'product_id' => $r['product_id'] ? (int) $r['product_id'] : null, 'hits' => (int) $r['hits'], ), (array) $daily ), 'recent' => array_map( static fn( $r ) => array( 'at' => Util::mysql_to_iso( $r['at'] ), 'bot' => $r['bot'], 'kind' => $r['kind'], 'ua' => $r['ua'], // Even if the daily cron has not run yet, never return a raw IP older than RAW_IP_DAYS. 'ip' => strtotime( $r['at'] . ' UTC' ) < $ip_cutoff ? self::truncate_ip( (string) $r['ip'] ) : $r['ip'], 'forwarded_ip' => '' !== $r['fwd_ip'] ? ( strtotime( $r['at'] . ' UTC' ) < $ip_cutoff ? self::truncate_ip( (string) $r['fwd_ip'] ) : $r['fwd_ip'] ) : null, 'path' => $r['path'], 'product_id' => $r['product_id'] ? (int) $r['product_id'] : null, 'status' => (int) $r['status'], ), (array) $recent ), ); } }