'Googlebot', 'purpose' => 'Google Search indexing and rendering', 'verification' => 'FCrDNS: *.googlebot.com', 'handling' => 'Automatic after verification' ), array( 'name' => 'Bingbot', 'purpose' => 'Microsoft Bing indexing', 'verification' => 'FCrDNS: *.search.msn.com', 'handling' => 'Automatic after verification' ), array( 'name' => 'Applebot', 'purpose' => 'Apple Search, Siri, Spotlight and Safari', 'verification' => 'FCrDNS: *.applebot.apple.com', 'handling' => 'Automatic after verification' ), array( 'name' => 'Yandex robots', 'purpose' => 'Yandex indexing and webmaster tools', 'verification' => 'FCrDNS: *.yandex.ru, *.yandex.net or *.yandex.com', 'handling' => 'Automatic after verification' ), array( 'name' => 'Lighthouse / local site audit', 'purpose' => 'Performance, accessibility and SEO testing', 'verification' => 'Administrator-started window bound to the current source IP', 'handling' => 'Temporary 15-minute exception' ), array( 'name' => 'PageSpeed Insights', 'purpose' => 'Remote Google Lighthouse testing', 'verification' => 'Google-published fetcher ranges are required; User-Agent is not stable proof', 'handling' => 'Listed, not automatically exempted' ), array( 'name' => 'DuckDuckBot', 'purpose' => 'DuckDuckGo indexing', 'verification' => 'Cached official HTTPS IP list and matching User-Agent', 'handling' => 'Opt-in GET/HEAD rate exception after verification' ), array( 'name' => 'AhrefsBot / AhrefsSiteAudit', 'purpose' => 'SEO crawling and site audits', 'verification' => 'Cached official HTTPS IP ranges and matching User-Agent', 'handling' => 'Opt-in GET/HEAD rate exception after verification' ), array( 'name' => 'SemrushBot / SiteAuditBot', 'purpose' => 'SEO crawling and site audits', 'verification' => 'Official ranges or Web Bot Auth', 'handling' => 'Listed, not automatically exempted' ), array( 'name' => 'UptimeRobot', 'purpose' => 'Availability monitoring', 'verification' => 'Cached official HTTPS IP list and matching User-Agent', 'handling' => 'Opt-in GET/HEAD rate exception after verification' ), array( 'name' => 'GTmetrix', 'purpose' => 'Remote performance testing', 'verification' => 'Provider-specific verification is still required', 'handling' => 'Listed, not automatically exempted' ), array( 'name' => 'Baiduspider, PetalBot, Bravebot, Qwantbot, Naver/Yeti and SeznamBot', 'purpose' => 'Regional and independent search indexing', 'verification' => 'Provider-specific verification is required', 'handling' => 'Visible in registry; no automatic trust' ), array( 'name' => 'GPTBot, ChatGPT-User, ClaudeBot, Claude-User, PerplexityBot, Perplexity-User, Amazonbot and Bytespider', 'purpose' => 'AI training, search, user-requested retrieval and assistant access', 'verification' => 'A User-Agent alone is not identity proof', 'handling' => 'Visible in registry; no automatic trust' ), array( 'name' => 'Facebook, LinkedIn, X, WhatsApp, Slack and Discord previews', 'purpose' => 'Social link previews', 'verification' => 'No common strong identity method', 'handling' => 'Visible in registry; no automatic trust' ), ); } public static function ip_sources(): array { return array( 'duck' => array( 'name' => 'DuckDuckBot', 'setting' => 'verified_bot_duck_enabled', 'url' => 'https://duckduckgo.com/duckduckbot.json', 'format' => 'prefixes' ), 'ahrefs' => array( 'name' => 'Ahrefs', 'setting' => 'verified_bot_ahrefs_enabled', 'url' => 'https://api.ahrefs.com/v3/public/crawler-ip-ranges', 'format' => 'prefixes' ), 'uptime' => array( 'name' => 'UptimeRobot', 'setting' => 'verified_bot_uptime_enabled', 'url' => 'https://cdn.uptimerobot.com/api/IPv4andIPv6.txt', 'format' => 'text' ), ); } public static function ip_candidate_provider( string $ua ): string { $ua = strtolower( substr( $ua, 0, 512 ) ); foreach ( array( 'duck' => 'duckduckbot', 'ahrefs' => 'ahrefs(?:bot|siteaudit)', 'uptime' => 'uptimerobot' ) as $provider => $pattern ) { if ( preg_match( '/(? self::IP_MAX_BYTES ) { return array(); } $raw = array(); if ( 'prefixes' === $source['format'] ) { $data = json_decode( $body, true, 16 ); if ( JSON_ERROR_NONE !== json_last_error() || ! is_array( $data ) || ! is_array( $data['prefixes'] ?? null ) || ! array_is_list( $data['prefixes'] ) ) { return array(); } if ( count( $data['prefixes'] ) > self::IP_MAX_RECORDS ) { return array(); } foreach ( $data['prefixes'] as $entry ) { if ( ! is_array( $entry ) || 1 !== count( $entry ) ) { return array(); } $key = array_key_first( $entry ); if ( ! in_array( $key, array( 'ipv4Prefix', 'ipv6Prefix' ), true ) || ! is_string( $entry[ $key ] ) ) { return array(); } $network = self::normalize_network( $entry[ $key ], $provider ); if ( '' === $network || ( 'ipv4Prefix' === $key ) !== ( false === strpos( $network, ':' ) ) ) { return array(); } $raw[] = $network; } } else { $lines = preg_split( '/\R/', trim( $body ) ) ?: array(); if ( count( $lines ) > self::IP_MAX_RECORDS ) { return array(); } foreach ( $lines as $line ) { $network = self::normalize_network( trim( $line ), $provider ); if ( '' === $network ) { return array(); } $raw[] = $network; } } return array_values( array_unique( $raw ) ); } private static function normalize_network( string $raw, string $provider ): string { $parts = explode( '/', $raw ); if ( count( $parts ) > 2 || ! self::public_ip( $parts[0] ) ) { return ''; } $packed = inet_pton( $parts[0] ); $width = strlen( $packed ) * 8; if ( isset( $parts[1] ) && ! preg_match( '/^(?:0|[1-9][0-9]{0,2})$/D', $parts[1] ) ) { return ''; } $bits = isset( $parts[1] ) ? (int) $parts[1] : $width; $minimum = 'ahrefs' === $provider ? ( 32 === $width ? 20 : 48 ) : $width; if ( $bits < $minimum || $bits > $width ) { return ''; } for ( $i = 0; $i < strlen( $packed ); $i++ ) { $keep = max( 0, min( 8, $bits - $i * 8 ) ); if ( ( ord( $packed[ $i ] ) & ( 255 >> $keep ) ) !== 0 ) { return ''; } } return inet_ntop( $packed ) . '/' . $bits; } public static function verified_list_identity( string $provider, string $ip ): bool { $source = self::ip_sources()[ $provider ] ?? null; if ( ! $source || ! NLSSL_Core::setting( $source['setting'], 0 ) || ! self::public_ip( $ip ) ) { return false; } $lists = get_option( self::IP_LIST_OPTION, array() ); $state = is_array( $lists ) ? ( $lists[ $provider ] ?? null ) : null; if ( ! is_array( $state ) ) { return false; } $fetched = (int) ( $state['fetched_at'] ?? 0 ); if ( $fetched <= 0 || $fetched > time() || $fetched + self::IP_LIST_TTL <= time() ) { return false; } $ranges = self::validated_stored_ranges( $provider, $state ); if ( ! $ranges ) { return false; } $packed = inet_pton( $ip ); $matched = false; foreach ( $ranges as $range ) { list( $network, $bits ) = explode( '/', $range ); $base = inet_pton( $network ); if ( strlen( $base ) !== strlen( $packed ) ) { continue; } $whole = intdiv( (int) $bits, 8 ); $rest = (int) $bits % 8; if ( substr( $base, 0, $whole ) !== substr( $packed, 0, $whole ) ) { continue; } if ( 0 === $rest || ( ord( $base[ $whole ] ) & ( 255 << ( 8 - $rest ) ) ) === ( ord( $packed[ $whole ] ) & ( 255 << ( 8 - $rest ) ) ) ) { $matched = true; } } return $matched; } public static function refresh_ip_lists(): void { $enabled = false; foreach ( self::ip_sources() as $source ) { $enabled = $enabled || (bool) NLSSL_Core::setting( $source['setting'], 0 ); } if ( ! $enabled ) { return; } if ( get_transient( 'nlssl_bot_ip_refresh_cooldown' ) ) { return; } if ( ! NLSSL_Core::write_transient_verified( 'nlssl_bot_ip_refresh_cooldown', 1, 300 ) ) { NLSSL_Core::degraded_storage( 'bot_refresh_cooldown_write' ); return; } foreach ( self::ip_sources() as $provider => $source ) { if ( ! NLSSL_Core::setting( $source['setting'], 0 ) ) { continue; } $result = 'network_error'; // WordPress's default user agent embeds the site address. A third-party IP-list provider has // no need to learn which site is asking, so the request identifies only the plugin. try { $response = wp_safe_remote_get( $source['url'], array( 'timeout' => 5, 'redirection' => 0, 'sslverify' => true, 'user-agent' => 'NeverLast-Security-Lite/' . ( defined( 'NLSSL_VERSION' ) ? NLSSL_VERSION : '1' ), 'limit_response_size' => self::IP_MAX_BYTES + 1, 'cookies' => array(), 'headers' => array( 'Accept' => 'prefixes' === $source['format'] ? 'application/json' : 'text/plain' ) ) ); } catch ( Throwable $error ) { $response = new WP_Error( 'nlssl_bot_http_exception', 'Bot list HTTP transport failed.' ); $result = 'network_exception'; } if ( ! is_wp_error( $response ) ) { $result = 'http_error'; if ( 200 === wp_remote_retrieve_response_code( $response ) ) { $ranges = self::parse_ip_list( $provider, wp_remote_retrieve_body( $response ) ); $result = 'invalid_list'; if ( $ranges ) { $lists = get_option( self::IP_LIST_OPTION, array() ); $lists = is_array( $lists ) ? $lists : array(); $lists[ $provider ] = array( 'schema' => 1, 'source' => $source['url'], 'fetched_at' => time(), 'ranges' => $ranges ); $result = NLSSL_Core::write_option_verified( self::IP_LIST_OPTION, $lists ) ? 'ready' : 'storage_error'; } } } $status = get_option( self::IP_STATUS_OPTION, array() ); $status = is_array( $status ) ? $status : array(); $status[ $provider ] = array( 'attempt_at' => time(), 'result' => $result ); if ( ! NLSSL_Core::write_option_verified( self::IP_STATUS_OPTION, $status ) ) { NLSSL_Core::degraded_storage( 'bot_refresh_status_write' ); } NLSSL_Core::log( 'bot_ip_list_refresh', 'ready' === $result ? 'info' : 'warning', 'Official bot IP list refresh completed.', array( 'provider' => $provider, 'result' => $result ) ); } } public static function ip_list_status(): array { $lists = get_option( self::IP_LIST_OPTION, array() ); $attempts = get_option( self::IP_STATUS_OPTION, array() ); $output = array(); foreach ( self::ip_sources() as $provider => $source ) { $state = is_array( $lists ) && is_array( $lists[ $provider ] ?? null ) ? $lists[ $provider ] : array(); $ranges = self::validated_stored_ranges( $provider, $state ); $valid_state = ! empty( $ranges ); $fetched = $valid_state ? (int) ( $state['fetched_at'] ?? 0 ) : 0; $enabled = (bool) NLSSL_Core::setting( $source['setting'], 0 ); $list_state = ! $enabled ? 'disabled' : ( ! empty( $state ) && ! $valid_state ? 'invalid' : ( $fetched > 0 && $fetched <= time() && $fetched + self::IP_LIST_TTL > time() ? 'ready' : ( $fetched ? 'expired' : 'waiting' ) ) ); $output[ $provider ] = array( 'name' => $source['name'], 'enabled' => $enabled, 'count' => count( $ranges ), 'fetched_at' => $fetched, 'state' => $list_state, 'last_result' => is_array( $attempts ) ? (string) ( $attempts[ $provider ]['result'] ?? '' ) : '' ); } return $output; } private static function validated_stored_ranges( string $provider, array $state ): array { $source = self::ip_sources()[ $provider ] ?? null; if ( ! $source || ( $state['source'] ?? '' ) !== $source['url'] || 1 !== ( $state['schema'] ?? 0 ) ) { return array(); } $ranges = $state['ranges'] ?? null; if ( ! is_array( $ranges ) || ! $ranges || count( $ranges ) > self::IP_MAX_RECORDS ) { return array(); } foreach ( $ranges as $range ) { if ( ! is_string( $range ) || $range !== self::normalize_network( $range, $provider ) ) { return array(); } } return array_values( array_unique( $ranges ) ); } public static function start_audit_window(): bool { $ip = NLSSL_Core::client_ip(); if ( '' === $ip ) { return false; } $key = self::audit_key( $ip ); $value = array( 'expires_at' => time() + self::AUDIT_WINDOW_TTL, 'actor' => get_current_user_id() ); if ( ! NLSSL_Core::write_transient_verified( $key, $value, self::AUDIT_WINDOW_TTL ) || ! self::audit_window_active() ) { NLSSL_Core::degraded_storage( 'audit_window_write' ); return false; } return true; } public static function stop_audit_window(): bool { $ip = NLSSL_Core::client_ip(); if ( '' === $ip ) { return false; } delete_transient( self::audit_key( $ip ) ); if ( self::audit_window_active() ) { NLSSL_Core::degraded_storage( 'audit_window_delete' ); return false; } return true; } public static function audit_window_active(): bool { $ip = NLSSL_Core::client_ip(); if ( '' === $ip ) { return false; } $state = get_transient( self::audit_key( $ip ) ); return is_array( $state ) && (int) ( $state['expires_at'] ?? 0 ) >= time(); } public static function resolver_available(): bool { return function_exists( 'gethostbyaddr' ) && ( function_exists( 'dns_get_record' ) || function_exists( 'gethostbynamel' ) ); } public static function verify_scheduled(): void { $queue = self::queued_jobs(); if ( ! NLSSL_Core::setting( 'verified_bot_rate_limit_bypass', 0 ) ) { if ( ! self::store_queue( array() ) ) { NLSSL_Core::degraded_storage( 'bot_queue_delete' ); } return; } if ( empty( $queue ) ) { if ( ! self::store_queue( array() ) ) { NLSSL_Core::degraded_storage( 'bot_queue_delete' ); } return; } $batch = array_slice( $queue, 0, self::BATCH_SIZE, true ); $queue = array_diff_key( $queue, $batch ); if ( ! self::store_queue( $queue ) ) { // Do not perform DNS work unless advancing the queue was confirmed. Otherwise // the same jobs can repeat indefinitely while transient storage is unhealthy. NLSSL_Core::degraded_storage( 'bot_queue_update' ); return; } foreach ( $batch as $job ) { $provider = sanitize_key( (string) ( $job['provider'] ?? '' ) ); $ip = (string) ( $job['ip'] ?? '' ); if ( ! isset( self::provider_suffixes()[ $provider ] ) || ! self::public_ip( $ip ) ) { continue; } $verified = self::verify_dns( $provider, $ip ); $cache_key = self::cache_key( $provider, $ip ); $cache_value = array( 'provider' => $provider, 'verified' => $verified ? 1 : 0, 'checked_at' => time() ); if ( ! NLSSL_Core::write_transient_verified( $cache_key, $cache_value, $verified ? self::SUCCESS_TTL : self::FAILURE_TTL ) ) { NLSSL_Core::degraded_storage( 'bot_verification_cache_write' ); continue; } if ( $verified ) { NLSSL_Core::log( 'crawler_identity_verified', 'info', 'A search crawler passed cached forward-confirmed reverse DNS.', array( 'provider' => $provider ) ); } } if ( ! empty( $queue ) && ! self::schedule_worker( 10 ) ) { NLSSL_Core::degraded_storage( 'bot_queue_schedule' ); } } /** * Pure verifier exposed for deterministic tests with synthetic DNS answers. */ public static function verified_resolved_identity( string $provider, string $ip, string $reverse_name, array $forward_addresses ): bool { $suffixes = self::provider_suffixes(); if ( ! isset( $suffixes[ $provider ] ) || ! self::public_ip( $ip ) ) { return false; } $reverse_name = strtolower( rtrim( trim( $reverse_name ), '.' ) ); if ( '' === $reverse_name || false === filter_var( $reverse_name, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME ) ) { return false; } $trusted_name = false; foreach ( $suffixes[ $provider ] as $suffix ) { if ( str_ends_with( $reverse_name, '.' . $suffix ) ) { $trusted_name = true; break; } } if ( ! $trusted_name ) { return false; } $expected = @inet_pton( $ip ); if ( false === $expected ) { return false; } foreach ( $forward_addresses as $address ) { $resolved = is_string( $address ) ? @inet_pton( $address ) : false; if ( false !== $resolved && hash_equals( $expected, $resolved ) ) { return true; } } return false; } private static function queue_verification( string $provider, string $ip ): void { if ( ! self::resolver_available() ) { return; } $job_id = self::identity_hash( $provider, $ip ); $queue = self::queued_jobs(); if ( isset( $queue[ $job_id ] ) ) { if ( ! self::schedule_worker( 1 ) ) { NLSSL_Core::degraded_storage( 'bot_queue_schedule' ); } return; } if ( count( $queue ) >= self::QUEUE_MAX ) { return; } if ( NLSSL_Core::rate_hit( 'crawler_verification_queue', 60, 30, false ) ) { return; } $queue[ $job_id ] = array( 'provider' => $provider, 'ip' => $ip, 'queued_at' => time() ); if ( ! self::store_queue( $queue ) ) { NLSSL_Core::degraded_storage( 'bot_queue_write' ); return; } if ( ! self::schedule_worker( 1 ) ) { NLSSL_Core::degraded_storage( 'bot_queue_schedule' ); } } private static function store_queue( array $queue ): bool { if ( empty( $queue ) ) { delete_transient( self::QUEUE_KEY ); return false === get_transient( self::QUEUE_KEY ); } return NLSSL_Core::write_transient_verified( self::QUEUE_KEY, $queue, self::JOB_TTL ); } private static function queued_jobs(): array { $stored = get_transient( self::QUEUE_KEY ); if ( ! is_array( $stored ) ) { return array(); } $minimum_time = time() - self::JOB_TTL; $queue = array(); foreach ( $stored as $job_id => $job ) { if ( ! is_string( $job_id ) || ! preg_match( '/^[a-f0-9]{64}$/', $job_id ) || ! is_array( $job ) ) { continue; } if ( (int) ( $job['queued_at'] ?? 0 ) < $minimum_time ) { continue; } $queue[ $job_id ] = $job; if ( count( $queue ) >= self::QUEUE_MAX ) { break; } } return $queue; } private static function schedule_worker( int $delay ): bool { if ( false !== wp_next_scheduled( self::CRON_HOOK ) ) { return true; } $scheduled = wp_schedule_single_event( time() + max( 1, $delay ), self::CRON_HOOK ); return true === $scheduled; } private static function verify_dns( string $provider, string $ip ): bool { if ( ! self::resolver_available() ) { return false; } $reverse = @gethostbyaddr( $ip ); if ( ! is_string( $reverse ) || '' === $reverse || $reverse === $ip ) { return false; } return self::verified_resolved_identity( $provider, $ip, $reverse, self::forward_addresses( $reverse ) ); } private static function forward_addresses( string $host ): array { $addresses = array(); if ( function_exists( 'dns_get_record' ) ) { $type = ( defined( 'DNS_A' ) ? DNS_A : 0 ) | ( defined( 'DNS_AAAA' ) ? DNS_AAAA : 0 ); $records = $type > 0 ? @dns_get_record( $host, $type ) : false; if ( is_array( $records ) ) { foreach ( $records as $record ) { if ( isset( $record['ip'] ) && is_string( $record['ip'] ) ) { $addresses[] = $record['ip']; } if ( isset( $record['ipv6'] ) && is_string( $record['ipv6'] ) ) { $addresses[] = $record['ipv6']; } } } } if ( empty( $addresses ) && function_exists( 'gethostbynamel' ) ) { $fallback = @gethostbynamel( $host ); if ( is_array( $fallback ) ) { $addresses = $fallback; } } return array_values( array_unique( array_filter( $addresses, 'is_string' ) ) ); } private static function public_ip( string $ip ): bool { return false !== filter_var( $ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE ); } private static function provider_suffixes(): array { return array( 'google' => array( 'googlebot.com' ), 'bing' => array( 'search.msn.com' ), 'apple' => array( 'applebot.apple.com' ), 'yandex' => array( 'yandex.ru', 'yandex.net', 'yandex.com' ), ); } private static function identity_hash( string $provider, string $ip ): string { return hash_hmac( 'sha256', $provider . '|' . $ip, wp_salt( 'nonce' ) ); } private static function cache_key( string $provider, string $ip ): string { return 'nlssl_bot_result_' . self::identity_hash( $provider, $ip ); } private static function audit_key( string $ip ): string { return 'nlssl_audit_window_' . hash_hmac( 'sha256', $ip, wp_salt( 'auth' ) ); } }