*/ public const PURPOSES = [ 'training' => 'AI training', 'ai-search' => 'AI search', 'user-fetch' => 'User triggered', 'dataset' => 'Open dataset', ]; /** * Crawler catalogue keyed by a stable slug. * * @var array */ private const CRAWLERS = [ 'gptbot' => [ 'label' => 'GPTBot (OpenAI)', 'token' => 'GPTBot', 'purpose' => 'training', 'default' => self::BLOCK, 'note' => 'Crawls content for OpenAI model training.', ], 'claudebot' => [ 'label' => 'ClaudeBot (Anthropic)', 'token' => 'ClaudeBot', 'purpose' => 'training', 'default' => self::BLOCK, 'note' => 'Crawls content for Anthropic model training.', ], 'google-extended' => [ 'label' => 'Google-Extended', 'token' => 'Google-Extended', 'purpose' => 'training', 'default' => self::BLOCK, 'note' => 'Controls Gemini training only. Blocking it does not affect Google Search or AI Overviews.', ], 'applebot-extended' => [ 'label' => 'Applebot-Extended', 'token' => 'Applebot-Extended', 'purpose' => 'training', 'default' => self::BLOCK, 'note' => 'Opts out of Apple Intelligence training. Pages stay in search results.', ], 'meta-externalagent' => [ 'label' => 'meta-externalagent (Meta)', 'token' => 'meta-externalagent', 'purpose' => 'training', 'default' => self::BLOCK, 'note' => 'Meta AI training and direct indexing.', ], 'amazonbot' => [ 'label' => 'Amazonbot', 'token' => 'Amazonbot', 'purpose' => 'training', 'default' => self::BLOCK, 'note' => 'May be used to train Amazon AI models.', ], 'bytespider' => [ 'label' => 'Bytespider (ByteDance)', 'token' => 'Bytespider', 'purpose' => 'training', 'default' => self::BLOCK, 'note' => 'Training crawler. Compliance with robots.txt is not verified by the operator.', ], 'ccbot' => [ 'label' => 'CCBot (Common Crawl)', 'token' => 'CCBot', 'purpose' => 'dataset', 'default' => self::BLOCK, 'note' => 'Builds an open dataset widely reused for model training.', ], 'oai-searchbot' => [ 'label' => 'OAI-SearchBot (OpenAI)', 'token' => 'OAI-SearchBot', 'purpose' => 'ai-search', 'default' => self::ALLOW, 'note' => 'Builds the ChatGPT search index. Blocking it removes you from ChatGPT results.', ], 'claude-searchbot' => [ 'label' => 'Claude-SearchBot (Anthropic)', 'token' => 'Claude-SearchBot', 'purpose' => 'ai-search', 'default' => self::ALLOW, 'note' => 'Builds the Claude search index.', ], 'perplexitybot' => [ 'label' => 'PerplexityBot', 'token' => 'PerplexityBot', 'purpose' => 'ai-search', 'default' => self::ALLOW, 'note' => 'Builds Perplexity search results. Not used for training.', ], 'chatgpt-user' => [ 'label' => 'ChatGPT-User (OpenAI)', 'token' => 'ChatGPT-User', 'purpose' => 'user-fetch', 'default' => self::ALLOW, 'note' => 'Fetches a page when a user asks. May ignore robots.txt by design.', ], 'claude-user' => [ 'label' => 'Claude-User (Anthropic)', 'token' => 'Claude-User', 'purpose' => 'user-fetch', 'default' => self::ALLOW, 'note' => 'Fetches a page at a user request.', ], 'perplexity-user' => [ 'label' => 'Perplexity-User', 'token' => 'Perplexity-User', 'purpose' => 'user-fetch', 'default' => self::ALLOW, 'note' => 'Fetches a page at a user request.', ], ]; /** * Crawler catalogue, filterable for new bots. * * The filter is a public hook, so its result is treated as untrusted. Every * entry must be an array carrying string label, token, purpose, default and * note values. A malformed entry for a catalogued slug falls back to the * built in entry, a malformed entry for an unknown slug is dropped, and only * well formed entries survive. This keeps a malformed token, including one * with a line break, from ever reaching the robots.txt builder. * * @return array */ public static function all(): array { /** * Filter the catalogued AI crawlers. * * @param array $crawlers Crawlers keyed by slug. */ $filtered = apply_filters( 'rankkernel/robots/crawlers', self::CRAWLERS ); // phpcs:ignore WordPress.NamingConventions.ValidHookName.UseUnderscores -- public hook name, part of the plugin API, must stay stable. if ( ! is_array( $filtered ) ) { return self::CRAWLERS; } $out = []; foreach ( $filtered as $slug => $crawler ) { if ( ! is_string( $slug ) ) { continue; } if ( self::isWellFormed( $crawler ) ) { $out[ $slug ] = $crawler; continue; } if ( isset( self::CRAWLERS[ $slug ] ) ) { $out[ $slug ] = self::CRAWLERS[ $slug ]; } } return ( [] !== $out ) ? $out : self::CRAWLERS; } /** * Whether a filtered crawler entry carries every required field. * * Four fields must be present strings. The token must also be a non empty * string, because it lands on the User-agent line of robots.txt. * * @param mixed $crawler Raw entry from the filter. * @return bool The result. */ private static function isWellFormed( mixed $crawler ): bool { if ( ! is_array( $crawler ) ) { return false; } foreach ( [ 'label', 'purpose', 'default', 'note' ] as $field ) { if ( ! isset( $crawler[ $field ] ) || ! is_string( $crawler[ $field ] ) ) { return false; } } return isset( $crawler['token'] ) && is_string( $crawler['token'] ) && '' !== $crawler['token']; } /** * Known crawler slugs. * * @return string[] */ public static function slugs(): array { return array_map( 'strval', array_keys( self::all() ) ); } /** * Get one crawler definition. * * @param string $slug Crawler slug. * @return array{label: string, token: string, purpose: string, default: string, note: string}|null The result. */ public static function get( string $slug ): ?array { $all = self::all(); return $all[ $slug ] ?? null; } /** * Default policy map, one entry per crawler. * * @return array The result. */ public static function defaults(): array { $out = []; foreach ( self::all() as $slug => $crawler ) { $out[ (string) $slug ] = (string) $crawler['default']; } return $out; } /** * Normalise a single policy value. * * @param mixed $value Raw value. * @return string The result. */ public static function sanitizePolicy( mixed $value ): string { $value = is_string( $value ) ? strtolower( $value ) : ''; return in_array( $value, [ self::ALLOW, self::BLOCK, self::CUSTOM ], true ) ? $value : self::CUSTOM; } /** * Normalise a policy map to known slugs, defaulting missing crawlers. * * @param mixed $value Raw map. * @return array The result. */ public static function sanitizeMap( mixed $value ): array { $stored = is_array( $value ) ? $value : []; $out = []; foreach ( self::all() as $slug => $crawler ) { $slug = (string) $slug; $out[ $slug ] = array_key_exists( $slug, $stored ) ? self::sanitizePolicy( $stored[ $slug ] ) : (string) $crawler['default']; } return $out; } }