<?php
/**
 * Known AI crawler User-Agent patterns.
 *
 * @package PressRoost\AIVE
 */

namespace PressRoost\AIVE\Utils;

/**
 * Registry of the AI crawlers AIVE recognises.
 *
 * Every entry carries a display name, a case-insensitive regular expression
 * fragment matched against the raw User-Agent header, and a type category.
 *
 * Type categories drive policy elsewhere in the plugin:
 *
 * - `search`    fetches pages in order to cite them in an AI answer. Allowing
 *               these is what produces AI visibility.
 * - `assistant` an interactive assistant fetching live content on behalf of a
 *               user. Also citation-bearing.
 * - `training`  collects data for model training with no citation or visibility
 *               benefit to the site.
 *
 * The data set is deliberately a flat array so that entries can be added,
 * renamed or removed in one place, and `all()` exposes the `aive_crawler_list`
 * filter so that site owners and add-ons can extend it without a plugin edit.
 *
 * Provenance: every entry below was read on its operator's own documentation on
 * 2026-10-11, and each carries the `source` URL it was read on. Section 5 sets
 * the bar: an entry ships only when the publisher states both the token and a
 * purpose that makes the crawler an AI crawler - it feeds an AI answer, an AI
 * assistant, or model training. A crawler whose publisher documents some other
 * purpose is not in this list, however real it is.
 * `docs/crawler-inventory.md` records the full inventory with the quoted
 * purpose behind each decision. Re-verify when that document's date goes stale -
 * operators rename, retire and reclassify their crawlers without notice.
 *
 * `pattern` is null when an entry must never be matched against a request
 * header, for either of two reasons. Google and Apple publish `-Extended`
 * tokens that exist only to carry a robots.txt decision and never appear in a
 * header at all; `anthropic-ai` and `cohere-ai` are legacy robots.txt names
 * their operators no longer document as crawlers; and `Bytespider` sends a
 * header its operator documents, but documents it as a conventional search
 * index crawler with no AI purpose stated. All five stay in the registry
 * because Section 12 specifies a robots.txt default for each and a site owner
 * still needs that rule, but a null pattern keeps them out of
 * `matchable()` - so no detector can report AI crawler traffic the publisher
 * does not support.
 *
 * @since 1.0.0
 */
final class CrawlerList {

	/**
	 * Bot type: fetches pages to cite in AI answers.
	 *
	 * @since 1.0.0
	 * @var string
	 */
	public const TYPE_SEARCH = 'search';

	/**
	 * Bot type: interactive assistant fetching live content.
	 *
	 * @since 1.0.0
	 * @var string
	 */
	public const TYPE_ASSISTANT = 'assistant';

	/**
	 * Bot type: collects training data, no citation benefit.
	 *
	 * @since 1.0.0
	 * @var string
	 */
	public const TYPE_TRAINING = 'training';

	/**
	 * Returns the built-in crawler list, unfiltered.
	 *
	 * Pure data with no WordPress dependency, which is what makes the crawler
	 * matching logic testable in the Unit suite.
	 *
	 * @since 1.0.0
	 *
	 * @return array<int, array{name: string, pattern: string|null, type: string, source: string}> Crawler definitions.
	 */
	public static function default_list() {
		$openai      = 'https://developers.openai.com/api/docs/bots';
		$anthropic   = 'https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler';
		$perplexity  = 'https://docs.perplexity.ai/guides/bots';
		$meta        = 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/';
		$commoncrawl = 'https://commoncrawl.org/ccbot';

		return array(
			// Search bots: fetch pages in order to cite them in an AI answer. Allowing these is what
			// produces AI visibility.
			array(
				// "used to surface websites in search results in ChatGPT's search features".
				'name'    => 'OAI-SearchBot',
				'pattern' => 'OAI-SearchBot',
				'type'    => self::TYPE_SEARCH,
				'source'  => $openai,
			),
			array(
				// "designed to surface and link websites in search results on Perplexity".
				'name'    => 'PerplexityBot',
				'pattern' => 'PerplexityBot',
				'type'    => self::TYPE_SEARCH,
				'source'  => $perplexity,
			),
			array(
				// "navigates the web to improve search result quality for users".
				'name'    => 'Claude-SearchBot',
				'pattern' => 'Claude-SearchBot',
				'type'    => self::TYPE_SEARCH,
				'source'  => $anthropic,
			),
			array(
				// "navigates the web to improve Meta AI search result quality for users".
				'name'    => 'Meta-WebIndexer',
				'pattern' => 'meta-webindexer',
				'type'    => self::TYPE_SEARCH,
				'source'  => $meta,
			),
			array(
				// "the web crawler that powers the You.com search engine", on the page the published
				// User-Agent itself points at. You.com documents that search product as "The Leading Web
				// Search APIs for AI", which "generate grounded answers for your agents and LLMs" - so the AI
				// purpose is the operator's own, stated on a second page of its own site rather than on the
				// crawler page. The inventory records both.
				'name'    => 'YouBot',
				'pattern' => 'YouBot',
				'type'    => self::TYPE_SEARCH,
				'source'  => 'https://docs.you.com/youbot',
			),
			array(
				// Registered for its robots.txt rule only, with no pattern. ByteDance's own webmaster
				// documentation confirms the token - Toutiao Search's crawler UA is "Bytespider", capitalised
				// - and states the purpose as a conventional search pipeline: crawl pages, build a keyword
				// index, then return matching pages with a title, URL and snippet. It states no AI purpose of
				// any kind, so Section 5 does not admit it as a detectable AI crawler. Section 12 still
				// specifies a default rule for the token, which is why the entry stays.
				'name'    => 'Bytespider',
				'pattern' => null,
				'type'    => self::TYPE_SEARCH,
				'source'  => 'https://zhanzhang.toutiao.com/docs/intro/26899',
			),

			/*
			 * Assistant bots: an interactive assistant fetching live content on
			 * behalf of a user. Also citation-bearing.
			 */
			array(
				// "certain user actions in ChatGPT and Custom GPTs".
				'name'    => 'ChatGPT-User',
				'pattern' => 'ChatGPT-User',
				'type'    => self::TYPE_ASSISTANT,
				'source'  => $openai,
			),
			array(
				// "supports user actions within Perplexity".
				'name'    => 'Perplexity-User',
				'pattern' => 'Perplexity-User',
				'type'    => self::TYPE_ASSISTANT,
				'source'  => $perplexity,
			),
			array(
				// "When individuals ask questions to Claude, it may access websites".
				'name'    => 'Claude-User',
				'pattern' => 'Claude-User',
				'type'    => self::TYPE_ASSISTANT,
				'source'  => $anthropic,
			),
			array(
				// "fetches individual links at a user's request".
				'name'    => 'Meta-ExternalFetcher',
				'pattern' => 'meta-externalfetcher',
				'type'    => self::TYPE_ASSISTANT,
				'source'  => $meta,
			),
			array(
				// Legacy robots.txt token, no User-Agent. Cohere's own crawler page states it does not use
				// "Cohere bots or user agents for the purpose of crawling or scraping web content" and lists no
				// active crawler, so nothing here may match a live request. The token is retained only so
				// Section 12's default rule for it still has something to apply to.
				'name'    => 'cohere-ai',
				'pattern' => null,
				'type'    => self::TYPE_ASSISTANT,
				'source'  => 'https://docs.cohere.com/docs/cohere-web-crawlers',
			),

			/*
			 * Training bots: collect data for model training, with no citation
			 * or visibility benefit to the site.
			 */
			array(
				// "crawl content that may be used in training our generative AI foundation models".
				'name'    => 'GPTBot',
				'pattern' => 'GPTBot',
				'type'    => self::TYPE_TRAINING,
				'source'  => $openai,
			),
			array(
				// "collecting web content that could potentially contribute to their training".
				'name'    => 'ClaudeBot',
				'pattern' => 'ClaudeBot',
				'type'    => self::TYPE_TRAINING,
				'source'  => $anthropic,
			),
			array(
				// "crawls the web for use cases such as training foundation AI models".
				'name'    => 'Meta-ExternalAgent',
				'pattern' => 'meta-externalagent',
				'type'    => self::TYPE_TRAINING,
				'source'  => $meta,
			),
			array(
				// Common Crawl publishes an open corpus rather than an answer engine, so a visit carries no
				// citation back to the site. The crawler page states the token; the AI purpose is the
				// foundation's own, but in its Terms of Use rather than on that page, which address "use of
				// Crawled Content in connection with artificial intelligence, machine learning, or other
				// similar technologies, including, without limitation, large language models ... for the
				// purpose of developing, training, or deploying AI Systems". See the inventory.
				'name'    => 'CCBot',
				'pattern' => 'CCBot',
				'type'    => self::TYPE_TRAINING,
				'source'  => $commoncrawl,
			),
			array(
				// robots.txt control token, no User-Agent: Google states it "doesn't have a separate HTTP
				// request user agent string" and that "the robots.txt user-agent token is used in a control
				// capacity". Crawling happens under Googlebot.
				'name'    => 'Google-Extended',
				'pattern' => null,
				'type'    => self::TYPE_TRAINING,
				'source'  => 'https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers',
			),
			array(
				// robots.txt control token, no User-Agent: Apple states it "does not crawl webpages" and is
				// "only used to determine how to use the data crawled by the Applebot user agent".
				'name'    => 'Applebot-Extended',
				'pattern' => null,
				'type'    => self::TYPE_TRAINING,
				'source'  => 'https://support.apple.com/en-us/119829',
			),
			array(
				// Legacy robots.txt token, no User-Agent. Anthropic's current crawler documentation names only
				// ClaudeBot, Claude-User and Claude-SearchBot; anthropic-ai is absent from it. Retained as a
				// robots.txt rule because Section 12 defaults it to Block and an old deployment may still
				// honour the name.
				'name'    => 'anthropic-ai',
				'pattern' => null,
				'type'    => self::TYPE_TRAINING,
				'source'  => $anthropic,
			),
		);
	}

	/**
	 * Returns the crawler list after the extensibility filter has run.
	 *
	 * @since 1.0.0
	 *
	 * @return array<int, array{name: string, pattern: string|null, type: string, source: string}> Crawler definitions.
	 */
	public static function all() {
		/**
		 * Filters the list of AI crawlers AIVE recognises.
		 *
		 * Each entry must be an array with a `name`, and may carry `pattern`,
		 * `type` and `source` keys. `pattern` is a regular expression fragment
		 * matched case-insensitively against the raw User-Agent header; it must
		 * not include delimiters. Pass null or omit it for a token that only
		 * ever appears in robots.txt and never in a request header. An entry
		 * with no usable name, or a pattern that does not compile, is discarded.
		 *
		 * @since 1.0.0
		 *
		 * @param array<int, array{name: string, pattern: string|null, type: string, source: string}> $crawlers Crawler definitions.
		 */
		$crawlers = apply_filters( 'aive_crawler_list', self::default_list() );

		return self::sanitize_list( is_array( $crawlers ) ? $crawlers : array() );
	}

	/**
	 * Discards entries that do not carry a usable name, pattern and type.
	 *
	 * Runs over filtered input, so it cannot assume well-formed data.
	 *
	 * @since 1.0.0
	 *
	 * @param array<int|string, mixed> $crawlers Candidate crawler definitions.
	 * @return array<int, array{name: string, pattern: string|null, type: string, source: string}> Valid crawler definitions, re-indexed.
	 */
	public static function sanitize_list( array $crawlers ) {
		$valid = array();

		foreach ( $crawlers as $crawler ) {
			if ( ! is_array( $crawler ) ) {
				continue;
			}

			if ( ! isset( $crawler['name'] ) ) {
				continue;
			}

			$name = trim( (string) $crawler['name'] );

			if ( '' === $name ) {
				continue;
			}

			/*
			 * A missing, null or empty pattern means "robots.txt token only":
			 * the entry still needs a rule in robots.txt but can never match a
			 * request. A pattern that does not compile is a different case - it
			 * was meant to match and cannot - so the whole entry is dropped
			 * rather than silently demoted to unmatchable.
			 */
			$pattern = isset( $crawler['pattern'] ) ? (string) $crawler['pattern'] : '';

			if ( '' === $pattern ) {
				$pattern = null;
			} elseif ( ! self::is_valid_pattern( $pattern ) ) {
				continue;
			}

			$type = isset( $crawler['type'] ) ? (string) $crawler['type'] : self::TYPE_SEARCH;

			if ( ! in_array( $type, self::types(), true ) ) {
				$type = self::TYPE_SEARCH;
			}

			$valid[] = array(
				'name'    => $name,
				'pattern' => $pattern,
				'type'    => $type,
				'source'  => isset( $crawler['source'] ) ? (string) $crawler['source'] : '',
			);
		}

		return $valid;
	}

	/**
	 * Returns only the entries that can match a live User-Agent.
	 *
	 * Callers that log or detect traffic want this rather than `all()`: a
	 * robots.txt control token such as Google-Extended has no User-Agent, so
	 * including it would invite a detector that reports visits no crawler made.
	 *
	 * @since 1.0.0
	 *
	 * @return array<int, array{name: string, pattern: string|null, type: string, source: string}> Matchable crawler definitions, re-indexed.
	 */
	public static function matchable() {
		$matchable = array();

		foreach ( self::all() as $crawler ) {
			if ( null !== $crawler['pattern'] && '' !== $crawler['pattern'] ) {
				$matchable[] = $crawler;
			}
		}

		return $matchable;
	}

	/**
	 * Reports whether a pattern fragment compiles as a regular expression.
	 *
	 * Filtered input can carry a malformed fragment. Compiling it once here,
	 * with diagnostics captured rather than emitted, keeps a bad third-party
	 * pattern from warning on every matched request.
	 *
	 * @since 1.0.0
	 *
	 * @param string $pattern Regular expression fragment without delimiters.
	 * @return bool True when the fragment compiles.
	 */
	private static function is_valid_pattern( $pattern ) {
		$regex = '#' . str_replace( '#', '\\#', (string) $pattern ) . '#i';

		set_error_handler( static fn() => true ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_set_error_handler -- Not debug code: scoped to one preg_match so a malformed filtered pattern is rejected instead of warning on every request.
		$compiles = false !== preg_match( $regex, '' );
		restore_error_handler();

		return $compiles;
	}

	/**
	 * Returns the recognised bot type categories.
	 *
	 * @since 1.0.0
	 *
	 * @return array<int, string> Type slugs.
	 */
	public static function types() {
		return array( self::TYPE_SEARCH, self::TYPE_ASSISTANT, self::TYPE_TRAINING );
	}

	/**
	 * Returns the crawler names keyed in list order.
	 *
	 * @since 1.0.0
	 *
	 * @return array<int, string> Crawler names.
	 */
	public static function names() {
		return array_column( self::all(), 'name' );
	}

	/**
	 * Finds the crawler in a given list that matches a User-Agent most closely.
	 *
	 * Pure logic with no WordPress dependency so the Unit suite can exercise it
	 * directly. Matching is case-insensitive and anchored nowhere, because
	 * crawler tokens appear in the middle of a longer User-Agent string.
	 *
	 * Where two entries both match, the one that matched more of the header
	 * wins, rather than whichever happens to be listed first. Overlapping
	 * tokens are a recurring shape in operators' documentation: a crawl token
	 * and a `-User` token for the same product, where the longer is a substring
	 * extension of the shorter. A header normally carries the operator's URL as
	 * well as its token, so a negative lookahead on the shorter pattern does not
	 * resolve it - the shorter token still appears, in the URL. Deciding on how
	 * much of the header matched means no entry can be mislabelled by a later
	 * edit that reorders the registry, or by a filter that adds an overlapping
	 * token of its own.
	 *
	 * @since 1.0.0
	 *
	 * @param string                                                                              $user_agent Raw User-Agent header value.
	 * @param array<int, array{name: string, pattern: string|null, type: string, source: string}> $crawlers   Crawler definitions to match against.
	 * @return array{name: string, pattern: string|null, type: string, source: string}|null Matching definition, or null when nothing matches.
	 */
	public static function match_in( $user_agent, array $crawlers ) {
		$user_agent = (string) $user_agent;

		if ( '' === trim( $user_agent ) ) {
			return null;
		}

		$best        = null;
		$best_length = 0;

		foreach ( $crawlers as $crawler ) {
			if ( ! isset( $crawler['pattern'] ) || '' === (string) $crawler['pattern'] ) {
				continue;
			}

			$regex   = '#' . str_replace( '#', '\\#', (string) $crawler['pattern'] ) . '#i';
			$matches = array();

			if ( 1 !== preg_match( $regex, $user_agent, $matches ) ) {
				continue;
			}

			$length = strlen( $matches[0] );

			// Strictly greater, so the earlier entry still wins an exact tie.
			if ( $length > $best_length ) {
				$best        = $crawler;
				$best_length = $length;
			}
		}

		return $best;
	}

	/**
	 * Finds the known crawler matching a User-Agent most closely.
	 *
	 * @since 1.0.0
	 *
	 * @param string $user_agent Raw User-Agent header value.
	 * @return array{name: string, pattern: string|null, type: string, source: string}|null Matching definition, or null when nothing matches.
	 */
	public static function match( $user_agent ) {
		return self::match_in( $user_agent, self::all() );
	}

	/**
	 * Returns a single crawler definition by name.
	 *
	 * @since 1.0.0
	 *
	 * @param string $name Crawler name as listed by names().
	 * @return array{name: string, pattern: string|null, type: string, source: string}|null Definition, or null when unknown.
	 */
	public static function get( $name ) {
		foreach ( self::all() as $crawler ) {
			if ( $crawler['name'] === $name ) {
				return $crawler;
			}
		}

		return null;
	}
}