Rule slugs. */ public static function bot_rules() { return array( self::RULE_ALLOW, self::RULE_BLOCK, self::RULE_NONE ); } /** * Reports whether a registry name is safe to emit as a robots.txt token. * * @since 1.2.0 * * @param string $name Registry name. * @return bool True when the name may be emitted and configured. */ public static function is_safe_bot_token( $name ) { return 1 === preg_match( self::BOT_TOKEN_PATTERN, (string) $name ); } /** * Returns the per-bot default rules the specification fixes by name. * * Section 12 states these fourteen defaults literally, so they are stated * literally here rather than derived. That matters because verifying the * crawler list against publisher documentation moved GPTBot, ClaudeBot and * CCBot from `search` to `training`: a purely type-driven default would have * silently flipped all three from Allow to Block, which is a product * decision about who may read the site, not a consequence of correcting a * classification. The type stays truthful; the default stays as specified. * * The same separation keeps `Bytespider` here. Verification left it with no * publisher-stated AI purpose, so it carries no User-Agent pattern and * cannot be detected - but Section 12 names it, so the robots.txt rule a * site owner needs is still generated. * * @since 1.0.0 * * @return array Rule keyed by crawler name. */ private static function specified_bot_rules() { return array( 'GPTBot' => self::RULE_ALLOW, 'OAI-SearchBot' => self::RULE_ALLOW, 'ChatGPT-User' => self::RULE_ALLOW, 'PerplexityBot' => self::RULE_ALLOW, 'Perplexity-User' => self::RULE_ALLOW, 'Google-Extended' => self::RULE_ALLOW, 'Applebot-Extended' => self::RULE_ALLOW, 'Meta-ExternalAgent' => self::RULE_ALLOW, 'CCBot' => self::RULE_ALLOW, 'Bytespider' => self::RULE_ALLOW, 'ClaudeBot' => self::RULE_ALLOW, 'Claude-User' => self::RULE_ALLOW, 'cohere-ai' => self::RULE_ALLOW, 'anthropic-ai' => self::RULE_BLOCK, ); } /** * Returns the default per-bot robots.txt rules. * * Keyed over the live crawler list, so a crawler added through the * `aive_crawler_list` filter receives a default without a second edit, and a * crawler removed from the list leaves no orphan rule behind. * * Two rules decide the value. A name Section 12 states a default for takes * that default verbatim. Anything else follows Section 5: a citation-bearing * crawler - search or assistant - is allowed, while a training crawler that * has not been reviewed is left unmanaged so the site owner decides, rather * than being blocked on the plugin's initiative. * * @since 1.0.0 * * @return array Rule keyed by crawler name. */ public static function default_bot_rules() { $specified = self::specified_bot_rules(); $rules = array(); foreach ( CrawlerList::all() as $crawler ) { $name = $crawler['name']; if ( isset( $specified[ $name ] ) ) { $rules[ $name ] = $specified[ $name ]; continue; } switch ( $crawler['type'] ) { case CrawlerList::TYPE_SEARCH: case CrawlerList::TYPE_ASSISTANT: $rules[ $name ] = self::RULE_ALLOW; break; default: // An unreviewed training crawler: the site owner decides. $rules[ $name ] = self::RULE_NONE; break; } } return $rules; } /** * Returns the per-bot rules in force, scoped to the current registry. * * This is the map the robots module emits from, the settings screen edits * and the audit compares against. Three things make it narrower than the * stored option, and each of them is a requirement rather than tidiness: * * - It is keyed over the live registry in registry order, so a token that * has been removed from the registry has no rule here and therefore no * editable row and no emitted group. The stored option is left alone: the * registry is filterable, so a token missing today may be back when a * filter is reactivated, and discarding the owner's saved choice on read * would lose it silently. The sanitiser is what rewrites the stored map, * on an explicit save. * - A name that cannot be emitted safely is excluded entirely. See * `is_safe_bot_token()`; `invalid_registry_tokens()` reports them so the * loss is visible to an administrator instead of silent. * - A stored value that is not one of the three recognised rules falls back * to that token's default rather than to Block, so a corrupted option * cannot quietly turn a site's AI access off. * * @since 1.2.0 * * @return array Rule keyed by registry name, in registry order. */ public static function effective_bot_rules() { $stored = self::get_setting( 'ai_bot_rules', array() ); $stored = is_array( $stored ) ? $stored : array(); $rules = array(); foreach ( self::default_bot_rules() as $name => $default ) { if ( ! self::is_safe_bot_token( $name ) ) { continue; } $value = isset( $stored[ $name ] ) && is_string( $stored[ $name ] ) ? $stored[ $name ] : ''; $rules[ $name ] = in_array( $value, self::bot_rules(), true ) ? $value : $default; } return $rules; } /** * Returns the registry names that cannot be emitted as robots.txt tokens. * * Always empty for the built-in registry. A name can only appear here by way * of the `aive_crawler_list` filter, and the point of reporting it is that * the entry is being dropped from the robots policy: silently skipping it * would leave an administrator believing a bot is managed when it is not. * * @since 1.2.0 * * @return array Registry names excluded from the robots policy. */ public static function invalid_registry_tokens() { $invalid = array(); foreach ( CrawlerList::names() as $name ) { if ( ! self::is_safe_bot_token( $name ) ) { $invalid[] = $name; } } return $invalid; } /** * Returns the full default settings array. * * @since 1.0.0 * * @return array Default settings. */ public static function default_settings() { return array( 'enable_crawler_log' => true, 'enable_robots_audit' => true, 'enable_schema' => true, 'enable_llms_txt' => true, 'enable_answer_density' => true, 'enable_conflict_detector' => true, 'log_retention_days' => 90, 'ai_bot_rules' => self::default_bot_rules(), 'llms_txt_description' => '', 'llms_txt_pages' => array(), 'answer_density_post_types' => array( 'post', 'page' ), ); } /** * Returns the stored settings merged over the defaults. * * Always returns every key, so callers never need an isset() dance. * * @since 1.0.0 * * @return array Effective settings. */ public static function get_settings() { $stored = get_option( self::OPTION_KEY, array() ); return self::merge_settings( is_array( $stored ) ? $stored : array() ); } /** * Merges a partial settings array over the defaults. * * Only known keys survive, so a stale key left by an older version does not * linger in the effective settings. The `ai_bot_rules` map is merged one * level deeper so that a stored rule for one bot does not drop the defaults * for the others. * * @since 1.0.0 * * @param array $settings Partial settings. * @return array Complete settings. */ public static function merge_settings( array $settings ) { $defaults = self::default_settings(); $merged = array_merge( $defaults, array_intersect_key( $settings, $defaults ) ); $stored_rules = isset( $settings['ai_bot_rules'] ) && is_array( $settings['ai_bot_rules'] ) ? $settings['ai_bot_rules'] : array(); $merged['ai_bot_rules'] = array_merge( $defaults['ai_bot_rules'], $stored_rules ); return $merged; } /** * Returns a single effective setting. * * @since 1.0.0 * * @param string $key Settings key. * @param mixed $fallback Value returned when the key is unknown. * @return mixed Setting value, or $fallback for an unknown key. */ public static function get_setting( $key, $fallback = null ) { $settings = self::get_settings(); return array_key_exists( $key, $settings ) ? $settings[ $key ] : $fallback; } /** * Reports whether a feature module is enabled in settings. * * @since 1.0.0 * * @param string $slug Module slug, for example `crawler_log`. * @return bool True when the module should run. */ public static function is_module_enabled( $slug ) { return (bool) self::get_setting( 'enable_' . $slug, false ); } /** * Reports whether the WordPress AI Client API is available. * * WordPress 7.0 introduced `wp_ai_client_prompt()`. Its absence means the * site runs an older release and AIVE must fall back to local algorithms. * This only reports availability of the API, not whether the site owner has * configured a provider; that is a separate per-prompt check. * * @since 1.0.0 * * @return bool True when the AI Client API is present. */ public static function has_ai_client() { return function_exists( 'wp_ai_client_prompt' ); } /** * Returns the crawler definition matching a User-Agent. * * @since 1.0.0 * * @param string $user_agent Raw User-Agent header value. * @return array{name: string, pattern: string|null, type: string, source: string}|null Definition, or null when not a known AI bot. */ public static function detect_ai_bot( $user_agent ) { return CrawlerList::match( $user_agent ); } /** * Reports whether a User-Agent belongs to a known AI crawler. * * @since 1.0.0 * * @param string $user_agent Raw User-Agent header value. * @return bool True when the User-Agent matches a known AI crawler. */ public static function is_ai_bot( $user_agent ) { return null !== self::detect_ai_bot( $user_agent ); } /** * Returns the question-detection patterns keyed by locale family. * * Exposed so that the Unit suite can assert coverage per language and so a * translation can extend detection without touching the matcher. * * @since 1.0.0 * * @return array, contains: array}> Patterns by language. */ public static function question_patterns() { return array( // Latin-script languages are matched on a leading interrogative word. 'en' => array( 'prefixes' => array( 'how', 'what', 'why', 'when', 'where', 'who', 'which', 'can', 'does', 'do', 'is', 'are', 'should', 'will' ), 'contains' => array(), ), 'de' => array( 'prefixes' => array( 'wie', 'was', 'warum', 'wann', 'wo', 'wer', 'welche', 'kann', 'sind', 'ist' ), 'contains' => array(), ), /* * CJK languages are matched on a substring because they are written * without word delimiters, so a leading-word test cannot apply. */ 'zh' => array( 'prefixes' => array(), 'contains' => array( '如何', '什么是', '为什么', '怎么办', '怎么', '哪里', '多少', '什么时候', '能不能', '是否', '为何', '何为' ), ), 'ja' => array( 'prefixes' => array(), 'contains' => array( '方法', 'とは', 'なぜ', 'どう', 'どこ', 'いくら', 'いつ' ), ), ); } /** * Reports whether a string reads as a question. * * Checks an explicit question mark first - ASCII or fullwidth - then the * per-language interrogative patterns. Pure logic with no WordPress * dependency so the Unit suite can exercise it directly. * * @since 1.0.0 * * @param string $text Candidate heading or summary text. * @return bool True when the text reads as a question. */ public static function is_question_text( $text ) { $text = self::normalize_text( $text ); if ( '' === $text ) { return false; } // An explicit terminator settles it in every supported language. if ( 1 === preg_match( '/[?\x{FF1F}]\s*$/u', $text ) ) { return true; } $lowered = self::lowercase( $text ); foreach ( self::question_patterns() as $patterns ) { foreach ( $patterns['contains'] as $needle ) { if ( false !== mb_strpos( $lowered, self::lowercase( $needle ) ) ) { return true; } } foreach ( $patterns['prefixes'] as $prefix ) { /* * The word boundary matters: "Island facts" must not match the * "is" prefix. preg_quote keeps a pattern inert if a translation * ever introduces a metacharacter. */ if ( 1 === preg_match( '/^' . preg_quote( self::lowercase( $prefix ), '/' ) . '\b/u', $lowered ) ) { return true; } } } return false; } /** * Collapses whitespace and trims a string. * * Normalises the non-breaking space that editors inject, so that text * comparison and length limits behave predictably. * * @since 1.0.0 * * @param string $text Raw text. * @return string Normalised text. */ public static function normalize_text( $text ) { $text = (string) $text; if ( '' === $text ) { return ''; } // Normalise UTF-8 and HTML-entity non-breaking spaces to plain spaces. $text = str_replace( array( "\xc2\xa0", ' ' ), ' ', $text ); $text = preg_replace( '/\s+/u', ' ', $text ); return trim( (string) $text ); } /** * Lowercases a string, multibyte-aware. * * @since 1.0.0 * * @param string $text Raw text. * @return string Lowercased text. */ public static function lowercase( $text ) { return mb_strtolower( (string) $text, 'UTF-8' ); } /** * Parses an HTML fragment into a DOMDocument. * * Post content is a fragment, not a document, so it is wrapped in a minimal * UTF-8 document before loading. libxml diagnostics are captured rather * than emitted because post content routinely contains markup libxml * dislikes and that is not a plugin error. * * @since 1.0.0 * * @param string $html HTML fragment, typically post content. * @return DOMDocument|null Loaded document, or null when the fragment is empty or unparsable. */ public static function parse_html_fragment( $html ) { $html = (string) $html; if ( '' === trim( $html ) ) { return null; } $document = new DOMDocument(); $previous_state = libxml_use_internal_errors( true ); /* * The XML declaration forces UTF-8 interpretation without libxml * guessing from a meta tag the fragment does not have. */ $loaded = $document->loadHTML( '' . $html . '', LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING ); libxml_clear_errors(); libxml_use_internal_errors( $previous_state ); return $loaded ? $document : null; } /** * Returns the heading elements of a document in document order. * * @since 1.0.0 * * @param DOMDocument $document Parsed document. * @param array $levels Heading levels to collect, defaults to h2-h6. * @return array Heading elements in document order. */ public static function get_headings( DOMDocument $document, array $levels = array( 2, 3, 4, 5, 6 ) ) { $wanted = array(); foreach ( $levels as $level ) { $level = (int) $level; if ( $level >= 1 && $level <= 6 ) { $wanted[ 'h' . $level ] = true; } } if ( array() === $wanted ) { return array(); } $headings = array(); /* * A single wildcard query walks the tree once and yields elements in * document order. Querying each level separately would return them * grouped by level, and restoring order afterwards would need * DOMNode::compareDocumentPosition(), which is not available on every * supported PHP build. */ foreach ( $document->getElementsByTagName( '*' ) as $element ) { if ( ! $element instanceof DOMElement ) { continue; } if ( isset( $wanted[ strtolower( $element->tagName ) ] ) ) { $headings[] = $element; } } return $headings; } /** * Returns the heading level of an element. * * @since 1.0.0 * * @param DOMElement $element Candidate heading element. * @return int Level 1-6, or 0 when the element is not a heading. */ public static function get_heading_level( DOMElement $element ) { if ( 1 !== preg_match( '/^h([1-6])$/i', $element->tagName, $matches ) ) { return 0; } return (int) $matches[1]; } /** * Returns the sibling nodes that make up a heading's section. * * A section runs from just after the heading up to the next heading at the * same or a higher level, which is how a reader - and an answer engine - * scopes the answer that belongs to a question heading. * * @since 1.0.0 * * @param DOMElement $heading Heading element that opens the section. * @return array Nodes belonging to the section, in document order. */ public static function get_section_nodes( DOMElement $heading ) { $level = self::get_heading_level( $heading ); $nodes = array(); for ( $node = $heading->nextSibling; null !== $node; $node = $node->nextSibling ) { if ( $node instanceof DOMElement ) { $node_level = self::get_heading_level( $node ); if ( $node_level > 0 && $node_level <= $level ) { break; } } $nodes[] = $node; } return $nodes; } /** * Writes a message to the PHP error log when debug logging is on. * * Silent unless both WP_DEBUG and WP_DEBUG_LOG are enabled, so a production * site never pays for instrumentation it did not ask for. * * @since 1.0.0 * * @param string $message Message to log. * @param array $context Optional structured context appended as JSON. * @return void */ public static function log( $message, array $context = array() ) { if ( ! defined( 'WP_DEBUG' ) || ! WP_DEBUG ) { return; } if ( ! defined( 'WP_DEBUG_LOG' ) || ! WP_DEBUG_LOG ) { return; } $line = '[AIVE] ' . (string) $message; if ( array() !== $context ) { $encoded = wp_json_encode( $context ); $line .= false === $encoded ? ' {uncodable context}' : ' ' . $encoded; } error_log( $line ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log -- Gated on WP_DEBUG and WP_DEBUG_LOG; this is the plugin's debug logger. } }