/search/` and `/cart/` to EVERY user-agent group (a bot that matches its own group * ignores `User-agent: *`). * * @package AooShop */ defined( 'ABSPATH' ) || exit; class AooShop_Llms { const TRANSIENT = 'aooshop_llms'; const STALE_TRANSIENT = 'aooshop_llms_stale'; const TTL = HOUR_IN_SECONDS; const MAX_POSTS = 100; /** @var AooShop_Settings */ private $settings; /** @var AooShop_Api_Client */ private $api; /** @var AooShop_Renderer */ private $renderer; /** @var AooShop_Template_Page */ private $template; public function __construct( AooShop_Settings $settings, AooShop_Api_Client $api, AooShop_Renderer $renderer, AooShop_Template_Page $template ) { $this->settings = $settings; $this->api = $api; $this->renderer = $renderer; $this->template = $template; } public function hooks() { add_action( 'parse_request', array( $this, 'maybe_serve' ), 50 ); add_filter( 'rank_math/llms_txt/extra_content', array( $this, 'rankmath_extra' ), 20 ); add_filter( 'robots_txt', array( $this, 'robots_txt' ), 99, 2 ); } public function live() { return ! $this->settings->is_preview() && $this->settings->is_configured(); } // ── who owns /llms.txt ────────────────────────────────────────────── public static function physical_file() { return file_exists( ABSPATH . 'llms.txt' ); } public static function rankmath_module() { if ( class_exists( '\RankMath\Helper' ) && method_exists( '\RankMath\Helper', 'is_module_active' ) ) { return (bool) \RankMath\Helper::is_module_active( 'llms-txt' ); } return false; } public static function yoast_llms() { if ( ! defined( 'WPSEO_VERSION' ) ) { return false; } $opt = get_option( 'wpseo', array() ); return is_array( $opt ) && ! empty( $opt['enable_llms_txt'] ); } /** '' (ours) | 'file' | 'rankmath' | 'yoast' */ public static function owner() { if ( self::physical_file() ) { return 'file'; } if ( self::rankmath_module() ) { return 'rankmath'; } if ( self::yoast_llms() ) { return 'yoast'; } return ''; } // ── serving ─────────────────────────────────────────────────────── public function maybe_serve( $wp ) { $path = isset( $_SERVER['REQUEST_URI'] ) ? (string) wp_parse_url( wp_unslash( $_SERVER['REQUEST_URI'] ), PHP_URL_PATH ) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized $home = (string) wp_parse_url( home_url( '/' ), PHP_URL_PATH ); if ( untrailingslashit( $home ) . '/llms.txt' !== $path ) { return; } if ( ! $this->live() || '' !== self::owner() ) { return; } $text = $this->full_text(); if ( '' === $text ) { return; // aoo has AI crawlers off / unreachable — let WordPress answer (404). } status_header( 200 ); header( 'Content-Type: text/plain; charset=utf-8' ); header( 'X-Robots-Tag: noindex' ); header( 'X-Aoo-Llms: 1' ); header( 'Cache-Control: public, max-age=3600' ); echo wp_kses( $text, array() ); // text/plain — no tags; entities normalized. exit; } /** aoo llms text with links on this site ('' when unavailable). */ public function shop_text() { $text = get_transient( self::TRANSIENT ); if ( ! is_string( $text ) ) { $res = $this->api->get( 'llms' ); if ( ! is_wp_error( $res ) && is_string( $res['data'] ) ) { $text = $res['data']; set_transient( self::TRANSIENT, $text, self::TTL ); set_transient( self::STALE_TRANSIENT, $text, WEEK_IN_SECONDS ); } elseif ( is_wp_error( $res ) && in_array( (int) ( $res->get_error_data()['status'] ?? 0 ), array( 404, 409 ), true ) ) { $text = ''; // Shop turned AI crawlers off / not set up — remember for an hour. set_transient( self::TRANSIENT, $text, self::TTL ); } else { $stale = get_transient( self::STALE_TRANSIENT ); $text = is_string( $stale ) ? $stale : ''; } } return '' !== $text ? $this->renderer->localize( $text ) : ''; } public function full_text() { $shop = trim( $this->shop_text() ); if ( '' === $shop ) { return ''; } $posts = $this->posts_section(); return $shop . "\n" . ( '' !== $posts ? "\n" . $posts : '' ); } /** "## บทความและหน้า" — recent published posts/pages of this site (template page and noindex excluded). */ public function posts_section() { // Central + per-type template pages are dropped in PHP (a few ids) instead of a NOT IN query. $skip = array_map( 'intval', (array) $this->template->ids() ); $ids = get_posts( array( 'post_type' => array( 'post', 'page' ), 'post_status' => 'publish', 'posts_per_page' => self::MAX_POSTS + count( $skip ), 'orderby' => 'modified', 'order' => 'DESC', 'has_password' => false, 'fields' => 'ids', 'suppress_filters' => false, ) + ( function_exists( 'pll_default_language' ) ? array( 'lang' => pll_default_language() ) : array() ) ); $ids = array_slice( array_values( array_diff( array_map( 'intval', $ids ), $skip ) ), 0, self::MAX_POSTS ); $lines = array(); foreach ( $ids as $id ) { $rm = get_post_meta( $id, 'rank_math_robots', true ); if ( ( is_array( $rm ) && in_array( 'noindex', $rm, true ) ) || '1' === (string) get_post_meta( $id, '_yoast_wpseo_meta-robots-noindex', true ) ) { continue; } $title = trim( wp_strip_all_tags( html_entity_decode( get_the_title( $id ), ENT_QUOTES, 'UTF-8' ) ) ); if ( '' === $title ) { continue; } $lines[] = '- [' . str_replace( array( '[', ']' ), array( '(', ')' ), $title ) . '](' . get_permalink( $id ) . ')'; } $lines = (array) apply_filters( 'aooshop_llms_posts', $lines, $ids ); if ( ! $lines ) { return ''; } return '## ' . __( 'Posts and pages', 'aoo-shop' ) . "\n\n" . implode( "\n", $lines ) . "\n"; } /** Rank Math module on + owner opted in → append the shop section to Rank Math's llms.txt. */ public function rankmath_extra( $extra ) { if ( ! $this->live() ) { return $extra; } $shop = trim( $this->shop_text() ); if ( '' === $shop ) { return $extra; } // Rank Math prints its own H1 / site summary — drop aoo's "# name / > summary" header lines. $shop = preg_replace( '/\A#\s[^\n]*\n+(>[^\n]*\n+)?/u', '', $shop ); return trim( (string) $extra . "\n\n" . $shop ); } // ── robots.txt ──────────────────────────────────────────────────── /** Paths shop crawlers should skip (live prefixes). */ public function disallow_paths() { $shop = $this->settings->configured_paths()['prefix_shop']; return (array) apply_filters( 'aooshop_robots_disallow', array( $shop . '/search/', $shop . '/cart/' ) ); } public function robots_txt( $output, $public = true ) { if ( ! $this->live() || ! $public ) { return $output; } return self::add_disallow( (string) $output, $this->disallow_paths() ); } /** Insert `Disallow:` lines into every user-agent group (or append a `User-agent: *` group). */ public static function add_disallow( $robots, array $paths ) { $lines = preg_split( '/\r\n|\n|\r/', $robots ); $count = count( $lines ); $is_ua = function ( $line ) { return (bool) preg_match( '/^\s*user-agent\s*:/i', $line ); }; $out = array(); $groups = 0; for ( $i = 0; $i < $count; $i++ ) { $out[] = $lines[ $i ]; if ( ! $is_ua( $lines[ $i ] ) || ( $i + 1 < $count && $is_ua( $lines[ $i + 1 ] ) ) ) { continue; } // Last user-agent line of a group: collect the group's rules (until the next group starts). $groups++; $rules = ''; for ( $j = $i + 1; $j < $count && ! $is_ua( $lines[ $j ] ); $j++ ) { $rules .= strtolower( preg_replace( '/\s+/', '', $lines[ $j ] ) ) . "\n"; } foreach ( $paths as $p ) { if ( false === strpos( $rules, 'disallow:' . strtolower( $p ) . "\n" ) ) { $out[] = 'Disallow: ' . $p; } } } if ( 0 === $groups ) { $out[] = ''; $out[] = 'User-agent: *'; foreach ( $paths as $p ) { $out[] = 'Disallow: ' . $p; } } return implode( "\n", $out ); } /** * Parse robots.txt → which of these AI crawlers may NOT fetch the site root. * * @return string[] Blocked bot names. */ public static function blocked_ai_bots( $robots ) { $bots = array( 'GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'Claude-SearchBot', 'PerplexityBot', 'Google-Extended', 'CCBot', 'Applebot-Extended', 'meta-externalagent' ); $groups = self::parse_robots( $robots ); $blocked = array(); foreach ( $bots as $bot ) { $rules = self::rules_for( $groups, $bot ); if ( ! self::allowed( $rules, '/' ) ) { $blocked[] = $bot; } } return $blocked; } /** Is a path allowed for '*' (used to check our Disallow lines are effective in a real robots.txt file). */ public static function path_allowed( $robots, $path, $agent = '*' ) { return self::allowed( self::rules_for( self::parse_robots( $robots ), $agent ), $path ); } /** @return array}> */ private static function parse_robots( $robots ) { $groups = array(); $cur = null; $last = ''; foreach ( preg_split( '/\r\n|\n|\r/', (string) $robots ) as $line ) { $line = trim( preg_replace( '/#.*$/', '', $line ) ); if ( '' === $line || false === strpos( $line, ':' ) ) { continue; } list( $field, $value ) = array_map( 'trim', explode( ':', $line, 2 ) ); $field = strtolower( $field ); if ( 'user-agent' === $field ) { if ( 'ua' !== $last || null === $cur ) { $groups[] = array( 'agents' => array(), 'rules' => array() ); $cur = count( $groups ) - 1; } $groups[ $cur ]['agents'][] = strtolower( $value ); $last = 'ua'; } elseif ( ( 'allow' === $field || 'disallow' === $field ) && null !== $cur ) { $groups[ $cur ]['rules'][] = array( $field, $value ); $last = 'rule'; } else { $last = 'other'; } } return $groups; } private static function rules_for( array $groups, $agent ) { $agent = strtolower( $agent ); $specific = array(); $star = array(); foreach ( $groups as $g ) { if ( in_array( $agent, $g['agents'], true ) ) { $specific = array_merge( $specific, $g['rules'] ); } if ( in_array( '*', $g['agents'], true ) ) { $star = array_merge( $star, $g['rules'] ); } } return ( '*' !== $agent && $specific ) ? $specific : $star; } /** Google semantics: longest matching rule wins, Allow wins a tie; empty Disallow = allow. */ private static function allowed( array $rules, $path ) { $best_len = -1; $allowed = true; foreach ( $rules as $rule ) { list( $type, $pattern ) = $rule; if ( '' === $pattern ) { continue; } $regex = '#^' . str_replace( array( '\*', '\$' ), array( '.*', '$' ), preg_quote( $pattern, '#' ) ) . '#'; if ( ! preg_match( $regex, $path ) ) { continue; } $len = strlen( $pattern ); if ( $len > $best_len || ( $len === $best_len && 'allow' === $type ) ) { $best_len = $len; $allowed = 'allow' === $type; } } return $allowed; } }