[ 'label' => 'ChatGPT (OpenAI)', 'description' => 'Used to train OpenAI models and power ChatGPT search citations.', 'icon' => '🟢', 'category' => 'training_and_search', ], 'ChatGPT-User' => [ 'label' => 'ChatGPT Browsing', 'description' => 'Used when a ChatGPT user asks it to browse your specific page live.', 'icon' => '🟢', 'category' => 'live_browsing', ], 'ClaudeBot' => [ 'label' => 'Claude (Anthropic)', 'description' => 'Used to train Anthropic\'s Claude models.', 'icon' => '🟠', 'category' => 'training_and_search', ], 'anthropic-ai' => [ 'label' => 'Anthropic AI (legacy)', 'description' => 'Older Anthropic crawler, still active on some systems.', 'icon' => '🟠', 'category' => 'training_and_search', ], 'Google-Extended' => [ 'label' => 'Google Gemini / AI Overviews', 'description' => 'Controls whether Google can use your content for Gemini and AI Overviews (separate from normal Googlebot search indexing).', 'icon' => '🔵', 'category' => 'training_and_search', ], 'PerplexityBot' => [ 'label' => 'Perplexity AI', 'description' => 'Crawls and cites content directly in Perplexity search answers.', 'icon' => '🟣', 'category' => 'live_browsing', ], 'CCBot' => [ 'label' => 'Common Crawl', 'description' => 'A shared dataset used to train many AI models including some open-source LLMs.', 'icon' => '⚪', 'category' => 'training_and_search', ], 'Bytespider' => [ 'label' => 'ByteDance (TikTok AI)', 'description' => 'Used by ByteDance for AI training, including TikTok-related AI features.', 'icon' => '⚫', 'category' => 'training_and_search', ], ]; } // ── ADMIN MENU ───────────────────────────────────────── public function add_menu() { add_submenu_page( 'shadowsync-seo-dashboard', __( 'AI Bot Access', 'shadowsync-seo' ), __( '👁️ AI Bot Access', 'shadowsync-seo' ), 'manage_options', 'shadowsync-seo-bot-manager', [ $this, 'render_page' ] ); } // ── GET SAVED RULES ──────────────────────────────────── private function get_rules() { $defaults = []; foreach ( array_keys( self::get_bots() ) as $bot ) { $defaults[ $bot ] = 'allow'; // default: allow all AI bots (good for GEO visibility) } $saved = get_option( 'shadowsync_seo_bot_rules', [] ); return wp_parse_args( $saved, $defaults ); } // ── APPLY RULES TO ROBOTS.TXT ────────────────────────── private function apply_rules_to_robots( $rules ) { if ( ! class_exists( 'ShadowSync_SEO_Robots_Txt' ) ) { return; } $base_content = get_option( 'shadowsync_seo_robots_txt_content', '' ); if ( empty( $base_content ) ) { $sitemap_url = home_url( '/shadowsync-seo-sitemap.xml' ); $base_content = "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n\nSitemap: " . $sitemap_url; } // Strip any previously auto-generated AI bot block $base_content = preg_replace( '/\n?# BEGIN ShadowSync SEO BOT RULES.*?# END ShadowSync SEO BOT RULES\n?/s', '', $base_content ); $bot_lines = []; $bot_lines[] = '# BEGIN ShadowSync SEO BOT RULES'; foreach ( $rules as $bot => $action ) { $bot_lines[] = ''; $bot_lines[] = 'User-agent: ' . $bot; $bot_lines[] = ( 'block' === $action ) ? 'Disallow: /' : 'Allow: /'; } $bot_lines[] = ''; $bot_lines[] = '# END ShadowSync SEO BOT RULES'; $new_content = rtrim( $base_content ) . "\n\n" . implode( "\n", $bot_lines ); update_option( 'shadowsync_seo_robots_txt_content', $new_content ); } // ── AJAX SAVE ─────────────────────────────────────────── public function save_bot_rules() { check_ajax_referer( 'shadowsync_seo_scan_nonce', 'nonce' ); ShadowSync_SEO_Security::require_admin(); $bots = array_keys( self::get_bots() ); // map_deep() sanitizes every leaf, so the array is clean at the point // of access rather than only where individual keys are read below. $raw_rules = isset( $_POST['rules'] ) ? map_deep( (array) wp_unslash( $_POST['rules'] ), 'sanitize_text_field' ) : []; $clean_rules = []; foreach ( $bots as $bot ) { $value = isset( $raw_rules[ $bot ] ) ? sanitize_text_field( $raw_rules[ $bot ] ) : 'allow'; $clean_rules[ $bot ] = in_array( $value, [ 'allow', 'block' ], true ) ? $value : 'allow'; } update_option( 'shadowsync_seo_bot_rules', $clean_rules ); $this->apply_rules_to_robots( $clean_rules ); // Clear llms.txt cache too in case bot policy affects messaging delete_transient( 'shadowsync_seo_llms_txt_cache' ); wp_send_json_success( [ 'message' => __( '✅ AI bot access rules saved and applied to robots.txt.', 'shadowsync-seo' ), ] ); } // ── RENDER ADMIN PAGE ───────────────────────────────── public function render_page() { $rules = $this->get_rules(); $bots = self::get_bots(); $training_bots = array_filter( $bots, function( $b ) { return 'training_and_search' === $b['category']; } ); $live_bots = array_filter( $bots, function( $b ) { return 'live_browsing' === $b['category']; } ); ?>