description */ function aiprofile_inside_crawlers() { return array( 'GPTBot' => 'OpenAI', 'OAI-SearchBot' => 'ChatGPT search', 'ChatGPT-User' => 'ChatGPT', 'ClaudeBot' => 'Anthropic', 'Claude-SearchBot' => 'Claude search', 'PerplexityBot' => 'Perplexity', 'Google-Extended' => 'Gemini', 'Googlebot' => 'Google Search', 'Bingbot' => 'Bing and Copilot', ); } /** * Parse robots.txt into groups. * * @param string $text robots.txt. * @return array { groups: list of { agents: string[], rules: list of [ type, path ] }, other: string[] } */ function aiprofile_inside_robots_parse( $text ) { $groups = array(); $other = array(); $current = null; $in_rules = false; foreach ( preg_split( '/\r\n|\r|\n/', (string) $text ) as $line ) { $line = trim( (string) preg_replace( '/#.*$/', '', $line ) ); if ( '' === $line || false === strpos( $line, ':' ) ) { continue; } list( $field, $value ) = array_map( 'trim', explode( ':', $line, 2 ) ); $field = strtolower( $field ); if ( 'user-agent' === $field ) { if ( null === $current || $in_rules ) { $groups[] = array( 'agents' => array(), 'names' => array(), 'rules' => array(), ); $current = count( $groups ) - 1; $in_rules = false; } $groups[ $current ]['agents'][] = strtolower( $value ); $groups[ $current ]['names'][] = $value; } elseif ( 'allow' === $field || 'disallow' === $field ) { if ( null === $current ) { continue; // A rule before any user-agent belongs to nobody. } $groups[ $current ]['rules'][] = array( $field, $value ); $in_rules = true; } else { $other[] = $line; // Sitemap and friends. if ( null !== $current ) { $in_rules = true; } } } return array( 'groups' => $groups, 'other' => $other, ); } /** * The rules that apply to a crawler: every group naming it (merged), else * every "*" group. * * @param array $parsed From aiprofile_inside_robots_parse(). * @param string $token Crawler token, such as GPTBot. * @return array { rules: list of [ type, path ], own: bool } */ function aiprofile_inside_robots_rules_for( $parsed, $token ) { $token = strtolower( $token ); $own = array(); $star = array(); foreach ( $parsed['groups'] as $group ) { foreach ( $group['agents'] as $agent ) { if ( $agent === $token ) { $own = array_merge( $own, $group['rules'] ); continue 2; } if ( '*' === $agent ) { $star = array_merge( $star, $group['rules'] ); } } } $named = false; foreach ( $parsed['groups'] as $group ) { if ( in_array( $token, $group['agents'], true ) ) { $named = true; } } return array( 'rules' => $named ? $own : $star, 'own' => $named, ); } /** * Does a robots.txt path pattern match a path? Supports * and a final $. * * @param string $pattern Rule path. * @param string $path URL path. * @return bool */ function aiprofile_inside_robots_match( $pattern, $path ) { if ( '' === $pattern ) { return false; } $anchored = '$' === substr( $pattern, -1 ); $core = $anchored ? substr( $pattern, 0, -1 ) : $pattern; $regex = '#^' . str_replace( '\*', '.*', preg_quote( $core, '#' ) ) . ( $anchored ? '$' : '' ) . '#'; return 1 === preg_match( $regex, $path ); } /** * Is a path blocked by these rules? Longest match wins, Allow wins a tie. * * @param array $rules List of [ type, path ]. * @param string $path URL path. * @return bool */ function aiprofile_inside_robots_blocked( $rules, $path ) { $best_len = -1; $best = 'allow'; foreach ( $rules as $rule ) { list( $type, $pattern ) = $rule; if ( ! aiprofile_inside_robots_match( $pattern, $path ) ) { continue; } $len = strlen( $pattern ); if ( $len > $best_len || ( $len === $best_len && 'allow' === $type ) ) { $best_len = $len; $best = $type; } } return 'disallow' === $best; } /** * The robots.txt the site serves, and where it comes from. * * @return array { text: string, physical: bool } */ function aiprofile_inside_robots_source() { $file = aiprofile_inside_home_path() . 'robots.txt'; if ( file_exists( $file ) ) { if ( ! class_exists( 'WP_Filesystem_Direct' ) ) { require_once ABSPATH . 'wp-admin/includes/class-wp-filesystem-base.php'; require_once ABSPATH . 'wp-admin/includes/class-wp-filesystem-direct.php'; } $fs = new WP_Filesystem_Direct( null ); return array( 'text' => (string) $fs->get_contents( $file ), 'physical' => true, ); } // What do_robots() prints, without printing it. $public = (string) get_option( 'blog_public' ); $output = "User-agent: *\n"; if ( '0' === $public ) { $output .= "Disallow: /\n"; } else { // From admin_url(), so a site with WordPress in a subfolder (or a // filtered admin URL) gets its real paths. $admin_path = (string) wp_parse_url( admin_url(), PHP_URL_PATH ); $ajax_path = (string) wp_parse_url( admin_url( 'admin-ajax.php' ), PHP_URL_PATH ); $output .= "Disallow: $admin_path\n"; $output .= "Allow: $ajax_path\n"; } return array( 'text' => (string) apply_filters( 'robots_txt', $output, $public ), // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WordPress core's own filter, as do_robots() applies it. 'physical' => false, ); } /** * The paths the check tests: the home page and the answers page. * * @return string[] */ function aiprofile_inside_robots_paths() { $paths = array( '/' === (string) wp_parse_url( home_url( '/' ), PHP_URL_PATH ) ? '/' : (string) wp_parse_url( home_url( '/' ), PHP_URL_PATH ) ); $settings = aiprofile_inside_settings(); if ( ! empty( $settings['page_id'] ) ) { $page = (string) wp_parse_url( (string) get_permalink( (int) $settings['page_id'] ), PHP_URL_PATH ); if ( '' !== $page ) { $paths[] = $page; } } return array_values( array_unique( $paths ) ); } /** * The check: which crawlers are blocked, and whether we may fix it. * * @return array { blocked: token => description, physical: bool, discouraged: bool, fixed: bool } */ function aiprofile_inside_robots_check() { $source = aiprofile_inside_robots_source(); $parsed = aiprofile_inside_robots_parse( $source['text'] ); $paths = aiprofile_inside_robots_paths(); $blocked = array(); foreach ( aiprofile_inside_crawlers() as $token => $what ) { $rules = aiprofile_inside_robots_rules_for( $parsed, $token ); foreach ( $paths as $path ) { if ( aiprofile_inside_robots_blocked( $rules['rules'], $path ) ) { $blocked[ $token ] = $what; break; } } } return array( 'blocked' => $blocked, 'physical' => $source['physical'], 'discouraged' => '0' === (string) get_option( 'blog_public' ), 'fixed' => ! $source['physical'] && (bool) aiprofile_inside_option( 'robots_fix' ), ); } /** * The fix, as a late `robots_txt` filter: every crawler that the rules block * from the home page or the answers page gets a group of its own, with the * rules it had minus the ones that block those pages. Its name leaves any * group it shared with others. Comments, sitemaps and every other crawler's * rules stay as they were. * * @param string $output robots.txt so far. * @param string $public blog_public. * @return string */ function aiprofile_inside_robots_fix( $output, $public ) { if ( '0' === (string) $public || ! aiprofile_inside_option( 'robots_fix' ) ) { return $output; } $parsed = aiprofile_inside_robots_parse( $output ); $paths = aiprofile_inside_robots_paths(); $extra = ''; $retake = array(); foreach ( array_keys( aiprofile_inside_crawlers() ) as $token ) { $rules = aiprofile_inside_robots_rules_for( $parsed, $token ); $blocked = false; foreach ( $paths as $path ) { if ( aiprofile_inside_robots_blocked( $rules['rules'], $path ) ) { $blocked = true; } } if ( ! $blocked ) { continue; } $kept = array(); foreach ( $rules['rules'] as $rule ) { $blocks = false; if ( 'disallow' === $rule[0] ) { foreach ( $paths as $path ) { if ( aiprofile_inside_robots_match( $rule[1], $path ) ) { $blocks = true; } } } if ( ! $blocks ) { $kept[] = ucfirst( $rule[0] ) . ': ' . $rule[1]; } } if ( empty( $kept ) ) { $kept[] = 'Allow: /'; } $extra .= "\n# AiProfile Inside: " . $token . " may read this site (the owner's choice).\nUser-agent: " . $token . "\n" . implode( "\n", $kept ) . "\n"; if ( $rules['own'] ) { $retake[] = strtolower( $token ); } } if ( '' === $extra ) { return $output; } if ( ! empty( $retake ) ) { // A crawler with a group of its own: write the file again without its // name in those groups. Removing only the line would hand the group's // rules to the group above it. $output = aiprofile_inside_robots_write( $parsed, $retake ); } return rtrim( $output ) . "\n" . $extra; } /** * Write parsed robots.txt back, leaving the given crawlers out of every * group. A group with no crawler left goes with its rules. * * @param array $parsed From aiprofile_inside_robots_parse(). * @param string[] $leave Lowercase tokens to leave out. * @return string */ function aiprofile_inside_robots_write( $parsed, $leave ) { $out = array(); foreach ( $parsed['groups'] as $group ) { $names = array(); foreach ( $group['agents'] as $i => $agent ) { if ( ! in_array( $agent, $leave, true ) ) { $names[] = $group['names'][ $i ]; } } if ( empty( $names ) ) { continue; } foreach ( $names as $name ) { $out[] = 'User-agent: ' . $name; } foreach ( $group['rules'] as $rule ) { $out[] = ucfirst( $rule[0] ) . ': ' . $rule[1]; } $out[] = ''; } foreach ( $parsed['other'] as $line ) { $out[] = $line; } return implode( "\n", $out ) . "\n"; } add_filter( 'robots_txt', 'aiprofile_inside_robots_fix', 9999, 2 );