sanitizeRules($custom)) . "\n"; } } $aiRules = Robots\AiCrawlers::robotsRules(); if ('' !== $aiRules) { $output = rtrim($output, "\n") . "\n" . $aiRules; } return $output; } /** * Strip anything that is not a plausible robots.txt directive. * * The field is admin-only, so this is not a security boundary — it is there * to stop a stray paste of HTML or a sitemap URL list turning the served file * into something no crawler will parse. */ protected function sanitizeRules($rules) { $allowed = ['user-agent', 'disallow', 'allow', 'crawl-delay', 'sitemap', 'host', 'clean-param', 'noindex', 'request-rate', 'visit-time']; $clean = []; foreach (preg_split('/\R/', (string) $rules) as $line) { $line = trim(wp_strip_all_tags($line)); if ('' === $line) { $clean[] = ''; continue; } if (0 === strpos($line, '#')) { $clean[] = $line; continue; } $directive = strtolower(trim(strtok($line, ':'))); if (in_array($directive, $allowed, true)) { $clean[] = $line; } } // Collapse the runs of blank lines that dropping invalid lines leaves. return preg_replace("/\n{3,}/", "\n\n", implode("\n", $clean)); } /** * Whether a real robots.txt exists on disk. * * WordPress only serves the virtual file when no physical one is present, so * a site with one silently ignores everything configured here. The settings * UI has to say so rather than letting the user edit a file nobody serves. */ public static function physicalFileExists() { $path = trailingslashit(ABSPATH) . 'robots.txt'; return file_exists($path) && is_file($path); } /** * The robots.txt as it will actually be served, for the settings preview. */ public static function preview() { // Deliberately not `do_robots()`. That function sends a text/plain // Content-Type header, and headers are not captured by output buffering — // calling it from the settings page, where HTML has already been sent, // logs a "headers already sent" warning on every page load. // // What is reproduced instead is exactly the rest of its body, so the // preview still runs through `robots_txt` and shows what will really be // served, including any other plugin's additions. do_action('do_robotstxt'); $public = (bool) get_option('blog_public'); $sitePath = (string) wp_parse_url(site_url(), PHP_URL_PATH); $output = "User-agent: *\n"; $output .= "Disallow: {$sitePath}/wp-admin/\n"; $output .= "Allow: {$sitePath}/wp-admin/admin-ajax.php\n"; return trim((string) apply_filters('robots_txt', $output, $public)); } }