$policies Crawler slug to policy map. * @param string $override Custom override, the whole document. * @return string The result. */ public function build( string $base, bool $isPublic, array $policies, string $override ): string { if ( ! $isPublic ) { return $base; } if ( '' !== trim( $override ) ) { return rtrim( trim( $override ) ) . "\n"; } $text = rtrim( $base ); $block = $this->crawlerBlock( $policies ); if ( '' !== $block ) { $text = $this->insertAboveWildcard( $text, $block ); } $text = trim( $text ); return ( '' === $text ) ? '' : $text . "\n"; } /** * Crawler directive groups for blocked and explicitly allowed crawlers. * * @param array $policies Crawler slug to policy map. * @return string The result. */ private function crawlerBlock( array $policies ): string { $groups = []; foreach ( CrawlerPolicy::all() as $slug => $crawler ) { $slug = (string) $slug; $policy = CrawlerPolicy::sanitizePolicy( $policies[ $slug ] ?? CrawlerPolicy::CUSTOM ); $token = $this->safeToken( (string) $crawler['token'] ); if ( CrawlerPolicy::BLOCK === $policy ) { $groups[] = 'User-agent: ' . $token . "\nDisallow: /"; } elseif ( CrawlerPolicy::ALLOW === $policy ) { $groups[] = 'User-agent: ' . $token . "\nAllow: /"; } } return implode( "\n\n", $groups ); } /** * Sanitise a crawler token for the User-agent line. * * The crawler catalogue is filterable through rankkernel/robots/crawlers, * so the sink validates the token even though CrawlerPolicy also normalises * the catalogue. A CR or LF would otherwise inject an arbitrary robots.txt * directive. Only the characters that appear in real user agent tokens * survive the class, so a stray line break can never start a new line. * * @param string $token Raw token. * @return string The result. */ private function safeToken( string $token ): string { $token = str_replace( [ "\r", "\n" ], '', $token ); $clean = preg_replace( '/[^A-Za-z0-9._* -]/', '', $token ); return is_string( $clean ) ? $clean : ''; } /** * Insert a block directly above the wildcard user-agent group. * * @param string $base Base text. * @param string $block Block to insert. * @return string The result. */ private function insertAboveWildcard( string $base, string $block ): string { $matched = preg_match( '/^User-agent:\s*\*/mi', $base, $matches, PREG_OFFSET_CAPTURE ); if ( 1 === $matched ) { $offset = (int) $matches[0][1]; $before = rtrim( substr( $base, 0, $offset ) ); $after = substr( $base, $offset ); $head = ( '' === $before ) ? $block : $before . "\n\n" . $block; return $head . "\n\n" . $after; } return ( '' === $base ) ? $block : $block . "\n\n" . $base; } }