<?php
/**
 * Manages AI-crawler access signals in robots.txt and parses robots.txt
 * files to tell whether a given crawler is blocked.
 *
 * @package Evoka
 */

if ( ! defined( 'ABSPATH' ) ) {
	exit;
}

/**
 * Adds robots.txt directives for the major AI crawlers based on the chosen
 * policy:
 *
 * - allow: every AI crawler may read the site.
 * - track: AI search / answer crawlers (which fetch pages so assistants can
 *          cite them) are allowed; model-training crawlers are disallowed.
 *          This is the only opt-out vendors actually honour - there is no
 *          standard meta tag for "index but don't train".
 * - block: every AI crawler is disallowed.
 */
class Evoka_Crawler_Controls {

	/**
	 * Marker line written into robots.txt so the audit can tell whether our
	 * rules are actually being served.
	 */
	const MARKER = '# Evoka: AI crawler policy';

	/**
	 * Crawlers that collect content for model training.
	 *
	 * @return array
	 */
	public static function training_bots() {
		return array( 'GPTBot', 'ClaudeBot', 'Google-Extended', 'Applebot-Extended', 'CCBot' );
	}

	/**
	 * Crawlers / fetchers that power AI search answers and citations.
	 *
	 * @return array
	 */
	public static function search_bots() {
		return array( 'OAI-SearchBot', 'ChatGPT-User', 'PerplexityBot', 'Perplexity-User', 'Claude-SearchBot', 'Claude-User' );
	}

	/**
	 * Every AI crawler we manage.
	 *
	 * @return array
	 */
	public static function all_bots() {
		return array_merge( self::training_bots(), self::search_bots() );
	}

	/**
	 * The robots.txt URL crawlers actually read: always the root of the host,
	 * even when WordPress lives in a subfolder (e.g. http://localhost/robots.txt
	 * for a site at http://localhost/demo).
	 *
	 * @return string
	 */
	public static function robots_url() {
		$parts = wp_parse_url( home_url() );
		$url   = ( isset( $parts['scheme'] ) ? $parts['scheme'] : 'http' ) . '://' . ( isset( $parts['host'] ) ? $parts['host'] : '' );
		if ( ! empty( $parts['port'] ) ) {
			$url .= ':' . $parts['port'];
		}
		return $url . '/robots.txt';
	}

	/**
	 * Whether WordPress itself can serve /robots.txt for this install. It
	 * cannot when it is installed in a subfolder (crawlers only read the host
	 * root) or when permalinks are "Plain" (no rewrite rule for robots.txt).
	 *
	 * @return bool
	 */
	public static function wp_serves_robots() {
		$path = (string) wp_parse_url( home_url(), PHP_URL_PATH );
		return ( '' === trim( $path, '/' ) ) && '' !== (string) get_option( 'permalink_structure' );
	}

	/**
	 * The robots.txt lines for a policy (without the leading blank line).
	 *
	 * @param string $policy allow | track | block.
	 * @return string
	 */
	public static function rules( $policy ) {
		$lines = array( self::MARKER . ' (' . $policy . ')' );

		foreach ( self::all_bots() as $bot ) {
			if ( 'block' === $policy ) {
				$allowed = false;
			} elseif ( 'track' === $policy ) {
				$allowed = in_array( $bot, self::search_bots(), true );
			} else {
				$allowed = true;
			}
			$lines[] = 'User-agent: ' . $bot;
			$lines[] = $allowed ? 'Allow: /' : 'Disallow: /';
			$lines[] = '';
		}

		return implode( "\n", $lines );
	}

	/**
	 * Append our rules to the virtual robots.txt WordPress serves.
	 *
	 * @param string $output    Existing robots.txt output.
	 * @param bool   $is_public Whether the site is public.
	 * @return string
	 */
	public function filter_robots_txt( $output, $is_public ) {
		if ( ! $is_public ) {
			return $output; // Respect "discourage search engines".
		}

		$policy = evoka_get_setting( 'crawler_policy', 'allow' );

		return rtrim( $output ) . "\n\n" . self::rules( $policy ) . "\n";
	}

	/*
	 * ---------------------------------------------------------------------
	 * robots.txt parsing (RFC 9309)
	 * ---------------------------------------------------------------------
	 */

	/**
	 * Parse robots.txt into groups of {agents: string[], rules: [type, path][]}.
	 *
	 * Consecutive user-agent lines share one group; a user-agent line that
	 * follows a rule starts a new group. Comments and unknown lines are ignored.
	 *
	 * @param string $robots robots.txt body.
	 * @return array
	 */
	public static function parse( $robots ) {
		$groups  = array();
		$current = null;
		$in_ua   = false;

		foreach ( preg_split( '/\r\n|\r|\n/', (string) $robots ) as $line ) {
			$line = trim( preg_replace( '/#.*$/', '', $line ) );
			if ( '' === $line || false === strpos( $line, ':' ) ) {
				continue;
			}
			list( $field, $value ) = array_map( 'trim', explode( ':', $line, 2 ) );
			$field                 = strtolower( $field );

			if ( 'user-agent' === $field ) {
				if ( ! $in_ua || null === $current ) {
					$groups[] = array(
						'agents' => array(),
						'rules'  => array(),
					);
					$current  = count( $groups ) - 1;
				}
				$groups[ $current ]['agents'][] = strtolower( $value );
				$in_ua                          = true;
			} elseif ( ( 'allow' === $field || 'disallow' === $field ) && null !== $current ) {
				$groups[ $current ]['rules'][] = array( $field, $value );
				$in_ua                         = false;
			}
		}

		return $groups;
	}

	/**
	 * Is a crawler disallowed from the whole site (the "/" path)?
	 *
	 * Uses the group(s) naming the bot, falling back to "*" groups. Among the
	 * rules that match "/", the longest path wins and Allow wins ties.
	 *
	 * @param string $robots robots.txt body ('' = no robots.txt = allowed).
	 * @param string $bot    User-agent token.
	 * @return bool
	 */
	public static function is_blocked( $robots, $bot ) {
		$groups = self::parse( $robots );
		$bot    = strtolower( $bot );

		$rules = array();
		foreach ( $groups as $g ) {
			if ( in_array( $bot, $g['agents'], true ) ) {
				$rules = array_merge( $rules, $g['rules'] );
			}
		}
		if ( empty( $rules ) ) {
			foreach ( $groups as $g ) {
				if ( in_array( '*', $g['agents'], true ) ) {
					$rules = array_merge( $rules, $g['rules'] );
				}
			}
		}

		$best_len   = -1;
		$best_allow = true;
		foreach ( $rules as $rule ) {
			list( $type, $path ) = $rule;
			if ( '' === $path ) {
				continue; // "Disallow:" with an empty value allows everything.
			}
			// Only rules covering every path matter for "whole site" ("/$" covers just the home page).
			if ( ! in_array( $path, array( '/', '/*', '*' ), true ) ) {
				continue;
			}
			$len   = strlen( $path );
			$allow = ( 'allow' === $type );
			if ( $len > $best_len || ( $len === $best_len && $allow ) ) {
				$best_len   = $len;
				$best_allow = $allow;
			}
		}

		return ! $best_allow;
	}
}