<?php
/**
 * UTF-8 word tokenizer with stop-word and stemming support.
 *
 * @package Postlinkly
 * @since   1.0.0
 */

declare(strict_types=1);

namespace Postlinkly\Text;

/**
 * UTF-8 word tokenizer with stop-word filtering and stemming.
 *
 * Wraps a Stemmer and (optionally) an IgnoreWords list. Provides three
 * levels of tokenisation:
 *
 *   1. {@see split()}            — raw words, lower-cased, stripped of
 *                                  trailing/leading punctuation.
 *   2. {@see split_meaningful()} — `split()` minus stop-words and numbers.
 *   3. {@see stem_sentence()}    — full sentence reduced to space-joined stems.
 *
 * The class is request-scoped and holds an internal cache for stemmed
 * sentences. The cache is bounded so a long-running scan can't consume
 * unbounded memory.
 *
 * @since 1.0.0
 */
final class WordTokenizer {

	/**
	 * Maximum number of stemmed sentences to keep cached. When the cache
	 * exceeds this size the oldest entry is evicted (FIFO).
	 */
	private const SENTENCE_CACHE_LIMIT = 1000;

	private Stemmer $stemmer;

	private IgnoreWords $ignore_words;

	private bool $skip_numbers;

	/**
	 * Cached stem-sentence results keyed by lower-cased input.
	 *
	 * @var array<string, string>
	 */
	private array $sentence_cache = array();

	/**
	 * @since 1.0.0
	 *
	 * @param Stemmer     $stemmer      The stemmer to apply during stemming.
	 * @param IgnoreWords $ignore_words The stop-word loader.
	 * @param bool        $skip_numbers Optional. Whether {@see split_meaningful()}
	 *                                  should drop numeric tokens. Default true.
	 */
	public function __construct(
		Stemmer $stemmer,
		IgnoreWords $ignore_words,
		bool $skip_numbers = true
	) {
		$this->stemmer      = $stemmer;
		$this->ignore_words = $ignore_words;
		$this->skip_numbers = $skip_numbers;
	}

	/**
	 * Splits a sentence into raw lower-cased words.
	 *
	 * Splits on Unicode whitespace and the punctuation set
	 * `. , ; : ! ? " ' ` ( ) [ ] { } | < > / \\`. Empty tokens are
	 * dropped. Quotation marks adjacent to letters (apostrophes within
	 * contractions like "don't") are preserved.
	 *
	 * @since 1.0.0
	 *
	 * @param string $sentence The input sentence.
	 *
	 * @return array<int, string> Lower-cased word tokens.
	 */
	public function split( string $sentence ): array {
		return $this->split_tokens( $sentence, true );
	}

	/**
	 * Splits a sentence into words exactly like {@see split()}, but keeps
	 * each word's original casing instead of lower-casing it.
	 *
	 * Same punctuation/whitespace rules as {@see split()}, so calling both
	 * methods on the same sentence yields two arrays of equal length whose
	 * items line up index-for-index — the only difference is casing. This
	 * exists for callers that need to match case-insensitively (via
	 * {@see split()} or a stem) but must then display or insert the
	 * matched text exactly as it appeared in the source, such as the
	 * anchor text a suggested link is built from.
	 *
	 * @since 1.6.0
	 *
	 * @param string $sentence The input sentence.
	 *
	 * @return array<int, string> Word tokens in their original casing.
	 */
	public function split_preserving_case( string $sentence ): array {
		return $this->split_tokens( $sentence, false );
	}

	/**
	 * Shared implementation behind {@see split()} and
	 * {@see split_preserving_case()}.
	 *
	 * @since 1.6.0
	 *
	 * @param string $sentence  The input sentence.
	 * @param bool   $lowercase Whether to lower-case before splitting.
	 *
	 * @return array<int, string> Word tokens.
	 */
	private function split_tokens( string $sentence, bool $lowercase ): array {
		if ( '' === trim( $sentence ) ) {
			return array();
		}

		$normalised = $this->normalise_whitespace( $sentence );
		if ( $lowercase ) {
			$normalised = mb_strtolower( $normalised, 'UTF-8' );
		}

		// Split on whitespace OR runs of unwanted punctuation.
		// Apostrophes inside words (e.g. don't, l'amour) are preserved
		// because they are not in the split character class.
		$parts = preg_split( '/[\s.,;:!?"`()\[\]{}|<>\/\\\\]+/u', $normalised );
		if ( false === $parts ) {
			return array();
		}

		$out = array();
		foreach ( $parts as $part ) {
			$part = trim( $part, "'\"`-_" );
			if ( '' === $part ) {
				continue;
			}
			$out[] = $part;
		}
		return $out;
	}

	/**
	 * Splits a sentence into meaningful words: raw words minus stop-words.
	 *
	 * Stop-words are looked up against the language code reported by the
	 * configured Stemmer. Numeric tokens are dropped when
	 * `$skip_numbers` was set in the constructor.
	 *
	 * @since 1.0.0
	 *
	 * @param string $sentence The input sentence.
	 *
	 * @return array<int, string> Meaningful word tokens, lower-cased.
	 */
	public function split_meaningful( string $sentence ): array {
		$words  = $this->split( $sentence );
		$lookup = $this->ignore_words->lookup_for_language( $this->stemmer->get_language_code() );

		$out = array();
		foreach ( $words as $word ) {
			if ( isset( $lookup[ $word ] ) ) {
				continue;
			}
			if ( $this->skip_numbers && self::is_numeric_token( $word ) ) {
				continue;
			}
			$out[] = $word;
		}
		return $out;
	}

	/**
	 * Counts words in a sentence using the same definition as {@see split()}.
	 *
	 * Empty input returns 0.
	 *
	 * @since 1.0.0
	 *
	 * @param string $sentence The input sentence.
	 *
	 * @return int Word count.
	 */
	public function count_words( string $sentence ): int {
		return count( $this->split( $sentence ) );
	}

	/**
	 * Returns the meaningful words of a sentence, each replaced by its stem.
	 *
	 * Order is preserved. Duplicates are kept (use array_unique on the
	 * result if you want a set).
	 *
	 * @since 1.0.0
	 *
	 * @param string $sentence The input sentence.
	 *
	 * @return array<int, string> Stemmed, meaningful words.
	 */
	public function stem_words( string $sentence ): array {
		$words = $this->split_meaningful( $sentence );
		$out   = array();
		foreach ( $words as $word ) {
			$out[] = $this->stemmer->stem( $word );
		}
		return $out;
	}

	/**
	 * Returns a sentence reduced to a space-joined string of stemmed words.
	 *
	 * Result is cached on the lower-cased input so repeated calls with the
	 * same sentence are free. The cache is bounded — when full, the oldest
	 * entry is evicted.
	 *
	 * @since 1.0.0
	 *
	 * @param string $sentence The input sentence.
	 *
	 * @return string Space-joined stems, lower-cased.
	 */
	public function stem_sentence( string $sentence ): string {
		$key = mb_strtolower( trim( $sentence ), 'UTF-8' );
		if ( '' === $key ) {
			return '';
		}

		if ( isset( $this->sentence_cache[ $key ] ) ) {
			return $this->sentence_cache[ $key ];
		}

		$stemmed = implode( ' ', $this->stem_words( $sentence ) );

		// Bound cache growth.
		if ( count( $this->sentence_cache ) >= self::SENTENCE_CACHE_LIMIT ) {
			array_shift( $this->sentence_cache );
		}
		$this->sentence_cache[ $key ] = $stemmed;
		return $stemmed;
	}

	/**
	 * Returns the unique stemmed words of a sentence as a count map.
	 *
	 * Useful as the input to set-overlap calculations in the suggestion
	 * engine. Keys are stems; values are occurrence counts.
	 *
	 * @since 1.0.0
	 *
	 * @param string $sentence The input sentence.
	 *
	 * @return array<string, int> Stemmed word counts.
	 */
	public function unique_stems( string $sentence ): array {
		$out = array();
		foreach ( $this->stem_words( $sentence ) as $stem ) {
			if ( '' === $stem ) {
				continue;
			}
			$out[ $stem ] = ( $out[ $stem ] ?? 0 ) + 1;
		}
		return $out;
	}

	/**
	 * Lower-cases a UTF-8 string in a Unicode-aware way.
	 *
	 * Wraps `mb_strtolower` so call sites don't have to remember the encoding.
	 *
	 * @since 1.0.0
	 *
	 * @param string $value Input string.
	 *
	 * @return string Lower-cased string.
	 */
	public static function strtolower( string $value ): string {
		return mb_strtolower( $value, 'UTF-8' );
	}

	/**
	 * Strips accents from a UTF-8 string, returning ASCII-friendly text.
	 *
	 * Used when building accent-normalised keyword variants. Falls back to
	 * an explicit character map when WordPress's `remove_accents()` is
	 * unavailable (e.g. unit tests) — the iconv path is unreliable on some
	 * Windows builds, so we use a deterministic table.
	 *
	 * @since 1.0.0
	 *
	 * @param string $value Input string.
	 *
	 * @return string Accent-stripped string.
	 */
	public static function remove_accents( string $value ): string {
		if ( function_exists( 'remove_accents' ) ) {
			return remove_accents( $value );
		}

		// Pure-PHP fallback covering Latin-1 Supplement and Latin Extended-A
		// characters common in Western European content. Used only when
		// WordPress is not loaded (test runs).
		return strtr( $value, self::accent_map() );
	}

	/**
	 * Returns the diacritic-to-ASCII fallback map used by remove_accents().
	 *
	 * Covers the Latin-1 Supplement and Latin Extended-A ranges common in
	 * Western European content. Used only when WordPress's own
	 * `remove_accents()` is not loaded (test runs).
	 *
	 * @since 1.0.0
	 *
	 * @return array<string, string>
	 */
	private static function accent_map(): array {
		return array(
			'À' => 'A',
			'Á' => 'A',
			'Â' => 'A',
			'Ã' => 'A',
			'Ä' => 'A',
			'Å' => 'A',
			'Æ' => 'AE',
			'Ç' => 'C',
			'È' => 'E',
			'É' => 'E',
			'Ê' => 'E',
			'Ë' => 'E',
			'Ì' => 'I',
			'Í' => 'I',
			'Î' => 'I',
			'Ï' => 'I',
			'Ð' => 'D',
			'Ñ' => 'N',
			'Ò' => 'O',
			'Ó' => 'O',
			'Ô' => 'O',
			'Õ' => 'O',
			'Ö' => 'O',
			'Ø' => 'O',
			'Ù' => 'U',
			'Ú' => 'U',
			'Û' => 'U',
			'Ü' => 'U',
			'Ý' => 'Y',
			'Þ' => 'TH',
			'ß' => 'ss',
			'à' => 'a',
			'á' => 'a',
			'â' => 'a',
			'ã' => 'a',
			'ä' => 'a',
			'å' => 'a',
			'æ' => 'ae',
			'ç' => 'c',
			'è' => 'e',
			'é' => 'e',
			'ê' => 'e',
			'ë' => 'e',
			'ì' => 'i',
			'í' => 'i',
			'î' => 'i',
			'ï' => 'i',
			'ð' => 'd',
			'ñ' => 'n',
			'ò' => 'o',
			'ó' => 'o',
			'ô' => 'o',
			'õ' => 'o',
			'ö' => 'o',
			'ø' => 'o',
			'ù' => 'u',
			'ú' => 'u',
			'û' => 'u',
			'ü' => 'u',
			'ý' => 'y',
			'ÿ' => 'y',
			'þ' => 'th',
			'Œ' => 'OE',
			'œ' => 'oe',
			'Š' => 'S',
			'š' => 's',
			'Ÿ' => 'Y',
			'Ž' => 'Z',
			'ž' => 'z',
		);
	}

	/**
	 * Normalises whitespace and decoded entities to single ASCII spaces.
	 *
	 * Replaces non-breaking spaces, soft hyphens, smart quotes, and runs
	 * of any whitespace with a single space.
	 *
	 * @since 1.0.0
	 *
	 * @param string $sentence Input.
	 *
	 * @return string Normalised string.
	 */
	private function normalise_whitespace( string $sentence ): string {
		// Smart quotes → straight quotes (helps the apostrophe-preserving split).
		$sentence = strtr(
			$sentence,
			array(
				"\u{2018}" => "'",
				"\u{2019}" => "'",
				"\u{201C}" => '"',
				"\u{201D}" => '"',
			)
		);
		// Non-breaking space + soft hyphen → space.
		$sentence = str_replace( array( "\u{00A0}", "\u{00AD}" ), ' ', $sentence );
		// HTML entity for non-breaking space, in case raw markup leaks through.
		$sentence = str_replace( '&nbsp;', ' ', $sentence );
		// Collapse runs of whitespace.
		$collapsed = preg_replace( '/\s+/u', ' ', $sentence );
		return null === $collapsed ? $sentence : trim( $collapsed );
	}

	/**
	 * Returns whether a token is a numeric value (integer, decimal, currency).
	 *
	 * Strips a leading dollar sign and any thousands separators, then asks
	 * `is_numeric`. Numeric tokens are dropped from match candidates to
	 * avoid linking on prices, dates, and other numeric strings.
	 *
	 * @since 1.0.0
	 *
	 * @param string $token A single word token.
	 *
	 * @return bool True when the token represents a number.
	 */
	private static function is_numeric_token( string $token ): bool {
		$cleaned = str_replace( array( '$', ',' ), '', $token );
		return is_numeric( $cleaned );
	}
}