experimental additional-term features (TermNormalizer); always empty for a production embedder */ private array $features = []; /** * The production embedder: original terms only. There is deliberately no way to switch features on through this constructor or through * any option, constant or filter; see experimental(). */ public function __construct(private int $dimension = 256) { if ($dimension < 16 || $dimension > 4096) { throw new \InvalidArgumentException('Invalid dimension.'); } } /** * EXPERIMENTAL, evaluation only (Stage C2): an embedder that adds the optional TermNormalizer terms. It is NOT used by the plugin and * is not approved for any site. Its model id differs from the production one on purpose, so an index built by one is never searched * with the other: a site could not silently mix vectors built with and without the extra terms (that would change ranking without * any re-index). Putting it into production needs a versioned index, a safe re-index and a rollback (see docs/stage-c2-multilingual-retrieval.md). * * @param list $features names from TermNormalizer::FEATURES * @internal */ public static function experimental(array $features, int $dimension = 256): self { $e = new self($dimension); foreach ($features as $f) { if (!in_array($f, TermNormalizer::FEATURES, true)) { throw new \InvalidArgumentException('Unknown lexical feature.'); } } $e->features = array_values(array_unique($features)); sort($e->features); return $e; } /** * The terms of a text for this embedder: the original terms, plus (only when features are on) additional normalized ones that the * original list does not already contain. * * @return list */ public function termList(string $text): array { $a = self::terms($text); if ($this->features === []) { return $a; } $have = array_flip($a); if (in_array('cjk', $this->features, true)) { // Whole runs of three or more Han characters are replaced by their two-character groups (a long run is a clause, not a word, and // would only dilute the vector); text indexed earlier keeps its run terms and still matches through the groups of short runs. $a = array_values(array_filter($a, static fn(string $w): bool => preg_match('/^\p{Han}{3,}$/u', $w) !== 1)); } $extra = []; foreach (TermNormalizer::extra($text, $this->features) as $w) { if (in_array($w, self::STOP, true)) { continue; } $w = self::stem($w); if (!isset($have[$w])) { $extra[] = $w; } } $all = array_merge($a, $extra); return in_array('stop', $this->features, true) ? array_values(array_filter($all, static fn(string $w): bool => !TermNormalizer::isStop($w))) : $all; } public function model(): string { return 'mock-hash-v1-' . $this->dimension . ($this->features === [] ? '' : '+exp-' . implode('.', $this->features)); } public function dimension(): int { return $this->dimension; } public function embed(array $texts): array { return array_map(fn(string $t): array => $this->vector($t), $texts); } public function sharesTerm(string $query, string $text): bool { return array_intersect($this->termList($query), $this->termList($text)) !== []; } /** @return list normalized terms, exposed for diagnostics and tests */ public static function terms(string $text): array { preg_match_all('/[\p{L}\p{N}]+/u', mb_strtolower($text), $m); $out = []; foreach ($m[0] as $w) { if (mb_strlen($w) < 2 || in_array($w, self::STOP, true)) { continue; } $out[] = self::stem($w); } return $out; } private static function stem(string $w): string { foreach (['ingly', 'ings', 'ing', 'edly', 'ies', 'ied', 'ed', 'ly', 's'] as $suffix) { if (mb_strlen($w) > mb_strlen($suffix) + 2 && str_ends_with($w, $suffix)) { return in_array($suffix, ['ies', 'ied'], true) ? mb_substr($w, 0, -3) . 'y' : mb_substr($w, 0, -mb_strlen($suffix)); } } return $w; } /** @return list */ private function vector(string $text): array { $tf = array_count_values($this->termList($text)); $v = array_fill(0, $this->dimension, 0.0); foreach ($tf as $term => $count) { $h = crc32((string) $term); $sign = ($h & 0x80000000) ? -1.0 : 1.0; $v[$h % $this->dimension] += $sign * (1.0 + log($count)); } $norm = sqrt(array_sum(array_map(static fn(float $x): float => $x * $x, $v))); return $norm > 0 ? array_map(static fn(float $x): float => $x / $norm, $v) : $v; } }