'c', 'ş' => 's', 'ğ' => 'g', 'ö' => 'o', 'ü' => 'u', 'ı' => 'i', 'İ' => 'i', 'â' => 'a', 'î' => 'i', 'û' => 'u', 'á' => 'a', 'à' => 'a', 'ä' => 'a', 'ã' => 'a', 'å' => 'a', 'é' => 'e', 'è' => 'e', 'ê' => 'e', 'ë' => 'e', 'í' => 'i', 'ì' => 'i', 'ï' => 'i', 'ó' => 'o', 'ò' => 'o', 'ô' => 'o', 'õ' => 'o', 'ú' => 'u', 'ù' => 'u', 'ñ' => 'n', 'ý' => 'y', 'ÿ' => 'y', 'ł' => 'l', 'ø' => 'o', 'œ' => 'oe', 'æ' => 'ae', 'ß' => 'ss', ]; private const ARABIC = [ 'ي' => 'ی', 'ى' => 'ی', 'ئ' => 'ی', 'ك' => 'ک', 'أ' => 'ا', 'إ' => 'ا', 'آ' => 'ا', 'ٱ' => 'ا', 'ة' => 'ه', 'ۀ' => 'ه', 'ؤ' => 'و', '٠' => '0', '١' => '1', '٢' => '2', '٣' => '3', '٤' => '4', '٥' => '5', '٦' => '6', '٧' => '7', '٨' => '8', '٩' => '9', '۰' => '0', '۱' => '1', '۲' => '2', '۳' => '3', '۴' => '4', '۵' => '5', '۶' => '6', '۷' => '7', '۸' => '8', '۹' => '9', ]; /** Normalized forms (after the arabic step where it is on) of common function words. */ private const STOP = [ // Turkish 've', 'bir', 'bu', 'şu', 'mi', 'mı', 'mu', 'mü', 'ne', 'ile', 'de', 'da', 'ya', // Arabic (unified spelling) 'من', 'في', 'علی', 'على', 'الی', 'إلى', 'عن', 'هل', 'ما', 'هو', 'هی', 'هي', 'ماذا', 'هذا', 'هذه', 'ان', 'أن', 'مع', 'كم', 'او', 'أو', 'التی', 'التي', 'الذی', 'الذي', // Persian 'است', 'را', 'در', 'به', 'از', 'که', 'با', 'می', 'آیا', 'این', 'آن', 'تا', 'برای', 'چه', 'چند', 'چقدر', 'کجا', 'کجاست', 'کیست', 'هم', ]; /** * Extra terms for the text (to be merged after the original terms by the caller, each only if the original list lacks it). * * @param list $features * @return list */ public static function extra(string $text, array $features): array { if ($features === []) { return []; } $on = array_flip($features); $t = $text; if (isset($on['fold'])) { $t = strtr($t, ['İ' => 'i', 'I' => 'i']); // the dotted capital would otherwise lower-case into i + combining dot and split the word } $t = mb_strtolower($t); if (isset($on['fold'])) { $t = strtr($t, self::FOLD); } if (isset($on['arabic'])) { $t = (string) preg_replace('/[\x{064B}-\x{065F}\x{0670}\x{0640}]/u', '', $t); $t = strtr($t, self::ARABIC); } $joiners = isset($on['zwnj']) ? '\x{200C}' : ''; preg_match_all('/[\p{L}\p{N}' . $joiners . ']+/u', $t, $m); $out = []; foreach ($m[0] as $w) { $w = str_replace("\u{200C}", '', $w); if (mb_strlen($w) < 2) { continue; } if (isset($on['cjk']) && preg_match('/^\p{Han}+$/u', $w) === 1) { $n = mb_strlen($w); for ($i = 0; $i + 1 < $n; $i++) { $out[] = mb_substr($w, $i, 2); } continue; } $out[] = $w; } return $out; } /** @return bool whether $term (already lower-cased) is on the extra stop list */ public static function isStop(string $term): bool { static $set = null; $set ??= array_flip(self::STOP); return isset($set[$term]); } }