embedder instanceof RemoteEmbedder; } /** * Brute-force cosine scan, bounded in memory: chunks are read in id order in batches of SCAN_BATCH * (vectors only), only the best k candidates are kept, and the text/metadata of those k is loaded * afterwards. Results, ordering and the MAX_SCAN/truncated contract are identical to a single full read. * * @return array{chunks: list, scanned: int, truncated: bool} */ public function retrieve(Principal $principal, int $botId, string $query, int $k = 5, float $minScore = 0.12, ?string $requestId = null): array { $gen = $this->generations?->activeFor($principal->siteId, $this->embedder); $params = $gen === null ? [$botId, $principal->siteId, $principal->siteId, $this->embedder->model(), $this->embedder->dimension()] : [$gen, $botId, $principal->siteId, $principal->siteId, $principal->siteId, $this->embedder->model(), $this->embedder->dimension()]; $first = $this->batch($params, 0, $gen); if ($first === [] || trim($query) === '') { return ['chunks' => [], 'scanned' => min(count($first), self::MAX_SCAN), 'truncated' => count($first) > self::MAX_SCAN]; } // The query is embedded only after at least one candidate exists, so an empty corpus never causes a remote call. // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- exception messages go to logs and callers, never echoed; public errors are fixed codes if ($this->embedder instanceof RemoteEmbedder && ($requestId === null || $principal->chatSessionId === null)) { throw new ExtractionException('embedding_context', 'Explicit query operation required.', RetrievalFailure::Configuration); } $q = $this->embedder instanceof RemoteEmbedder ? $this->embedder->embedFor([$query], 'query:' . $principal->siteId . ':' . $principal->chatSessionId . ':' . $requestId, ['site', 'session:' . $principal->chatSessionId])[0] : $this->embedder->embed([$query])[0]; $dim = $this->embedder->dimension(); $verify = $this->verifyTerms && $this->embedder instanceof LexicalEmbedder; $keep = $verify ? $k * self::VERIFY_POOL_FACTOR : $k; /** @var list $best score, chunk id (best first) */ $best = []; $seen = 0; $truncated = false; $rows = $first; while ($rows !== []) { foreach ($rows as $r) { if ($seen >= self::MAX_SCAN) { $truncated = true; // one more matching chunk exists beyond the cap break 2; } $seen++; $score = VectorCodec::dot($q, VectorCodec::decode((string) $r['embedding'], $dim)); if ($score >= $minScore) { $best[] = [$score, (int) $r['id']]; } } usort($best, static fn(array $a, array $b): int => $b[0] <=> $a[0] ?: $a[1] <=> $b[1]); $best = array_slice($best, 0, $keep); $lastId = (int) $rows[count($rows) - 1]['id']; $rows = count($rows) < self::SCAN_BATCH ? [] : $this->batch($params, $lastId, $gen); } if ($best === []) { return ['chunks' => [], 'scanned' => $seen, 'truncated' => $truncated]; } // Text and metadata for the winners only, under the same visibility rules. $ids = array_column($best, 1); $in = implode(', ', array_fill(0, count($ids), '?')); $meta = []; $metaSql = $gen === null ? "SELECT c.id, c.source_id, c.version_id, c.anchor, c.heading, c.text, s.title, s.url FROM {knowledge_chunks} c JOIN {knowledge_sources} s ON s.id = c.source_id AND s.current_version_id = c.version_id JOIN {knowledge_versions} v ON v.id = c.version_id JOIN {bot_sources} b ON b.source_id = s.id AND b.bot_id = ? AND b.site_id = s.site_id WHERE s.site_id = ? AND c.site_id = ? AND s.status = 'ready' AND s.visibility = 'public' AND v.state = 'ready' AND v.embedding_model = ? AND v.dimension = ? AND c.id IN ($in)" : "SELECT c.id, c.source_id, c.version_id, c.anchor, c.heading, c.text, s.title, s.url FROM {knowledge_chunks} c JOIN {knowledge_versions} v ON v.id = c.version_id AND v.generation_id = ? JOIN {knowledge_sources} s ON s.id = c.source_id JOIN {knowledge_versions} cv ON cv.id = s.current_version_id AND cv.content_hash = v.content_hash JOIN {bot_sources} b ON b.source_id = s.id AND b.bot_id = ? AND b.site_id = s.site_id WHERE s.site_id = ? AND c.site_id = ? AND v.site_id = ? AND s.status = 'ready' AND s.visibility = 'public' AND v.state = 'ready' AND v.embedding_model = ? AND v.dimension = ? AND c.id IN ($in)"; foreach ($this->db->select($metaSql, array_merge($params, $ids)) as $r) { $meta[(int) $r['id']] = $r; } $out = []; foreach ($best as [$score, $id]) { if (count($out) >= $k) { break; } if (!isset($meta[$id])) { continue; // changed between the scan and this read: never returned } $r = $meta[$id]; if ($verify && !$this->embedder->sharesTerm($query, ((string) $r['heading'] !== '' ? $r['heading'] . " " : '') . $r['text'])) { continue; // matched only through a hash-bucket collision: no common word } $out[] = new RetrievedChunk($id, (int) $r['source_id'], (int) $r['version_id'], (string) $r['title'], $r['url'] === null ? null : (string) $r['url'], (string) $r['anchor'], (string) $r['heading'], (string) $r['text'], round($score, 4)); } return ['chunks' => $out, 'scanned' => $seen, 'truncated' => $truncated]; } /** * One batch of candidate vectors (id and vector only) with an id greater than $afterId, in id order. * The MAX_SCAN cap is enforced by the caller. * * @param list $params bot, site, site, embedding model, dimension (generation mode: generation, bot, site x3, model, dimension) * @return list> */ private function batch(array $params, int $afterId, ?int $gen = null): array { if ($gen !== null) { return $this->db->select( "SELECT c.id, c.embedding FROM {knowledge_chunks} c JOIN {knowledge_versions} v ON v.id = c.version_id AND v.generation_id = ? JOIN {knowledge_sources} s ON s.id = c.source_id JOIN {knowledge_versions} cv ON cv.id = s.current_version_id AND cv.content_hash = v.content_hash JOIN {bot_sources} b ON b.source_id = s.id AND b.bot_id = ? AND b.site_id = s.site_id WHERE s.site_id = ? AND c.site_id = ? AND v.site_id = ? AND s.status = 'ready' AND s.visibility = 'public' AND v.state = 'ready' AND v.embedding_model = ? AND v.dimension = ? AND c.id > ? ORDER BY c.id LIMIT ?", array_merge($params, [$afterId, self::SCAN_BATCH]) ); } return $this->db->select( "SELECT c.id, c.embedding FROM {knowledge_chunks} c JOIN {knowledge_sources} s ON s.id = c.source_id AND s.current_version_id = c.version_id JOIN {knowledge_versions} v ON v.id = c.version_id JOIN {bot_sources} b ON b.source_id = s.id AND b.bot_id = ? AND b.site_id = s.site_id WHERE s.site_id = ? AND c.site_id = ? AND s.status = 'ready' AND s.visibility = 'public' AND v.state = 'ready' AND v.embedding_model = ? AND v.dimension = ? AND c.id > ? ORDER BY c.id LIMIT ?", array_merge($params, [$afterId, self::SCAN_BATCH]) ); } /** * Re-validates chunks immediately before they are sent to a model: drops * any whose source was privatized, deleted, unpublished or re-versioned * since retrieval. * * @param list $chunks * @return list */ public function recheck(int $siteId, int $botId, array $chunks): array { if ($chunks === []) { return []; } $ids = array_values(array_unique(array_map(static fn(RetrievedChunk $c): int => $c->sourceId, $chunks))); $in = implode(', ', array_fill(0, count($ids), '?')); $valid = []; $gen = $this->generations?->activeFor($siteId, $this->embedder); if ($gen !== null) { // Same rules against the generation that is active NOW: a chunk from a generation that was deactivated meanwhile is dropped. foreach ($this->db->select( "SELECT s.id, v.id AS vid FROM {knowledge_sources} s JOIN {bot_sources} b ON b.source_id = s.id AND b.bot_id = ? AND b.site_id = s.site_id JOIN {knowledge_versions} cv ON cv.id = s.current_version_id JOIN {knowledge_versions} v ON v.source_id = s.id AND v.site_id = s.site_id AND v.generation_id = ? AND v.state = 'ready' AND v.content_hash = cv.content_hash WHERE s.site_id = ? AND s.status = 'ready' AND s.visibility = 'public' AND s.id IN ($in)", array_merge([$botId, $gen, $siteId], $ids) ) as $r) { $valid[(int) $r['id']] = (int) $r['vid']; } return array_values(array_filter($chunks, static fn(RetrievedChunk $c): bool => ($valid[$c->sourceId] ?? null) === $c->versionId)); } foreach ($this->db->select( "SELECT s.id, s.current_version_id FROM {knowledge_sources} s JOIN {bot_sources} b ON b.source_id = s.id AND b.bot_id = ? WHERE s.site_id = ? AND s.status = 'ready' AND s.visibility = 'public' AND s.id IN ($in)", array_merge([$botId, $siteId], $ids) ) as $r) { $valid[(int) $r['id']] = (int) $r['current_version_id']; } return array_values(array_filter($chunks, static fn(RetrievedChunk $c): bool => ($valid[$c->sourceId] ?? null) === $c->versionId)); } }