mode !== 'off'; } /** * Everything the screen shows, for one site. * * @return array */ public function status(int $siteId): array { $config = $this->configs->get($siteId); $probe = $config->isSemantic() ? $config : EmbeddingConfig::semantic($siteId, 'openai', 'text-embedding-3-small', 1536, null, 0); $pre = $this->builder->preflight($siteId, $probe); $usage = (new EmbeddingUsage($this->db))->summary($siteId); $state = $this->generations->peekState($siteId); $current = $this->db->selectOne("SELECT * FROM {knowledge_generations} WHERE site_id = ? AND state IN ('building', 'ready') ORDER BY id DESC LIMIT 1", [$siteId]) ?? $this->db->selectOne("SELECT * FROM {knowledge_generations} WHERE site_id = ? AND state = 'failed' ORDER BY id DESC LIMIT 1", [$siteId]); $active = $state['active'] === null ? null : $this->generations->find($siteId, $state['active']); $previous = $state['previous'] === null ? null : $this->generations->find($siteId, $state['previous']); $blocked = []; if ($current !== null && $current['state'] === IndexGenerations::BUILDING) { foreach ($this->claims->blocked($siteId, (int) $current['id']) as $sid) { $row = $this->db->selectOne('SELECT title FROM {knowledge_sources} WHERE id = ? AND site_id = ?', [$sid, $siteId]); $blocked[] = ['source_id' => $sid, 'title' => (string) ($row['title'] ?? '')]; } } $jobs = $this->db->select("SELECT state, COUNT(*) AS n FROM {jobs} WHERE site_id = ? AND type = ? GROUP BY state", [$siteId, GenerationJobs::BUILD]); $jobCounts = []; foreach ($jobs as $j) { $jobCounts[(string) $j['state']] = (int) $j['n']; } $building = $current !== null && $current['state'] === IndexGenerations::BUILDING; return [ 'mode' => $this->mode, 'execution_available' => $this->executionAvailable(), 'config' => ['semantic' => $config->isSemantic(), 'model' => $config->model, 'dimension' => $config->dimension, 'corrupt' => $config->corrupt], 'preflight' => $pre, 'budget' => $usage, 'current' => $current === null ? null : $this->view($current) + ['paused_by_admin' => $this->control->isPaused($siteId, (int) $current['id'])], 'blocked' => $blocked, 'active' => $active === null ? null : $this->view($active), 'previous' => $previous === null ? null : $this->view($previous) + ['retain_until' => $previous['retain_until'] === null ? null : (int) $previous['retain_until']], 'revision' => $state['revision'], 'jobs' => $jobCounts, 'auto_sync' => $this->control->autoSync($siteId), // sources the active (or else the complete, not yet active) index is missing: changed or new content it does not cover yet 'pending' => $state['active'] !== null ? $this->generations->countMissing($siteId, $state['active'], 1000) : ($current !== null && $current['state'] === IndexGenerations::READY ? $this->generations->countMissing($siteId, (int) $current['id'], 1000) : 0), 'actions' => $this->actions($siteId, $config, $pre, $usage, $current, $building, $state, $previous), ]; } /** @param array $g @return array */ private function view(array $g): array { return ['id' => (int) $g['id'], 'state' => (string) $g['state'], 'model' => (string) $g['model'], 'dimension' => (int) $g['dimension'], 'sources_total' => (int) $g['sources_total'], 'sources_built' => (int) $g['sources_built'], 'sources_failed' => (int) $g['sources_failed'], 'code' => $g['error'] === null ? null : (string) $g['error']]; } /** * @param array $pre * @param array $usage * @param array|null $current * @param array{revision:int, active:?int, previous:?int} $state * @param array|null $previous * @return array */ private function actions(int $siteId, EmbeddingConfig $config, array $pre, array $usage, ?array $current, bool $building, array $state, ?array $previous): array { $ok = static fn(): array => ['allowed' => true, 'reason' => null]; $no = static fn(string $why): array => ['allowed' => false, 'reason' => $why]; $start = match (true) { !$this->executionAvailable() => $no('unavailable'), !$config->isSemantic() => $no('not_semantic'), $current !== null && in_array($current['state'], ['building', 'ready'], true) => $no('build_in_progress'), (int) $pre['sources'] === 0 => $no('nothing_to_index'), !$pre['ok'] => $no((string) $pre['code']), $usage['budget'] === null => $no('no_budget'), (int) $usage['budget']['available'] < (int) $pre['units'] => $no('budget_too_small'), $state['active'] !== null && $this->generations->find($siteId, $state['active'])['fingerprint'] === $config->fingerprint() => $no('already_active'), default => $ok(), }; $paused = $building && $this->control->isPaused($siteId, (int) $current['id']); $activate = match (true) { !$this->executionAvailable() => $no('unavailable'), $current === null || $current['state'] !== IndexGenerations::READY => $no('not_ready'), !$config->isSemantic() || !hash_equals((string) $current['fingerprint'], (string) $config->fingerprint()) => $no('config_mismatch'), default => $ok(), }; return [ 'start' => $start, 'pause' => $building && !$paused ? $ok() : $no($building ? 'already_paused' : 'no_build'), 'resume' => !$this->executionAvailable() ? $no('unavailable') : ($building ? $ok() : $no('no_build')), 'cancel' => $current !== null && in_array($current['state'], ['building', 'ready'], true) ? $ok() : $no('no_build'), 'activate' => $activate, 'rollback' => match (true) { !$this->executionAvailable() => $no('unavailable'), $previous === null => $no('no_rollback_target'), $previous['retain_until'] !== null && (int) $previous['retain_until'] <= $this->clock->now() => $no('rollback_expired'), default => $ok(), }, 'deactivate' => $state['active'] !== null ? $ok() : $no('nothing_active'), ]; } /** * The standing approval for automatic updates after source changes (off by default, per site). While it is on, a changed or new * public source in an active or complete index is embedded in the background within the embedding budget; switching it off stops * that at once (queued automatic jobs do nothing). Turning it on needs an explicit confirmation, a chosen model and a budget. * * @return array{ok:bool, code:string} */ public function setAutoSync(int $siteId, bool $enabled, bool $confirmed = false): array { if (!$enabled) { $this->control->setAutoSync($siteId, false); return $this->ok('auto_sync_off'); } if (!$this->executionAvailable()) { return $this->fail('unavailable'); } if (!$confirmed) { return $this->fail('confirm_required'); } if (!$this->configs->get($siteId)->isSemantic()) { return $this->fail('not_semantic'); } if ($this->ledger->balance($siteId, EmbeddingUsage::BUCKET) === null) { return $this->fail('no_budget'); } $this->control->setAutoSync($siteId, true); return $this->ok('auto_sync_on'); } /** * The administrator's own "update the index now" (works whether or not automatic updates are approved): queues the background work * that embeds the sources an active or complete index is missing, within the embedding budget. * * @return array{ok:bool, code:string} */ public function syncNow(int $siteId): array { if (!$this->executionAvailable()) { return $this->fail('unavailable'); } if (!$this->configs->get($siteId)->isSemantic()) { return $this->fail('not_semantic'); } $balance = $this->ledger->balance($siteId, EmbeddingUsage::BUCKET); if ($balance === null) { return $this->fail('no_budget'); } if ($balance['available'] <= 0) { return $this->fail('budget_too_small'); } $queued = false; $active = $this->generations->peekState($siteId)['active']; if ($active !== null && $this->generations->countMissing($siteId, $active, 1) > 0) { $this->jobs->requestManualSync($siteId); $queued = true; } $open = $this->openBuild($siteId); if ($open !== null && $open['state'] === IndexGenerations::READY && $this->generations->countMissing($siteId, (int) $open['id'], 1) > 0) { try { $this->generations->reopen($siteId, (int) $open['id']); $this->control->setPaused($siteId, 0); $this->jobs->enqueueBuild($siteId, (int) $open['id'], 'sync-' . $this->clock->now() . '-' . bin2hex(random_bytes(3)), $balance['available']); $queued = true; } catch (GenerationException) { // activated or reopened by someone else meanwhile } } return $queued ? $this->ok('sync_queued') : $this->fail('nothing_to_update'); } /** Sets the site's embedding token limit (its own bucket; never the chat budget). @return array{ok:bool, code:string} */ public function setBudget(int $siteId, int $units): array { if ($units < 0 || $units > self::BUDGET_MAX) { return $this->fail('invalid'); } $this->ledger->setLimit($siteId, EmbeddingUsage::BUCKET, $units); return $this->ok('budget_saved'); } /** Chooses the embedding model for this site (or word matching only). Takes effect for the next build. @return array{ok:bool, code:string} */ public function setModel(int $siteId, string $model): array { if ($this->openBuild($siteId) !== null) { return $this->fail('build_in_progress'); } if ($model === 'lexical') { $this->configs->save(EmbeddingConfig::lexical($siteId)); return $this->ok('model_saved'); } $dimension = EmbeddingConfig::supported()['openai'][$model] ?? null; if ($dimension === null) { return $this->fail('invalid'); } $this->configs->save(EmbeddingConfig::semantic($siteId, 'openai', $model, $dimension)); return $this->ok('model_saved'); } /** @return array{ok:bool, code:string, generation?:int} */ public function start(int $siteId, bool $confirmed): array { if (!$this->executionAvailable()) { return $this->fail('unavailable'); } if (!$confirmed) { return $this->fail('confirm_required'); } $config = $this->configs->get($siteId); if (!$config->isSemantic()) { return $this->fail('not_semantic'); } $pre = $this->builder->preflight($siteId, $config); if ($pre['sources'] === 0) { return $this->fail('nothing_to_index'); } if (!$pre['ok']) { return $this->fail((string) $pre['code']); } $balance = $this->ledger->balance($siteId, EmbeddingUsage::BUCKET); if ($balance === null) { return $this->fail('no_budget'); } if ($balance['available'] < $pre['units']) { return $this->fail('budget_too_small'); } try { $generation = $this->generations->begin($config); } catch (GenerationException $e) { return $this->fail($e->getMessage()); } $this->control->setPaused($siteId, 0); $this->jobs->enqueueBuild($siteId, $generation, 'start', $balance['available']); return $this->ok('started') + ['generation' => $generation]; } /** Stops the build chain after the step that is running now. Nothing is released or settled. @return array{ok:bool, code:string} */ public function pause(int $siteId): array { $g = $this->openBuild($siteId); if ($g === null || $g['state'] !== IndexGenerations::BUILDING) { return $this->fail('no_build'); } if ($this->control->isPaused($siteId, (int) $g['id'])) { return $this->fail('already_paused'); } $this->control->setPaused($siteId, (int) $g['id']); return $this->ok('paused'); } /** Continues a paused or builder-paused build with a fresh job (new operation ids). Blocked tasks stay blocked. @return array{ok:bool, code:string} */ public function resume(int $siteId): array { if (!$this->executionAvailable()) { return $this->fail('unavailable'); } $g = $this->openBuild($siteId); if ($g === null || $g['state'] !== IndexGenerations::BUILDING) { return $this->fail('no_build'); } $balance = $this->ledger->balance($siteId, EmbeddingUsage::BUCKET); $this->control->setPaused($siteId, 0); $this->jobs->enqueueBuild($siteId, (int) $g['id'], 'resume-' . $this->clock->now() . '-' . bin2hex(random_bytes(3)), $balance['available'] ?? null); return $this->ok('resumed'); } /** Gives up the open build; its vectors are deleted at once. Never touches the active generation. @return array{ok:bool, code:string} */ public function cancel(int $siteId): array { $g = $this->openBuild($siteId); if ($g === null) { return $this->fail('no_build'); } try { $this->generations->fail($siteId, (int) $g['id'], 'owner_cancelled'); } catch (GenerationException) { return $this->fail('no_build'); } $this->control->setPaused($siteId, 0); return $this->ok('cancelled'); } /** * Allows a source whose earlier request may have reached the provider to be built again, but only once none of that operation's * ledger holds is still open: an uncertain hold must first be resolved on the Usage page (charged as an estimate or replaced by * the usage the provider reports). Nothing here releases or settles a hold. * * @return array{ok:bool, code:string} */ public function unblock(int $siteId, int $generationId, int $sourceId): array { $op = $this->claims->blockedOperation($siteId, $generationId, $sourceId); if ($op !== null && $this->openHolds($siteId, $op['operation'], $op['batches']) > 0) { return $this->fail('unresolved_hold'); } return $this->claims->unblock($siteId, $generationId, $sourceId) ? $this->ok('unblocked') : $this->fail('not_found'); } /** @return array{ok:bool, code:string} */ public function activate(int $siteId, int $generationId): array { if (!$this->executionAvailable()) { return $this->fail('unavailable'); } $g = $this->generations->find($siteId, $generationId); if ($g === null) { return $this->fail('not_found'); } $config = $this->configs->get($siteId); if (!$config->isSemantic() || !hash_equals((string) $g['fingerprint'], (string) $config->fingerprint())) { return $this->fail('config_mismatch'); } return $this->swap(fn(int $rev) => $this->generations->activate($siteId, $generationId, $rev), $siteId, 'activated'); } /** @return array{ok:bool, code:string} */ public function rollback(int $siteId): array { return $this->executionAvailable() ? $this->swap(fn(int $rev) => $this->generations->rollback($siteId, $rev), $siteId, 'rolled_back') : $this->fail('unavailable'); } /** Back to word matching: always available (it only removes semantic answers). @return array{ok:bool, code:string} */ public function deactivate(int $siteId): array { return $this->swap(fn(int $rev) => $this->generations->deactivate($siteId, $rev), $siteId, 'deactivated'); } /** @param callable(int): void $change @return array{ok:bool, code:string} */ private function swap(callable $change, int $siteId, string $done): array { try { $change($this->generations->state($siteId)['revision']); } catch (GenerationConflictException) { return $this->fail('conflict'); } catch (GenerationException $e) { return $this->fail($e->getMessage()); } return $this->ok($done); } /** @return array|null the building or ready generation of the site */ private function openBuild(int $siteId): ?array { return $this->db->selectOne("SELECT * FROM {knowledge_generations} WHERE site_id = ? AND state IN ('building', 'ready') ORDER BY id DESC LIMIT 1", [$siteId]); } private function openHolds(int $siteId, string $operation, int $batches): int { $ids = []; for ($n = 0; $n < max(1, $batches); $n++) { $ids[] = RemoteEmbedder::requestId(Ingestor::batchOperation($operation, $n)); } return (int) $this->db->selectOne( "SELECT COUNT(*) AS n FROM {reservations} WHERE site_id = ? AND state IN ('reserved', 'dispatched', 'uncertain') AND request_id IN (" . implode(', ', array_fill(0, count($ids), '?')) . ')', array_merge([$siteId], $ids) )['n']; } /** @return array{ok:bool, code:string} */ private function ok(string $code): array { return ['ok' => true, 'code' => $code]; } /** @return array{ok:bool, code:string} */ private function fail(string $code): array { return ['ok' => false, 'code' => preg_match('/^[a-z_]{1,40}$/', $code) === 1 ? $code : 'failed']; } }