From 411e357c89c6edc62061bef035fa95a1771f4937 Mon Sep 17 00:00:00 2001 From: max Date: Mon, 3 Aug 2026 13:03:50 +0300 Subject: [PATCH] =?UTF-8?q?feat(store):=20the=20index=20knows=20its=20embe?= =?UTF-8?q?dding=20space=20=E2=80=94=20search=20refuses=20to=20mix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every file row already carried the embed fingerprint in its signature (that is what drives clean re-embeds), but SEARCH never checked: swap to a same-dimension model and the query lands in a different vector space than the corpus — silently garbage ranking until the next index run. Now a clean index pass records provider/model/dim in a meta table, and recall_search compares before embedding: on mismatch it skips KNN entirely and returns the FTS path with an honest degraded note naming both spaces and the fix. The marker moves only after a CLEAN pass — a failed file keeps rolled-back old-space vectors, so the corpus stays officially mixed until healed. Existing indexes have no marker and behave exactly as before until their next index run. Co-Authored-By: Claude Fable 5 --- src/session_recall/config.py | 7 ++++ src/session_recall/index.py | 8 ++++- src/session_recall/retrieve.py | 41 ++++++++++++++--------- src/session_recall/store.py | 13 ++++++++ tests/test_index.py | 11 +++++++ tests/test_retrieve.py | 60 ++++++++++++++++++++++++++++++++++ 6 files changed, 124 insertions(+), 16 deletions(-) diff --git a/src/session_recall/config.py b/src/session_recall/config.py index 15b2e70..a64eef4 100644 --- a/src/session_recall/config.py +++ b/src/session_recall/config.py @@ -177,3 +177,10 @@ def resolve_embed(env: dict | None = None, probe=None) -> EmbedSettings: # KNN + FTS only (not every embedding provider ships a reranker). RERANK_PROVIDER = _EMBED.rerank_provider RERANK_MODEL = _EMBED.rerank_model + + +def embed_fingerprint() -> str: + """Which embedding space vectors live in right now. Read at call time (not + frozen above) so tests and long processes see configuration changes. The + format is part of file signatures — change it and every file re-embeds.""" + return f"{EMBED_PROVIDER}/{EMBED_MODEL}/{EMBED_DIM}" diff --git a/src/session_recall/index.py b/src/session_recall/index.py index 97298dc..6e24de7 100644 --- a/src/session_recall/index.py +++ b/src/session_recall/index.py @@ -12,7 +12,7 @@ def _embed_fp() -> str: # Which embedding space the vectors live in. Same-dim provider/model swaps # produce incompatible spaces, so this must invalidate files AND the reuse # cache. WHY: docs/decisions/2026-07-02-post-review-hardening.md - return f"{config.EMBED_PROVIDER}/{config.EMBED_MODEL}/{config.EMBED_DIM}" + return config.embed_fingerprint() def _file_sig(path: Path, source: str = "claude") -> str: st = path.stat() @@ -183,4 +183,10 @@ def index_corpus(store: Store, embedder: Embedder, projects_dir: Path | None, if failed: print(f"session-recall: {len(failed)} file(s) failed to index (will retry " f"next run):\n " + "\n ".join(failed[:10]), file=sys.stderr) + else: + # The index-wide space marker moves only after a CLEAN pass: a failed + # file keeps its rolled-back old-space vectors, and search must keep + # treating the corpus as mixed until a full run heals it. + store.set_meta("embed_fp", _embed_fp()) + store.commit() return new_count diff --git a/src/session_recall/retrieve.py b/src/session_recall/retrieve.py index 8291659..4c74b0c 100644 --- a/src/session_recall/retrieve.py +++ b/src/session_recall/retrieve.py @@ -2,6 +2,7 @@ import json import time from collections import deque +from . import config from .store import Store from .embed import Embedder from .rerank import Reranker @@ -72,21 +73,31 @@ def recall_search(self, query: str, k: int = 10, candidates: int = 100, order: list[int] = [] dist: dict[int, float | None] = {} degraded: str | None = None - try: - qv = self.embedder.embed_query(query) - for cid, d in self.store.knn( - qv, candidates, scope_root=root, source=source, - start_ts=start_ts, end_ts=end_ts): - order.append(cid) - dist[cid] = d - except Exception as exc: - # WHY: docs/decisions/2026-07-26-voyage-403-egress-via-netcup.md - # Embedding unavailable -> FTS-only. Never hard-fail: keyword hits still - # beat nothing. But say so — a silent fallback looks identical to a - # healthy search that found little, and the caller stops trusting recall - # instead of fixing the embedder. - degraded = (f"fts-only: embeddings unavailable, semantic ranking is off — " - f"only literal word matches are returned ({type(exc).__name__}: {exc})") + stored_fp = self.store.get_meta("embed_fp") + if stored_fp and stored_fp != config.embed_fingerprint(): + # A same-dim model swap passes the schema check but puts the query + # in a different vector space than the corpus — matches would be + # silent noise. Refuse the mix: words still work, `index` heals it. + degraded = (f"embedder changed: the index was built with {stored_fp}, " + f"the current config is {config.embed_fingerprint()} — " + "semantic ranking is off until `session-recall index` " + "re-embeds") + else: + try: + qv = self.embedder.embed_query(query) + for cid, d in self.store.knn( + qv, candidates, scope_root=root, source=source, + start_ts=start_ts, end_ts=end_ts): + order.append(cid) + dist[cid] = d + except Exception as exc: + # WHY: docs/decisions/2026-07-26-voyage-403-egress-via-netcup.md + # Embedding unavailable -> FTS-only. Never hard-fail: keyword hits still + # beat nothing. But say so — a silent fallback looks identical to a + # healthy search that found little, and the caller stops trusting recall + # instead of fixing the embedder. + degraded = (f"fts-only: embeddings unavailable, semantic ranking is off — " + f"only literal word matches are returned ({type(exc).__name__}: {exc})") for cid in self.store.fts( query, candidates, scope_root=root, source=source, start_ts=start_ts, end_ts=end_ts): diff --git a/src/session_recall/store.py b/src/session_recall/store.py index 1f7d967..bbe4f5f 100644 --- a/src/session_recall/store.py +++ b/src/session_recall/store.py @@ -68,6 +68,9 @@ def _schema(self): self.db.execute( "CREATE TABLE IF NOT EXISTS indexed_files(" "path TEXT PRIMARY KEY, sig TEXT, source TEXT NOT NULL DEFAULT 'claude')") + # index-wide facts, e.g. which embedding space the vectors live in — + # per-file sigs drive re-embedding, this one lets SEARCH refuse to mix + self.db.execute("CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT)") indexed_cols = {row[1] for row in self.db.execute("PRAGMA table_info(indexed_files)")} if "source" not in indexed_cols: self.db.execute( @@ -279,6 +282,16 @@ def first_user_text(self, session_id: str, source: str | None = None) -> str: f"{source_sql} ORDER BY turn_index LIMIT 1", params).fetchone() return row[0] if row else "" + def get_meta(self, key: str) -> str | None: + row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone() + return row[0] if row else None + + def set_meta(self, key: str, value: str) -> None: + # Not committed here — rides the caller's transaction, like mark_indexed + self.db.execute( + "INSERT INTO meta(key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value)) + def mark_indexed(self, path: str, sig: str, source: str = "claude"): # Not committed here — joins the caller's per-file transaction, so the # "indexed" marker can never outlive a rolled-back set of chunks. diff --git a/tests/test_index.py b/tests/test_index.py index bddf085..6c3c4b0 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -266,3 +266,14 @@ def test_reindex_changed_file_does_not_accumulate_duplicate_rows(tmp_path): assert total == 3, f"duplicate accumulation: {total} chunk rows (expected 3)" assert vec == 3 and fts == 3, f"vec/fts out of sync with chunks: vec={vec} fts={fts}" store.close() + + +def test_clean_pass_records_the_embed_space(tmp_path): + """After a clean pass the index knows which vector space it holds — the + marker search uses to refuse mixing after a same-dim model swap.""" + from session_recall import config + projects = _corpus(tmp_path) + store = Store(tmp_path / "i.db") + index_corpus(store, FakeEmbedder(), projects) + assert store.get_meta("embed_fp") == config.embed_fingerprint() + store.close() diff --git a/tests/test_retrieve.py b/tests/test_retrieve.py index 8c76748..c7ea06d 100644 --- a/tests/test_retrieve.py +++ b/tests/test_retrieve.py @@ -568,3 +568,63 @@ def knn_then_delete(*args, **kwargs): r.store.knn = knn_then_delete hits = r.recall_search("cache embeddings", k=5) # must not raise assert all(a.uuid for a in hits) + + +def _chunk(text): + from session_recall.models import Chunk + return Chunk(session_id="s", uuid="u1", role="user", text=text, + project="p", cwd="/p", git_branch="main", ts=1, + file_path="/p/f.jsonl", byte_offset=0, byte_len=len(text), + turn_index=0, content_hash="h1", source="claude") + + +def test_embedder_swap_degrades_instead_of_mixing_spaces(tmp_path, monkeypatch): + """A same-dim model swap passes the schema check, so without the meta + guard the query lands in a different vector space than the corpus and the + ranking is silent noise. The guard must refuse to mix: no embed call, + words still work, and the note names both spaces.""" + from session_recall import config + from session_recall.embed import FakeEmbedder + from session_recall.retrieve import Recall + from session_recall.store import Store + from session_recall.models import Chunk + + store = Store(tmp_path / "i.db") + store.add(_chunk("the relay transport was rebuilt"), + FakeEmbedder().embed_query("the relay transport was rebuilt")) + store.commit() + store.set_meta("embed_fp", "voyage/old-model/1024") + store.commit() + + class NeverEmbed(FakeEmbedder): + def embed_query(self, text): + raise AssertionError("mixed-space KNN must not run at all") + + monkeypatch.setattr(config, "EMBED_MODEL", "brand-new-model") + res = Recall(store, NeverEmbed()).recall_search("relay transport") + assert res.degraded and "embedder changed" in res.degraded + assert "voyage/old-model/1024" in res.degraded + assert res, "FTS must still return the word match" + assert all(a.score is None for a in res) + store.close() + + +def test_matching_fingerprint_keeps_the_vector_path(tmp_path): + from session_recall import config + from session_recall.embed import FakeEmbedder + from session_recall.retrieve import Recall + from session_recall.store import Store + from session_recall.models import Chunk + + store = Store(tmp_path / "i.db") + e = FakeEmbedder() + store.add(_chunk("the relay transport was rebuilt"), + e.embed_query("the relay transport was rebuilt")) + store.commit() + store.set_meta("embed_fp", config.embed_fingerprint()) + store.commit() + + res = Recall(store, e).recall_search("the relay transport was rebuilt") + assert res.degraded is None + assert res and res[0].score is not None + store.close()