SpillwaveSolutions · RichardHightower · Aug 30, 2026 · Aug 30, 2026
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
@@ -11,7 +11,7 @@
       "name": "data-engineering-knowledge-capture",
       "source": "./",
       "description": "Capture data-platform knowledge \u2014 schemas, lineage, medallion, semantic layer, business glossary \u2014 into OKF. Extends PKC; depends on OKF.",
-      "version": "0.4.3",
+      "version": "0.5.0",
       "author": {
         "name": "Rick Hightower"
       },

diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json
@@ -1,6 +1,6 @@
 {
   "name": "data-engineering-knowledge-capture",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "Data Engineering Knowledge Capture (DEKC) \u2014 extends PKC + OKF for schemas, lineage, medallion layers, SQL/DAX, semantic models, business objects, and an indexed second brain. Agents walk data lakes. Works in Claude Code, Grok Build, Codex, and OpenCode.",
   "author": {
     "name": "Rick Hightower",

diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json
@@ -1,6 +1,6 @@
 {
   "name": "data-engineering-knowledge-capture",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "Data Engineering Knowledge Capture (DEKC) \u2014 schemas, lineage, medallion layers, semantic models, business glossary. Extends PKC; depends on OKF. Codex-native skills + hooks port.",
   "author": {
     "name": "Rick Hightower",

diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json
@@ -1,6 +1,6 @@
 {
   "name": "data-engineering-knowledge-capture",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "Data Engineering Knowledge Capture: schemas, lineage, medallion, semantic models, and glossary into a durable OKF knowledge graph. Multi-host bindings and write isolation.",
   "author": {
     "name": "Rick Hightower"

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -17,23 +17,12 @@ jobs:
           python-version: "3.12"
 
       - name: Unit tests
-        run: python3 tests/test_dekc.py && python3 tests/test_isolation.py
+        run: python3 tests/test_dekc.py && python3 tests/test_retrieval_ladder.py && python3 tests/test_isolation.py
 
+      # Glob, not a list. The hand-maintained list drifted whenever a new
+      # dekc_*.py landed. PKC 0.9.0 made the same change.
       - name: Compile scripts
-        run: |
-          python3 -m py_compile \
-            scripts/dekc_common.py \
-            scripts/dekc_capture.py \
-            scripts/dekc_walk.py \
-            scripts/dekc_lineage.py \
-            scripts/dekc_business.py \
-            scripts/dekc_index.py \
-            scripts/dekc_validate.py \
-            scripts/dekc_pack.py \
-            scripts/dekc_search.py \
-            scripts/dekc_doctor.py \
-            scripts/dekc_link.py \
-            scripts/brain_session.py
+        run: python3 -m py_compile scripts/dekc_*.py scripts/brain_session.py
 
       - name: Validate + doctor sample-knowledge
         run: |
@@ -43,5 +32,7 @@ jobs:
       - name: Index + search
         run: |
           python3 scripts/dekc_index.py --repo . --bundle sample-knowledge build
+          python3 scripts/dekc_index.py status --repo . --bundle sample-knowledge --json | \
+            python3 -c "import json,sys; d=json.load(sys.stdin); assert d['fts5'] is True; assert d['nodes']>=1, d"
           python3 scripts/dekc_search.py revenue --repo . --bundle sample-knowledge --json | \
-            python3 -c "import json,sys; d=json.load(sys.stdin); assert d['count']>=1, d"
+            python3 -c "import json,sys; d=json.load(sys.stdin); assert d['count']>=1, d; assert d['engine'] in ('index','rg','scan')"
diff --git a/.gitignore b/.gitignore
@@ -34,3 +34,12 @@ src/routeTree.gen.ts
 # Isolation session bookkeeping
 .brain-sessions/
 .brain-worktrees/
+
+# Disposable retrieval index (docs/designs/retrieval-ladder.md). Git + Markdown
+# stay source of truth; deleting these files is always valid recovery.
+# Do not ignore the whole `.dekc/` directory — repo-level `.dekc/config.yml` lives there.
+**/.dekc/index.sqlite
+**/.dekc/index.sqlite-journal
+**/.dekc/index.sqlite-wal
+**/.dekc/index.sqlite-shm
+
diff --git a/.grok-plugin/marketplace.json b/.grok-plugin/marketplace.json
@@ -1,13 +1,13 @@
 {
   "name": "dekc-plugin-marketplace",
   "description": "Optional native Grok marketplace metadata. Grok Build loads Claude plugins with zero config; this pins identity for listings.",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "plugins": [
     {
       "name": "data-engineering-knowledge-capture",
       "source": ".",
       "description": "DEKC \u2014 data engineering knowledge capture on PKC + OKF. Claude-compatible.",
-      "version": "0.4.3",
+      "version": "0.5.0",
       "compatibility": {
         "claude_plugin": true,
         "zero_config": true

diff --git a/.opencode/plugin/dekc.json b/.opencode/plugin/dekc.json
@@ -1,6 +1,6 @@
 {
   "name": "data-engineering-knowledge-capture",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "OpenCode port of DEKC \u2014 same skills/agents/scripts as Claude & Codex. Policy via AGENTS.md.",
   "skills": "../../skills",
   "agents": "../../agents",

diff --git a/AGENTS.md b/AGENTS.md
@@ -21,6 +21,7 @@ One plugin tree, multiple hosts. Do not diverge packaging without updating [PORT
 ## Docs
 
 - [User guide](./docs/user_guide/user-guide.md) — install, walks, streams/jobs, multi-cloud recipes
+- [Retrieval ladder](./docs/designs/retrieval-ladder.md) — SQLite index → rg → scan; Git stays truth
 - [Design doc](./docs/designs/current_design_doc.md) — AGER agent loops, adversarial rubrics, Azure Fabric / AWS / GCP
 - [Typed edges](./docs/typed-edges.md)
 - [PORTS](./PORTS.md)
@@ -64,11 +65,12 @@ Plugin root: `${CLAUDE_PLUGIN_ROOT}`.
 2. Prefer deterministic scripts; agents extract structure from free text / SQL.
 3. Idempotent writes; never invent lineage edges.
 4. Direction matters: bronze → silver → gold; business objects `derived_from` tables.
-5. After walks: grade (`dekc_grade.py` + adversarial judges) then `dekc_index.py build` + `dekc_doctor.py`.
+5. After walks: grade (`dekc_grade.py` + adversarial judges) then `dekc_doctor.py`. Search/pack refresh `knowledge/.dekc/index.sqlite` themselves; `dekc_index.py build` is optional (`refresh --force`).
 6. Progressive disclosure: 2-hop packs (~20 nodes).
 7. Scrub secrets/PII on capture.
 8. **No RE success without re-adversary-judge pass** (threshold 0.75) unless user waives.
 9. On fail: capture missing evidence or **retract** unproven claims — never invent to raise scores.
+10. Never install ripgrep (or any package) from a hook. rg is optional. `DEKC_NO_INDEX=1` / `--no-index` fall through to rg then scan.
 
 ## Orchestrators
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,8 +2,50 @@
 
 ## Unreleased
 
+## 0.5.0 — 2026-08-30
+
+Retrieval ladder: Git + Markdown stays source of truth. Accelerators are disposable.
+Replaces the JSON `knowledge/.index/` inverted index (which was committed by accident).
+
+### Added
+
+- **ripgrep accelerator** for search and pack. `dekc_search.py` uses `rg -l` as a
+  candidate prefilter when `rg` is on PATH (or `DEKC_RG_PATH` / `PKC_RG_PATH` /
+  `OKF_RG_PATH`). Ranking stays in Python, so scores match a full scan.
+  `--no-rg` forces the linear walk. Missing rg is not an error. Never installed
+  from a hook.
+- **rg-backed reverse index** in `dekc_pack.py`. Inbound discovery is
+  `rg -lF` of the concept path; the current file is always parsed for outbound
+  typed-flow + SQL lineage (the file almost never contains its own path).
+- `dekc_doctor.py` reports a **toolchain** section (rg found/missing, FTS5, index).
+- **SQLite/FTS5 incremental index.** `scripts/dekc_index.py` writes
+  `knowledge/.dekc/index.sqlite` (gitignored). Search and pack self-heal via
+  mtime+size on every call. LIKE on a stored haystack is the default candidate
+  prefilter so Python scores stay identical to a scan. `--engine fts` is the
+  FTS5 MATCH opt-in. `--no-index` / `DEKC_NO_INDEX=1` fall through to rg then
+  scan. `/dekc-index` skill for status/refresh/drop. `build` remains an alias
+  for `refresh --force` (CI / existing skills).
+- SQL lineage edges store `origin` (the file that authored them) so incremental
+  delete does not leak table-to-table edges after the source file vanishes.
+- Retrieval ladder locked in [`docs/designs/retrieval-ladder.md`](docs/designs/retrieval-ladder.md).
+
+### Changed
+
+- Host manifests (`.opencode/plugin/dekc.json`, `public/data/catalog.json`)
+  lockstep with root `plugin.json` **0.5.0**. They were still labeled 0.4.2.
+
+### Notes
+
+- Pack identity: lineage is undirected for every visited node; extra non-lineage
+  `links[]` still attach to the focus only. Graphs match a scan unless
+  `--engine fts`.
+- `dekc_pack.py --mermaid` still calls `build_graph` (full scan). Accepted leftover.
+- Do not ignore the whole `.dekc/` directory — `.dekc/config.example.yml` is
+  committed. Gitignore only `**/.dekc/index.sqlite*`.
+
 ## 0.4.3 — 2026-08-30
 
+
 - `dekc_doctor.py` reports toolchain (ripgrep / SQLite FTS5). Search already
   goes through `knowledge/.index`; rg is optional, not a second index.
 

diff --git a/CLAUDE.md b/CLAUDE.md
@@ -12,14 +12,16 @@ Depends on **PKC** + **OKF**. Dual/quad host: Claude, Grok, Codex, OpenCode —
 
 ```bash
 python3 tests/test_dekc.py
+python3 tests/test_retrieval_ladder.py
 python3 scripts/dekc_validate.py --bundle sample-knowledge
 python3 scripts/dekc_doctor.py --bundle sample-knowledge
+npm run test
 npm run dev          # explorer on :8080
 npm run typecheck
 npm run build
 ```
 
-Add new `scripts/dekc_*.py` to `package.json` typecheck/test lists and CI.
+Add new `scripts/dekc_*.py` is covered the moment it lands: `npm run py:compile` and CI both `py_compile scripts/dekc_*.py`.
 
 ## Layout
 
@@ -34,6 +36,7 @@ Add new `scripts/dekc_*.py` to `package.json` typecheck/test lists and CI.
 - Absolute in-bundle links; typed `links[].rel` for lineage/business edges
 - Never invent edges; prefer scripts over freehand Markdown when possible
 - Reverse engineering: orchestrators + adversarial skeptics/rubrics (`dekc_grade.py`, re-adversary-judge); no success without pass
+- Retrieval: Git + Markdown is truth. `knowledge/.dekc/index.sqlite` is a disposable SQLite/FTS5 accelerator (gitignored, mtime+size self-heal). Ripgrep is optional (`DEKC_RG_PATH`). Search and pack must keep working when rg or FTS5 is absent. Never install packages from a hook. See `docs/designs/retrieval-ladder.md`. Do not resurrect JSON `.index/`.
 
 ## Docs
 

diff --git a/README.md b/README.md
@@ -8,14 +8,15 @@ DEKC **extends [Project Knowledge Capture (PKC)](https://github.com/SpillwaveSol
 |---|---|
 | **Plugin name** | `data-engineering-knowledge-capture` |
 | **Repo** | [SpillwaveSolutions/data-engineering-knowledge-capture](https://github.com/SpillwaveSolutions/data-engineering-knowledge-capture) |
-| **Version** | 0.4.3 |
+| **Version** | 0.5.0 |
 | **License** | MIT |
 | **Hosts** | Claude Code · Grok Build · Codex · OpenCode · Agent Plugins 1.0 · Grok Bot · LangChain Deep Agents |
 
 ## Docs
 
 | Doc | Audience |
 |-----|----------|
+| **[Retrieval ladder](./docs/designs/retrieval-ladder.md)** | Search/pack: SQLite index → rg → scan. Git stays truth. |
 | **[Noun-ownership migration](./docs/user_guide/noun-ownership-migration.md)** | Existing brains: `Workflow` → `IngestionJob`; mixed AGER/SAC types |
 | **[User guide](./docs/user_guide/user-guide.md)** | Install, walk a lake, promote business objects, multi-cloud recipes |
 | **[Design doc](./docs/designs/current_design_doc.md)** | Agent graph loops (AGER), Azure Fabric / AWS / GCP reverse engineering, streams & jobs |
@@ -81,7 +82,8 @@ Standard concept schemas: [`schemas/okf-concepts/`](./schemas/okf-concepts/) (Ta
 ```bash
 python3 scripts/dekc_schemas.py list
 python3 scripts/dekc_schemas.py validate --bundle sample-knowledge
-python3 scripts/dekc_index.py --bundle sample-knowledge build   # refresh search index
+python3 scripts/dekc_index.py status --bundle sample-knowledge
+python3 scripts/dekc_index.py refresh --bundle sample-knowledge   # self-heals; search/pack also refresh
 ```
 
 
@@ -200,8 +202,8 @@ python3 scripts/dekc_walk.py path/to/lake --repo . --bundle knowledge
 python3 scripts/dekc_lineage.py --repo . --bundle knowledge materialize
 python3 scripts/dekc_business.py --repo . --bundle knowledge promote-layer --layer gold
 
-# Index second brain + health
-python3 scripts/dekc_index.py --repo . --bundle knowledge build
+# Index second brain + health (search/pack refresh the index themselves)
+python3 scripts/dekc_index.py status --repo . --bundle knowledge
 python3 scripts/dekc_doctor.py --repo . --bundle knowledge
 python3 scripts/dekc_search.py "revenue" --repo . --bundle knowledge
 ```
@@ -252,31 +254,41 @@ python3 scripts/dekc_doctor.py --bundle sample-knowledge
 
 ## Second-brain index
 
-> **Do not commit the index.** It is fully derived and rebuilds in seconds;
-> `search.json` alone runs to megabytes and would churn every diff. Add this to
-> your `.gitignore` (a copy ships at `templates/gitignore-fragment`):
+Git + Markdown is the source of truth. Search and pack use a disposable ladder
+(SQLite FTS5 index → ripgrep → full scan). Deleting the accelerators is always
+valid recovery. See [retrieval ladder](./docs/designs/retrieval-ladder.md).
+
+> **Do not commit the index.** Add this to your `.gitignore` (a copy ships at
+> `templates/gitignore-fragment`):
 >
 > ```gitignore
 > **/.index/
+> **/.dekc/index.sqlite
+> **/.dekc/index.sqlite-journal
+> **/.dekc/index.sqlite-wal
+> **/.dekc/index.sqlite-shm
 > ```
 >
-> `**/.index/` rather than `*/.index/`, so it holds at any nesting depth and for
-> every bundle rather than one hardcoded name. A repo with two bundles is where
-> the narrower pattern shows: ignoring `knowledge/.index/` leaves the second
-> bundle's index sitting in `git status`.
-
+> Do not ignore the whole `.dekc/` directory — repo-level config lives there.
+> `**/.index/` still ignores the old JSON inverted index from 0.4.x so it
+> cannot be re-committed.
 
-`dekc_index.py build` writes:
+`dekc_index.py` writes:
 
 ```text
-<bundle>/.index/
-  inventory.json
-  search.json
-  graph.json
-  embeddings.jsonl   # local bag-of-tokens (no API key)
-  manifest.json
+<bundle>/.dekc/index.sqlite   # incremental, mtime+size self-heal
+```
+
+```bash
+python3 scripts/dekc_index.py status --bundle knowledge
+python3 scripts/dekc_index.py refresh --bundle knowledge
+python3 scripts/dekc_index.py drop --bundle knowledge
 ```
 
+Search, pack, and doctor refresh the index themselves. `build` is an alias
+for `refresh --force`. Ranking stays in Python so scores match a full scan
+unless you pass `--engine fts`. `DEKC_NO_INDEX=1` / `--no-index` fall through
+to rg then scan. Missing `rg` is not an error.
 ## Config
 
 See [`.dekc/config.example.yml`](./.dekc/config.example.yml).

diff --git a/agents/data-lake-walker.md b/agents/data-lake-walker.md
@@ -14,7 +14,7 @@ DEKC extends **PKC** and **OKF**. Multi-agent loops follow **AGER** ([okf-agent-
 3. Scrub secrets/PII before writing concepts.
 4. Progressive disclosure: 2-hop packs (~20 nodes).
 5. **No success claim without re-adversary-judge pass** (or explicit user waiver).
-6. After accepted walks: `dekc_index.py build`.
+6. After accepted walks: `dekc_doctor.py`. Search/pack refresh the SQLite index themselves; `dekc_index.py build` is optional (`refresh --force`).
 
 ## Producer subagents (Workers)
 

diff --git a/agents/reverse-engineering-orchestrator.md b/agents/reverse-engineering-orchestrator.md
@@ -67,8 +67,8 @@ python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_lineage.py" --repo . --bundle knowle
 python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_business.py" --repo . --bundle knowledge promote-layer --layer gold
 python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_grade.py" --repo . --bundle knowledge --json
 python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_grade.py" --repo . --bundle knowledge --prefix semantic,tables/gold-,reports,dashboards
-python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" --repo . --bundle knowledge build
 python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_doctor.py" --repo . --bundle knowledge
+# optional: python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" refresh --force --repo . --bundle knowledge
 ```
 
 Write judgments under `knowledge/agents/judgment-<run>.md` (or `agents/` if colocated) with scores, failed criteria, and required revisions.

diff --git a/commands/dekc-index.md b/commands/dekc-index.md
@@ -1,6 +1,6 @@
 ---
 name: dekc-index
-description: Rebuild the local second-brain index (inventory, inverted search, graph, embeddings).
+description: Refresh, inspect, or drop the disposable SQLite/FTS5 knowledge index.
 ---
 
 Run the **dekc-index** skill.

diff --git a/commands/dekc-search.md b/commands/dekc-search.md
@@ -1,6 +1,6 @@
 ---
 name: dekc-search
-description: Search the indexed DEKC second brain.
+description: Full-text search over the DEKC knowledge bundle (AND terms, type filters).
 ---
 
 Run the **dekc-search** skill.

diff --git a/docs/designs/current_design_doc.md b/docs/designs/current_design_doc.md
@@ -21,7 +21,9 @@ Related systems:
 | PKC | Project reasoning capture | [project-knowledge-capture](https://github.com/SpillwaveSolutions/project-knowledge-capture) |
 | DEKC | Data-domain catalogs + walk scripts + agents | this repo |
 
-AGER spec version referenced: **0.3.0** ([AGER_SPEC](https://github.com/SpillwaveSolutions/okf-agent-graph/blob/main/docs/AGER_SPEC.md)). DEKC plugin **0.2.0**.
+AGER spec version referenced: **0.3.0** ([AGER_SPEC](https://github.com/SpillwaveSolutions/okf-agent-graph/blob/main/docs/AGER_SPEC.md)). DEKC plugin **0.5.0**.
+
+**v0.5.0 retrieval:** search/pack use a disposable ladder (SQLite FTS5 index → ripgrep → scan). Git + Markdown is still the source of truth. See [`docs/designs/retrieval-ladder.md`](retrieval-ladder.md) — the `.index/` JSON layout later in this snapshot is historical (0.4.x) and is not how 0.5.0 retrieves.
 
 ---