diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c991fda..81d042b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "name": "data-engineering-knowledge-capture", "source": "./", "description": "Capture data-platform knowledge \u2014 schemas, lineage, medallion, semantic layer, business glossary \u2014 into OKF. Extends PKC; depends on OKF.", - "version": "0.4.3", + "version": "0.5.0", "author": { "name": "Rick Hightower" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index f88d46d..93c8d87 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.4.3", + "version": "0.5.0", "description": "Data Engineering Knowledge Capture (DEKC) \u2014 extends PKC + OKF for schemas, lineage, medallion layers, SQL/DAX, semantic models, business objects, and an indexed second brain. Agents walk data lakes. Works in Claude Code, Grok Build, Codex, and OpenCode.", "author": { "name": "Rick Hightower", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 177d430..db5eae0 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.4.3", + "version": "0.5.0", "description": "Data Engineering Knowledge Capture (DEKC) \u2014 schemas, lineage, medallion layers, semantic models, business glossary. Extends PKC; depends on OKF. Codex-native skills + hooks port.", "author": { "name": "Rick Hightower", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index d33aafe..619989f 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.4.3", + "version": "0.5.0", "description": "Data Engineering Knowledge Capture: schemas, lineage, medallion, semantic models, and glossary into a durable OKF knowledge graph. Multi-host bindings and write isolation.", "author": { "name": "Rick Hightower" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d407dae..a7aaf82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,23 +17,12 @@ jobs: python-version: "3.12" - name: Unit tests - run: python3 tests/test_dekc.py && python3 tests/test_isolation.py + run: python3 tests/test_dekc.py && python3 tests/test_retrieval_ladder.py && python3 tests/test_isolation.py + # Glob, not a list. The hand-maintained list drifted whenever a new + # dekc_*.py landed. PKC 0.9.0 made the same change. - name: Compile scripts - run: | - python3 -m py_compile \ - scripts/dekc_common.py \ - scripts/dekc_capture.py \ - scripts/dekc_walk.py \ - scripts/dekc_lineage.py \ - scripts/dekc_business.py \ - scripts/dekc_index.py \ - scripts/dekc_validate.py \ - scripts/dekc_pack.py \ - scripts/dekc_search.py \ - scripts/dekc_doctor.py \ - scripts/dekc_link.py \ - scripts/brain_session.py + run: python3 -m py_compile scripts/dekc_*.py scripts/brain_session.py - name: Validate + doctor sample-knowledge run: | @@ -43,5 +32,7 @@ jobs: - name: Index + search run: | python3 scripts/dekc_index.py --repo . --bundle sample-knowledge build + python3 scripts/dekc_index.py status --repo . --bundle sample-knowledge --json | \ + python3 -c "import json,sys; d=json.load(sys.stdin); assert d['fts5'] is True; assert d['nodes']>=1, d" python3 scripts/dekc_search.py revenue --repo . --bundle sample-knowledge --json | \ - python3 -c "import json,sys; d=json.load(sys.stdin); assert d['count']>=1, d" + python3 -c "import json,sys; d=json.load(sys.stdin); assert d['count']>=1, d; assert d['engine'] in ('index','rg','scan')" diff --git a/.gitignore b/.gitignore index 48fa9c2..e485f5e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,12 @@ src/routeTree.gen.ts # Isolation session bookkeeping .brain-sessions/ .brain-worktrees/ + +# Disposable retrieval index (docs/designs/retrieval-ladder.md). Git + Markdown +# stay source of truth; deleting these files is always valid recovery. +# Do not ignore the whole `.dekc/` directory — repo-level `.dekc/config.yml` lives there. +**/.dekc/index.sqlite +**/.dekc/index.sqlite-journal +**/.dekc/index.sqlite-wal +**/.dekc/index.sqlite-shm + diff --git a/.grok-plugin/marketplace.json b/.grok-plugin/marketplace.json index 49eb3f6..3b6c9da 100644 --- a/.grok-plugin/marketplace.json +++ b/.grok-plugin/marketplace.json @@ -1,13 +1,13 @@ { "name": "dekc-plugin-marketplace", "description": "Optional native Grok marketplace metadata. Grok Build loads Claude plugins with zero config; this pins identity for listings.", - "version": "0.4.3", + "version": "0.5.0", "plugins": [ { "name": "data-engineering-knowledge-capture", "source": ".", "description": "DEKC \u2014 data engineering knowledge capture on PKC + OKF. Claude-compatible.", - "version": "0.4.3", + "version": "0.5.0", "compatibility": { "claude_plugin": true, "zero_config": true diff --git a/.opencode/plugin/dekc.json b/.opencode/plugin/dekc.json index 9c84d7a..0c2ba17 100644 --- a/.opencode/plugin/dekc.json +++ b/.opencode/plugin/dekc.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.4.2", + "version": "0.5.0", "description": "OpenCode port of DEKC \u2014 same skills/agents/scripts as Claude & Codex. Policy via AGENTS.md.", "skills": "../../skills", "agents": "../../agents", diff --git a/AGENTS.md b/AGENTS.md index 57ef975..0d9dd53 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ One plugin tree, multiple hosts. Do not diverge packaging without updating [PORT ## Docs - [User guide](./docs/user_guide/user-guide.md) — install, walks, streams/jobs, multi-cloud recipes +- [Retrieval ladder](./docs/designs/retrieval-ladder.md) — SQLite index → rg → scan; Git stays truth - [Design doc](./docs/designs/current_design_doc.md) — AGER agent loops, adversarial rubrics, Azure Fabric / AWS / GCP - [Typed edges](./docs/typed-edges.md) - [PORTS](./PORTS.md) @@ -64,11 +65,12 @@ Plugin root: `${CLAUDE_PLUGIN_ROOT}`. 2. Prefer deterministic scripts; agents extract structure from free text / SQL. 3. Idempotent writes; never invent lineage edges. 4. Direction matters: bronze → silver → gold; business objects `derived_from` tables. -5. After walks: grade (`dekc_grade.py` + adversarial judges) then `dekc_index.py build` + `dekc_doctor.py`. +5. After walks: grade (`dekc_grade.py` + adversarial judges) then `dekc_doctor.py`. Search/pack refresh `knowledge/.dekc/index.sqlite` themselves; `dekc_index.py build` is optional (`refresh --force`). 6. Progressive disclosure: 2-hop packs (~20 nodes). 7. Scrub secrets/PII on capture. 8. **No RE success without re-adversary-judge pass** (threshold 0.75) unless user waives. 9. On fail: capture missing evidence or **retract** unproven claims — never invent to raise scores. +10. Never install ripgrep (or any package) from a hook. rg is optional. `DEKC_NO_INDEX=1` / `--no-index` fall through to rg then scan. ## Orchestrators diff --git a/CHANGELOG.md b/CHANGELOG.md index e0b07ac..ec2fcdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,50 @@ ## Unreleased +## 0.5.0 — 2026-08-30 + +Retrieval ladder: Git + Markdown stays source of truth. Accelerators are disposable. +Replaces the JSON `knowledge/.index/` inverted index (which was committed by accident). + +### Added + +- **ripgrep accelerator** for search and pack. `dekc_search.py` uses `rg -l` as a + candidate prefilter when `rg` is on PATH (or `DEKC_RG_PATH` / `PKC_RG_PATH` / + `OKF_RG_PATH`). Ranking stays in Python, so scores match a full scan. + `--no-rg` forces the linear walk. Missing rg is not an error. Never installed + from a hook. +- **rg-backed reverse index** in `dekc_pack.py`. Inbound discovery is + `rg -lF` of the concept path; the current file is always parsed for outbound + typed-flow + SQL lineage (the file almost never contains its own path). +- `dekc_doctor.py` reports a **toolchain** section (rg found/missing, FTS5, index). +- **SQLite/FTS5 incremental index.** `scripts/dekc_index.py` writes + `knowledge/.dekc/index.sqlite` (gitignored). Search and pack self-heal via + mtime+size on every call. LIKE on a stored haystack is the default candidate + prefilter so Python scores stay identical to a scan. `--engine fts` is the + FTS5 MATCH opt-in. `--no-index` / `DEKC_NO_INDEX=1` fall through to rg then + scan. `/dekc-index` skill for status/refresh/drop. `build` remains an alias + for `refresh --force` (CI / existing skills). +- SQL lineage edges store `origin` (the file that authored them) so incremental + delete does not leak table-to-table edges after the source file vanishes. +- Retrieval ladder locked in [`docs/designs/retrieval-ladder.md`](docs/designs/retrieval-ladder.md). + +### Changed + +- Host manifests (`.opencode/plugin/dekc.json`, `public/data/catalog.json`) + lockstep with root `plugin.json` **0.5.0**. They were still labeled 0.4.2. + +### Notes + +- Pack identity: lineage is undirected for every visited node; extra non-lineage + `links[]` still attach to the focus only. Graphs match a scan unless + `--engine fts`. +- `dekc_pack.py --mermaid` still calls `build_graph` (full scan). Accepted leftover. +- Do not ignore the whole `.dekc/` directory — `.dekc/config.example.yml` is + committed. Gitignore only `**/.dekc/index.sqlite*`. + ## 0.4.3 — 2026-08-30 + - `dekc_doctor.py` reports toolchain (ripgrep / SQLite FTS5). Search already goes through `knowledge/.index`; rg is optional, not a second index. diff --git a/CLAUDE.md b/CLAUDE.md index 3dc4190..be352e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,14 +12,16 @@ Depends on **PKC** + **OKF**. Dual/quad host: Claude, Grok, Codex, OpenCode — ```bash python3 tests/test_dekc.py +python3 tests/test_retrieval_ladder.py python3 scripts/dekc_validate.py --bundle sample-knowledge python3 scripts/dekc_doctor.py --bundle sample-knowledge +npm run test npm run dev # explorer on :8080 npm run typecheck npm run build ``` -Add new `scripts/dekc_*.py` to `package.json` typecheck/test lists and CI. +Add new `scripts/dekc_*.py` is covered the moment it lands: `npm run py:compile` and CI both `py_compile scripts/dekc_*.py`. ## Layout @@ -34,6 +36,7 @@ Add new `scripts/dekc_*.py` to `package.json` typecheck/test lists and CI. - Absolute in-bundle links; typed `links[].rel` for lineage/business edges - Never invent edges; prefer scripts over freehand Markdown when possible - Reverse engineering: orchestrators + adversarial skeptics/rubrics (`dekc_grade.py`, re-adversary-judge); no success without pass +- Retrieval: Git + Markdown is truth. `knowledge/.dekc/index.sqlite` is a disposable SQLite/FTS5 accelerator (gitignored, mtime+size self-heal). Ripgrep is optional (`DEKC_RG_PATH`). Search and pack must keep working when rg or FTS5 is absent. Never install packages from a hook. See `docs/designs/retrieval-ladder.md`. Do not resurrect JSON `.index/`. ## Docs diff --git a/README.md b/README.md index 7184dee..c395bf5 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ DEKC **extends [Project Knowledge Capture (PKC)](https://github.com/SpillwaveSol |---|---| | **Plugin name** | `data-engineering-knowledge-capture` | | **Repo** | [SpillwaveSolutions/data-engineering-knowledge-capture](https://github.com/SpillwaveSolutions/data-engineering-knowledge-capture) | -| **Version** | 0.4.3 | +| **Version** | 0.5.0 | | **License** | MIT | | **Hosts** | Claude Code · Grok Build · Codex · OpenCode · Agent Plugins 1.0 · Grok Bot · LangChain Deep Agents | @@ -16,6 +16,7 @@ DEKC **extends [Project Knowledge Capture (PKC)](https://github.com/SpillwaveSol | Doc | Audience | |-----|----------| +| **[Retrieval ladder](./docs/designs/retrieval-ladder.md)** | Search/pack: SQLite index → rg → scan. Git stays truth. | | **[Noun-ownership migration](./docs/user_guide/noun-ownership-migration.md)** | Existing brains: `Workflow` → `IngestionJob`; mixed AGER/SAC types | | **[User guide](./docs/user_guide/user-guide.md)** | Install, walk a lake, promote business objects, multi-cloud recipes | | **[Design doc](./docs/designs/current_design_doc.md)** | Agent graph loops (AGER), Azure Fabric / AWS / GCP reverse engineering, streams & jobs | @@ -81,7 +82,8 @@ Standard concept schemas: [`schemas/okf-concepts/`](./schemas/okf-concepts/) (Ta ```bash python3 scripts/dekc_schemas.py list python3 scripts/dekc_schemas.py validate --bundle sample-knowledge -python3 scripts/dekc_index.py --bundle sample-knowledge build # refresh search index +python3 scripts/dekc_index.py status --bundle sample-knowledge +python3 scripts/dekc_index.py refresh --bundle sample-knowledge # self-heals; search/pack also refresh ``` @@ -200,8 +202,8 @@ python3 scripts/dekc_walk.py path/to/lake --repo . --bundle knowledge python3 scripts/dekc_lineage.py --repo . --bundle knowledge materialize python3 scripts/dekc_business.py --repo . --bundle knowledge promote-layer --layer gold -# Index second brain + health -python3 scripts/dekc_index.py --repo . --bundle knowledge build +# Index second brain + health (search/pack refresh the index themselves) +python3 scripts/dekc_index.py status --repo . --bundle knowledge python3 scripts/dekc_doctor.py --repo . --bundle knowledge python3 scripts/dekc_search.py "revenue" --repo . --bundle knowledge ``` @@ -252,31 +254,41 @@ python3 scripts/dekc_doctor.py --bundle sample-knowledge ## Second-brain index -> **Do not commit the index.** It is fully derived and rebuilds in seconds; -> `search.json` alone runs to megabytes and would churn every diff. Add this to -> your `.gitignore` (a copy ships at `templates/gitignore-fragment`): +Git + Markdown is the source of truth. Search and pack use a disposable ladder +(SQLite FTS5 index → ripgrep → full scan). Deleting the accelerators is always +valid recovery. See [retrieval ladder](./docs/designs/retrieval-ladder.md). + +> **Do not commit the index.** Add this to your `.gitignore` (a copy ships at +> `templates/gitignore-fragment`): > > ```gitignore > **/.index/ +> **/.dekc/index.sqlite +> **/.dekc/index.sqlite-journal +> **/.dekc/index.sqlite-wal +> **/.dekc/index.sqlite-shm > ``` > -> `**/.index/` rather than `*/.index/`, so it holds at any nesting depth and for -> every bundle rather than one hardcoded name. A repo with two bundles is where -> the narrower pattern shows: ignoring `knowledge/.index/` leaves the second -> bundle's index sitting in `git status`. - +> Do not ignore the whole `.dekc/` directory — repo-level config lives there. +> `**/.index/` still ignores the old JSON inverted index from 0.4.x so it +> cannot be re-committed. -`dekc_index.py build` writes: +`dekc_index.py` writes: ```text -/.index/ - inventory.json - search.json - graph.json - embeddings.jsonl # local bag-of-tokens (no API key) - manifest.json +/.dekc/index.sqlite # incremental, mtime+size self-heal +``` + +```bash +python3 scripts/dekc_index.py status --bundle knowledge +python3 scripts/dekc_index.py refresh --bundle knowledge +python3 scripts/dekc_index.py drop --bundle knowledge ``` +Search, pack, and doctor refresh the index themselves. `build` is an alias +for `refresh --force`. Ranking stays in Python so scores match a full scan +unless you pass `--engine fts`. `DEKC_NO_INDEX=1` / `--no-index` fall through +to rg then scan. Missing `rg` is not an error. ## Config See [`.dekc/config.example.yml`](./.dekc/config.example.yml). diff --git a/agents/data-lake-walker.md b/agents/data-lake-walker.md index aaf7bd1..fc00086 100644 --- a/agents/data-lake-walker.md +++ b/agents/data-lake-walker.md @@ -14,7 +14,7 @@ DEKC extends **PKC** and **OKF**. Multi-agent loops follow **AGER** ([okf-agent- 3. Scrub secrets/PII before writing concepts. 4. Progressive disclosure: 2-hop packs (~20 nodes). 5. **No success claim without re-adversary-judge pass** (or explicit user waiver). -6. After accepted walks: `dekc_index.py build`. +6. After accepted walks: `dekc_doctor.py`. Search/pack refresh the SQLite index themselves; `dekc_index.py build` is optional (`refresh --force`). ## Producer subagents (Workers) diff --git a/agents/reverse-engineering-orchestrator.md b/agents/reverse-engineering-orchestrator.md index 1bac641..7ef7dc7 100644 --- a/agents/reverse-engineering-orchestrator.md +++ b/agents/reverse-engineering-orchestrator.md @@ -67,8 +67,8 @@ python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_lineage.py" --repo . --bundle knowle python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_business.py" --repo . --bundle knowledge promote-layer --layer gold python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_grade.py" --repo . --bundle knowledge --json python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_grade.py" --repo . --bundle knowledge --prefix semantic,tables/gold-,reports,dashboards -python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" --repo . --bundle knowledge build python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_doctor.py" --repo . --bundle knowledge +# optional: python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" refresh --force --repo . --bundle knowledge ``` Write judgments under `knowledge/agents/judgment-.md` (or `agents/` if colocated) with scores, failed criteria, and required revisions. diff --git a/commands/dekc-index.md b/commands/dekc-index.md index d0e63ac..543dfb4 100644 --- a/commands/dekc-index.md +++ b/commands/dekc-index.md @@ -1,6 +1,6 @@ --- name: dekc-index -description: Rebuild the local second-brain index (inventory, inverted search, graph, embeddings). +description: Refresh, inspect, or drop the disposable SQLite/FTS5 knowledge index. --- Run the **dekc-index** skill. diff --git a/commands/dekc-search.md b/commands/dekc-search.md index 59fe773..f72c893 100644 --- a/commands/dekc-search.md +++ b/commands/dekc-search.md @@ -1,6 +1,6 @@ --- name: dekc-search -description: Search the indexed DEKC second brain. +description: Full-text search over the DEKC knowledge bundle (AND terms, type filters). --- Run the **dekc-search** skill. diff --git a/docs/designs/current_design_doc.md b/docs/designs/current_design_doc.md index ca3c3eb..1b6be67 100644 --- a/docs/designs/current_design_doc.md +++ b/docs/designs/current_design_doc.md @@ -21,7 +21,9 @@ Related systems: | PKC | Project reasoning capture | [project-knowledge-capture](https://github.com/SpillwaveSolutions/project-knowledge-capture) | | DEKC | Data-domain catalogs + walk scripts + agents | this repo | -AGER spec version referenced: **0.3.0** ([AGER_SPEC](https://github.com/SpillwaveSolutions/okf-agent-graph/blob/main/docs/AGER_SPEC.md)). DEKC plugin **0.2.0**. +AGER spec version referenced: **0.3.0** ([AGER_SPEC](https://github.com/SpillwaveSolutions/okf-agent-graph/blob/main/docs/AGER_SPEC.md)). DEKC plugin **0.5.0**. + +**v0.5.0 retrieval:** search/pack use a disposable ladder (SQLite FTS5 index → ripgrep → scan). Git + Markdown is still the source of truth. See [`docs/designs/retrieval-ladder.md`](retrieval-ladder.md) — the `.index/` JSON layout later in this snapshot is historical (0.4.x) and is not how 0.5.0 retrieves. --- diff --git a/docs/designs/retrieval-ladder.md b/docs/designs/retrieval-ladder.md new file mode 100644 index 0000000..b05f22f --- /dev/null +++ b/docs/designs/retrieval-ladder.md @@ -0,0 +1,127 @@ +--- +wiki_key: design/retrieval-ladder +doc_type: design +truth_state: current +--- + +# Retrieval ladder + +Git + Markdown is the source of truth on every rung. Everything above Git is +a disposable accelerator you can delete and rebuild. Same philosophy as PKC +(`docs/designs/retrieval-ladder.md` there) and research-graph (`rg_project.py`: +"the index can be destroyed and rebuilt"). + +Each rung attacks a different cost, buys roughly an order of magnitude of +scale, and keeps the rung below it as a runtime fallback: + +``` +index present → use it +else rg on PATH → prefilter +else → pure Python scan +``` + +Three tiers, one behavior. `--no-rg` / `--no-index` force a lower rung. + +0.4.x wrote JSON `knowledge/.index/` (inventory + inverted tokens) and +rebuilt the whole thing. That was a full scan plus a JSON parse, and it +got committed by accident. 0.5.0 replaced it with PKC's ladder: SQLite FTS5 +at `knowledge/.dekc/index.sqlite`, gitignored, incremental, self-healing. + +## Rung 1 — ripgrep prefilter (now) + +**Cost it attacks:** reading and lowercasing files that cannot possibly match. + +**Mechanics:** `find_rg()` / `rg_list_files()` in `dekc_common.py`. Search +AND-intersects `rg -l` per term, then runs the existing Python scorer +(`title×10 / description×5 / tags×4 / min(body,8)`) over only those +candidates. Pack inbound discovery is `rg -lF` of the concept path, then +the current file is always parsed for outbound typed-flow + SQL lineage +(the file almost never contains its own path). Override with `DEKC_RG_PATH` +/ `PKC_RG_PATH` / `OKF_RG_PATH`. Missing rg is not an error. + +**Why scores stay identical:** rg only decides which files get read. +Over-selection is harmless (Python re-checks). It cannot under-select for +plain substring terms. + +**Why it is first:** no state on disk, so no cache-staleness class of bugs. +Zero new dependencies. Claude Code / Grok hosts usually ship `rg`. + +**Ceiling:** still an O(corpus) scan per query — compiled, parallel, +memory-mapped. It only accelerates "which files contain X." Parsed-graph +commands (validate, doctor orphans, mermaid via `build_graph`) still walk +the tree. The curate hook does not install rg and does not build the index. + +| Hot path | Rung 1 helps? | Why | +|---|---|---| +| `dekc_search.py` | yes | candidate prefilter | +| `dekc_pack.py` inbound | yes | reverse index via literal path + self SQL | +| `dekc_validate.py` / `dekc-curate.sh` | **no** | must parse frontmatter + resolve links | +| orphans / doctor listings | **no** | need the parsed graph | +| mermaid (`build_graph`) | **no** | leftover full scan; accepted | + +## Rung 2 — stdlib SQLite + FTS5 incremental index (now) + +**Cost it attacks:** re-parsing files that have not changed since the last +invocation. + +**Why SQLite:** `sqlite3` is in the stdlib. FTS5 is compiled into CPython +(doctor reports it). Honors the hard "no pip dependencies" rule. One file, +atomic transactions, concurrent readers, trivially deletable. + +**Shape:** + +| Table | Role | +|---|---| +| `meta(schema_version)` | drop-rebuild on mismatch | +| `nodes(path, type, title, description, status, tags, layer, mtime, size, fm_json, body, hay)` | concept cards | +| `edges(src, dst, rel, label, origin)` | typed flow + SQL lineage; `origin` is the file that authored the edge so incremental delete does not leak | +| `fts` (FTS5 over title / description / tags / body) | lexical retrieval (`--engine fts`) | + +Path: `knowledge/.dekc/index.sqlite`. Gitignore `**/.dekc/index.sqlite*`. +Do not ignore the whole `.dekc/` directory — repo-level +`.dekc/config.example.yml` lives there. Never a hook install, never a pip +dep. `scripts/dekc_index.py` + `/dekc-index`. + +**Incremental refresh is the whole trick.** On each invocation the reader +stats the tree, compares `mtime+size` against stored rows, re-parses only +what changed, deletes vanished files. Every reader does this sweep itself +rather than trusting `dekc-curate.sh` — that makes it self-healing against +hand edits, `git checkout`, branch switches, and a disabled hook. Cold +rebuild costs one full scan. Steady-state queries are 1–10 ms. + +**Scoring identity.** FTS5 `bm25()` with column weights is *not* the same +function as `title.count×10`. Same pattern as rg: + +- Default (`engine=index`): SQL LIKE on the stored haystack decides + candidates. Existing Python scorer ranks, so `--no-index` stays + score-identical. +- `--engine fts` exposes FTS5 MATCH with prefix tokens as an opt-in. + +**Pack identity.** Lineage is undirected for every visited node (typed +`FORWARD_FLOW` / `REVERSE_FLOW` + SQL `sql_from`). Extra non-lineage +`links[]` are added on the focus only. Scores/graphs match a scan unless +`--engine fts`. + +**Trigger.** Implemented. `DEKC_NO_INDEX=1` / `--no-index` disables it. +`DEKC_INDEX_PATH` overrides the sqlite location (tests). + +## Rung 3 — okfcli (if ever) + +The fast-native slot is already assigned by okf-plugin working rule #1: +prefer `okf` / `okfcli` when installed, else `python3`. A single static +binary (Rust/Go) with the Python scripts as the portable fallback. Do not +invent a second runtime (Ruby closed-won't-do, PKC #60). + +## Runtime degradation (load this, do not skip it) + +``` +def retrieve(bundle, query): + if index_ok(bundle): # rung 2 + return from_index(bundle, query) + if find_rg(): # rung 1 + return from_rg(bundle, query) + return from_scan(bundle, query) # rung 0 +``` + +Git remains the only durable store. Deleting `knowledge/.dekc/index.sqlite` +is always valid recovery. diff --git a/docs/user_guide/user-guide.md b/docs/user_guide/user-guide.md index 51d4f2b..a3636c8 100644 --- a/docs/user_guide/user-guide.md +++ b/docs/user_guide/user-guide.md @@ -10,7 +10,7 @@ truth_state: current **Data Engineering Knowledge Capture** turns data platforms into a durable, Git-native [OKF](https://github.com/SpillwaveSolutions/okf-plugin) knowledge graph, with multi-agent walk loops designed using [AGER](https://github.com/SpillwaveSolutions/okf-agent-graph) (OKF Agent Graph Engineering Runtime). -Plugin release **0.4.0**. Storage format is OKF **0.2**. Agent loops follow AGER **0.3** roles (orchestrator / worker / judge / synthesizer) even when you run DEKC skills without a separate AGER bundle. +Plugin release **0.5.0**. Storage format is OKF **0.2**. Agent loops follow AGER **0.3** roles (orchestrator / worker / judge / synthesizer) even when you run DEKC skills without a separate AGER bundle. Search and pack use a disposable [retrieval ladder](../designs/retrieval-ladder.md) (SQLite index → ripgrep → scan); Git + Markdown stays source of truth. Existing second brains: [noun-ownership migration](./noun-ownership-migration.md) (`Workflow` jobs → `IngestionJob`; diagrams stay SAC). @@ -193,8 +193,11 @@ python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_business.py" --repo . --bundle knowl ### 4. Index the second brain +Git + Markdown is the source of truth. Search and pack refresh a disposable +SQLite index themselves (mtime+size). You do not need to run this first. + ```bash -python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" --repo . --bundle knowledge build +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" status --repo . --bundle knowledge python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" "GMV" --repo . --bundle knowledge python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_doctor.py" --repo . --bundle knowledge ``` @@ -202,14 +205,12 @@ python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_doctor.py" --repo . --bundle knowled Index layout: ```text -knowledge/.index/ - inventory.json # all concepts - search.json # inverted tokens - graph.json # lineage adjacency - embeddings.jsonl # local bag-of-tokens vectors - manifest.json +knowledge/.dekc/index.sqlite # gitignored; delete anytime ``` +`build` is an alias for `refresh --force`. `--no-index` / `DEKC_NO_INDEX=1` +fall through to ripgrep then a full scan. `--engine fts` opts into FTS5 MATCH +(not score-identical). See [retrieval ladder](../designs/retrieval-ladder.md). ### 5. Progressive disclosure packs When an agent works on one table: @@ -417,7 +418,7 @@ Explorer UI (this repo): `npm run dev` → open the live preview catalog tabs. | `dekc_capture.py ` | source/table/view/query/lineage/… | | `dekc_lineage.py` | graph / upstream / mermaid / materialize | | `dekc_business.py` | promote / promote-layer | -| `dekc_index.py` | build / search | +| `dekc_index.py` | status / refresh / drop / build | | `dekc_pack.py` | context packs | | `dekc_search.py` | full-text + index | | `dekc_validate.py` / `dekc_doctor.py` | quality | diff --git a/marketplace.json b/marketplace.json index eaffb2c..22f7576 100644 --- a/marketplace.json +++ b/marketplace.json @@ -11,7 +11,7 @@ "name": "data-engineering-knowledge-capture", "source": "./", "description": "Data lake second brain: schemas, lineage, medallion, semantic layer, glossary. Depends on PKC + OKF.", - "version": "0.4.3", + "version": "0.5.0", "author": { "name": "Rick Hightower" }, diff --git a/package.json b/package.json index e1688cb..698603e 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build:dev": "vite build --mode development", "preview": "vite preview --host 0.0.0.0 --port 8080", "typecheck": "tsc --noEmit", - "test": "python3 tests/test_dekc.py && python3 tests/test_fabric_walk_fixes.py && python3 tests/test_isolation.py", + "test": "python3 tests/test_dekc.py && python3 tests/test_retrieval_ladder.py && python3 tests/test_fabric_walk_fixes.py && python3 tests/test_isolation.py", "lint": "eslint .", "format": "prettier --write .", "test:js": "node --test 'scripts/**/*.test.mjs'", @@ -23,7 +23,7 @@ "brain": "python3 scripts/dekc_brain.py revenue --intent design-report --bundle sample-knowledge", "schemas": "python3 scripts/dekc_schemas.py list", "index": "python3 scripts/dekc_index.py --repo . --bundle sample-knowledge build", - "py:compile": "python3 -m py_compile scripts/dekc_common.py scripts/dekc_capture.py scripts/dekc_walk.py scripts/dekc_lineage.py scripts/dekc_business.py scripts/dekc_index.py scripts/dekc_validate.py scripts/dekc_pack.py scripts/dekc_search.py scripts/dekc_doctor.py scripts/dekc_link.py scripts/dekc_grade.py scripts/dekc_brain.py scripts/dekc_schemas.py scripts/dekc_diagram.py scripts/dekc_platform.py scripts/brain_session.py" + "py:compile": "python3 -m py_compile scripts/dekc_*.py scripts/brain_session.py" }, "dependencies": { "@electric-sql/pglite": "^0.5.4", @@ -99,6 +99,6 @@ "vite": "^8.2.0" }, "description": "Data Engineering Knowledge Capture (DEKC) \u2014 Claude/Grok/Codex/OpenCode plugin + explorer", - "version": "0.4.3", + "version": "0.5.0", "license": "MIT" } diff --git a/plugin.json b/plugin.json index 28d92db..7972dd1 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "data-engineering-knowledge-capture", - "version": "0.4.3", + "version": "0.5.0", "description": "Data Engineering Knowledge Capture: schemas, lineage, medallion, semantic models, and glossary into a durable OKF knowledge graph. Multi-host bindings and write isolation.", "author": { "name": "Rick Hightower", diff --git a/public/data/catalog.json b/public/data/catalog.json index bf80ab8..4d46abd 100644 --- a/public/data/catalog.json +++ b/public/data/catalog.json @@ -1,6 +1,6 @@ { "title": "Retail Lake Sample Knowledge", - "version": "0.4.2", + "version": "0.5.0", "depends_on": [ "project-knowledge-capture", "okf-graph-eng" diff --git a/public/data/manifest.json b/public/data/manifest.json index 698fa79..aed7957 100644 --- a/public/data/manifest.json +++ b/public/data/manifest.json @@ -4,11 +4,8 @@ "token_count": 211, "edge_count": 4, "bundle": "/workspace/sample-knowledge", - "engine": "dekc-local-bow-v1", + "engine": "dekc-sqlite-fts5-v1", "files": [ - ".index/inventory.json", - ".index/search.json", - ".index/graph.json", - ".index/embeddings.jsonl" + ".dekc/index.sqlite" ] } \ No newline at end of file diff --git a/sample-knowledge/.index/embeddings.jsonl b/sample-knowledge/.index/embeddings.jsonl deleted file mode 100644 index 6ddc5ce..0000000 --- a/sample-knowledge/.index/embeddings.jsonl +++ /dev/null @@ -1,83 +0,0 @@ -{"path": "agents/walk-retail-lake-walk.md", "type": "AgentNode", "title": "Lake walk: retail-lake-walk", "vector": {"lake": 0.580042, "walk": 0.580042, "dekc": 0.193347, "fixtures": 0.193347, "retail": 0.193347, "sample": 0.193347, "tests": 0.193347, "workspace": 0.193347, "14": 0.096674, "agent": 0.096674, "agentnode": 0.096674, "created": 0.096674, "errors": 0.096674, "json": 0.096674, "of": 0.096674, "root": 0.096674, "skipped": 0.096674, "stats": 0.096674, "updated": 0.096674}} -{"path": "business-objects/customer-lifetime-value.md", "type": "BusinessObject", "title": "Customer Lifetime Value", "vector": {"customer": 0.545455, "value": 0.363636, "amounts": 0.181818, "based": 0.181818, "business": 0.181818, "gold": 0.181818, "gross": 0.181818, "historical": 0.181818, "lifetime": 0.181818, "ltv": 0.181818, "md": 0.181818, "of": 0.181818, "on": 0.181818, "order": 0.181818, "predicted": 0.181818, "realized": 0.181818, "views": 0.181818, "analytics": 0.090909, "businessobject": 0.090909, "definition": 0.090909, "dekc": 0.090909, "growth": 0.090909, "object": 0.090909, "owner": 0.090909, "sources": 0.090909, "technical": 0.090909}} -{"path": "business-objects/customer-order.md", "type": "BusinessObject", "title": "Customer Order", "vector": {"customer": 0.39036, "order": 0.39036, "data": 0.29277, "after": 0.19518, "business": 0.19518, "canonical": 0.19518, "facts": 0.19518, "md": 0.19518, "orders": 0.19518, "purchase": 0.19518, "quality": 0.19518, "rules": 0.19518, "silver": 0.19518, "single": 0.19518, "source": 0.19518, "tables": 0.19518, "transaction": 0.19518, "businessobject": 0.09759, "commerce": 0.09759, "definition": 0.09759, "dekc": 0.09759, "object": 0.09759, "owner": 0.09759, "sources": 0.09759, "technical": 0.09759}} -{"path": "business-objects/daily-order-summary.md", "type": "BusinessObject", "title": "Daily Order Summary", "vector": {"daily": 0.436436, "order": 0.436436, "business": 0.290957, "by": 0.290957, "summary": 0.290957, "currency": 0.145479, "facing": 0.145479, "finance": 0.145479, "forecasting": 0.145479, "gold": 0.145479, "gross": 0.145479, "growth": 0.145479, "md": 0.145479, "merchandise": 0.145479, "of": 0.145479, "pacing": 0.145479, "tables": 0.145479, "used": 0.145479, "value": 0.145479, "volume": 0.145479, "analytics": 0.072739, "businessobject": 0.072739, "definition": 0.072739, "dekc": 0.072739, "engineering": 0.072739, "object": 0.072739, "owner": 0.072739, "sources": 0.072739, "technical": 0.072739}} -{"path": "catalogs/retail-unity-style-catalog.md", "type": "DataCatalog", "title": "Retail Unity-style Catalog", "vector": {"catalog": 0.548821, "retail": 0.439057, "schemas": 0.329293, "unity": 0.329293, "fabric": 0.219529, "style": 0.219529, "workspace": 0.219529, "data": 0.109764, "datacatalog": 0.109764, "datahub": 0.109764, "dekc": 0.109764, "engine": 0.109764, "glue": 0.109764, "of": 0.109764, "purview": 0.109764, "record": 0.109764, "system": 0.109764, "tables": 0.109764, "uri": 0.109764}} -{"path": "columns/bronze-orders-raw-currency.md", "type": "Column", "title": "orders_raw.currency", "vector": {"currency": 0.666667, "code": 0.333333, "column": 0.333333, "iso": 0.333333, "orders_raw": 0.333333, "bronze": 0.166667, "dekc": 0.166667, "nullable": 0.166667, "string": 0.166667}} -{"path": "columns/bronze-orders-raw-customer-id.md", "type": "Column", "title": "orders_raw.customer_id", "vector": {"column": 0.408248, "customer": 0.408248, "customer_id": 0.408248, "key": 0.408248, "orders_raw": 0.408248, "bronze": 0.204124, "dekc": 0.204124, "nullable": 0.204124, "string": 0.204124}} -{"path": "columns/bronze-orders-raw-gross-amount.md", "type": "Column", "title": "orders_raw.gross_amount", "vector": {"column": 0.377964, "gross": 0.377964, "gross_amount": 0.377964, "merchandise": 0.377964, "orders_raw": 0.377964, "value": 0.377964, "bronze": 0.188982, "decimal": 0.188982, "dekc": 0.188982, "nullable": 0.188982}} -{"path": "columns/bronze-orders-raw-order-id.md", "type": "Column", "title": "orders_raw.order_id", "vector": {"column": 0.408248, "identifier": 0.408248, "order": 0.408248, "order_id": 0.408248, "orders_raw": 0.408248, "bronze": 0.204124, "dekc": 0.204124, "nullable": 0.204124, "string": 0.204124}} -{"path": "columns/bronze-orders-raw-order-ts.md", "type": "Column", "title": "orders_raw.order_ts", "vector": {"timestamp": 0.53033, "column": 0.353553, "order": 0.353553, "order_ts": 0.353553, "orders_raw": 0.353553, "utc": 0.353553, "bronze": 0.176777, "dekc": 0.176777, "nullable": 0.176777}} -{"path": "columns/bronze-orders-raw-status.md", "type": "Column", "title": "orders_raw.status", "vector": {"column": 0.5, "order": 0.5, "orders_raw": 0.5, "bronze": 0.25, "dekc": 0.25, "nullable": 0.25, "string": 0.25}} -{"path": "columns/gold-order-daily-currency.md", "type": "Column", "title": "order_daily.currency", "vector": {"column": 0.53033, "currency": 0.53033, "order_daily": 0.53033, "dekc": 0.176777, "gold": 0.176777, "nullable": 0.176777, "on": 0.176777, "string": 0.176777}} -{"path": "columns/gold-order-daily-gross-revenue.md", "type": "Column", "title": "order_daily.gross_revenue", "vector": {"column": 0.53033, "gross_revenue": 0.53033, "order_daily": 0.53033, "decimal": 0.176777, "dekc": 0.176777, "gold": 0.176777, "nullable": 0.176777, "on": 0.176777}} -{"path": "columns/gold-order-daily-order-count.md", "type": "Column", "title": "order_daily.order_count", "vector": {"column": 0.53033, "order_count": 0.53033, "order_daily": 0.53033, "bigint": 0.176777, "dekc": 0.176777, "gold": 0.176777, "nullable": 0.176777, "on": 0.176777}} -{"path": "columns/gold-order-daily-order-date.md", "type": "Column", "title": "order_daily.order_date", "vector": {"column": 0.53033, "order_daily": 0.53033, "order_date": 0.53033, "date": 0.176777, "dekc": 0.176777, "gold": 0.176777, "nullable": 0.176777, "on": 0.176777}} -{"path": "columns/silver-orders-currency.md", "type": "Column", "title": "orders.currency", "vector": {"column": 0.53033, "currency": 0.53033, "orders": 0.53033, "dekc": 0.176777, "nullable": 0.176777, "on": 0.176777, "silver": 0.176777, "string": 0.176777}} -{"path": "columns/silver-orders-customer-id.md", "type": "Column", "title": "orders.customer_id", "vector": {"column": 0.53033, "customer_id": 0.53033, "orders": 0.53033, "dekc": 0.176777, "nullable": 0.176777, "on": 0.176777, "silver": 0.176777, "string": 0.176777}} -{"path": "columns/silver-orders-gross-amount.md", "type": "Column", "title": "orders.gross_amount", "vector": {"column": 0.53033, "gross_amount": 0.53033, "orders": 0.53033, "decimal": 0.176777, "dekc": 0.176777, "nullable": 0.176777, "on": 0.176777, "silver": 0.176777}} -{"path": "columns/silver-orders-order-id.md", "type": "Column", "title": "orders.order_id", "vector": {"column": 0.53033, "order_id": 0.53033, "orders": 0.53033, "dekc": 0.176777, "nullable": 0.176777, "on": 0.176777, "silver": 0.176777, "string": 0.176777}} -{"path": "columns/silver-orders-order-ts.md", "type": "Column", "title": "orders.order_ts", "vector": {"column": 0.53033, "order_ts": 0.53033, "orders": 0.53033, "dekc": 0.176777, "nullable": 0.176777, "on": 0.176777, "silver": 0.176777, "timestamp": 0.176777}} -{"path": "columns/silver-orders-status.md", "type": "Column", "title": "orders.status", "vector": {"column": 0.625543, "orders": 0.625543, "dekc": 0.208514, "nullable": 0.208514, "on": 0.208514, "silver": 0.208514, "string": 0.208514}} -{"path": "dashboards/executive-revenue.md", "type": "Dashboard", "title": "Executive Revenue", "vector": {"revenue": 0.589768, "daily": 0.294884, "dashboard": 0.294884, "executive": 0.294884, "level": 0.294884, "order": 0.294884, "powerbi": 0.294884, "trends": 0.294884, "dekc": 0.147442, "tool": 0.147442}} -{"path": "dax/total-revenue-dax.md", "type": "DaxArtifact", "title": "total-revenue-dax (DAX)", "vector": {"dax": 0.675053, "revenue": 0.450035, "total": 0.450035, "order_daily": 0.225018, "bi": 0.112509, "daxartifact": 0.112509, "dekc": 0.112509, "gross_revenue": 0.112509, "measure": 0.112509, "power": 0.112509, "sumx": 0.112509}} -{"path": "dax/total-revenue.md", "type": "DaxArtifact", "title": "total_revenue (DAX)", "vector": {"dax": 0.842701, "total_revenue": 0.361158, "order_daily": 0.240772, "daxartifact": 0.120386, "dekc": 0.120386, "discovered": 0.120386, "gross_revenue": 0.120386, "revenue": 0.120386, "sumx": 0.120386, "total": 0.120386}} -{"path": "diagrams/activity-daily-medallion-orders-activity.md", "type": "Diagram", "title": "daily-medallion-orders-activity", "vector": {"activity": 0.372822, "daily": 0.372822, "medallion": 0.372822, "orders": 0.372822, "diagram": 0.310685, "mermaid": 0.248548, "md": 0.186411, "workflows": 0.186411, "dq": 0.124274, "language": 0.124274, "no": 0.124274, "workflow": 0.124274, "yes": 0.124274, "_edit": 0.062137, "above": 0.062137, "alert": 0.062137, "as": 0.062137, "bronze": 0.062137, "data": 0.062137, "dead": 0.062137, "dekc": 0.062137, "fail": 0.062137, "fenced": 0.062137, "flowchart": 0.062137, "gold": 0.062137, "keep": 0.062137, "kind": 0.062137, "land": 0.062137, "letter": 0.062137, "listing": 0.062137, "notes": 0.062137, "notify": 0.062137, "ok": 0.062137, "or": 0.062137, "pass": 0.062137, "plantuml": 0.062137, "publish": 0.062137, "quarantine": 0.062137, "rules": 0.062137, "run": 0.062137, "silver": 0.062137, "source": 0.062137, "start": 0.062137, "subjects": 0.062137, "success": 0.062137, "tag": 0.062137, "td": 0.062137, "transform": 0.062137, "validate": 0.062137}} -{"path": "diagrams/activity-orders-stream-landing-activity.md", "type": "Diagram", "title": "orders-stream-landing-activity", "vector": {"activity": 0.361814, "landing": 0.361814, "orders": 0.361814, "stream": 0.361814, "diagram": 0.301511, "ingestion": 0.301511, "mermaid": 0.241209, "md": 0.180907, "dq": 0.120605, "job": 0.120605, "language": 0.120605, "no": 0.120605, "yes": 0.120605, "_edit": 0.060302, "above": 0.060302, "alert": 0.060302, "as": 0.060302, "bronze": 0.060302, "data": 0.060302, "dead": 0.060302, "dekc": 0.060302, "fail": 0.060302, "fenced": 0.060302, "flowchart": 0.060302, "gold": 0.060302, "keep": 0.060302, "kind": 0.060302, "land": 0.060302, "letter": 0.060302, "listing": 0.060302, "notes": 0.060302, "notify": 0.060302, "ok": 0.060302, "or": 0.060302, "pass": 0.060302, "plantuml": 0.060302, "publish": 0.060302, "quarantine": 0.060302, "rules": 0.060302, "run": 0.060302, "silver": 0.060302, "source": 0.060302, "start": 0.060302, "subjects": 0.060302, "success": 0.060302, "tag": 0.060302, "td": 0.060302, "transform": 0.060302, "validate": 0.060302}} -{"path": "diagrams/architecture-commerce-lake-architecture.md", "type": "Diagram", "title": "Commerce lake architecture", "vector": {"architecture": 0.416025, "commerce": 0.346688, "lake": 0.346688, "mermaid": 0.346688, "diagram": 0.27735, "brz": 0.208013, "gld": 0.208013, "job": 0.208013, "slv": 0.208013, "language": 0.138675, "retail": 0.138675, "sem": 0.138675, "_edit": 0.069338, "above": 0.069338, "as": 0.069338, "bronze": 0.069338, "cleansed": 0.069338, "dashboards": 0.069338, "dekc": 0.069338, "fenced": 0.069338, "flowchart": 0.069338, "gold": 0.069338, "jobs": 0.069338, "keep": 0.069338, "kind": 0.069338, "lakes": 0.069338, "landing": 0.069338, "listing": 0.069338, "lr": 0.069338, "marts": 0.069338, "md": 0.069338, "model": 0.069338, "notes": 0.069338, "or": 0.069338, "pipelines": 0.069338, "plantuml": 0.069338, "reports": 0.069338, "rpt": 0.069338, "semantic": 0.069338, "silver": 0.069338, "sources": 0.069338, "src": 0.069338, "streams": 0.069338, "subjects": 0.069338, "tag": 0.069338}} -{"path": "diagrams/class-daily-medallion-orders-class.md", "type": "Diagram", "title": "daily-medallion-orders-class", "vector": {"class": 0.543928, "daily": 0.326357, "medallion": 0.326357, "orders": 0.326357, "diagram": 0.271964, "mermaid": 0.217571, "workflow": 0.217571, "md": 0.163178, "table": 0.163178, "task": 0.163178, "workflows": 0.163178, "dqrule": 0.108786, "language": 0.108786, "name": 0.108786, "_edit": 0.054393, "above": 0.054393, "as": 0.054393, "classdiagram": 0.054393, "contains": 0.054393, "dekc": 0.054393, "fenced": 0.054393, "fqn": 0.054393, "keep": 0.054393, "kind": 0.054393, "layer": 0.054393, "listing": 0.054393, "mode": 0.054393, "notes": 0.054393, "or": 0.054393, "orchestrator": 0.054393, "plantuml": 0.054393, "reads_writes": 0.054393, "retries": 0.054393, "rule_type": 0.054393, "schedule": 0.054393, "severity": 0.054393, "subjects": 0.054393, "tag": 0.054393, "validates": 0.054393}} -{"path": "diagrams/component-daily-medallion-orders-component.md", "type": "Diagram", "title": "daily-medallion-orders-component", "vector": {"component": 0.340777, "daily": 0.340777, "medallion": 0.340777, "orders": 0.340777, "diagram": 0.283981, "mermaid": 0.227185, "wf": 0.227185, "end": 0.170389, "md": 0.170389, "s2": 0.170389, "subgraph": 0.170389, "task": 0.170389, "workflow": 0.170389, "workflows": 0.170389, "language": 0.113592, "s1": 0.113592, "t1": 0.113592, "t2": 0.113592, "t3": 0.113592, "_edit": 0.056796, "above": 0.056796, "as": 0.056796, "compute": 0.056796, "curated": 0.056796, "dekc": 0.056796, "extract": 0.056796, "fenced": 0.056796, "flowchart": 0.056796, "keep": 0.056796, "kind": 0.056796, "landing": 0.056796, "listing": 0.056796, "load": 0.056796, "notes": 0.056796, "or": 0.056796, "orchestrator": 0.056796, "path": 0.056796, "pipeline": 0.056796, "plantuml": 0.056796, "storage": 0.056796, "subjects": 0.056796, "tables": 0.056796, "tag": 0.056796, "tb": 0.056796, "transform": 0.056796}} -{"path": "diagrams/component-orders-stream-landing-component.md", "type": "Diagram", "title": "orders-stream-landing-component", "vector": {"landing": 0.383023, "component": 0.328305, "orders": 0.328305, "stream": 0.328305, "diagram": 0.273588, "ingestion": 0.273588, "mermaid": 0.21887, "wf": 0.21887, "end": 0.164153, "md": 0.164153, "s2": 0.164153, "subgraph": 0.164153, "task": 0.164153, "job": 0.109435, "language": 0.109435, "s1": 0.109435, "t1": 0.109435, "t2": 0.109435, "t3": 0.109435, "_edit": 0.054718, "above": 0.054718, "as": 0.054718, "compute": 0.054718, "curated": 0.054718, "dekc": 0.054718, "extract": 0.054718, "fenced": 0.054718, "flowchart": 0.054718, "keep": 0.054718, "kind": 0.054718, "listing": 0.054718, "load": 0.054718, "notes": 0.054718, "or": 0.054718, "orchestrator": 0.054718, "path": 0.054718, "pipeline": 0.054718, "plantuml": 0.054718, "storage": 0.054718, "subjects": 0.054718, "tables": 0.054718, "tag": 0.054718, "tb": 0.054718, "transform": 0.054718, "workflow": 0.054718}} -{"path": "diagrams/erd-orders-erd.md", "type": "Diagram", "title": "Orders ERD", "vector": {"string": 0.491436, "erd": 0.368577, "order": 0.307148, "mermaid": 0.245718, "orders": 0.245718, "diagram": 0.184289, "fk": 0.184289, "order_line": 0.184289, "pk": 0.184289, "as": 0.122859, "commerce": 0.122859, "customer": 0.122859, "customer_id": 0.122859, "daily": 0.122859, "gold": 0.122859, "language": 0.122859, "logical": 0.122859, "number": 0.122859, "order_id": 0.122859, "product": 0.122859, "product_id": 0.122859, "_edit": 0.06143, "above": 0.06143, "contains": 0.06143, "date": 0.06143, "dekc": 0.06143, "erdiagram": 0.06143, "fenced": 0.06143, "gross_amount": 0.06143, "keep": 0.06143, "kind": 0.06143, "listing": 0.06143, "md": 0.06143, "name": 0.06143, "notes": 0.06143, "or": 0.06143, "order_ts": 0.06143, "places": 0.06143, "plantuml": 0.06143, "qty": 0.06143, "segment": 0.06143, "sold": 0.06143, "subjects": 0.06143, "tables": 0.06143, "tag": 0.06143}} -{"path": "diagrams/sequence-orders-stream-landing-sequence.md", "type": "Diagram", "title": "orders-stream-landing-sequence", "vector": {"job": 0.418718, "landing": 0.372194, "stream": 0.325669, "as": 0.279145, "orders": 0.279145, "sequence": 0.279145, "diagram": 0.232621, "ingestion": 0.232621, "participant": 0.232621, "dq": 0.186097, "mermaid": 0.186097, "md": 0.139573, "brz": 0.093048, "language": 0.093048, "pass": 0.093048, "slv": 0.093048, "src": 0.093048, "_edit": 0.046524, "above": 0.046524, "batch": 0.046524, "bronze": 0.046524, "dekc": 0.046524, "event": 0.046524, "fail": 0.046524, "fenced": 0.046524, "keep": 0.046524, "kind": 0.046524, "listing": 0.046524, "notes": 0.046524, "on": 0.046524, "or": 0.046524, "plantuml": 0.046524, "promote": 0.046524, "rules": 0.046524, "sequencediagram": 0.046524, "silver": 0.046524, "source": 0.046524, "subjects": 0.046524, "tag": 0.046524, "validate": 0.046524, "write": 0.046524}} -{"path": "diagrams/state-daily-medallion-orders-state.md", "type": "Diagram", "title": "daily-medallion-orders-state", "vector": {"daily": 0.365826, "medallion": 0.365826, "orders": 0.365826, "state": 0.365826, "diagram": 0.304855, "mermaid": 0.243884, "running": 0.243884, "failed": 0.182913, "md": 0.182913, "workflows": 0.182913, "deadletter": 0.121942, "idle": 0.121942, "language": 0.121942, "succeeded": 0.121942, "workflow": 0.121942, "_edit": 0.060971, "above": 0.060971, "as": 0.060971, "complete": 0.060971, "dekc": 0.060971, "error": 0.060971, "fenced": 0.060971, "keep": 0.060971, "kind": 0.060971, "listing": 0.060971, "max": 0.060971, "notes": 0.060971, "or": 0.060971, "plantuml": 0.060971, "retries": 0.060971, "retry": 0.060971, "statediagram": 0.060971, "subjects": 0.060971, "tag": 0.060971, "trigger": 0.060971, "v2": 0.060971}} -{"path": "diagrams/state-orders-stream-landing-state.md", "type": "Diagram", "title": "orders-stream-landing-state", "vector": {"landing": 0.355409, "orders": 0.355409, "state": 0.355409, "stream": 0.355409, "diagram": 0.296174, "ingestion": 0.296174, "mermaid": 0.23694, "running": 0.23694, "failed": 0.177705, "md": 0.177705, "deadletter": 0.11847, "idle": 0.11847, "job": 0.11847, "language": 0.11847, "succeeded": 0.11847, "_edit": 0.059235, "above": 0.059235, "as": 0.059235, "complete": 0.059235, "dekc": 0.059235, "error": 0.059235, "fenced": 0.059235, "keep": 0.059235, "kind": 0.059235, "listing": 0.059235, "max": 0.059235, "notes": 0.059235, "or": 0.059235, "plantuml": 0.059235, "retries": 0.059235, "retry": 0.059235, "statediagram": 0.059235, "subjects": 0.059235, "tag": 0.059235, "trigger": 0.059235, "v2": 0.059235}} -{"path": "domains/commerce.md", "type": "DataDomain", "title": "Commerce", "vector": {"cash": 0.3849, "commerce": 0.3849, "merchandising": 0.3849, "order": 0.3849, "to": 0.3849, "analytics": 0.19245, "data": 0.19245, "datadomain": 0.19245, "dekc": 0.19245, "domain": 0.19245, "engineering": 0.19245, "owner": 0.19245}} -{"path": "glossary/customer-lifetime-value.md", "type": "GlossaryTerm", "title": "Customer Lifetime Value", "vector": {"customer": 0.452911, "value": 0.452911, "amounts": 0.226455, "based": 0.226455, "gross": 0.226455, "historical": 0.226455, "lifetime": 0.226455, "of": 0.226455, "on": 0.226455, "order": 0.226455, "predicted": 0.226455, "realized": 0.226455, "customer_ltv": 0.113228, "definition": 0.113228, "dekc": 0.113228, "glossary": 0.113228, "glossaryterm": 0.113228, "synonyms": 0.113228}} -{"path": "glossary/customer-order.md", "type": "GlossaryTerm", "title": "Customer Order", "vector": {"customer": 0.452911, "order": 0.452911, "after": 0.226455, "canonical": 0.226455, "data": 0.226455, "facts": 0.226455, "purchase": 0.226455, "quality": 0.226455, "rules": 0.226455, "single": 0.226455, "source": 0.226455, "transaction": 0.226455, "definition": 0.113228, "dekc": 0.113228, "glossary": 0.113228, "glossaryterm": 0.113228, "orders": 0.113228, "synonyms": 0.113228}} -{"path": "glossary/daily-order-summary.md", "type": "GlossaryTerm", "title": "Daily Order Summary", "vector": {"by": 0.362143, "daily": 0.362143, "order": 0.362143, "summary": 0.362143, "business": 0.181071, "currency": 0.181071, "facing": 0.181071, "finance": 0.181071, "forecasting": 0.181071, "gross": 0.181071, "growth": 0.181071, "merchandise": 0.181071, "of": 0.181071, "pacing": 0.181071, "used": 0.181071, "value": 0.181071, "volume": 0.181071, "definition": 0.090536, "dekc": 0.090536, "glossary": 0.090536, "glossaryterm": 0.090536, "order_daily": 0.090536, "synonyms": 0.090536}} -{"path": "glossary/gmv.md", "type": "GlossaryTerm", "title": "GMV", "vector": {"gross": 0.458831, "merchandise": 0.344124, "value": 0.344124, "before": 0.229416, "cancellations": 0.229416, "discounts": 0.229416, "dollar": 0.229416, "gmv": 0.229416, "returns": 0.229416, "sales": 0.229416, "total": 0.229416, "volume": 0.229416, "definition": 0.114708, "dekc": 0.114708, "glossary": 0.114708, "glossaryterm": 0.114708, "revenue": 0.114708, "synonyms": 0.114708}} -{"path": "ingestion/orders-stream-landing.md", "type": "IngestionJob", "title": "orders-stream-landing", "vector": {"bronze": 0.510343, "orders": 0.340229, "stream": 0.340229, "job": 0.283524, "delta": 0.226819, "event": 0.170114, "events": 0.170114, "ingestion": 0.170114, "landing": 0.170114, "lands": 0.170114, "md": 0.170114, "dq": 0.11341, "hubs": 0.11341, "order": 0.11341, "storage": 0.11341, "streaming": 0.11341, "tables": 0.11341, "15": 0.056705, "as": 0.056705, "checkpoint": 0.056705, "checkpoints": 0.056705, "connector": 0.056705, "dekc": 0.056705, "eventhubs": 0.056705, "fabric": 0.056705, "flow": 0.056705, "flowchart": 0.056705, "formats": 0.056705, "hub": 0.056705, "idempotent": 0.056705, "ingestionjob": 0.056705, "json": 0.056705, "land": 0.056705, "layer": 0.056705, "lr": 0.056705, "mermaid": 0.056705, "minutes": 0.056705, "mode": 0.056705, "optional": 0.056705, "orchestrator": 0.056705, "pattern": 0.056705, "pipeline": 0.056705, "prefix": 0.056705, "raw": 0.056705, "read": 0.056705, "sla": 0.056705, "source": 0.056705, "sources": 0.056705, "src": 0.056705, "steps": 0.056705, "streams": 0.056705, "target": 0.056705, "to": 0.056705, "update": 0.056705, "watermark": 0.056705, "write": 0.056705}} -{"path": "ingestion/product-catalog-sftp-load.md", "type": "IngestionJob", "title": "product-catalog-sftp-load", "vector": {"sftp": 0.475551, "bronze": 0.416107, "catalog": 0.297219, "job": 0.297219, "product": 0.297219, "to": 0.237775, "ingestion": 0.178331, "dq": 0.118888, "extract": 0.118888, "file_drop": 0.118888, "full": 0.118888, "landing": 0.118888, "load": 0.118888, "nightly": 0.118888, "of": 0.118888, "source": 0.118888, "sources": 0.118888, "adf": 0.059444, "connector": 0.059444, "copy": 0.059444, "csv": 0.059444, "dekc": 0.059444, "delta": 0.059444, "files": 0.059444, "flow": 0.059444, "flowchart": 0.059444, "formats": 0.059444, "idempotent": 0.059444, "ingestionjob": 0.059444, "land": 0.059444, "layer": 0.059444, "list": 0.059444, "lr": 0.059444, "md": 0.059444, "mermaid": 0.059444, "mode": 0.059444, "optional": 0.059444, "orchestrator": 0.059444, "pattern": 0.059444, "products": 0.059444, "register": 0.059444, "schedule": 0.059444, "src": 0.059444, "steps": 0.059444, "stream": 0.059444, "table": 0.059444, "target": 0.059444}} -{"path": "lakes/retail-commerce-lake.md", "type": "DataLake", "title": "Retail Commerce Lake", "vector": {"commerce": 0.357771, "lake": 0.357771, "layers": 0.357771, "bronze": 0.268328, "gold": 0.268328, "md": 0.268328, "onelake": 0.268328, "retail": 0.268328, "silver": 0.268328, "fabric": 0.178885, "lakehouse": 0.178885, "primary": 0.178885, "abfss": 0.089443, "architecture": 0.089443, "data": 0.089443, "datalake": 0.089443, "dekc": 0.089443, "flowchart": 0.089443, "lr": 0.089443, "mermaid": 0.089443, "platform": 0.089443, "sources": 0.089443, "uri": 0.089443}} -{"path": "layers/bronze.md", "type": "Layer", "title": "Bronze layer", "vector": {"bronze": 0.662266, "layer": 0.529813, "medallion": 0.39736, "zone": 0.264906, "curated": 0.132453, "data": 0.132453, "products": 0.132453}} -{"path": "layers/gold.md", "type": "Layer", "title": "Gold layer", "vector": {"gold": 0.662266, "layer": 0.529813, "medallion": 0.39736, "zone": 0.264906, "curated": 0.132453, "data": 0.132453, "products": 0.132453}} -{"path": "layers/silver.md", "type": "Layer", "title": "Silver layer", "vector": {"silver": 0.662266, "layer": 0.529813, "medallion": 0.39736, "zone": 0.264906, "curated": 0.132453, "data": 0.132453, "products": 0.132453}} -{"path": "lineage/orders-to-revenue.md", "type": "LineagePath", "title": "orders-to-revenue", "vector": {"orders": 0.525226, "gold": 0.328266, "revenue": 0.262613, "to": 0.262613, "bronze": 0.19696, "daily": 0.19696, "lineage": 0.19696, "md": 0.19696, "n1": 0.19696, "order": 0.19696, "raw": 0.19696, "silver": 0.19696, "tables": 0.19696, "commerce": 0.131306, "core": 0.131306, "landing": 0.131306, "mart": 0.131306, "n0": 0.131306, "n2": 0.131306, "dekc": 0.065653, "flowchart": 0.065653, "lineagepath": 0.065653, "lr": 0.065653, "mermaid": 0.065653, "path": 0.065653}} -{"path": "marts/revenue-mart.md", "type": "DataMart", "title": "Revenue Mart", "vector": {"gold": 0.548821, "mart": 0.439057, "revenue": 0.439057, "aggregates": 0.219529, "exec": 0.219529, "reporting": 0.219529, "tables": 0.219529, "business": 0.109764, "curated": 0.109764, "daily": 0.109764, "data": 0.109764, "datamart": 0.109764, "dekc": 0.109764, "facing": 0.109764, "md": 0.109764, "order": 0.109764, "typically": 0.109764}} -{"path": "metrics/gross-merchandise-value.md", "type": "Metric", "title": "Gross Merchandise Value", "vector": {"gross": 0.444444, "sum": 0.333333, "amounts": 0.222222, "before": 0.222222, "currency": 0.222222, "discounts": 0.222222, "in": 0.222222, "merchandise": 0.222222, "metric": 0.222222, "of": 0.222222, "order": 0.222222, "returns": 0.222222, "sql": 0.222222, "transaction": 0.222222, "value": 0.222222, "definition": 0.111111, "dekc": 0.111111, "expression": 0.111111, "gross_revenue": 0.111111}} -{"path": "metrics/order-count.md", "type": "Metric", "title": "Order Count", "vector": {"count": 0.668153, "completed": 0.267261, "distinct": 0.267261, "metric": 0.267261, "of": 0.267261, "order": 0.267261, "orders": 0.267261, "sql": 0.267261, "definition": 0.133631, "dekc": 0.133631, "expression": 0.133631}} -{"path": "packs/pack-gold-order-daily.md", "type": "ContextPack", "title": "Pack: gold-order-daily", "vector": {"gold": 0.400445, "bronze": 0.367075, "table": 0.300334, "orders": 0.266963, "sql": 0.233593, "as": 0.200223, "md": 0.200223, "orders_raw": 0.200223, "daily": 0.166852, "order": 0.166852, "tables": 0.166852, "by": 0.133482, "inferred": 0.133482, "n0": 0.133482, "order_daily": 0.133482, "pack": 0.133482, "path": 0.133482, "silver": 0.133482, "customer": 0.100111, "layer": 0.100111, "n1": 0.100111, "select": 0.100111, "view": 0.100111, "api": 0.066741, "count": 0.066741, "create": 0.066741, "customer_id": 0.066741, "gross_amount": 0.066741, "group": 0.066741, "lifetime": 0.066741, "marts": 0.066741, "n2": 0.066741, "or": 0.066741, "order_count": 0.066741, "orders_api": 0.066741, "per": 0.066741, "raw": 0.066741, "referenced": 0.066741, "replace": 0.066741, "sales": 0.066741, "sum": 0.066741, "value": 0.066741, "cast": 0.03337, "context": 0.03337, "contextpack": 0.03337, "currency": 0.03337, "customer_ltv": 0.03337, "date": 0.03337, "dekc": 0.03337, "diagram": 0.03337, "disclosure": 0.03337, "flowchart": 0.03337, "gross_revenue": 0.03337, "hops": 0.03337, "landing": 0.03337, "lifetime_value": 0.03337, "lineage": 0.03337, "lr": 0.03337, "ltv": 0.03337, "mermaid": 0.03337, "nodes": 0.03337, "order_date": 0.03337, "order_ts": 0.03337, "progressive": 0.03337}} -{"path": "products/daily-revenue-product.md", "type": "DataProduct", "title": "Daily Revenue Product", "vector": {"product": 0.6, "daily": 0.4, "data": 0.4, "revenue": 0.4, "bi": 0.2, "certified": 0.2, "apps": 0.1, "consumers": 0.1, "dataproduct": 0.1, "dekc": 0.1, "domains": 0.1, "downstream": 0.1, "publishable": 0.1, "reports": 0.1}} -{"path": "quality/gold-order-daily-freshness.md", "type": "DQRule", "title": "gold-order-daily-freshness", "vector": {"gold": 0.51031, "freshness": 0.408248, "daily": 0.306186, "order": 0.306186, "be": 0.204124, "day": 0.204124, "fresh": 0.204124, "mart": 0.204124, "must": 0.204124, "within": 0.204124, "current_date": 0.102062, "dekc": 0.102062, "dq": 0.102062, "dqrule": 0.102062, "error": 0.102062, "expression": 0.102062, "max": 0.102062, "md": 0.102062, "order_date": 0.102062, "quality": 0.102062, "severity": 0.102062, "tables": 0.102062, "target": 0.102062}} -{"path": "queries/customer-ltv.md", "type": "Query", "title": "customer_ltv", "vector": {"sql": 0.631169, "customer_ltv": 0.450835, "as": 0.270501, "gold": 0.270501, "customer_id": 0.180334, "discovered": 0.180334, "marts": 0.180334, "query": 0.180334, "by": 0.090167, "count": 0.090167, "create": 0.090167, "dekc": 0.090167, "gross_amount": 0.090167, "group": 0.090167, "lifetime_value": 0.090167, "or": 0.090167, "order_count": 0.090167, "orders": 0.090167, "replace": 0.090167, "select": 0.090167, "silver": 0.090167, "sum": 0.090167, "view": 0.090167}} -{"path": "queries/daily-medallion.md", "type": "Query", "title": "daily_medallion", "vector": {"sql": 0.7298, "daily_medallion": 0.417029, "discovered": 0.208514, "query": 0.208514, "workflows": 0.208514, "aggregate": 0.104257, "api": 0.104257, "bronze": 0.104257, "clean": 0.104257, "dekc": 0.104257, "documented": 0.104257, "gold": 0.104257, "land": 0.104257, "orchestrated": 0.104257, "order_daily": 0.104257, "orders": 0.104257, "orders_raw": 0.104257, "select": 0.104257, "silver": 0.104257, "steps": 0.104257}} -{"path": "queries/order-daily.md", "type": "Query", "title": "order_daily", "vector": {"sql": 0.591608, "as": 0.422577, "order_daily": 0.422577, "gold": 0.253546, "discovered": 0.169031, "marts": 0.169031, "query": 0.169031, "by": 0.084515, "cast": 0.084515, "count": 0.084515, "create": 0.084515, "currency": 0.084515, "date": 0.084515, "dekc": 0.084515, "gross_amount": 0.084515, "gross_revenue": 0.084515, "group": 0.084515, "or": 0.084515, "order_count": 0.084515, "order_date": 0.084515, "order_ts": 0.084515, "orders": 0.084515, "replace": 0.084515, "select": 0.084515, "silver": 0.084515, "sum": 0.084515, "table": 0.084515}} -{"path": "queries/orders-raw.md", "type": "Query", "title": "orders_raw", "vector": {"sql": 0.686406, "orders_raw": 0.49029, "bronze": 0.294174, "discovered": 0.196116, "query": 0.196116, "sales": 0.196116, "as": 0.098058, "create": 0.098058, "dekc": 0.098058, "landing": 0.098058, "or": 0.098058, "orders_api": 0.098058, "replace": 0.098058, "select": 0.098058, "table": 0.098058}} -{"path": "queries/orders.md", "type": "Query", "title": "orders", "vector": {"sql": 0.664411, "orders": 0.474579, "silver": 0.284747, "discovered": 0.189832, "query": 0.189832, "sales": 0.189832, "as": 0.094916, "bronze": 0.094916, "create": 0.094916, "currency": 0.094916, "customer_id": 0.094916, "dekc": 0.094916, "gross_amount": 0.094916, "is": 0.094916, "or": 0.094916, "order_id": 0.094916, "order_ts": 0.094916, "orders_raw": 0.094916, "replace": 0.094916, "select": 0.094916, "table": 0.094916, "where": 0.094916}} -{"path": "queries/total-revenue-dax.md", "type": "Query", "title": "total-revenue-dax", "vector": {"dax": 0.505076, "revenue": 0.505076, "total": 0.505076, "bi": 0.202031, "measure": 0.202031, "order_daily": 0.202031, "power": 0.202031, "query": 0.202031, "dekc": 0.101015, "gross_revenue": 0.101015, "sumx": 0.101015}} -{"path": "queries/total-revenue.md", "type": "Query", "title": "total_revenue", "vector": {"dax": 0.842927, "total_revenue": 0.374634, "discovered": 0.187317, "order_daily": 0.187317, "query": 0.187317, "dekc": 0.093659, "gross_revenue": 0.093659, "revenue": 0.093659, "sumx": 0.093659, "total": 0.093659}} -{"path": "reports/weekly-commerce-digest.md", "type": "Report", "title": "Weekly Commerce Digest", "vector": {"weekly": 0.549442, "automated": 0.274721, "commerce": 0.274721, "digest": 0.274721, "ltv": 0.274721, "of": 0.274721, "order": 0.274721, "pdf": 0.274721, "report": 0.274721, "trends": 0.274721, "dekc": 0.137361}} -{"path": "schemas/bronze.md", "type": "Schema", "title": "bronze", "vector": {"bronze": 0.609994, "schema": 0.609994, "tables": 0.304997, "dekc": 0.152499, "in": 0.152499, "md": 0.152499, "orders": 0.152499, "orders_raw": 0.152499, "raw": 0.152499, "views": 0.152499}} -{"path": "schemas/gold.md", "type": "Schema", "title": "gold", "vector": {"gold": 0.609994, "schema": 0.609994, "tables": 0.304997, "daily": 0.152499, "dekc": 0.152499, "in": 0.152499, "md": 0.152499, "order": 0.152499, "order_daily": 0.152499, "views": 0.152499}} -{"path": "schemas/silver.md", "type": "Schema", "title": "silver", "vector": {"schema": 0.603023, "silver": 0.603023, "orders": 0.301511, "tables": 0.301511, "dekc": 0.150756, "in": 0.150756, "md": 0.150756, "views": 0.150756}} -{"path": "semantic/retail-commerce-model.md", "type": "SemanticModel", "title": "Retail Commerce Model", "vector": {"semantic": 0.481543, "layer": 0.361158, "bi": 0.240772, "commerce": 0.240772, "gold": 0.240772, "marts": 0.240772, "model": 0.240772, "order": 0.240772, "over": 0.240772, "retail": 0.240772, "tools": 0.240772, "binding": 0.120386, "business": 0.120386, "dekc": 0.120386, "metrics": 0.120386, "semanticmodel": 0.120386, "tables": 0.120386, "technical": 0.120386, "to": 0.120386}} -{"path": "sources/product-catalog-sftp.md", "type": "SourceSystem", "title": "product-catalog-sftp", "vector": {"sftp": 0.594089, "catalog": 0.396059, "product": 0.396059, "files": 0.297044, "file": 0.19803, "flat": 0.19803, "nightly": 0.19803, "via": 0.19803, "com": 0.099015, "dekc": 0.099015, "example": 0.099015, "kind": 0.099015, "notes": 0.099015, "products": 0.099015, "source": 0.099015, "sourcesystem": 0.099015, "uri": 0.099015}} -{"path": "sources/retail-lake-walk.md", "type": "SourceSystem", "title": "retail-lake-walk", "vector": {"lake": 0.752618, "fixtures": 0.250873, "sample": 0.250873, "tests": 0.250873, "workspace": 0.250873, "at": 0.167248, "data": 0.167248, "retail": 0.167248, "walk": 0.167248, "walked": 0.167248, "dekc": 0.083624, "kind": 0.083624, "notes": 0.083624, "source": 0.083624, "sourcesystem": 0.083624, "uri": 0.083624}} -{"path": "sources/retail-lake.md", "type": "SourceSystem", "title": "retail-lake", "vector": {"lake": 0.6799, "retail": 0.485643, "analytics": 0.194257, "data": 0.194257, "delta": 0.194257, "models": 0.194257, "primary": 0.194257, "sql": 0.194257, "acme": 0.097129, "dekc": 0.097129, "kind": 0.097129, "notes": 0.097129, "s3": 0.097129, "source": 0.097129, "sourcesystem": 0.097129, "uri": 0.097129}} -{"path": "sql/customer-ltv.md", "type": "SqlArtifact", "title": "customer_ltv (SQL)", "vector": {"sql": 0.643268, "customer_ltv": 0.428845, "as": 0.321634, "customer_id": 0.214423, "gold": 0.214423, "by": 0.107211, "count": 0.107211, "create": 0.107211, "dekc": 0.107211, "discovered": 0.107211, "gross_amount": 0.107211, "group": 0.107211, "lifetime_value": 0.107211, "marts": 0.107211, "or": 0.107211, "order_count": 0.107211, "orders": 0.107211, "replace": 0.107211, "select": 0.107211, "silver": 0.107211, "sqlartifact": 0.107211, "sum": 0.107211, "view": 0.107211}} -{"path": "sql/daily-medallion.md", "type": "SqlArtifact", "title": "daily_medallion (SQL)", "vector": {"sql": 0.755929, "daily_medallion": 0.377964, "aggregate": 0.125988, "api": 0.125988, "bronze": 0.125988, "clean": 0.125988, "dekc": 0.125988, "discovered": 0.125988, "documented": 0.125988, "gold": 0.125988, "land": 0.125988, "orchestrated": 0.125988, "order_daily": 0.125988, "orders": 0.125988, "orders_raw": 0.125988, "select": 0.125988, "silver": 0.125988, "sqlartifact": 0.125988, "steps": 0.125988, "workflows": 0.125988}} -{"path": "sql/order-daily.md", "type": "SqlArtifact", "title": "order_daily (SQL)", "vector": {"sql": 0.588348, "as": 0.49029, "order_daily": 0.392232, "gold": 0.196116, "by": 0.098058, "cast": 0.098058, "count": 0.098058, "create": 0.098058, "currency": 0.098058, "date": 0.098058, "dekc": 0.098058, "discovered": 0.098058, "gross_amount": 0.098058, "gross_revenue": 0.098058, "group": 0.098058, "marts": 0.098058, "or": 0.098058, "order_count": 0.098058, "order_date": 0.098058, "order_ts": 0.098058, "orders": 0.098058, "replace": 0.098058, "select": 0.098058, "silver": 0.098058, "sqlartifact": 0.098058, "sum": 0.098058, "table": 0.098058}} -{"path": "sql/orders-raw.md", "type": "SqlArtifact", "title": "orders_raw (SQL)", "vector": {"sql": 0.727607, "orders_raw": 0.485071, "bronze": 0.242536, "as": 0.121268, "create": 0.121268, "dekc": 0.121268, "discovered": 0.121268, "landing": 0.121268, "or": 0.121268, "orders_api": 0.121268, "replace": 0.121268, "sales": 0.121268, "select": 0.121268, "sqlartifact": 0.121268, "table": 0.121268}} -{"path": "sql/orders.md", "type": "SqlArtifact", "title": "orders (SQL)", "vector": {"sql": 0.69282, "orders": 0.46188, "silver": 0.23094, "as": 0.11547, "bronze": 0.11547, "create": 0.11547, "currency": 0.11547, "customer_id": 0.11547, "dekc": 0.11547, "discovered": 0.11547, "gross_amount": 0.11547, "is": 0.11547, "or": 0.11547, "order_id": 0.11547, "order_ts": 0.11547, "orders_raw": 0.11547, "replace": 0.11547, "sales": 0.11547, "select": 0.11547, "sqlartifact": 0.11547, "table": 0.11547, "where": 0.11547}} -{"path": "storage/bronze-orders-prefix.md", "type": "StorageLocation", "title": "bronze-orders-prefix", "vector": {"bronze": 0.520756, "orders": 0.520756, "prefix": 0.390567, "path": 0.260378, "storage": 0.260378, "abfss": 0.130189, "dekc": 0.130189, "delta": 0.130189, "format": 0.130189, "kind": 0.130189, "lake": 0.130189, "onelake": 0.130189, "retail": 0.130189, "storagelocation": 0.130189, "uri": 0.130189}} -{"path": "streams/orders-events.md", "type": "Stream", "title": "orders-events", "vector": {"events": 0.452911, "orders": 0.452911, "eventhubs": 0.339683, "stream": 0.339683, "bronze": 0.226455, "order": 0.226455, "placement": 0.226455, "as": 0.113228, "dekc": 0.113228, "endpoints": 0.113228, "flowchart": 0.113228, "format": 0.113228, "json": 0.113228, "landing": 0.113228, "lands": 0.113228, "lr": 0.113228, "md": 0.113228, "mermaid": 0.113228, "platform": 0.113228, "producers": 0.113228, "raw": 0.113228, "tables": 0.113228, "uri": 0.113228}} -{"path": "tables/bronze-orders-api.md", "type": "Table", "title": "orders_api", "vector": {"bronze": 0.392232, "by": 0.392232, "orders_api": 0.392232, "orders_raw": 0.392232, "referenced": 0.392232, "table": 0.392232, "dekc": 0.196116, "layer": 0.196116}} -{"path": "tables/bronze-orders-raw.md", "type": "Table", "title": "orders_raw", "vector": {"bronze": 0.481125, "orders_raw": 0.481125, "table": 0.481125, "sql": 0.3849, "inferred": 0.19245, "sales": 0.19245, "as": 0.096225, "create": 0.096225, "dekc": 0.096225, "landing": 0.096225, "layer": 0.096225, "or": 0.096225, "orders_api": 0.096225, "replace": 0.096225, "select": 0.096225}} -{"path": "tables/gold-order-daily.md", "type": "Table", "title": "order_daily", "vector": {"as": 0.416667, "gold": 0.416667, "order_daily": 0.416667, "table": 0.416667, "sql": 0.333333, "inferred": 0.166667, "marts": 0.166667, "by": 0.083333, "cast": 0.083333, "count": 0.083333, "create": 0.083333, "currency": 0.083333, "date": 0.083333, "dekc": 0.083333, "gross_amount": 0.083333, "gross_revenue": 0.083333, "group": 0.083333, "layer": 0.083333, "or": 0.083333, "order_count": 0.083333, "order_date": 0.083333, "order_ts": 0.083333, "orders": 0.083333, "replace": 0.083333, "select": 0.083333, "silver": 0.083333, "sum": 0.083333}} -{"path": "tables/silver-orders.md", "type": "Table", "title": "orders", "vector": {"orders": 0.466252, "silver": 0.466252, "table": 0.466252, "sql": 0.373002, "inferred": 0.186501, "sales": 0.186501, "as": 0.09325, "bronze": 0.09325, "create": 0.09325, "currency": 0.09325, "customer_id": 0.09325, "dekc": 0.09325, "gross_amount": 0.09325, "is": 0.09325, "layer": 0.09325, "or": 0.09325, "order_id": 0.09325, "order_ts": 0.09325, "orders_raw": 0.09325, "replace": 0.09325, "select": 0.09325, "where": 0.09325}} -{"path": "transformations/aggregate-order-daily.md", "type": "Transformation", "title": "aggregate-order-daily", "vector": {"daily": 0.549442, "aggregate": 0.274721, "aggregation": 0.274721, "gold": 0.274721, "grain": 0.274721, "order": 0.274721, "reporting": 0.274721, "revenue": 0.274721, "silver": 0.274721, "transformation": 0.274721, "dekc": 0.137361}} -{"path": "transformations/cleanse-orders.md", "type": "Transformation", "title": "cleanse-orders", "vector": {"cleanse": 0.508001, "bronze": 0.381, "coercion": 0.254, "commerce": 0.254, "drop": 0.254, "order": 0.254, "orders": 0.254, "silver": 0.254, "sql": 0.254, "transformation": 0.254, "dekc": 0.127, "is": 0.127, "orders_raw": 0.127, "select": 0.127, "where": 0.127}} -{"path": "views/gold-customer-ltv.md", "type": "View", "title": "customer_ltv", "vector": {"view": 0.46188, "orders": 0.34641, "silver": 0.34641, "customer": 0.23094, "customer_id": 0.23094, "customer_ltv": 0.23094, "lifetime": 0.23094, "per": 0.23094, "sql": 0.23094, "value": 0.23094, "by": 0.11547, "count": 0.11547, "dekc": 0.11547, "gold": 0.11547, "gross_amount": 0.11547, "group": 0.11547, "lifetime_value": 0.11547, "md": 0.11547, "order_count": 0.11547, "reads": 0.11547, "select": 0.11547, "sum": 0.11547, "tables": 0.11547}} -{"path": "wireframes/wireframe-executive-revenue-wireframe.md", "type": "Wireframe", "title": "Executive Revenue Wireframe", "vector": {"revenue": 0.492366, "wireframe": 0.492366, "plantuml": 0.369274, "executive": 0.307729, "salt": 0.246183, "dashboard": 0.184637, "diagram": 0.123091, "exec": 0.123091, "language": 0.123091, "142": 0.061546, "2m": 0.061546, "30": 0.061546, "420": 0.061546, "_edit": 0.061546, "above": 0.061546, "aov": 0.061546, "area": 0.061546, "as": 0.061546, "by": 0.061546, "chart": 0.061546, "currency": 0.061546, "dashboards": 0.061546, "day": 0.061546, "days": 0.061546, "dekc": 0.061546, "enduml": 0.061546, "export": 0.061546, "fenced": 0.061546, "gmv": 0.061546, "include": 0.061546, "keep": 0.061546, "kind": 0.061546, "last": 0.061546, "listing": 0.061546, "md": 0.061546, "mermaid": 0.061546, "notes": 0.061546, "or": 0.061546, "orders": 0.061546, "period": 0.061546, "products": 0.061546, "refresh": 0.061546, "startuml": 0.061546, "subjects": 0.061546, "table": 0.061546, "tag": 0.061546, "top": 0.061546}} -{"path": "workflows/daily-medallion-orders.md", "type": "Workflow", "title": "daily-medallion-orders", "vector": {"orders": 0.512989, "bronze": 0.307794, "gold": 0.307794, "silver": 0.307794, "airflow": 0.205196, "daily": 0.205196, "medallion": 0.205196, "nightly": 0.205196, "pipeline": 0.205196, "to": 0.205196, "workflow": 0.205196, "aggregate": 0.102598, "api": 0.102598, "bi": 0.102598, "cleanse": 0.102598, "commerce": 0.102598, "dekc": 0.102598, "land": 0.102598, "model": 0.102598, "orchestrator": 0.102598, "order_daily": 0.102598, "orders_raw": 0.102598, "power": 0.102598, "refresh": 0.102598, "semantic": 0.102598, "steps": 0.102598}} diff --git a/sample-knowledge/.index/graph.json b/sample-knowledge/.index/graph.json deleted file mode 100644 index 21cc91e..0000000 --- a/sample-knowledge/.index/graph.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "/views/gold-customer-ltv.md": [ - "/business-objects/customer-lifetime-value.md", - "/reports/weekly-commerce-digest.md" - ], - "/tables/silver-orders.md": [ - "/business-objects/customer-order.md", - "/tables/gold-order-daily.md", - "/queries/customer-ltv.md", - "/views/gold-customer-ltv.md", - "/queries/order-daily.md", - "/transformations/aggregate-order-daily.md" - ], - "/tables/gold-order-daily.md": [ - "/business-objects/daily-order-summary.md", - "/dashboards/executive-revenue.md", - "/queries/total-revenue-dax.md", - "/reports/weekly-commerce-digest.md" - ], - "/metrics/gross-merchandise-value.md": [ - "/dashboards/executive-revenue.md" - ], - "/metrics/order-count.md": [ - "/dashboards/executive-revenue.md" - ], - "/ingestion/orders-stream-landing.md": [ - "/layers/bronze.md", - "/tables/bronze-orders-raw.md", - "/storage/bronze-orders-prefix.md" - ], - "/streams/orders-events.md": [ - "/ingestion/orders-stream-landing.md", - "/tables/bronze-orders-raw.md" - ], - "/ingestion/product-catalog-sftp-load.md": [ - "/layers/bronze.md" - ], - "/sources/product-catalog-sftp.md": [ - "/ingestion/product-catalog-sftp-load.md" - ], - "/lineage/orders-to-revenue.md": [ - "/tables/silver-orders.md", - "/tables/gold-order-daily.md" - ], - "/tables/bronze-orders-api.md": [ - "/queries/orders-raw.md", - "/tables/bronze-orders-raw.md" - ], - "/tables/bronze-orders-raw.md": [ - "/queries/orders.md", - "/tables/silver-orders.md", - "/transformations/cleanse-orders.md" - ], - "/sources/retail-lake-walk.md": [ - "/tables/bronze-orders-api.md", - "/tables/bronze-orders-raw.md", - "/tables/gold-order-daily.md", - "/tables/silver-orders.md" - ], - "/layers/silver.md": [ - "/transformations/aggregate-order-daily.md" - ], - "/transformations/aggregate-order-daily.md": [ - "/layers/gold.md", - "/layers/silver.md", - "/tables/gold-order-daily.md" - ], - "/layers/bronze.md": [ - "/transformations/cleanse-orders.md" - ], - "/transformations/cleanse-orders.md": [ - "/layers/silver.md", - "/layers/bronze.md", - "/tables/silver-orders.md" - ] -} \ No newline at end of file diff --git a/sample-knowledge/.index/inventory.json b/sample-knowledge/.index/inventory.json deleted file mode 100644 index 72d0b04..0000000 --- a/sample-knowledge/.index/inventory.json +++ /dev/null @@ -1,1408 +0,0 @@ -[ - { - "path": "agents/walk-retail-lake-walk.md", - "type": "AgentNode", - "title": "Lake walk: retail-lake-walk", - "description": "DEKC walk of /workspace/tests/fixtures/sample-lake", - "layer": null, - "tags": [ - "agent", - "walk", - "dekc" - ], - "verified": true, - "status": "completed", - "wiki_key": "walk-retail-lake-walk", - "token_count": 35, - "hash": "35343d82f99ef3ee" - }, - { - "path": "business-objects/customer-lifetime-value.md", - "type": "BusinessObject", - "title": "Customer Lifetime Value", - "description": "Predicted and realized value of a customer based on historical order gross amounts.", - "layer": null, - "tags": [ - "business-object", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "bo-customer-lifetime-value", - "token_count": 49, - "hash": "ce75a058db2643d0" - }, - { - "path": "business-objects/customer-order.md", - "type": "BusinessObject", - "title": "Customer Order", - "description": "A single customer purchase transaction after data quality rules. Canonical source for order facts.", - "layer": null, - "tags": [ - "business-object", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "bo-customer-order", - "token_count": 47, - "hash": "45f0a512e5d73c36" - }, - { - "path": "business-objects/daily-order-summary.md", - "type": "BusinessObject", - "title": "Daily Order Summary", - "description": "Business-facing daily summary of order volume and gross merchandise value by currency. Used by finance and growth for pacing and forecasting.", - "layer": null, - "tags": [ - "business-object", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "bo-daily-order-summary", - "token_count": 63, - "hash": "27ea4fff16f1b66e" - }, - { - "path": "catalogs/retail-unity-style-catalog.md", - "type": "DataCatalog", - "title": "Retail Unity-style Catalog", - "description": "Workspace catalog for retail schemas", - "layer": null, - "tags": [ - "data-catalog", - "dekc", - "fabric" - ], - "verified": true, - "status": "active", - "wiki_key": "catalog-retail-unity-style-catalog", - "token_count": 33, - "hash": "427d21af0e154fb8" - }, - { - "path": "columns/bronze-orders-raw-currency.md", - "type": "Column", - "title": "orders_raw.currency", - "description": "ISO currency code", - "layer": null, - "tags": [ - "column", - "dekc", - "bronze" - ], - "verified": false, - "status": "active", - "wiki_key": "col-bronze-orders-raw-currency", - "token_count": 16, - "hash": "64714aeca81971ad" - }, - { - "path": "columns/bronze-orders-raw-customer-id.md", - "type": "Column", - "title": "orders_raw.customer_id", - "description": "Customer key", - "layer": null, - "tags": [ - "column", - "dekc", - "bronze" - ], - "verified": false, - "status": "active", - "wiki_key": "col-bronze-orders-raw-customer-id", - "token_count": 14, - "hash": "f8fc85b24eaa91f9" - }, - { - "path": "columns/bronze-orders-raw-gross-amount.md", - "type": "Column", - "title": "orders_raw.gross_amount", - "description": "Gross merchandise value", - "layer": null, - "tags": [ - "column", - "dekc", - "bronze" - ], - "verified": false, - "status": "active", - "wiki_key": "col-bronze-orders-raw-gross-amount", - "token_count": 16, - "hash": "8a1c57b663bf258f" - }, - { - "path": "columns/bronze-orders-raw-order-id.md", - "type": "Column", - "title": "orders_raw.order_id", - "description": "Order identifier", - "layer": null, - "tags": [ - "column", - "dekc", - "bronze" - ], - "verified": false, - "status": "active", - "wiki_key": "col-bronze-orders-raw-order-id", - "token_count": 14, - "hash": "f7e2d70ba7be0c60" - }, - { - "path": "columns/bronze-orders-raw-order-ts.md", - "type": "Column", - "title": "orders_raw.order_ts", - "description": "Order timestamp UTC", - "layer": null, - "tags": [ - "column", - "dekc", - "bronze" - ], - "verified": false, - "status": "active", - "wiki_key": "col-bronze-orders-raw-order-ts", - "token_count": 16, - "hash": "976cf5ffe46a1af4" - }, - { - "path": "columns/bronze-orders-raw-status.md", - "type": "Column", - "title": "orders_raw.status", - "description": "Order status", - "layer": null, - "tags": [ - "column", - "dekc", - "bronze" - ], - "verified": false, - "status": "active", - "wiki_key": "col-bronze-orders-raw-status", - "token_count": 10, - "hash": "a84578575782fcb5" - }, - { - "path": "columns/gold-order-daily-currency.md", - "type": "Column", - "title": "order_daily.currency", - "description": "Column currency on order_daily", - "layer": null, - "tags": [ - "column", - "dekc", - "gold" - ], - "verified": false, - "status": "active", - "wiki_key": "col-gold-order-daily-currency", - "token_count": 14, - "hash": "e92c51b0ba83fad3" - }, - { - "path": "columns/gold-order-daily-gross-revenue.md", - "type": "Column", - "title": "order_daily.gross_revenue", - "description": "Column gross_revenue on order_daily", - "layer": null, - "tags": [ - "column", - "dekc", - "gold" - ], - "verified": false, - "status": "active", - "wiki_key": "col-gold-order-daily-gross-revenue", - "token_count": 14, - "hash": "cf6684d4994041a7" - }, - { - "path": "columns/gold-order-daily-order-count.md", - "type": "Column", - "title": "order_daily.order_count", - "description": "Column order_count on order_daily", - "layer": null, - "tags": [ - "column", - "dekc", - "gold" - ], - "verified": false, - "status": "active", - "wiki_key": "col-gold-order-daily-order-count", - "token_count": 14, - "hash": "39fcfd48898a382a" - }, - { - "path": "columns/gold-order-daily-order-date.md", - "type": "Column", - "title": "order_daily.order_date", - "description": "Column order_date on order_daily", - "layer": null, - "tags": [ - "column", - "dekc", - "gold" - ], - "verified": false, - "status": "active", - "wiki_key": "col-gold-order-daily-order-date", - "token_count": 14, - "hash": "d290ffd17d005eb8" - }, - { - "path": "columns/silver-orders-currency.md", - "type": "Column", - "title": "orders.currency", - "description": "Column currency on orders", - "layer": null, - "tags": [ - "column", - "dekc", - "silver" - ], - "verified": false, - "status": "active", - "wiki_key": "col-silver-orders-currency", - "token_count": 14, - "hash": "c710bea98ff18665" - }, - { - "path": "columns/silver-orders-customer-id.md", - "type": "Column", - "title": "orders.customer_id", - "description": "Column customer_id on orders", - "layer": null, - "tags": [ - "column", - "dekc", - "silver" - ], - "verified": false, - "status": "active", - "wiki_key": "col-silver-orders-customer-id", - "token_count": 14, - "hash": "b32f431712f78e13" - }, - { - "path": "columns/silver-orders-gross-amount.md", - "type": "Column", - "title": "orders.gross_amount", - "description": "Column gross_amount on orders", - "layer": null, - "tags": [ - "column", - "dekc", - "silver" - ], - "verified": false, - "status": "active", - "wiki_key": "col-silver-orders-gross-amount", - "token_count": 14, - "hash": "f19fea44f03fed29" - }, - { - "path": "columns/silver-orders-order-id.md", - "type": "Column", - "title": "orders.order_id", - "description": "Column order_id on orders", - "layer": null, - "tags": [ - "column", - "dekc", - "silver" - ], - "verified": false, - "status": "active", - "wiki_key": "col-silver-orders-order-id", - "token_count": 14, - "hash": "4af9e278ef515174" - }, - { - "path": "columns/silver-orders-order-ts.md", - "type": "Column", - "title": "orders.order_ts", - "description": "Column order_ts on orders", - "layer": null, - "tags": [ - "column", - "dekc", - "silver" - ], - "verified": false, - "status": "active", - "wiki_key": "col-silver-orders-order-ts", - "token_count": 14, - "hash": "f99d78f71cf8ac32" - }, - { - "path": "columns/silver-orders-status.md", - "type": "Column", - "title": "orders.status", - "description": "Column status on orders", - "layer": null, - "tags": [ - "column", - "dekc", - "silver" - ], - "verified": false, - "status": "active", - "wiki_key": "col-silver-orders-status", - "token_count": 11, - "hash": "6764f2d2cb1439e4" - }, - { - "path": "dashboards/executive-revenue.md", - "type": "Dashboard", - "title": "Executive Revenue", - "description": "C-level daily revenue and order trends", - "layer": null, - "tags": [ - "dashboard", - "powerbi", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "dashboard-executive-revenue", - "token_count": 20, - "hash": "73b0773a7906541f" - }, - { - "path": "dax/total-revenue-dax.md", - "type": "DaxArtifact", - "title": "total-revenue-dax (DAX)", - "description": "Power BI total revenue measure", - "layer": null, - "tags": [ - "dax", - "dekc" - ], - "verified": false, - "status": "active", - "wiki_key": "dax-total-revenue-dax", - "token_count": 23, - "hash": "ce6b586476ef61c5" - }, - { - "path": "dax/total-revenue.md", - "type": "DaxArtifact", - "title": "total_revenue (DAX)", - "description": "Discovered DAX from dax/total_revenue.dax", - "layer": null, - "tags": [ - "dax", - "dekc" - ], - "verified": false, - "status": "active", - "wiki_key": "dax-total-revenue", - "token_count": 19, - "hash": "ad75df6a5d25bedf" - }, - { - "path": "diagrams/activity-daily-medallion-orders-activity.md", - "type": "Diagram", - "title": "daily-medallion-orders-activity", - "description": "activity diagram for workflow /workflows/daily-medallion-orders.md", - "layer": null, - "tags": [ - "diagram", - "activity", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-activity-daily-medallion-orders-activity", - "token_count": 85, - "hash": "fc2f7a203acaf16e" - }, - { - "path": "diagrams/activity-orders-stream-landing-activity.md", - "type": "Diagram", - "title": "orders-stream-landing-activity", - "description": "activity diagram for ingestion job /ingestion/orders-stream-landing.md", - "layer": null, - "tags": [ - "diagram", - "activity", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-activity-orders-stream-landing-activity", - "token_count": 87, - "hash": "feb216108d3b95ef" - }, - { - "path": "diagrams/architecture-commerce-lake-architecture.md", - "type": "Diagram", - "title": "Commerce lake architecture", - "description": "architecture diagram (mermaid): Commerce lake architecture", - "layer": null, - "tags": [ - "diagram", - "architecture", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-architecture-commerce-lake-architecture", - "token_count": 76, - "hash": "28109e7bd39071ed" - }, - { - "path": "diagrams/class-daily-medallion-orders-class.md", - "type": "Diagram", - "title": "daily-medallion-orders-class", - "description": "class diagram for workflow /workflows/daily-medallion-orders.md", - "layer": null, - "tags": [ - "diagram", - "class", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-class-daily-medallion-orders-class", - "token_count": 84, - "hash": "9d402f6084942bc5" - }, - { - "path": "diagrams/component-daily-medallion-orders-component.md", - "type": "Diagram", - "title": "daily-medallion-orders-component", - "description": "component diagram for workflow /workflows/daily-medallion-orders.md", - "layer": null, - "tags": [ - "diagram", - "component", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-component-daily-medallion-orders-component", - "token_count": 94, - "hash": "0b909bced48429b3" - }, - { - "path": "diagrams/component-orders-stream-landing-component.md", - "type": "Diagram", - "title": "orders-stream-landing-component", - "description": "component diagram for ingestion job /ingestion/orders-stream-landing.md", - "layer": null, - "tags": [ - "diagram", - "component", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-component-orders-stream-landing-component", - "token_count": 96, - "hash": "f607d5edba93ac32" - }, - { - "path": "diagrams/erd-orders-erd.md", - "type": "Diagram", - "title": "Orders ERD", - "description": "Logical ERD for commerce orders", - "layer": null, - "tags": [ - "diagram", - "erd", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-erd-orders-erd", - "token_count": 87, - "hash": "9dc80564a05d8d8f" - }, - { - "path": "diagrams/sequence-orders-stream-landing-sequence.md", - "type": "Diagram", - "title": "orders-stream-landing-sequence", - "description": "sequence diagram for ingestion job /ingestion/orders-stream-landing.md", - "layer": null, - "tags": [ - "diagram", - "sequence", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-sequence-orders-stream-landing-sequence", - "token_count": 102, - "hash": "a0a8ba32a8d16724" - }, - { - "path": "diagrams/state-daily-medallion-orders-state.md", - "type": "Diagram", - "title": "daily-medallion-orders-state", - "description": "state diagram for workflow /workflows/daily-medallion-orders.md", - "layer": null, - "tags": [ - "diagram", - "state", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-state-daily-medallion-orders-state", - "token_count": 77, - "hash": "aeedf75758d27777" - }, - { - "path": "diagrams/state-orders-stream-landing-state.md", - "type": "Diagram", - "title": "orders-stream-landing-state", - "description": "state diagram for ingestion job /ingestion/orders-stream-landing.md", - "layer": null, - "tags": [ - "diagram", - "state", - "mermaid", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "diagram-state-orders-stream-landing-state", - "token_count": 79, - "hash": "9906b8f8698a4c8d" - }, - { - "path": "domains/commerce.md", - "type": "DataDomain", - "title": "Commerce", - "description": "Order-to-cash and merchandising", - "layer": null, - "tags": [ - "data-domain", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "domain-commerce", - "token_count": 17, - "hash": "65aa1b97f57203ed" - }, - { - "path": "glossary/customer-lifetime-value.md", - "type": "GlossaryTerm", - "title": "Customer Lifetime Value", - "description": "Predicted and realized value of a customer based on historical order gross amounts.", - "layer": null, - "tags": [ - "glossary", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "term-customer-lifetime-value", - "token_count": 34, - "hash": "9ec79c6d5dc5f578" - }, - { - "path": "glossary/customer-order.md", - "type": "GlossaryTerm", - "title": "Customer Order", - "description": "A single customer purchase transaction after data quality rules. Canonical source for order facts.", - "layer": null, - "tags": [ - "glossary", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "term-customer-order", - "token_count": 34, - "hash": "323cf55039393e3a" - }, - { - "path": "glossary/daily-order-summary.md", - "type": "GlossaryTerm", - "title": "Daily Order Summary", - "description": "Business-facing daily summary of order volume and gross merchandise value by currency. Used by finance and growth for pacing and forecasting.", - "layer": null, - "tags": [ - "glossary", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "term-daily-order-summary", - "token_count": 48, - "hash": "28bb48392ac43a0a" - }, - { - "path": "glossary/gmv.md", - "type": "GlossaryTerm", - "title": "GMV", - "description": "Gross Merchandise Value \u2014 total sales dollar volume before returns, cancellations, and discounts.", - "layer": null, - "tags": [ - "glossary", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "term-gmv", - "token_count": 34, - "hash": "74645969b7523b8a" - }, - { - "path": "ingestion/orders-stream-landing.md", - "type": "IngestionJob", - "title": "orders-stream-landing", - "description": "Lands order events from Event Hubs into bronze Delta", - "layer": null, - "tags": [ - "ingestion", - "job", - "streaming", - "bronze", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "ingest-orders-stream-landing", - "token_count": 99, - "hash": "692ad1c03a428c03" - }, - { - "path": "ingestion/product-catalog-sftp-load.md", - "type": "IngestionJob", - "title": "product-catalog-sftp-load", - "description": "Nightly full extract of product catalog from SFTP to bronze", - "layer": null, - "tags": [ - "ingestion", - "job", - "file_drop", - "bronze", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "ingest-product-catalog-sftp-load", - "token_count": 87, - "hash": "b95e662b7d533d64" - }, - { - "path": "lakes/retail-commerce-lake.md", - "type": "DataLake", - "title": "Retail Commerce Lake", - "description": "Primary commerce lakehouse", - "layer": null, - "tags": [ - "data-lake", - "dekc", - "fabric-onelake" - ], - "verified": true, - "status": "active", - "wiki_key": "lake-retail-commerce-lake", - "token_count": 47, - "hash": "f80925d436946c36" - }, - { - "path": "layers/bronze.md", - "type": "Layer", - "title": "Bronze layer", - "description": "Medallion bronze zone", - "layer": "bronze", - "tags": [ - "layer", - "medallion", - "bronze" - ], - "verified": true, - "status": "active", - "wiki_key": "layer-bronze", - "token_count": 17, - "hash": "01d9cc8a30049fec" - }, - { - "path": "layers/gold.md", - "type": "Layer", - "title": "Gold layer", - "description": "Medallion gold zone", - "layer": "gold", - "tags": [ - "layer", - "medallion", - "gold" - ], - "verified": true, - "status": "active", - "wiki_key": "layer-gold", - "token_count": 17, - "hash": "643fcdcc5ab757e6" - }, - { - "path": "layers/silver.md", - "type": "Layer", - "title": "Silver layer", - "description": "Medallion silver zone", - "layer": "silver", - "tags": [ - "layer", - "medallion", - "silver" - ], - "verified": true, - "status": "active", - "wiki_key": "layer-silver", - "token_count": 17, - "hash": "71e90fa74df4dd89" - }, - { - "path": "lineage/orders-to-revenue.md", - "type": "LineagePath", - "title": "orders-to-revenue", - "description": "Core commerce revenue lineage from landing to gold mart", - "layer": null, - "tags": [ - "lineage", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "lineage-orders-to-revenue", - "token_count": 66, - "hash": "b9086afc65f1f833" - }, - { - "path": "marts/revenue-mart.md", - "type": "DataMart", - "title": "Revenue Mart", - "description": "Gold revenue aggregates for exec reporting", - "layer": null, - "tags": [ - "data-mart", - "gold", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "mart-revenue-mart", - "token_count": 31, - "hash": "7778e529dce9a5ef" - }, - { - "path": "metrics/gross-merchandise-value.md", - "type": "Metric", - "title": "Gross Merchandise Value", - "description": "Sum of gross order amounts before returns and discounts, in transaction currency.", - "layer": null, - "tags": [ - "metric", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "metric-gross-merchandise-value", - "token_count": 37, - "hash": "f62cb56165d14936" - }, - { - "path": "metrics/order-count.md", - "type": "Metric", - "title": "Order Count", - "description": "Count of distinct completed orders.", - "layer": null, - "tags": [ - "metric", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "metric-order-count", - "token_count": 22, - "hash": "e4ce6b8955104470" - }, - { - "path": "packs/pack-gold-order-daily.md", - "type": "ContextPack", - "title": "Pack: gold-order-daily", - "description": "Progressive disclosure pack for /tables/gold-order-daily.md", - "layer": null, - "tags": [ - "pack", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "pack-gold-order-daily", - "token_count": 184, - "hash": "ca022c342a713da5" - }, - { - "path": "products/daily-revenue-product.md", - "type": "DataProduct", - "title": "Daily Revenue Product", - "description": "Certified daily revenue data product for BI", - "layer": null, - "tags": [ - "data-product", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "product-daily-revenue-product", - "token_count": 30, - "hash": "888adaf58e46ca42" - }, - { - "path": "quality/gold-order-daily-freshness.md", - "type": "DQRule", - "title": "gold-order-daily-freshness", - "description": "Gold mart must be fresh within 1 day", - "layer": null, - "tags": [ - "dq", - "quality", - "freshness", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "dq-gold-order-daily-freshness", - "token_count": 40, - "hash": "6efbbfb153e0e767" - }, - { - "path": "queries/customer-ltv.md", - "type": "Query", - "title": "customer_ltv", - "description": "Discovered SQL from gold/marts/customer_ltv.sql", - "layer": null, - "tags": [ - "query", - "sql", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "query-customer-ltv", - "token_count": 41, - "hash": "2265c7676ab68feb" - }, - { - "path": "queries/daily-medallion.md", - "type": "Query", - "title": "daily_medallion", - "description": "Discovered SQL from workflows/daily_medallion.sql", - "layer": null, - "tags": [ - "query", - "sql", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "query-daily-medallion", - "token_count": 32, - "hash": "a0906c56918444fc" - }, - { - "path": "queries/order-daily.md", - "type": "Query", - "title": "order_daily", - "description": "Discovered SQL from gold/marts/order_daily.sql", - "layer": null, - "tags": [ - "query", - "sql", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "query-order-daily", - "token_count": 46, - "hash": "35e6274680727d2c" - }, - { - "path": "queries/orders-raw.md", - "type": "Query", - "title": "orders_raw", - "description": "Discovered SQL from bronze/sales/orders_raw.sql", - "layer": null, - "tags": [ - "query", - "sql", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "query-orders-raw", - "token_count": 30, - "hash": "517f606afb09e632" - }, - { - "path": "queries/orders.md", - "type": "Query", - "title": "orders", - "description": "Discovered SQL from silver/sales/orders.sql", - "layer": null, - "tags": [ - "query", - "sql", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "query-orders", - "token_count": 37, - "hash": "2d8611ea9747d211" - }, - { - "path": "queries/total-revenue-dax.md", - "type": "Query", - "title": "total-revenue-dax", - "description": "Power BI total revenue measure", - "layer": null, - "tags": [ - "query", - "dax", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "query-total-revenue-dax", - "token_count": 28, - "hash": "44e6133ea33b7d1c" - }, - { - "path": "queries/total-revenue.md", - "type": "Query", - "title": "total_revenue", - "description": "Discovered DAX from dax/total_revenue.dax", - "layer": null, - "tags": [ - "query", - "dax", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "query-total-revenue", - "token_count": 24, - "hash": "ddbb3b5d637cbda9" - }, - { - "path": "reports/weekly-commerce-digest.md", - "type": "Report", - "title": "Weekly Commerce Digest", - "description": "Automated weekly PDF of order and LTV trends", - "layer": null, - "tags": [ - "report", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "report-weekly-commerce-digest", - "token_count": 23, - "hash": "77788c967fd321f0" - }, - { - "path": "schemas/bronze.md", - "type": "Schema", - "title": "bronze", - "description": "Schema bronze", - "layer": null, - "tags": [ - "schema", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "schema-bronze", - "token_count": 17, - "hash": "40403a02c0f6f9d1" - }, - { - "path": "schemas/gold.md", - "type": "Schema", - "title": "gold", - "description": "Schema gold", - "layer": null, - "tags": [ - "schema", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "schema-gold", - "token_count": 17, - "hash": "9e192f47166a3834" - }, - { - "path": "schemas/silver.md", - "type": "Schema", - "title": "silver", - "description": "Schema silver", - "layer": null, - "tags": [ - "schema", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "schema-silver", - "token_count": 16, - "hash": "d44bb9a2fe85542c" - }, - { - "path": "semantic/retail-commerce-model.md", - "type": "SemanticModel", - "title": "Retail Commerce Model", - "description": "Semantic layer over gold order marts for BI tools", - "layer": null, - "tags": [ - "semantic", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "semantic-retail-commerce-model", - "token_count": 33, - "hash": "550d87663c9245f3" - }, - { - "path": "sources/product-catalog-sftp.md", - "type": "SourceSystem", - "title": "product-catalog-sftp", - "description": "Nightly product catalog flat files via SFTP", - "layer": null, - "tags": [ - "source", - "file", - "sftp", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "source-product-catalog-sftp", - "token_count": 34, - "hash": "1a813c72682fc8ba" - }, - { - "path": "sources/retail-lake-walk.md", - "type": "SourceSystem", - "title": "retail-lake-walk", - "description": "Walked data lake at /workspace/tests/fixtures/sample-lake", - "layer": null, - "tags": [ - "source", - "lake", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "source-retail-lake-walk", - "token_count": 37, - "hash": "1d073150427b2982" - }, - { - "path": "sources/retail-lake.md", - "type": "SourceSystem", - "title": "retail-lake", - "description": "Primary retail analytics data lake (Delta + SQL models)", - "layer": null, - "tags": [ - "source", - "lake", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "source-retail-lake", - "token_count": 32, - "hash": "6bd09e5e2b4db764" - }, - { - "path": "sql/customer-ltv.md", - "type": "SqlArtifact", - "title": "customer_ltv (SQL)", - "description": "Discovered SQL from gold/marts/customer_ltv.sql", - "layer": null, - "tags": [ - "sql", - "dekc" - ], - "verified": false, - "status": "active", - "wiki_key": "sql-customer-ltv", - "token_count": 35, - "hash": "ab20e86f7cab5b4b" - }, - { - "path": "sql/daily-medallion.md", - "type": "SqlArtifact", - "title": "daily_medallion (SQL)", - "description": "Discovered SQL from workflows/daily_medallion.sql", - "layer": null, - "tags": [ - "sql", - "dekc" - ], - "verified": false, - "status": "active", - "wiki_key": "sql-daily-medallion", - "token_count": 27, - "hash": "99576be96c53571b" - }, - { - "path": "sql/order-daily.md", - "type": "SqlArtifact", - "title": "order_daily (SQL)", - "description": "Discovered SQL from gold/marts/order_daily.sql", - "layer": null, - "tags": [ - "sql", - "dekc" - ], - "verified": false, - "status": "active", - "wiki_key": "sql-order-daily", - "token_count": 40, - "hash": "e76aa14c0a32ff02" - }, - { - "path": "sql/orders-raw.md", - "type": "SqlArtifact", - "title": "orders_raw (SQL)", - "description": "Discovered SQL from bronze/sales/orders_raw.sql", - "layer": null, - "tags": [ - "sql", - "dekc" - ], - "verified": false, - "status": "active", - "wiki_key": "sql-orders-raw", - "token_count": 24, - "hash": "c30d112805133a7e" - }, - { - "path": "sql/orders.md", - "type": "SqlArtifact", - "title": "orders (SQL)", - "description": "Discovered SQL from silver/sales/orders.sql", - "layer": null, - "tags": [ - "sql", - "dekc" - ], - "verified": false, - "status": "active", - "wiki_key": "sql-orders", - "token_count": 31, - "hash": "7e9c59adbe60e09a" - }, - { - "path": "storage/bronze-orders-prefix.md", - "type": "StorageLocation", - "title": "bronze-orders-prefix", - "description": "Storage: bronze-orders-prefix", - "layer": "bronze", - "tags": [ - "storage", - "path", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "storage-bronze-orders-prefix", - "token_count": 25, - "hash": "bdd3351b3984b112" - }, - { - "path": "streams/orders-events.md", - "type": "Stream", - "title": "orders-events", - "description": "Order placement events", - "layer": null, - "tags": [ - "stream", - "dekc", - "eventhubs" - ], - "verified": true, - "status": "active", - "wiki_key": "stream-orders-events", - "token_count": 36, - "hash": "edc2d682b54e669f" - }, - { - "path": "tables/bronze-orders-api.md", - "type": "Table", - "title": "orders_api", - "description": "Referenced by orders_raw", - "layer": "bronze", - "tags": [ - "table", - "dekc", - "bronze" - ], - "verified": true, - "status": "active", - "wiki_key": "table-bronze-orders-api", - "token_count": 14, - "hash": "fa3229885b25fa8e" - }, - { - "path": "tables/bronze-orders-raw.md", - "type": "Table", - "title": "orders_raw", - "description": "Inferred table from bronze/sales/orders_raw.sql", - "layer": "bronze", - "tags": [ - "table", - "dekc", - "bronze" - ], - "verified": true, - "status": "active", - "wiki_key": "table-bronze-orders-raw", - "token_count": 32, - "hash": "d62a7300ddd1c4d6" - }, - { - "path": "tables/gold-order-daily.md", - "type": "Table", - "title": "order_daily", - "description": "Inferred table from gold/marts/order_daily.sql", - "layer": "gold", - "tags": [ - "table", - "dekc", - "gold" - ], - "verified": true, - "status": "active", - "wiki_key": "table-gold-order-daily", - "token_count": 48, - "hash": "4edeea9928bae2e1" - }, - { - "path": "tables/silver-orders.md", - "type": "Table", - "title": "orders", - "description": "Inferred table from silver/sales/orders.sql", - "layer": "silver", - "tags": [ - "table", - "dekc", - "silver" - ], - "verified": true, - "status": "active", - "wiki_key": "table-silver-orders", - "token_count": 39, - "hash": "159032be5030ac47" - }, - { - "path": "transformations/aggregate-order-daily.md", - "type": "Transformation", - "title": "aggregate-order-daily", - "description": "Daily grain aggregation for revenue reporting", - "layer": null, - "tags": [ - "transformation", - "dekc", - "silver", - "gold" - ], - "verified": true, - "status": "active", - "wiki_key": "xform-aggregate-order-daily", - "token_count": 23, - "hash": "280a378f8b3a796f" - }, - { - "path": "transformations/cleanse-orders.md", - "type": "Transformation", - "title": "cleanse-orders", - "description": "Type coercion, null status drop, commerce order cleanse", - "layer": null, - "tags": [ - "transformation", - "dekc", - "bronze", - "silver" - ], - "verified": true, - "status": "active", - "wiki_key": "xform-cleanse-orders", - "token_count": 28, - "hash": "3b0f268cf8769e3a" - }, - { - "path": "views/gold-customer-ltv.md", - "type": "View", - "title": "customer_ltv", - "description": "Per-customer lifetime value view", - "layer": "gold", - "tags": [ - "view", - "dekc", - "gold" - ], - "verified": true, - "status": "active", - "wiki_key": "view-gold-customer-ltv", - "token_count": 37, - "hash": "e469046fc96462a2" - }, - { - "path": "wireframes/wireframe-executive-revenue-wireframe.md", - "type": "Wireframe", - "title": "Executive Revenue Wireframe", - "description": "PlantUML salt wireframe for exec revenue dashboard", - "layer": null, - "tags": [ - "diagram", - "wireframe", - "plantuml", - "dekc", - "wireframe" - ], - "verified": true, - "status": "active", - "wiki_key": "wireframe-wireframe-executive-revenue-wireframe", - "token_count": 78, - "hash": "432c49a33002bb4a" - }, - { - "path": "workflows/daily-medallion-orders.md", - "type": "Workflow", - "title": "daily-medallion-orders", - "description": "Nightly bronze\u2192silver\u2192gold orders pipeline", - "layer": null, - "tags": [ - "workflow", - "airflow", - "dekc" - ], - "verified": true, - "status": "active", - "wiki_key": "workflow-daily-medallion-orders", - "token_count": 43, - "hash": "db063f6ba6a0bfba" - } -] \ No newline at end of file diff --git a/sample-knowledge/.index/manifest.json b/sample-knowledge/.index/manifest.json deleted file mode 100644 index 2624fef..0000000 --- a/sample-knowledge/.index/manifest.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "built_at": "2026-08-13T19:46:37Z", - "concept_count": 83, - "token_count": 424, - "edge_count": 40, - "bundle": "/workspace/repos/data-engineering-knowledge-capture/sample-knowledge", - "engine": "dekc-local-bow-v1", - "files": [ - ".index/inventory.json", - ".index/search.json", - ".index/graph.json", - ".index/embeddings.jsonl" - ] -} \ No newline at end of file diff --git a/sample-knowledge/.index/search.json b/sample-knowledge/.index/search.json deleted file mode 100644 index eb834a5..0000000 --- a/sample-knowledge/.index/search.json +++ /dev/null @@ -1,2545 +0,0 @@ -{ - "14": [ - "agents/walk-retail-lake-walk.md" - ], - "142": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "15": [ - "ingestion/orders-stream-landing.md" - ], - "2m": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "30": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "420": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "_edit": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "abfss": [ - "lakes/retail-commerce-lake.md", - "storage/bronze-orders-prefix.md" - ], - "above": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "acme": [ - "sources/retail-lake.md" - ], - "activity": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "adf": [ - "ingestion/product-catalog-sftp-load.md" - ], - "after": [ - "business-objects/customer-order.md", - "glossary/customer-order.md" - ], - "agent": [ - "agents/walk-retail-lake-walk.md" - ], - "agentnode": [ - "agents/walk-retail-lake-walk.md" - ], - "aggregate": [ - "queries/daily-medallion.md", - "sql/daily-medallion.md", - "transformations/aggregate-order-daily.md", - "workflows/daily-medallion-orders.md" - ], - "aggregates": [ - "marts/revenue-mart.md" - ], - "aggregation": [ - "transformations/aggregate-order-daily.md" - ], - "airflow": [ - "workflows/daily-medallion-orders.md" - ], - "alert": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "amounts": [ - "business-objects/customer-lifetime-value.md", - "glossary/customer-lifetime-value.md", - "metrics/gross-merchandise-value.md" - ], - "analytics": [ - "business-objects/customer-lifetime-value.md", - "business-objects/daily-order-summary.md", - "domains/commerce.md", - "sources/retail-lake.md" - ], - "aov": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "api": [ - "packs/pack-gold-order-daily.md", - "queries/daily-medallion.md", - "sql/daily-medallion.md", - "workflows/daily-medallion-orders.md" - ], - "apps": [ - "products/daily-revenue-product.md" - ], - "architecture": [ - "diagrams/architecture-commerce-lake-architecture.md", - "lakes/retail-commerce-lake.md" - ], - "area": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "as": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "ingestion/orders-stream-landing.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "streams/orders-events.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "at": [ - "sources/retail-lake-walk.md" - ], - "automated": [ - "reports/weekly-commerce-digest.md" - ], - "based": [ - "business-objects/customer-lifetime-value.md", - "glossary/customer-lifetime-value.md" - ], - "batch": [ - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "be": [ - "quality/gold-order-daily-freshness.md" - ], - "before": [ - "glossary/gmv.md", - "metrics/gross-merchandise-value.md" - ], - "bi": [ - "dax/total-revenue-dax.md", - "products/daily-revenue-product.md", - "queries/total-revenue-dax.md", - "semantic/retail-commerce-model.md", - "workflows/daily-medallion-orders.md" - ], - "bigint": [ - "columns/gold-order-daily-order-count.md" - ], - "binding": [ - "semantic/retail-commerce-model.md" - ], - "bronze": [ - "columns/bronze-orders-raw-currency.md", - "columns/bronze-orders-raw-customer-id.md", - "columns/bronze-orders-raw-gross-amount.md", - "columns/bronze-orders-raw-order-id.md", - "columns/bronze-orders-raw-order-ts.md", - "columns/bronze-orders-raw-status.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lakes/retail-commerce-lake.md", - "layers/bronze.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "queries/daily-medallion.md", - "queries/orders-raw.md", - "queries/orders.md", - "schemas/bronze.md", - "sql/daily-medallion.md", - "sql/orders-raw.md", - "sql/orders.md", - "storage/bronze-orders-prefix.md", - "streams/orders-events.md", - "tables/bronze-orders-api.md", - "tables/bronze-orders-raw.md", - "tables/silver-orders.md", - "transformations/cleanse-orders.md", - "workflows/daily-medallion-orders.md" - ], - "brz": [ - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "business": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md", - "marts/revenue-mart.md", - "semantic/retail-commerce-model.md" - ], - "businessobject": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md" - ], - "by": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "tables/bronze-orders-api.md", - "tables/gold-order-daily.md", - "views/gold-customer-ltv.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "cancellations": [ - "glossary/gmv.md" - ], - "canonical": [ - "business-objects/customer-order.md", - "glossary/customer-order.md" - ], - "cash": [ - "domains/commerce.md" - ], - "cast": [ - "packs/pack-gold-order-daily.md", - "queries/order-daily.md", - "sql/order-daily.md", - "tables/gold-order-daily.md" - ], - "catalog": [ - "catalogs/retail-unity-style-catalog.md", - "ingestion/product-catalog-sftp-load.md", - "sources/product-catalog-sftp.md" - ], - "certified": [ - "products/daily-revenue-product.md" - ], - "chart": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "checkpoint": [ - "ingestion/orders-stream-landing.md" - ], - "checkpoints": [ - "ingestion/orders-stream-landing.md" - ], - "class": [ - "diagrams/class-daily-medallion-orders-class.md" - ], - "classdiagram": [ - "diagrams/class-daily-medallion-orders-class.md" - ], - "clean": [ - "queries/daily-medallion.md", - "sql/daily-medallion.md" - ], - "cleanse": [ - "transformations/cleanse-orders.md", - "workflows/daily-medallion-orders.md" - ], - "cleansed": [ - "diagrams/architecture-commerce-lake-architecture.md" - ], - "code": [ - "columns/bronze-orders-raw-currency.md" - ], - "coercion": [ - "transformations/cleanse-orders.md" - ], - "column": [ - "columns/bronze-orders-raw-currency.md", - "columns/bronze-orders-raw-customer-id.md", - "columns/bronze-orders-raw-gross-amount.md", - "columns/bronze-orders-raw-order-id.md", - "columns/bronze-orders-raw-order-ts.md", - "columns/bronze-orders-raw-status.md", - "columns/gold-order-daily-currency.md", - "columns/gold-order-daily-gross-revenue.md", - "columns/gold-order-daily-order-count.md", - "columns/gold-order-daily-order-date.md", - "columns/silver-orders-currency.md", - "columns/silver-orders-customer-id.md", - "columns/silver-orders-gross-amount.md", - "columns/silver-orders-order-id.md", - "columns/silver-orders-order-ts.md", - "columns/silver-orders-status.md" - ], - "com": [ - "sources/product-catalog-sftp.md" - ], - "commerce": [ - "business-objects/customer-order.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/erd-orders-erd.md", - "domains/commerce.md", - "lakes/retail-commerce-lake.md", - "lineage/orders-to-revenue.md", - "reports/weekly-commerce-digest.md", - "semantic/retail-commerce-model.md", - "transformations/cleanse-orders.md", - "workflows/daily-medallion-orders.md" - ], - "complete": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "completed": [ - "metrics/order-count.md" - ], - "component": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "compute": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "connector": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "consumers": [ - "products/daily-revenue-product.md" - ], - "contains": [ - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/erd-orders-erd.md" - ], - "context": [ - "packs/pack-gold-order-daily.md" - ], - "contextpack": [ - "packs/pack-gold-order-daily.md" - ], - "copy": [ - "ingestion/product-catalog-sftp-load.md" - ], - "core": [ - "lineage/orders-to-revenue.md" - ], - "count": [ - "metrics/order-count.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "tables/gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "create": [ - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md" - ], - "created": [ - "agents/walk-retail-lake-walk.md" - ], - "csv": [ - "ingestion/product-catalog-sftp-load.md" - ], - "curated": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "layers/bronze.md", - "layers/gold.md", - "layers/silver.md", - "marts/revenue-mart.md" - ], - "currency": [ - "business-objects/daily-order-summary.md", - "columns/bronze-orders-raw-currency.md", - "columns/gold-order-daily-currency.md", - "columns/silver-orders-currency.md", - "glossary/daily-order-summary.md", - "metrics/gross-merchandise-value.md", - "packs/pack-gold-order-daily.md", - "queries/order-daily.md", - "queries/orders.md", - "sql/order-daily.md", - "sql/orders.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "current_date": [ - "quality/gold-order-daily-freshness.md" - ], - "customer": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "columns/bronze-orders-raw-customer-id.md", - "diagrams/erd-orders-erd.md", - "glossary/customer-lifetime-value.md", - "glossary/customer-order.md", - "packs/pack-gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "customer_id": [ - "columns/bronze-orders-raw-customer-id.md", - "columns/silver-orders-customer-id.md", - "diagrams/erd-orders-erd.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/orders.md", - "sql/customer-ltv.md", - "sql/orders.md", - "tables/silver-orders.md", - "views/gold-customer-ltv.md" - ], - "customer_ltv": [ - "glossary/customer-lifetime-value.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "sql/customer-ltv.md", - "views/gold-customer-ltv.md" - ], - "daily": [ - "business-objects/daily-order-summary.md", - "dashboards/executive-revenue.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/state-daily-medallion-orders-state.md", - "glossary/daily-order-summary.md", - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "packs/pack-gold-order-daily.md", - "products/daily-revenue-product.md", - "quality/gold-order-daily-freshness.md", - "schemas/gold.md", - "transformations/aggregate-order-daily.md", - "workflows/daily-medallion-orders.md" - ], - "daily_medallion": [ - "queries/daily-medallion.md", - "sql/daily-medallion.md" - ], - "dashboard": [ - "dashboards/executive-revenue.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "dashboards": [ - "diagrams/architecture-commerce-lake-architecture.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "data": [ - "business-objects/customer-order.md", - "catalogs/retail-unity-style-catalog.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "domains/commerce.md", - "glossary/customer-order.md", - "lakes/retail-commerce-lake.md", - "layers/bronze.md", - "layers/gold.md", - "layers/silver.md", - "marts/revenue-mart.md", - "products/daily-revenue-product.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md" - ], - "datacatalog": [ - "catalogs/retail-unity-style-catalog.md" - ], - "datadomain": [ - "domains/commerce.md" - ], - "datahub": [ - "catalogs/retail-unity-style-catalog.md" - ], - "datalake": [ - "lakes/retail-commerce-lake.md" - ], - "datamart": [ - "marts/revenue-mart.md" - ], - "dataproduct": [ - "products/daily-revenue-product.md" - ], - "date": [ - "columns/gold-order-daily-order-date.md", - "diagrams/erd-orders-erd.md", - "packs/pack-gold-order-daily.md", - "queries/order-daily.md", - "sql/order-daily.md", - "tables/gold-order-daily.md" - ], - "dax": [ - "dax/total-revenue-dax.md", - "dax/total-revenue.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md" - ], - "daxartifact": [ - "dax/total-revenue-dax.md", - "dax/total-revenue.md" - ], - "day": [ - "quality/gold-order-daily-freshness.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "days": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "dead": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "deadletter": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "decimal": [ - "columns/bronze-orders-raw-gross-amount.md", - "columns/gold-order-daily-gross-revenue.md", - "columns/silver-orders-gross-amount.md" - ], - "definition": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "glossary/customer-lifetime-value.md", - "glossary/customer-order.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md", - "metrics/gross-merchandise-value.md", - "metrics/order-count.md" - ], - "dekc": [ - "agents/walk-retail-lake-walk.md", - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "catalogs/retail-unity-style-catalog.md", - "columns/bronze-orders-raw-currency.md", - "columns/bronze-orders-raw-customer-id.md", - "columns/bronze-orders-raw-gross-amount.md", - "columns/bronze-orders-raw-order-id.md", - "columns/bronze-orders-raw-order-ts.md", - "columns/bronze-orders-raw-status.md", - "columns/gold-order-daily-currency.md", - "columns/gold-order-daily-gross-revenue.md", - "columns/gold-order-daily-order-count.md", - "columns/gold-order-daily-order-date.md", - "columns/silver-orders-currency.md", - "columns/silver-orders-customer-id.md", - "columns/silver-orders-gross-amount.md", - "columns/silver-orders-order-id.md", - "columns/silver-orders-order-ts.md", - "columns/silver-orders-status.md", - "dashboards/executive-revenue.md", - "dax/total-revenue-dax.md", - "dax/total-revenue.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "domains/commerce.md", - "glossary/customer-lifetime-value.md", - "glossary/customer-order.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lakes/retail-commerce-lake.md", - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "metrics/gross-merchandise-value.md", - "metrics/order-count.md", - "packs/pack-gold-order-daily.md", - "products/daily-revenue-product.md", - "quality/gold-order-daily-freshness.md", - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md", - "reports/weekly-commerce-digest.md", - "schemas/bronze.md", - "schemas/gold.md", - "schemas/silver.md", - "semantic/retail-commerce-model.md", - "sources/product-catalog-sftp.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md", - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "storage/bronze-orders-prefix.md", - "streams/orders-events.md", - "tables/bronze-orders-api.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "transformations/aggregate-order-daily.md", - "transformations/cleanse-orders.md", - "views/gold-customer-ltv.md", - "wireframes/wireframe-executive-revenue-wireframe.md", - "workflows/daily-medallion-orders.md" - ], - "delta": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "sources/retail-lake.md", - "storage/bronze-orders-prefix.md" - ], - "diagram": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "packs/pack-gold-order-daily.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "digest": [ - "reports/weekly-commerce-digest.md" - ], - "disclosure": [ - "packs/pack-gold-order-daily.md" - ], - "discounts": [ - "glossary/gmv.md", - "metrics/gross-merchandise-value.md" - ], - "discovered": [ - "dax/total-revenue.md", - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "queries/total-revenue.md", - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md" - ], - "distinct": [ - "metrics/order-count.md" - ], - "documented": [ - "queries/daily-medallion.md", - "sql/daily-medallion.md" - ], - "dollar": [ - "glossary/gmv.md" - ], - "domain": [ - "domains/commerce.md" - ], - "domains": [ - "products/daily-revenue-product.md" - ], - "downstream": [ - "products/daily-revenue-product.md" - ], - "dq": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "quality/gold-order-daily-freshness.md" - ], - "dqrule": [ - "diagrams/class-daily-medallion-orders-class.md", - "quality/gold-order-daily-freshness.md" - ], - "drop": [ - "transformations/cleanse-orders.md" - ], - "end": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "endpoints": [ - "streams/orders-events.md" - ], - "enduml": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "engine": [ - "catalogs/retail-unity-style-catalog.md" - ], - "engineering": [ - "business-objects/daily-order-summary.md", - "domains/commerce.md" - ], - "erd": [ - "diagrams/erd-orders-erd.md" - ], - "erdiagram": [ - "diagrams/erd-orders-erd.md" - ], - "error": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "quality/gold-order-daily-freshness.md" - ], - "errors": [ - "agents/walk-retail-lake-walk.md" - ], - "event": [ - "diagrams/sequence-orders-stream-landing-sequence.md", - "ingestion/orders-stream-landing.md" - ], - "eventhubs": [ - "ingestion/orders-stream-landing.md", - "streams/orders-events.md" - ], - "events": [ - "ingestion/orders-stream-landing.md", - "streams/orders-events.md" - ], - "example": [ - "sources/product-catalog-sftp.md" - ], - "exec": [ - "marts/revenue-mart.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "executive": [ - "dashboards/executive-revenue.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "export": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "expression": [ - "metrics/gross-merchandise-value.md", - "metrics/order-count.md", - "quality/gold-order-daily-freshness.md" - ], - "extract": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "ingestion/product-catalog-sftp-load.md" - ], - "fabric": [ - "catalogs/retail-unity-style-catalog.md", - "ingestion/orders-stream-landing.md", - "lakes/retail-commerce-lake.md" - ], - "facing": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md", - "marts/revenue-mart.md" - ], - "facts": [ - "business-objects/customer-order.md", - "glossary/customer-order.md" - ], - "fail": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "failed": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "fenced": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "file": [ - "sources/product-catalog-sftp.md" - ], - "file_drop": [ - "ingestion/product-catalog-sftp-load.md" - ], - "files": [ - "ingestion/product-catalog-sftp-load.md", - "sources/product-catalog-sftp.md" - ], - "finance": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md" - ], - "fixtures": [ - "agents/walk-retail-lake-walk.md", - "sources/retail-lake-walk.md" - ], - "fk": [ - "diagrams/erd-orders-erd.md" - ], - "flat": [ - "sources/product-catalog-sftp.md" - ], - "flow": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "flowchart": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lakes/retail-commerce-lake.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "streams/orders-events.md" - ], - "forecasting": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md" - ], - "format": [ - "storage/bronze-orders-prefix.md", - "streams/orders-events.md" - ], - "formats": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "fqn": [ - "diagrams/class-daily-medallion-orders-class.md" - ], - "fresh": [ - "quality/gold-order-daily-freshness.md" - ], - "freshness": [ - "quality/gold-order-daily-freshness.md" - ], - "full": [ - "ingestion/product-catalog-sftp-load.md" - ], - "gld": [ - "diagrams/architecture-commerce-lake-architecture.md" - ], - "glossary": [ - "glossary/customer-lifetime-value.md", - "glossary/customer-order.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md" - ], - "glossaryterm": [ - "glossary/customer-lifetime-value.md", - "glossary/customer-order.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md" - ], - "glue": [ - "catalogs/retail-unity-style-catalog.md" - ], - "gmv": [ - "glossary/gmv.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "gold": [ - "business-objects/customer-lifetime-value.md", - "business-objects/daily-order-summary.md", - "columns/gold-order-daily-currency.md", - "columns/gold-order-daily-gross-revenue.md", - "columns/gold-order-daily-order-count.md", - "columns/gold-order-daily-order-date.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/erd-orders-erd.md", - "lakes/retail-commerce-lake.md", - "layers/gold.md", - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "packs/pack-gold-order-daily.md", - "quality/gold-order-daily-freshness.md", - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "schemas/gold.md", - "semantic/retail-commerce-model.md", - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "tables/gold-order-daily.md", - "transformations/aggregate-order-daily.md", - "views/gold-customer-ltv.md", - "workflows/daily-medallion-orders.md" - ], - "grain": [ - "transformations/aggregate-order-daily.md" - ], - "gross": [ - "business-objects/customer-lifetime-value.md", - "business-objects/daily-order-summary.md", - "columns/bronze-orders-raw-gross-amount.md", - "glossary/customer-lifetime-value.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md", - "metrics/gross-merchandise-value.md" - ], - "gross_amount": [ - "columns/bronze-orders-raw-gross-amount.md", - "columns/silver-orders-gross-amount.md", - "diagrams/erd-orders-erd.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "queries/orders.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "sql/orders.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "views/gold-customer-ltv.md" - ], - "gross_revenue": [ - "columns/gold-order-daily-gross-revenue.md", - "dax/total-revenue-dax.md", - "dax/total-revenue.md", - "metrics/gross-merchandise-value.md", - "packs/pack-gold-order-daily.md", - "queries/order-daily.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md", - "sql/order-daily.md", - "tables/gold-order-daily.md" - ], - "group": [ - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "tables/gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "growth": [ - "business-objects/customer-lifetime-value.md", - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md" - ], - "historical": [ - "business-objects/customer-lifetime-value.md", - "glossary/customer-lifetime-value.md" - ], - "hops": [ - "packs/pack-gold-order-daily.md" - ], - "hub": [ - "ingestion/orders-stream-landing.md" - ], - "hubs": [ - "ingestion/orders-stream-landing.md" - ], - "idempotent": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "identifier": [ - "columns/bronze-orders-raw-order-id.md" - ], - "idle": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "in": [ - "metrics/gross-merchandise-value.md", - "schemas/bronze.md", - "schemas/gold.md", - "schemas/silver.md" - ], - "include": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "inferred": [ - "packs/pack-gold-order-daily.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md" - ], - "ingestion": [ - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-orders-stream-landing-state.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "ingestionjob": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "is": [ - "queries/orders.md", - "sql/orders.md", - "tables/silver-orders.md", - "transformations/cleanse-orders.md" - ], - "iso": [ - "columns/bronze-orders-raw-currency.md" - ], - "job": [ - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-orders-stream-landing-state.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "jobs": [ - "diagrams/architecture-commerce-lake-architecture.md" - ], - "json": [ - "agents/walk-retail-lake-walk.md", - "ingestion/orders-stream-landing.md", - "streams/orders-events.md" - ], - "keep": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "key": [ - "columns/bronze-orders-raw-customer-id.md" - ], - "kind": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "sources/product-catalog-sftp.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md", - "storage/bronze-orders-prefix.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "lake": [ - "agents/walk-retail-lake-walk.md", - "diagrams/architecture-commerce-lake-architecture.md", - "lakes/retail-commerce-lake.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md", - "storage/bronze-orders-prefix.md" - ], - "lakehouse": [ - "lakes/retail-commerce-lake.md" - ], - "lakes": [ - "diagrams/architecture-commerce-lake-architecture.md" - ], - "land": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "queries/daily-medallion.md", - "sql/daily-medallion.md", - "workflows/daily-medallion-orders.md" - ], - "landing": [ - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-orders-stream-landing-state.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "queries/orders-raw.md", - "sql/orders-raw.md", - "streams/orders-events.md", - "tables/bronze-orders-raw.md" - ], - "lands": [ - "ingestion/orders-stream-landing.md", - "streams/orders-events.md" - ], - "language": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "last": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "layer": [ - "diagrams/class-daily-medallion-orders-class.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "layers/bronze.md", - "layers/gold.md", - "layers/silver.md", - "packs/pack-gold-order-daily.md", - "semantic/retail-commerce-model.md", - "tables/bronze-orders-api.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md" - ], - "layers": [ - "lakes/retail-commerce-lake.md" - ], - "letter": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "level": [ - "dashboards/executive-revenue.md" - ], - "lifetime": [ - "business-objects/customer-lifetime-value.md", - "glossary/customer-lifetime-value.md", - "packs/pack-gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "lifetime_value": [ - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "sql/customer-ltv.md", - "views/gold-customer-ltv.md" - ], - "lineage": [ - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md" - ], - "lineagepath": [ - "lineage/orders-to-revenue.md" - ], - "list": [ - "ingestion/product-catalog-sftp-load.md" - ], - "listing": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "load": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "ingestion/product-catalog-sftp-load.md" - ], - "logical": [ - "diagrams/erd-orders-erd.md" - ], - "lr": [ - "diagrams/architecture-commerce-lake-architecture.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lakes/retail-commerce-lake.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "streams/orders-events.md" - ], - "ltv": [ - "business-objects/customer-lifetime-value.md", - "packs/pack-gold-order-daily.md", - "reports/weekly-commerce-digest.md" - ], - "mart": [ - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "quality/gold-order-daily-freshness.md" - ], - "marts": [ - "diagrams/architecture-commerce-lake-architecture.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "semantic/retail-commerce-model.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "tables/gold-order-daily.md" - ], - "max": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "quality/gold-order-daily-freshness.md" - ], - "md": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lakes/retail-commerce-lake.md", - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "packs/pack-gold-order-daily.md", - "quality/gold-order-daily-freshness.md", - "schemas/bronze.md", - "schemas/gold.md", - "schemas/silver.md", - "streams/orders-events.md", - "views/gold-customer-ltv.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "measure": [ - "dax/total-revenue-dax.md", - "queries/total-revenue-dax.md" - ], - "medallion": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/state-daily-medallion-orders-state.md", - "layers/bronze.md", - "layers/gold.md", - "layers/silver.md", - "workflows/daily-medallion-orders.md" - ], - "merchandise": [ - "business-objects/daily-order-summary.md", - "columns/bronze-orders-raw-gross-amount.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md", - "metrics/gross-merchandise-value.md" - ], - "merchandising": [ - "domains/commerce.md" - ], - "mermaid": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lakes/retail-commerce-lake.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "streams/orders-events.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "metric": [ - "metrics/gross-merchandise-value.md", - "metrics/order-count.md" - ], - "metrics": [ - "semantic/retail-commerce-model.md" - ], - "minutes": [ - "ingestion/orders-stream-landing.md" - ], - "mode": [ - "diagrams/class-daily-medallion-orders-class.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "model": [ - "diagrams/architecture-commerce-lake-architecture.md", - "semantic/retail-commerce-model.md", - "workflows/daily-medallion-orders.md" - ], - "models": [ - "sources/retail-lake.md" - ], - "must": [ - "quality/gold-order-daily-freshness.md" - ], - "n0": [ - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md" - ], - "n1": [ - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md" - ], - "n2": [ - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md" - ], - "name": [ - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/erd-orders-erd.md" - ], - "nightly": [ - "ingestion/product-catalog-sftp-load.md", - "sources/product-catalog-sftp.md", - "workflows/daily-medallion-orders.md" - ], - "no": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "nodes": [ - "packs/pack-gold-order-daily.md" - ], - "notes": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "sources/product-catalog-sftp.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "notify": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "nullable": [ - "columns/bronze-orders-raw-currency.md", - "columns/bronze-orders-raw-customer-id.md", - "columns/bronze-orders-raw-gross-amount.md", - "columns/bronze-orders-raw-order-id.md", - "columns/bronze-orders-raw-order-ts.md", - "columns/bronze-orders-raw-status.md", - "columns/gold-order-daily-currency.md", - "columns/gold-order-daily-gross-revenue.md", - "columns/gold-order-daily-order-count.md", - "columns/gold-order-daily-order-date.md", - "columns/silver-orders-currency.md", - "columns/silver-orders-customer-id.md", - "columns/silver-orders-gross-amount.md", - "columns/silver-orders-order-id.md", - "columns/silver-orders-order-ts.md", - "columns/silver-orders-status.md" - ], - "number": [ - "diagrams/erd-orders-erd.md" - ], - "object": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md" - ], - "of": [ - "agents/walk-retail-lake-walk.md", - "business-objects/customer-lifetime-value.md", - "business-objects/daily-order-summary.md", - "catalogs/retail-unity-style-catalog.md", - "glossary/customer-lifetime-value.md", - "glossary/daily-order-summary.md", - "ingestion/product-catalog-sftp-load.md", - "metrics/gross-merchandise-value.md", - "metrics/order-count.md", - "reports/weekly-commerce-digest.md" - ], - "ok": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "on": [ - "business-objects/customer-lifetime-value.md", - "columns/gold-order-daily-currency.md", - "columns/gold-order-daily-gross-revenue.md", - "columns/gold-order-daily-order-count.md", - "columns/gold-order-daily-order-date.md", - "columns/silver-orders-currency.md", - "columns/silver-orders-customer-id.md", - "columns/silver-orders-gross-amount.md", - "columns/silver-orders-order-id.md", - "columns/silver-orders-order-ts.md", - "columns/silver-orders-status.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "glossary/customer-lifetime-value.md" - ], - "onelake": [ - "lakes/retail-commerce-lake.md", - "storage/bronze-orders-prefix.md" - ], - "optional": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "or": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "orchestrated": [ - "queries/daily-medallion.md", - "sql/daily-medallion.md" - ], - "orchestrator": [ - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "workflows/daily-medallion-orders.md" - ], - "order": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "columns/bronze-orders-raw-order-id.md", - "columns/bronze-orders-raw-order-ts.md", - "columns/bronze-orders-raw-status.md", - "dashboards/executive-revenue.md", - "diagrams/erd-orders-erd.md", - "domains/commerce.md", - "glossary/customer-lifetime-value.md", - "glossary/customer-order.md", - "glossary/daily-order-summary.md", - "ingestion/orders-stream-landing.md", - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "metrics/gross-merchandise-value.md", - "metrics/order-count.md", - "packs/pack-gold-order-daily.md", - "quality/gold-order-daily-freshness.md", - "reports/weekly-commerce-digest.md", - "schemas/gold.md", - "semantic/retail-commerce-model.md", - "streams/orders-events.md", - "transformations/aggregate-order-daily.md", - "transformations/cleanse-orders.md" - ], - "order_count": [ - "columns/gold-order-daily-order-count.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "tables/gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "order_daily": [ - "columns/gold-order-daily-currency.md", - "columns/gold-order-daily-gross-revenue.md", - "columns/gold-order-daily-order-count.md", - "columns/gold-order-daily-order-date.md", - "dax/total-revenue-dax.md", - "dax/total-revenue.md", - "glossary/daily-order-summary.md", - "packs/pack-gold-order-daily.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md", - "schemas/gold.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "tables/gold-order-daily.md", - "workflows/daily-medallion-orders.md" - ], - "order_date": [ - "columns/gold-order-daily-order-date.md", - "packs/pack-gold-order-daily.md", - "quality/gold-order-daily-freshness.md", - "queries/order-daily.md", - "sql/order-daily.md", - "tables/gold-order-daily.md" - ], - "order_id": [ - "columns/bronze-orders-raw-order-id.md", - "columns/silver-orders-order-id.md", - "diagrams/erd-orders-erd.md", - "queries/orders.md", - "sql/orders.md", - "tables/silver-orders.md" - ], - "order_line": [ - "diagrams/erd-orders-erd.md" - ], - "order_ts": [ - "columns/bronze-orders-raw-order-ts.md", - "columns/silver-orders-order-ts.md", - "diagrams/erd-orders-erd.md", - "packs/pack-gold-order-daily.md", - "queries/order-daily.md", - "queries/orders.md", - "sql/order-daily.md", - "sql/orders.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md" - ], - "orders": [ - "business-objects/customer-order.md", - "columns/silver-orders-currency.md", - "columns/silver-orders-customer-id.md", - "columns/silver-orders-gross-amount.md", - "columns/silver-orders-order-id.md", - "columns/silver-orders-order-ts.md", - "columns/silver-orders-status.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "glossary/customer-order.md", - "ingestion/orders-stream-landing.md", - "lineage/orders-to-revenue.md", - "metrics/order-count.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/orders.md", - "schemas/bronze.md", - "schemas/silver.md", - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "sql/orders.md", - "storage/bronze-orders-prefix.md", - "streams/orders-events.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "transformations/cleanse-orders.md", - "views/gold-customer-ltv.md", - "wireframes/wireframe-executive-revenue-wireframe.md", - "workflows/daily-medallion-orders.md" - ], - "orders_api": [ - "packs/pack-gold-order-daily.md", - "queries/orders-raw.md", - "sql/orders-raw.md", - "tables/bronze-orders-api.md", - "tables/bronze-orders-raw.md" - ], - "orders_raw": [ - "columns/bronze-orders-raw-currency.md", - "columns/bronze-orders-raw-customer-id.md", - "columns/bronze-orders-raw-gross-amount.md", - "columns/bronze-orders-raw-order-id.md", - "columns/bronze-orders-raw-order-ts.md", - "columns/bronze-orders-raw-status.md", - "packs/pack-gold-order-daily.md", - "queries/daily-medallion.md", - "queries/orders-raw.md", - "queries/orders.md", - "schemas/bronze.md", - "sql/daily-medallion.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-api.md", - "tables/bronze-orders-raw.md", - "tables/silver-orders.md", - "transformations/cleanse-orders.md", - "workflows/daily-medallion-orders.md" - ], - "over": [ - "semantic/retail-commerce-model.md" - ], - "owner": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "domains/commerce.md" - ], - "pacing": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md" - ], - "pack": [ - "packs/pack-gold-order-daily.md" - ], - "participant": [ - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "pass": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "path": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "storage/bronze-orders-prefix.md" - ], - "pattern": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "pdf": [ - "reports/weekly-commerce-digest.md" - ], - "per": [ - "packs/pack-gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "period": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "pipeline": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "ingestion/orders-stream-landing.md", - "workflows/daily-medallion-orders.md" - ], - "pipelines": [ - "diagrams/architecture-commerce-lake-architecture.md" - ], - "pk": [ - "diagrams/erd-orders-erd.md" - ], - "placement": [ - "streams/orders-events.md" - ], - "places": [ - "diagrams/erd-orders-erd.md" - ], - "plantuml": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "platform": [ - "lakes/retail-commerce-lake.md", - "streams/orders-events.md" - ], - "power": [ - "dax/total-revenue-dax.md", - "queries/total-revenue-dax.md", - "workflows/daily-medallion-orders.md" - ], - "powerbi": [ - "dashboards/executive-revenue.md" - ], - "predicted": [ - "business-objects/customer-lifetime-value.md", - "glossary/customer-lifetime-value.md" - ], - "prefix": [ - "ingestion/orders-stream-landing.md", - "storage/bronze-orders-prefix.md" - ], - "primary": [ - "lakes/retail-commerce-lake.md", - "sources/retail-lake.md" - ], - "producers": [ - "streams/orders-events.md" - ], - "product": [ - "diagrams/erd-orders-erd.md", - "ingestion/product-catalog-sftp-load.md", - "products/daily-revenue-product.md", - "sources/product-catalog-sftp.md" - ], - "product_id": [ - "diagrams/erd-orders-erd.md" - ], - "products": [ - "ingestion/product-catalog-sftp-load.md", - "layers/bronze.md", - "layers/gold.md", - "layers/silver.md", - "sources/product-catalog-sftp.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "progressive": [ - "packs/pack-gold-order-daily.md" - ], - "promote": [ - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "publish": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "publishable": [ - "products/daily-revenue-product.md" - ], - "purchase": [ - "business-objects/customer-order.md", - "glossary/customer-order.md" - ], - "purview": [ - "catalogs/retail-unity-style-catalog.md" - ], - "qty": [ - "diagrams/erd-orders-erd.md" - ], - "quality": [ - "business-objects/customer-order.md", - "glossary/customer-order.md", - "quality/gold-order-daily-freshness.md" - ], - "quarantine": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "query": [ - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md" - ], - "raw": [ - "ingestion/orders-stream-landing.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "schemas/bronze.md", - "streams/orders-events.md" - ], - "read": [ - "ingestion/orders-stream-landing.md" - ], - "reads": [ - "views/gold-customer-ltv.md" - ], - "reads_writes": [ - "diagrams/class-daily-medallion-orders-class.md" - ], - "realized": [ - "business-objects/customer-lifetime-value.md", - "glossary/customer-lifetime-value.md" - ], - "record": [ - "catalogs/retail-unity-style-catalog.md" - ], - "referenced": [ - "packs/pack-gold-order-daily.md", - "tables/bronze-orders-api.md" - ], - "refresh": [ - "wireframes/wireframe-executive-revenue-wireframe.md", - "workflows/daily-medallion-orders.md" - ], - "register": [ - "ingestion/product-catalog-sftp-load.md" - ], - "replace": [ - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md" - ], - "report": [ - "reports/weekly-commerce-digest.md" - ], - "reporting": [ - "marts/revenue-mart.md", - "transformations/aggregate-order-daily.md" - ], - "reports": [ - "diagrams/architecture-commerce-lake-architecture.md", - "products/daily-revenue-product.md" - ], - "retail": [ - "agents/walk-retail-lake-walk.md", - "catalogs/retail-unity-style-catalog.md", - "diagrams/architecture-commerce-lake-architecture.md", - "lakes/retail-commerce-lake.md", - "semantic/retail-commerce-model.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md", - "storage/bronze-orders-prefix.md" - ], - "retries": [ - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "retry": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "returns": [ - "glossary/gmv.md", - "metrics/gross-merchandise-value.md" - ], - "revenue": [ - "dashboards/executive-revenue.md", - "dax/total-revenue-dax.md", - "dax/total-revenue.md", - "glossary/gmv.md", - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "products/daily-revenue-product.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md", - "transformations/aggregate-order-daily.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "root": [ - "agents/walk-retail-lake-walk.md" - ], - "rpt": [ - "diagrams/architecture-commerce-lake-architecture.md" - ], - "rule_type": [ - "diagrams/class-daily-medallion-orders-class.md" - ], - "rules": [ - "business-objects/customer-order.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "glossary/customer-order.md" - ], - "run": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "running": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "s1": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "s2": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "s3": [ - "sources/retail-lake.md" - ], - "sales": [ - "glossary/gmv.md", - "packs/pack-gold-order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-raw.md", - "tables/silver-orders.md" - ], - "salt": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "sample": [ - "agents/walk-retail-lake-walk.md", - "sources/retail-lake-walk.md" - ], - "schedule": [ - "diagrams/class-daily-medallion-orders-class.md", - "ingestion/product-catalog-sftp-load.md" - ], - "schema": [ - "schemas/bronze.md", - "schemas/gold.md", - "schemas/silver.md" - ], - "schemas": [ - "catalogs/retail-unity-style-catalog.md" - ], - "segment": [ - "diagrams/erd-orders-erd.md" - ], - "select": [ - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "transformations/cleanse-orders.md", - "views/gold-customer-ltv.md" - ], - "sem": [ - "diagrams/architecture-commerce-lake-architecture.md" - ], - "semantic": [ - "diagrams/architecture-commerce-lake-architecture.md", - "semantic/retail-commerce-model.md", - "workflows/daily-medallion-orders.md" - ], - "semanticmodel": [ - "semantic/retail-commerce-model.md" - ], - "sequence": [ - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "sequencediagram": [ - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "severity": [ - "diagrams/class-daily-medallion-orders-class.md", - "quality/gold-order-daily-freshness.md" - ], - "sftp": [ - "ingestion/product-catalog-sftp-load.md", - "sources/product-catalog-sftp.md" - ], - "silver": [ - "business-objects/customer-order.md", - "columns/silver-orders-currency.md", - "columns/silver-orders-customer-id.md", - "columns/silver-orders-gross-amount.md", - "columns/silver-orders-order-id.md", - "columns/silver-orders-order-ts.md", - "columns/silver-orders-status.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "lakes/retail-commerce-lake.md", - "layers/silver.md", - "lineage/orders-to-revenue.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/orders.md", - "schemas/silver.md", - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "sql/orders.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "transformations/aggregate-order-daily.md", - "transformations/cleanse-orders.md", - "views/gold-customer-ltv.md", - "workflows/daily-medallion-orders.md" - ], - "single": [ - "business-objects/customer-order.md", - "glossary/customer-order.md" - ], - "skipped": [ - "agents/walk-retail-lake-walk.md" - ], - "sla": [ - "ingestion/orders-stream-landing.md" - ], - "slv": [ - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "sold": [ - "diagrams/erd-orders-erd.md" - ], - "source": [ - "business-objects/customer-order.md", - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "glossary/customer-order.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "sources/product-catalog-sftp.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md" - ], - "sources": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "diagrams/architecture-commerce-lake-architecture.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lakes/retail-commerce-lake.md" - ], - "sourcesystem": [ - "sources/product-catalog-sftp.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md" - ], - "sql": [ - "metrics/gross-merchandise-value.md", - "metrics/order-count.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/daily-medallion.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sources/retail-lake.md", - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "transformations/cleanse-orders.md", - "views/gold-customer-ltv.md" - ], - "sqlartifact": [ - "sql/customer-ltv.md", - "sql/daily-medallion.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md" - ], - "src": [ - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md" - ], - "start": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "startuml": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "state": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "statediagram": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "stats": [ - "agents/walk-retail-lake-walk.md" - ], - "steps": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "queries/daily-medallion.md", - "sql/daily-medallion.md", - "workflows/daily-medallion-orders.md" - ], - "storage": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "ingestion/orders-stream-landing.md", - "storage/bronze-orders-prefix.md" - ], - "storagelocation": [ - "storage/bronze-orders-prefix.md" - ], - "stream": [ - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-orders-stream-landing-state.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "streams/orders-events.md" - ], - "streaming": [ - "ingestion/orders-stream-landing.md" - ], - "streams": [ - "diagrams/architecture-commerce-lake-architecture.md", - "ingestion/orders-stream-landing.md" - ], - "string": [ - "columns/bronze-orders-raw-currency.md", - "columns/bronze-orders-raw-customer-id.md", - "columns/bronze-orders-raw-order-id.md", - "columns/bronze-orders-raw-status.md", - "columns/gold-order-daily-currency.md", - "columns/silver-orders-currency.md", - "columns/silver-orders-customer-id.md", - "columns/silver-orders-order-id.md", - "columns/silver-orders-status.md", - "diagrams/erd-orders-erd.md" - ], - "style": [ - "catalogs/retail-unity-style-catalog.md" - ], - "subgraph": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "subjects": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "succeeded": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "success": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "sum": [ - "metrics/gross-merchandise-value.md", - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "queries/order-daily.md", - "sql/customer-ltv.md", - "sql/order-daily.md", - "tables/gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "summary": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md" - ], - "sumx": [ - "dax/total-revenue-dax.md", - "dax/total-revenue.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md" - ], - "synonyms": [ - "glossary/customer-lifetime-value.md", - "glossary/customer-order.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md" - ], - "system": [ - "catalogs/retail-unity-style-catalog.md" - ], - "t1": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "t2": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "t3": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "table": [ - "diagrams/class-daily-medallion-orders-class.md", - "ingestion/product-catalog-sftp-load.md", - "packs/pack-gold-order-daily.md", - "queries/order-daily.md", - "queries/orders-raw.md", - "queries/orders.md", - "sql/order-daily.md", - "sql/orders-raw.md", - "sql/orders.md", - "tables/bronze-orders-api.md", - "tables/bronze-orders-raw.md", - "tables/gold-order-daily.md", - "tables/silver-orders.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "tables": [ - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "catalogs/retail-unity-style-catalog.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "ingestion/orders-stream-landing.md", - "lineage/orders-to-revenue.md", - "marts/revenue-mart.md", - "packs/pack-gold-order-daily.md", - "quality/gold-order-daily-freshness.md", - "schemas/bronze.md", - "schemas/gold.md", - "schemas/silver.md", - "semantic/retail-commerce-model.md", - "streams/orders-events.md", - "views/gold-customer-ltv.md" - ], - "tag": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/architecture-commerce-lake-architecture.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/erd-orders-erd.md", - "diagrams/sequence-orders-stream-landing-sequence.md", - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md", - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "target": [ - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "quality/gold-order-daily-freshness.md" - ], - "task": [ - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "tb": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "td": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "technical": [ - "business-objects/customer-lifetime-value.md", - "business-objects/customer-order.md", - "business-objects/daily-order-summary.md", - "semantic/retail-commerce-model.md" - ], - "tests": [ - "agents/walk-retail-lake-walk.md", - "sources/retail-lake-walk.md" - ], - "timestamp": [ - "columns/bronze-orders-raw-order-ts.md", - "columns/silver-orders-order-ts.md" - ], - "to": [ - "domains/commerce.md", - "ingestion/orders-stream-landing.md", - "ingestion/product-catalog-sftp-load.md", - "lineage/orders-to-revenue.md", - "semantic/retail-commerce-model.md", - "workflows/daily-medallion-orders.md" - ], - "tool": [ - "dashboards/executive-revenue.md" - ], - "tools": [ - "semantic/retail-commerce-model.md" - ], - "top": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "total": [ - "dax/total-revenue-dax.md", - "dax/total-revenue.md", - "glossary/gmv.md", - "queries/total-revenue-dax.md", - "queries/total-revenue.md" - ], - "total_revenue": [ - "dax/total-revenue.md", - "queries/total-revenue.md" - ], - "transaction": [ - "business-objects/customer-order.md", - "glossary/customer-order.md", - "metrics/gross-merchandise-value.md" - ], - "transform": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "transformation": [ - "transformations/aggregate-order-daily.md", - "transformations/cleanse-orders.md" - ], - "trends": [ - "dashboards/executive-revenue.md", - "reports/weekly-commerce-digest.md" - ], - "trigger": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "typically": [ - "marts/revenue-mart.md" - ], - "unity": [ - "catalogs/retail-unity-style-catalog.md" - ], - "update": [ - "ingestion/orders-stream-landing.md" - ], - "updated": [ - "agents/walk-retail-lake-walk.md" - ], - "uri": [ - "catalogs/retail-unity-style-catalog.md", - "lakes/retail-commerce-lake.md", - "sources/product-catalog-sftp.md", - "sources/retail-lake-walk.md", - "sources/retail-lake.md", - "storage/bronze-orders-prefix.md", - "streams/orders-events.md" - ], - "used": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md" - ], - "utc": [ - "columns/bronze-orders-raw-order-ts.md" - ], - "v2": [ - "diagrams/state-daily-medallion-orders-state.md", - "diagrams/state-orders-stream-landing-state.md" - ], - "validate": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md", - "diagrams/sequence-orders-stream-landing-sequence.md" - ], - "validates": [ - "diagrams/class-daily-medallion-orders-class.md" - ], - "value": [ - "business-objects/customer-lifetime-value.md", - "business-objects/daily-order-summary.md", - "columns/bronze-orders-raw-gross-amount.md", - "glossary/customer-lifetime-value.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md", - "metrics/gross-merchandise-value.md", - "packs/pack-gold-order-daily.md", - "views/gold-customer-ltv.md" - ], - "via": [ - "sources/product-catalog-sftp.md" - ], - "view": [ - "packs/pack-gold-order-daily.md", - "queries/customer-ltv.md", - "sql/customer-ltv.md", - "views/gold-customer-ltv.md" - ], - "views": [ - "business-objects/customer-lifetime-value.md", - "packs/pack-gold-order-daily.md", - "schemas/bronze.md", - "schemas/gold.md", - "schemas/silver.md" - ], - "volume": [ - "business-objects/daily-order-summary.md", - "glossary/daily-order-summary.md", - "glossary/gmv.md" - ], - "walk": [ - "agents/walk-retail-lake-walk.md", - "sources/retail-lake-walk.md" - ], - "walked": [ - "sources/retail-lake-walk.md" - ], - "watermark": [ - "ingestion/orders-stream-landing.md" - ], - "weekly": [ - "reports/weekly-commerce-digest.md" - ], - "wf": [ - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md" - ], - "where": [ - "queries/orders.md", - "sql/orders.md", - "tables/silver-orders.md", - "transformations/cleanse-orders.md" - ], - "wireframe": [ - "wireframes/wireframe-executive-revenue-wireframe.md" - ], - "within": [ - "quality/gold-order-daily-freshness.md" - ], - "workflow": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/component-orders-stream-landing-component.md", - "diagrams/state-daily-medallion-orders-state.md", - "workflows/daily-medallion-orders.md" - ], - "workflows": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/class-daily-medallion-orders-class.md", - "diagrams/component-daily-medallion-orders-component.md", - "diagrams/state-daily-medallion-orders-state.md", - "queries/daily-medallion.md", - "sql/daily-medallion.md" - ], - "workspace": [ - "agents/walk-retail-lake-walk.md", - "catalogs/retail-unity-style-catalog.md", - "sources/retail-lake-walk.md" - ], - "write": [ - "diagrams/sequence-orders-stream-landing-sequence.md", - "ingestion/orders-stream-landing.md" - ], - "yes": [ - "diagrams/activity-daily-medallion-orders-activity.md", - "diagrams/activity-orders-stream-landing-activity.md" - ], - "zone": [ - "layers/bronze.md", - "layers/gold.md", - "layers/silver.md" - ] -} \ No newline at end of file diff --git a/scripts/dekc_brain.py b/scripts/dekc_brain.py index 7322002..f8fc418 100644 --- a/scripts/dekc_brain.py +++ b/scripts/dekc_brain.py @@ -25,7 +25,7 @@ utc_now, write_knowledge, ) -from dekc_index import build_index, search_index # noqa: E402 +from dekc_index import refresh, search_index # noqa: E402 from dekc_lineage import build_graph, mermaid # noqa: E402 from dekc_pack import pack as graph_pack # noqa: E402 @@ -165,8 +165,7 @@ def load_registry_intents() -> dict: def ensure_index(bundle: Path) -> None: - if not (bundle / ".index" / "search.json").is_file(): - build_index(bundle) + refresh(bundle) def tokenize(q: str) -> list[str]: diff --git a/scripts/dekc_common.py b/scripts/dekc_common.py index e9940e4..3fa564c 100755 --- a/scripts/dekc_common.py +++ b/scripts/dekc_common.py @@ -16,6 +16,8 @@ import os import re import secrets +import shutil +import subprocess import sys from datetime import datetime, timezone from pathlib import Path @@ -1166,6 +1168,159 @@ def ensure_bundle(bundle: Path, title: str = "Data Engineering Knowledge") -> No append_log(bundle, "Bundle created for Data Engineering Knowledge Capture") +# ── ripgrep accelerator (optional; never a dependency) ────────────────── + +RG_ENV_VARS = ("DEKC_RG_PATH", "PKC_RG_PATH", "OKF_RG_PATH", "SECOND_BRAIN_RG_PATH") + +RG_INSTALL_HINTS = { + "darwin": ["brew install ripgrep"], + "linux": [ + "sudo apt-get install -y ripgrep", + "sudo dnf install -y ripgrep", + "cargo install ripgrep", + ], + "win32": ["winget install BurntSushi.ripgrep.MSVC"], +} + +RG_DEFAULT_GLOBS = ["*.md", "!**/packs/**"] + + +def find_rg(*, env_vars: tuple[str, ...] = RG_ENV_VARS) -> str | None: + """Return an rg binary path, or None. Override with DEKC_RG_PATH / OKF_RG_PATH.""" + for var in env_vars: + override = (os.environ.get(var) or "").strip() + if not override: + continue + p = Path(override) + if p.is_file() and os.access(p, os.X_OK): + return str(p.resolve()) + found = shutil.which(override) + if found: + return found + return shutil.which("rg") + + +def rg_install_hints() -> list[str]: + if sys.platform == "darwin": + return list(RG_INSTALL_HINTS["darwin"]) + if sys.platform.startswith("linux"): + return list(RG_INSTALL_HINTS["linux"]) + if sys.platform == "win32": + return list(RG_INSTALL_HINTS["win32"]) + return ["cargo install ripgrep"] + + +def toolchain_report() -> dict[str, Any]: + """Doctor payload: python, rg, sqlite FTS5. Zero pip deps.""" + rg = find_rg() + fts5 = False + sqlite_version = None + try: + import sqlite3 + + sqlite_version = sqlite3.sqlite_version + con = sqlite3.connect(":memory:") + opts = [row[0] for row in con.execute("pragma compile_options")] + fts5 = any("FTS5" in opt.upper() for opt in opts) + con.close() + except Exception: + pass + return { + "python": sys.version.split()[0], + "rg": { + "found": bool(rg), + "path": rg, + "hints": rg_install_hints(), + }, + "sqlite": {"version": sqlite_version, "fts5": fts5}, + } + + +def is_concept_path(bundle: Path, path: Path) -> bool: + """Same skip rules as iter_concept_paths: not index.md, log.md, packs/, or cache dirs.""" + if path.suffix.lower() not in {".md", ".markdown"}: + return False + if path.name in {"index.md", "log.md"}: + return False + try: + parts = path.resolve().relative_to(bundle.resolve()).parts + except ValueError: + return False + if any(part in {".index", ".dekc", ".pkc", "packs"} for part in parts): + return False + return True + + +def iter_concept_paths(bundle: Path) -> list[Path]: + """Markdown concept files. Skips catalogs (index.md), logs, packs, disposable caches.""" + files: list[Path] = [] + skip_names = {"index.md", "log.md"} + skip_dirs = {".index", ".dekc", ".pkc", "packs"} + for p in sorted(bundle.rglob("*.md")): + if p.name in skip_names or p.name.startswith("."): + continue + if any(part in skip_dirs for part in p.parts): + continue + files.append(p) + return files + + +def rg_list_files( + root: Path, + patterns: list[str], + *, + fixed_string: bool = False, + ignore_case: bool = True, + globs: list[str] | None = None, + timeout: float = 30.0, +) -> list[Path] | None: + """AND-intersect `rg -l` results. + + Returns None when rg is missing or the process fails (caller must full-scan). + Returns [] when rg ran and matched nothing — that is a real empty hit set, + not a fallback. + """ + rg = find_rg() + if not rg: + return None + terms = [p for p in patterns if p] + if not terms: + return None + root = root.resolve() + use_globs = list(RG_DEFAULT_GLOBS if globs is None else globs) + matched: set[Path] | None = None + for pat in terms: + cmd = [rg, "-l", "--no-messages", "--color", "never"] + if ignore_case: + cmd.append("-i") + if fixed_string: + cmd.append("-F") + for g in use_globs: + cmd.extend(["--glob", g]) + cmd.extend(["--", pat, str(root)]) + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, check=False + ) + except (OSError, subprocess.TimeoutExpired): + return None + # 0 = hits, 1 = no matches, anything else = rg error → fall back + if proc.returncode not in (0, 1): + return None + files: set[Path] = set() + for line in proc.stdout.splitlines(): + line = line.strip() + if not line: + continue + p = Path(line) + p = p.resolve() if p.is_absolute() else (root / p).resolve() + files.add(p) + matched = files if matched is None else (matched & files) + if not matched: + return [] + return sorted(matched or []) + + def list_concepts( bundle: Path, *, @@ -1183,7 +1338,7 @@ def list_concepts( rel = path.relative_to(bundle).as_posix() if rel == "index.md": continue - if ".index" in path.parts: + if any(part in {".index", ".dekc", ".pkc"} for part in path.parts): continue if prefix_list and not any(rel.startswith(p) for p in prefix_list): continue diff --git a/scripts/dekc_doctor.py b/scripts/dekc_doctor.py index c8490ee..bcb8069 100755 --- a/scripts/dekc_doctor.py +++ b/scripts/dekc_doctor.py @@ -10,45 +10,17 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from dekc_common import DEKC_OWNED_TYPES, list_concepts, resolve_knowledge_root # noqa: E402 +from dekc_common import DEKC_OWNED_TYPES, list_concepts, resolve_knowledge_root, toolchain_report # noqa: E402 from dekc_validate import validate_bundle # noqa: E402 from dekc_lineage import build_graph # noqa: E402 +from dekc_index import status as index_status # noqa: E402 -def _toolchain() -> dict: - import os - import shutil - import sys - - rg = None - for var in ("DEKC_RG_PATH", "PKC_RG_PATH", "OKF_RG_PATH"): - override = (os.environ.get(var) or "").strip() - if override and Path(override).is_file(): - rg = override - break - found = shutil.which(override) if override else None - if found: - rg = found - break - if rg is None: - rg = shutil.which("rg") - fts5 = False - sqlite_version = None - try: - import sqlite3 - - sqlite_version = sqlite3.sqlite_version - con = sqlite3.connect(":memory:") - fts5 = any("FTS5" in row[0].upper() for row in con.execute("pragma compile_options")) - con.close() - except Exception: - pass - return { - "python": sys.version.split()[0], - "rg": {"found": bool(rg), "path": rg}, - "sqlite": {"version": sqlite_version, "fts5": fts5}, - "index": "dekc already builds knowledge/.index (inventory + inverted tokens)", - } +def _toolchain(bundle: Path) -> dict: + report = toolchain_report() + idx = index_status(bundle) + report["index"] = idx + return report def doctor( @@ -79,7 +51,9 @@ def doctor( gloss = types_c.get("GlossaryTerm", 0) tables = types_c.get("Table", 0) + types_c.get("View", 0) coverage = round(bo_count / tables, 2) if tables else 0.0 - index_ok = (bundle / ".index" / "manifest.json").is_file() + toolchain = _toolchain(bundle) + idx = toolchain.get("index") or {} + index_ok = bool(idx.get("present")) return { "bundle": str(bundle), @@ -92,10 +66,11 @@ def doctor( "business_objects": bo_count, "glossary_terms": gloss, "index_built": index_ok, + "index": idx, "validation_ok": validation["ok"], "errors": validation["errors"][:10], "warnings": validation["warnings"][:10], - "toolchain": _toolchain(), + "toolchain": toolchain, } @@ -131,10 +106,12 @@ def main(argv: list[str] | None = None) -> int: print(f" edges: {report['edge_count']}") print(f" tables→BO: {report['business_coverage']} ({report['business_objects']} objects)") print(f" glossary: {report['glossary_terms']}") - print(f" index: {'yes' if report['index_built'] else 'NO — run dekc_index.py build'}") + print(f" index: {'yes · ' + str((report.get('index') or {}).get('path')) if report['index_built'] else 'NO — next search/pack will build knowledge/.dekc/index.sqlite'}") print(f" validation: {'OK' if report['validation_ok'] else 'FAILED'}") rg = (report.get("toolchain") or {}).get("rg") or {} - print(f" rg: {'found ' + str(rg.get('path')) if rg.get('found') else 'missing (index is the search path; rg is optional)'}") + sqlite = (report.get("toolchain") or {}).get("sqlite") or {} + print(f" rg: {'found ' + str(rg.get('path')) if rg.get('found') else 'missing (optional; install ripgrep or pass --no-rg)'}") + print(f" sqlite FTS5: {'yes' if sqlite.get('fts5') else 'no'}") print(" types:") for t, n in sorted(report["types"].items(), key=lambda kv: -kv[1]): print(f" {t:20} {n}") diff --git a/scripts/dekc_index.py b/scripts/dekc_index.py index 7c4d9b6..4d873a7 100755 --- a/scripts/dekc_index.py +++ b/scripts/dekc_index.py @@ -1,201 +1,669 @@ #!/usr/bin/env python3 -"""Build a local indexed second-brain over the DEKC OKF bundle. +"""Stdlib SQLite + FTS5 incremental index for a DEKC bundle. -Produces: - knowledge/.index/inventory.json — all concepts - knowledge/.index/search.json — inverted token index - knowledge/.index/graph.json — adjacency for lineage packs - knowledge/.index/embeddings.jsonl — bag-of-tokens pseudo-vectors (no API key required) - knowledge/.index/manifest.json — build metadata +Git + Markdown stays the source of truth. `knowledge/.dekc/index.sqlite` is a +disposable accelerator: delete it and the next reader rebuilds. Every reader +self-heals via mtime+size rather than trusting a hook. -Optional: if OPENAI_API_KEY or XAI_API_KEY is set and --embed-api is passed, -writes richer embeddings (not required for default operation). +Do not copy the old JSON `.index/` (inventory + inverted tokens). That was a +full rebuild plus a JSON parse, and it got committed by accident. + +Ladder (one behavior, three tiers): + + index present → use it + else rg on PATH → prefilter + else → pure Python scan + +FTS5 MATCH is the `--engine fts` opt-in. Default candidate selection is +SQL LIKE on a stored haystack so ranking stays identical to a full scan +(Python still scores). + +Usage: + python3 scripts/dekc_index.py status --bundle sample-knowledge + python3 scripts/dekc_index.py refresh --bundle sample-knowledge + python3 scripts/dekc_index.py drop --bundle sample-knowledge + python3 scripts/dekc_index.py build --bundle sample-knowledge # alias: refresh --force """ from __future__ import annotations import argparse -import hashlib import json -import math +import os import re +import sqlite3 import sys -from collections import defaultdict -from datetime import datetime, timezone +import time +from dataclasses import dataclass from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parent)) -from dekc_common import list_concepts, resolve_knowledge_root # noqa: E402 -from dekc_lineage import FORWARD_FLOW, REVERSE_FLOW, build_graph # noqa: E402 - -TOKEN_RE = re.compile(r"[a-z0-9_]{2,}", re.I) -STOP = { - "the", - "and", - "for", - "with", - "from", - "this", - "that", - "are", - "was", - "were", - "have", - "has", - "not", - "but", - "you", - "all", - "can", - "into", - "type", - "title", - "description", - "tags", - "status", - "true", - "false", - "null", - "markdown", - "none", -} - - -def tokenize(text: str) -> list[str]: - return [t.lower() for t in TOKEN_RE.findall(text) if t.lower() not in STOP] +from dekc_common import ( # noqa: E402 + iter_concept_paths, + iter_typed_edges, + parse_frontmatter, + resolve_knowledge_root, + toolchain_report, +) +from dekc_lineage import ( # noqa: E402 + FORWARD_FLOW, + REVERSE_FLOW, + extract_edges_from_sql, + _resolve_name, +) +SCHEMA_VERSION = 1 +INDEX_NAME = "index.sqlite" +LINEAGE_RELS = frozenset(FORWARD_FLOW + REVERSE_FLOW) | {"sql_from"} -def build_index(bundle: Path) -> dict[str, Any]: - index_dir = bundle / ".index" - index_dir.mkdir(parents=True, exist_ok=True) - - inventory: list[dict[str, Any]] = [] - inverted: dict[str, list[str]] = defaultdict(list) - emb_lines: list[str] = [] - - for path, fm, body in list_concepts(bundle): - rel = path.relative_to(bundle).as_posix() - text = f"{fm.get('title','')} {fm.get('description','')} {fm.get('type','')} {' '.join(fm.get('tags') or [])} {body}" - tokens = tokenize(text) - # term frequencies - tf: dict[str, int] = defaultdict(int) - for t in tokens: - tf[t] += 1 - # L2-normalized sparse bag as pseudo embedding (deterministic, local) - norm = math.sqrt(sum(v * v for v in tf.values())) or 1.0 - sparse = {k: round(v / norm, 6) for k, v in sorted(tf.items()) if v > 0} - # keep top 64 dims by weight - sparse = dict(sorted(sparse.items(), key=lambda kv: -kv[1])[:64]) - - entry = { - "path": rel, - "type": fm.get("type"), - "title": fm.get("title") or path.stem, - "description": fm.get("description") or "", - "layer": fm.get("layer"), - "tags": fm.get("tags") or [], - "verified": bool(fm.get("verified")), - "status": fm.get("status"), - "wiki_key": fm.get("wiki_key"), - "token_count": len(tokens), - "hash": hashlib.sha256(text.encode()).hexdigest()[:16], + +@dataclass +class NodeRow: + path: str + type: str + title: str + description: str + status: str + tags: str + layer: str + mtime: int + size: int + fm_json: str + body: str + hay: str + + def frontmatter(self) -> dict[str, Any]: + try: + data = json.loads(self.fm_json) + except json.JSONDecodeError: + return {} + return data if isinstance(data, dict) else {} + + +@dataclass +class RefreshStats: + ok: bool + path: str + nodes: int = 0 + edges: int = 0 + parsed: int = 0 + unchanged: int = 0 + deleted: int = 0 + rebuilt: bool = False + ms: float = 0.0 + error: str = "" + + def as_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "path": self.path, + "nodes": self.nodes, + "edges": self.edges, + "parsed": self.parsed, + "unchanged": self.unchanged, + "deleted": self.deleted, + "rebuilt": self.rebuilt, + "ms": round(self.ms, 2), + "error": self.error, } - inventory.append(entry) - for tok in set(tokens): - inverted[tok].append(rel) - emb_lines.append( - json.dumps({"path": rel, "type": entry["type"], "title": entry["title"], "vector": sparse}) - ) - graph = build_graph(bundle) - (index_dir / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") - (index_dir / "search.json").write_text( - json.dumps({k: v for k, v in sorted(inverted.items())}, indent=2), encoding="utf-8" +def fts5_available() -> bool: + report = toolchain_report() + sqlite = report.get("sqlite") or {} + return bool(sqlite.get("fts5")) + + +def index_enabled() -> bool: + """DEKC_NO_INDEX=1 disables the index for the process (tests, --no-index).""" + flag = os.environ.get("DEKC_NO_INDEX", "").strip().lower() + if flag in {"1", "true", "yes", "on"}: + return False + return fts5_available() + + +def index_path(bundle: Path) -> Path: + override = os.environ.get("DEKC_INDEX_PATH", "").strip() + if override: + return Path(override) + return bundle / ".dekc" / INDEX_NAME + + +def _connect(path: Path, *, create: bool) -> sqlite3.Connection | None: + if not fts5_available(): + return None + if not create and not path.is_file(): + return None + path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(str(path), timeout=5.0) + con.row_factory = sqlite3.Row + con.execute("PRAGMA journal_mode=WAL") + con.execute("PRAGMA busy_timeout=5000") + con.execute("PRAGMA synchronous=NORMAL") + con.execute("PRAGMA foreign_keys=ON") + return con + + +def _schema_version(con: sqlite3.Connection) -> int | None: + try: + row = con.execute( + "SELECT value FROM meta WHERE key = 'schema_version'" + ).fetchone() + except sqlite3.DatabaseError: + return None + if not row: + return None + try: + return int(row[0]) + except (TypeError, ValueError): + return None + + +def _init_schema(con: sqlite3.Connection) -> None: + con.executescript( + """ + CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS nodes ( + path TEXT PRIMARY KEY, + type TEXT, + title TEXT, + description TEXT, + status TEXT, + tags TEXT, + layer TEXT, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL, + fm_json TEXT NOT NULL, + body TEXT NOT NULL, + hay TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS edges ( + src TEXT NOT NULL, + dst TEXT NOT NULL, + rel TEXT NOT NULL, + label TEXT, + origin TEXT NOT NULL DEFAULT '', + PRIMARY KEY (src, dst, rel, origin) + ); + CREATE INDEX IF NOT EXISTS edges_dst ON edges(dst); + CREATE INDEX IF NOT EXISTS edges_src ON edges(src); + CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5( + path, + title, + description, + tags, + body, + tokenize = 'unicode61 remove_diacritics 2' + ); + """ + ) + con.execute( + "INSERT OR REPLACE INTO meta(key, value) VALUES ('schema_version', ?)", + (str(SCHEMA_VERSION),), ) - _edge_count = sum(len(v) for v in graph.values()) - if _edge_count == 0 and concepts: - # An empty lineage graph is indistinguishable from a failed build unless - # we say why. graph.json is lineage adjacency, not the general OKF graph, - # so a bundle authored with a non-lineage vocabulary legitimately yields - # nothing -- but silently writing 0 makes that look broken. - print( - f"dekc: no lineage relations found across {len(concepts)} concepts; " - "graph.json is empty. Lineage edges come from " - f"{', '.join(FORWARD_FLOW + REVERSE_FLOW)} — a bundle using a " - "non-lineage vocabulary will produce none.", - file=sys.stderr, + con.commit() + + +def _drop_schema(con: sqlite3.Connection) -> None: + con.executescript( + """ + DROP TABLE IF EXISTS fts; + DROP TABLE IF EXISTS edges; + DROP TABLE IF EXISTS nodes; + DROP TABLE IF EXISTS meta; + """ + ) + con.commit() + + +def _ensure_schema(con: sqlite3.Connection) -> bool: + """Return True if the schema was rebuilt from scratch.""" + version = _schema_version(con) + if version == SCHEMA_VERSION: + return False + _drop_schema(con) + _init_schema(con) + return True + + +def _sig(path: Path) -> tuple[int, int]: + st = path.stat() + mtime = getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000)) + return int(mtime), int(st.st_size) + + +def _hay(title: str, description: str, tags: str, body: str) -> str: + return f"{title}\n{description}\n{tags}\n{body}".lower() + + +def _norm_target(tgt: str) -> str: + tgt = str(tgt or "").strip() + if not tgt: + return "" + if not tgt.startswith("/"): + tgt = "/" + tgt.lstrip("./") + return tgt + + +def extract_dekc_edges( + bundle: Path, rel: str, fm: dict[str, Any], body: str +) -> list[tuple[str, str, str, str]]: + """Return (src, dst, rel, label) including SQL lineage edges.""" + out: list[tuple[str, str, str, str]] = [] + seen: set[tuple[str, str, str]] = set() + for tgt, r in iter_typed_edges(fm): + tgt = _norm_target(tgt) + if not tgt: + continue + key = (rel, tgt, str(r)) + if key in seen: + continue + seen.add(key) + out.append((rel, tgt, str(r), Path(tgt).stem)) + if "```sql" in body.lower(): + m = re.search(r"```sql\n(.*?)```", body, re.DOTALL | re.IGNORECASE) + if m: + for up, down in extract_edges_from_sql(m.group(1)): + up_ref = _resolve_name(bundle, up) + down_ref = _resolve_name(bundle, down) + if not up_ref or not down_ref: + continue + key = (up_ref, down_ref, "sql_from") + if key in seen: + continue + seen.add(key) + out.append((up_ref, down_ref, "sql_from", down)) + return out + + +def _upsert_node(con: sqlite3.Connection, bundle: Path, rel: str, path: Path) -> None: + text = path.read_text(encoding="utf-8") + fm, body = parse_frontmatter(text) + title = str(fm.get("title") or path.stem) + description = str(fm.get("description") or "") + status = str(fm.get("status") or "") + layer = str(fm.get("layer") or "") + tags_val = fm.get("tags") or [] + if isinstance(tags_val, list): + tags = " ".join(str(t) for t in tags_val) + else: + tags = str(tags_val) + ctype = str(fm.get("type") or "Unknown") + mtime, size = _sig(path) + fm_json = json.dumps(fm, default=str, ensure_ascii=False) + hay = _hay(title, description, tags, body) + con.execute( + """ + INSERT OR REPLACE INTO nodes( + path, type, title, description, status, tags, layer, + mtime, size, fm_json, body, hay + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (rel, ctype, title, description, status, tags, layer, mtime, size, fm_json, body, hay), + ) + con.execute("DELETE FROM edges WHERE origin = ? OR (origin = '' AND src = ?)", (rel, rel)) + con.execute("DELETE FROM fts WHERE path = ?", (rel,)) + edges = extract_dekc_edges(bundle, rel, fm, body) + if edges: + con.executemany( + "INSERT OR IGNORE INTO edges(src, dst, rel, label, origin) VALUES (?, ?, ?, ?, ?)", + [(src, dst, r, label, rel) for src, dst, r, label in edges], + ) + con.execute( + "INSERT INTO fts(path, title, description, tags, body) VALUES (?, ?, ?, ?, ?)", + (rel, title, description, tags, body), + ) + + +def refresh(bundle: Path, *, force: bool = False) -> RefreshStats | None: + """Self-healing incremental refresh. None = index unavailable (no FTS5).""" + if not index_enabled() and not force: + return None + if not fts5_available(): + return None + started = time.perf_counter() + path = index_path(bundle) + stats = RefreshStats(ok=True, path=str(path)) + con = _connect(path, create=True) + if con is None: + stats.ok = False + stats.error = "sqlite/fts5 unavailable" + return stats + try: + rebuilt = _ensure_schema(con) + if force: + _drop_schema(con) + _init_schema(con) + rebuilt = True + stats.rebuilt = rebuilt + + disk: dict[str, Path] = {} + disk_sig: dict[str, tuple[int, int]] = {} + for p in iter_concept_paths(bundle): + rel = "/" + p.relative_to(bundle).as_posix() + disk[rel] = p + try: + disk_sig[rel] = _sig(p) + except OSError: + continue + + stored = { + row["path"]: (int(row["mtime"]), int(row["size"])) + for row in con.execute("SELECT path, mtime, size FROM nodes") + } + + vanished = set(stored) - set(disk_sig) + for rel in vanished: + con.execute("DELETE FROM nodes WHERE path = ?", (rel,)) + con.execute("DELETE FROM edges WHERE origin = ? OR src = ?", (rel, rel)) + con.execute("DELETE FROM fts WHERE path = ?", (rel,)) + stats.deleted += 1 + + for rel, p in disk.items(): + sig = disk_sig.get(rel) + if sig is None: + continue + if not rebuilt and stored.get(rel) == sig: + stats.unchanged += 1 + continue + try: + _upsert_node(con, bundle, rel, p) + except (OSError, UnicodeDecodeError): + continue + stats.parsed += 1 + + con.commit() + stats.nodes = int(con.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]) + stats.edges = int(con.execute("SELECT COUNT(*) FROM edges").fetchone()[0]) + stats.ms = (time.perf_counter() - started) * 1000 + con.execute( + "INSERT OR REPLACE INTO meta(key, value) VALUES ('refreshed_ms', ?)", + (str(int(time.time() * 1000)),), ) - (index_dir / "graph.json").write_text(json.dumps(graph, indent=2), encoding="utf-8") - (index_dir / "embeddings.jsonl").write_text("\n".join(emb_lines) + ("\n" if emb_lines else ""), encoding="utf-8") - - manifest = { - "built_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "concept_count": len(inventory), - "token_count": len(inverted), - "edge_count": _edge_count, - "bundle": str(bundle), - "engine": "dekc-local-bow-v1", - "files": [ - ".index/inventory.json", - ".index/search.json", - ".index/graph.json", - ".index/embeddings.jsonl", - ], + con.commit() + return stats + except sqlite3.DatabaseError as exc: + stats.ok = False + stats.error = str(exc) + stats.ms = (time.perf_counter() - started) * 1000 + return stats + finally: + con.close() + + +class GraphIndex: + """Open connection after a refresh. One per search/pack call.""" + + def __init__(self, bundle: Path, con: sqlite3.Connection, stats: RefreshStats): + self.bundle = bundle + self.con = con + self.stats = stats + + def close(self) -> None: + try: + self.con.close() + except sqlite3.Error: + pass + + def __enter__(self) -> "GraphIndex": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def inbound(self, target: str) -> list[tuple[str, str, str]]: + rows = self.con.execute( + "SELECT rel, src, label FROM edges WHERE dst = ?", (target,) + ).fetchall() + return [(row["rel"], row["src"], row["label"] or "") for row in rows] + + def neighbors(self, node: str, *, lineage_only: bool = True) -> list[str]: + """Undirected neighbors. lineage_only matches dekc_pack's build_graph walk.""" + rows = self.con.execute( + "SELECT src, dst, rel FROM edges WHERE src = ? OR dst = ?", + (node, node), + ).fetchall() + out: list[str] = [] + seen: set[str] = set() + for row in rows: + rel = row["rel"] or "" + if lineage_only and rel not in LINEAGE_RELS: + continue + other = row["dst"] if row["src"] == node else row["src"] + if other == node or other in seen: + continue + seen.add(other) + out.append(other) + return out + + def node(self, rel: str) -> NodeRow | None: + row = self.con.execute( + """ + SELECT path, type, title, description, status, tags, layer, + mtime, size, fm_json, body, hay + FROM nodes WHERE path = ? + """, + (rel,), + ).fetchone() + if not row: + return None + return _row_to_node(row) + + def candidates(self, terms: list[str], *, engine: str = "index") -> list[Path]: + terms = [t for t in terms if t] + if not terms: + return [] + if engine == "fts": + query = " AND ".join('"' + t.replace('"', '""') + '"*' for t in terms) + rows = self.con.execute( + "SELECT path FROM fts WHERE fts MATCH ?", (query,) + ).fetchall() + else: + clause = " AND ".join("hay LIKE ? ESCAPE '\\'" for _ in terms) + params = [_like_term(t.lower()) for t in terms] + rows = self.con.execute( + f"SELECT path FROM nodes WHERE {clause}", params + ).fetchall() + return [self.bundle / row["path"].lstrip("/") for row in rows] + + def iter_nodes(self) -> list[NodeRow]: + rows = self.con.execute( + """ + SELECT path, type, title, description, status, tags, layer, + mtime, size, fm_json, body, hay + FROM nodes ORDER BY path + """ + ).fetchall() + return [_row_to_node(row) for row in rows] + + +def _row_to_node(row: sqlite3.Row) -> NodeRow: + return NodeRow( + path=row["path"], + type=row["type"] or "Unknown", + title=row["title"] or "", + description=row["description"] or "", + status=row["status"] or "", + tags=row["tags"] or "", + layer=row["layer"] or "", + mtime=int(row["mtime"]), + size=int(row["size"]), + fm_json=row["fm_json"] or "{}", + body=row["body"] or "", + hay=row["hay"] or "", + ) + + +def open_graph(bundle: Path, *, force: bool = False) -> GraphIndex | None: + stats = refresh(bundle, force=force) + if stats is None or not stats.ok: + return None + con = _connect(index_path(bundle), create=False) + if con is None: + return None + return GraphIndex(bundle, con, stats) + + +def drop_index(bundle: Path) -> bool: + path = index_path(bundle) + gone = False + for suffix in ("", "-journal", "-wal", "-shm"): + p = Path(str(path) + suffix) if suffix else path + if p.is_file(): + p.unlink() + gone = True + return gone + + +def _like_term(term: str) -> str: + return "%" + term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" + + +def candidates(bundle: Path, terms: list[str], *, engine: str = "index") -> list[Path] | None: + graph = open_graph(bundle) + if graph is None: + return None + try: + return graph.candidates(terms, engine=engine) + finally: + graph.close() + + +def inbound(bundle: Path, target: str) -> list[tuple[str, str, str]] | None: + graph = open_graph(bundle) + if graph is None: + return None + try: + return graph.inbound(target) + finally: + graph.close() + + +def status(bundle: Path) -> dict[str, Any]: + path = index_path(bundle) + payload = { + "path": str(path), + "present": path.is_file(), + "fts5": fts5_available(), + "enabled": index_enabled(), + "schema_version": SCHEMA_VERSION, + "nodes": 0, + "edges": 0, + "on_disk_schema": None, + "engine": "dekc-sqlite-fts5-v1", } - (index_dir / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") - return manifest + if not path.is_file(): + return payload + con = _connect(path, create=False) + if con is None: + return payload + try: + payload["on_disk_schema"] = _schema_version(con) + payload["nodes"] = int(con.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]) + payload["edges"] = int(con.execute("SELECT COUNT(*) FROM edges").fetchone()[0]) + except sqlite3.DatabaseError as exc: + payload["error"] = str(exc) + finally: + con.close() + return payload + + +# ── compatibility wrappers (dekc_brain / existing tests) ──────────────── + + +def build_index(bundle: Path) -> dict[str, Any]: + """Force-rebuild. Replaces the old JSON `.index/` writer.""" + stats = refresh(bundle, force=True) + if stats is None: + return {"ok": False, "error": "sqlite/fts5 unavailable", "engine": "dekc-sqlite-fts5-v1"} + payload = stats.as_dict() + payload["engine"] = "dekc-sqlite-fts5-v1" + payload["files"] = [".dekc/index.sqlite"] + payload["concept_count"] = stats.nodes + payload["edge_count"] = stats.edges + return payload def search_index(bundle: Path, query: str, limit: int = 20) -> list[dict[str, Any]]: - index_dir = bundle / ".index" - inv_path = index_dir / "inventory.json" - search_path = index_dir / "search.json" - if not inv_path.is_file() or not search_path.is_file(): - build_index(bundle) - inventory = {e["path"]: e for e in json.loads(inv_path.read_text(encoding="utf-8"))} - inverted = json.loads(search_path.read_text(encoding="utf-8")) - q_tokens = tokenize(query) - scores: dict[str, float] = defaultdict(float) - for tok in q_tokens: - for path in inverted.get(tok, []): - scores[path] += 1.0 - # prefix soft match - for k, paths in inverted.items(): - if k.startswith(tok) and k != tok: - for path in paths: - scores[path] += 0.25 - ranked = sorted(scores.items(), key=lambda kv: -kv[1])[:limit] - out = [] - for path, score in ranked: - e = dict(inventory.get(path) or {"path": path}) - e["score"] = score - out.append(e) - return out + """Ranked hits. Delegates to dekc_search so scores match a full scan.""" + from dekc_search import search as _search + + hits, _engine = _search(bundle, query, limit=limit) + return hits def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="DEKC second-brain index") + parser = argparse.ArgumentParser(description="DEKC SQLite/FTS5 incremental index") + parser.add_argument( + "action", + choices=("status", "refresh", "drop", "build", "search"), + help="build is an alias for refresh --force (kept for CI / skills)", + ) parser.add_argument("--repo", default=".") parser.add_argument("--bundle", default=None) - sub = parser.add_subparsers(dest="cmd", required=True) - sub.add_parser("build") - p = sub.add_parser("search") - p.add_argument("query") - p.add_argument("--limit", type=int, default=20) + parser.add_argument("--force", action="store_true", help="Drop and rebuild (refresh only)") + parser.add_argument("--json", action="store_true") + parser.add_argument("query", nargs="?", default="", help="Search query (search action)") + parser.add_argument("--limit", type=int, default=20) args = parser.parse_args(argv) - repo = Path(args.repo).resolve() - bundle = resolve_knowledge_root(repo, args.bundle) + bundle = resolve_knowledge_root(Path(args.repo).resolve(), args.bundle) + if not bundle.is_dir(): + print(f"error: bundle not found: {bundle}", file=sys.stderr) + return 1 + + if args.action == "drop": + dropped = drop_index(bundle) + payload = {"dropped": dropped, "path": str(index_path(bundle))} + if args.json: + print(json.dumps(payload, indent=2)) + else: + print(f"dropped {payload['path']}" if dropped else f"no index at {payload['path']}") + return 0 + + if args.action == "search": + if not args.query: + print("error: search requires a query", file=sys.stderr) + return 2 + hits = search_index(bundle, args.query, args.limit) + print(json.dumps(hits, indent=2)) + return 0 + + if args.action in {"refresh", "build"}: + stats = refresh(bundle, force=args.force or args.action == "build") + if stats is None: + print("error: SQLite FTS5 unavailable; index disabled", file=sys.stderr) + return 1 + payload = stats.as_dict() + payload["engine"] = "dekc-sqlite-fts5-v1" + if args.json: + print(json.dumps(payload, indent=2)) + else: + print( + f"index {stats.path}: nodes={stats.nodes} edges={stats.edges} " + f"parsed={stats.parsed} unchanged={stats.unchanged} " + f"deleted={stats.deleted} rebuilt={stats.rebuilt} {stats.ms:.1f}ms" + ) + return 0 if stats.ok else 1 - if args.cmd == "build": - manifest = build_index(bundle) - print(json.dumps(manifest, indent=2)) - elif args.cmd == "search": - print(json.dumps(search_index(bundle, args.query, args.limit), indent=2)) + payload = status(bundle) + if args.json: + print(json.dumps(payload, indent=2)) + else: + state = "present" if payload["present"] else "absent" + print(f"index {payload['path']}: {state}") + print(f" FTS5: {'yes' if payload['fts5'] else 'no'}") + print(f" enabled: {payload['enabled']}") + if payload["present"]: + print(f" nodes: {payload['nodes']}") + print(f" edges: {payload['edges']}") + print(f" schema: {payload['on_disk_schema']} (code {SCHEMA_VERSION})") return 0 if __name__ == "__main__": - raise SystemExit(main()) + sys.exit(main()) diff --git a/scripts/dekc_pack.py b/scripts/dekc_pack.py index 2e49c62..2a9c030 100755 --- a/scripts/dekc_pack.py +++ b/scripts/dekc_pack.py @@ -17,14 +17,20 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from dekc_common import ( # noqa: E402 append_log, + find_rg, + is_concept_path, + iter_typed_edges, list_concepts, + parse_frontmatter, path_for_type, refresh_catalog_index, resolve_knowledge_root, + rg_list_files, utc_now, write_knowledge, ) -from dekc_lineage import build_graph, mermaid # noqa: E402 +from dekc_lineage import FORWARD_FLOW, REVERSE_FLOW, build_graph, mermaid # noqa: E402 +from dekc_index import LINEAGE_RELS, open_graph # noqa: E402 DEFAULT_WINDOW_TOKENS = 128_000 PACK_BUDGET_DENOMINATOR = 4 @@ -77,12 +83,158 @@ def load_map(bundle: Path) -> dict[str, tuple[dict, str]]: return out +def _parse_rel(bundle: Path, rel: str) -> tuple[dict, str]: + fp = bundle / rel.lstrip("/") + if not fp.is_file(): + return {}, "" + try: + return parse_frontmatter(fp.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError): + return {}, "" + + +def _lineage_neighbors_from_file(bundle: Path, rel: str, fm: dict, body: str) -> list[str]: + """Undirected lineage neighbors authored on this file (typed edges + SQL).""" + from dekc_index import extract_dekc_edges + + out: list[str] = [] + seen: set[str] = set() + for src, dst, r, _label in extract_dekc_edges(bundle, rel, fm, body): + if r not in LINEAGE_RELS and r not in FORWARD_FLOW and r not in REVERSE_FLOW: + continue + other = dst if src == rel else src + if other == rel or other in seen: + continue + seen.add(other) + out.append(other) + return out + + +def _inbound_via_rg(bundle: Path, target: str) -> list[str] | None: + """Undirected lineage neighbors via rg. + + rg only finds files that *mention* `target`. The target file itself almost + never contains its own path (SQL lineage lives in a fenced block of table + names), so we always parse it. None = rg missing/failed → fall back. + """ + needles = [target] + if target.startswith("/"): + needles.append(target.lstrip("/")) + hits = rg_list_files(bundle, needles[:1], fixed_string=True, ignore_case=False) + if hits is None: + return None + neighbors: list[str] = [] + seen: set[str] = set() + + def _consider(src: str, nxt: str) -> None: + if nxt == target and src != target and src not in seen: + seen.add(src) + neighbors.append(src) + elif src == target and nxt != target and nxt not in seen: + seen.add(nxt) + neighbors.append(nxt) + + target_path = bundle / target.lstrip("/") + files: list[Path] = [] + seen_paths: set[Path] = set() + if target_path.is_file(): + files.append(target_path.resolve()) + seen_paths.add(target_path.resolve()) + for path in hits: + try: + resolved = path.resolve() + except OSError: + continue + if resolved in seen_paths: + continue + seen_paths.add(resolved) + files.append(path) + + for path in files: + if not is_concept_path(bundle, path) and path.resolve() != target_path.resolve(): + continue + try: + src = "/" + path.relative_to(bundle).as_posix() + except ValueError: + continue + fm, body = _parse_rel(bundle, src) + for nxt in _lineage_neighbors_from_file(bundle, src, fm, body): + _consider(src, nxt) + return neighbors + + +class ReverseIndex: + """Lineage neighbors: SQLite index → rg → full scan.""" + + def __init__( + self, + bundle: Path, + *, + use_rg: bool | None = None, + use_index: bool | None = None, + ): + self.bundle = bundle + self._full: dict[str, list[str]] | None = None + self._memo: dict[str, list[str]] = {} + self._graph = None + if use_index is False: + self._index = False + else: + self._graph = open_graph(bundle) + self._index = self._graph is not None + if use_index is True and not self._index: + self._index = False + if use_rg is False: + self._rg = False + else: + self._rg = bool(find_rg()) + + @property + def engine(self) -> str: + if self._index: + return "index" + return "rg" if self._rg else "scan" + + def close(self) -> None: + if self._graph is not None: + self._graph.close() + self._graph = None + + def get(self, target: str) -> list[str]: + if target in self._memo: + return self._memo[target] + if self._index and self._graph is not None: + found = self._graph.neighbors(target, lineage_only=True) + self._memo[target] = found + return found + if self._rg: + found = _inbound_via_rg(self.bundle, target) + if found is not None: + self._memo[target] = found + return found + self._rg = False + if self._full is None: + graph = build_graph(self.bundle) + undirected: dict[str, list[str]] = {} + for s, tgts in graph.items(): + undirected.setdefault(s, []) + for t in tgts: + undirected.setdefault(s, []).append(t) + undirected.setdefault(t, []).append(s) + self._full = {k: list(dict.fromkeys(v)) for k, v in undirected.items()} + edges = self._full.get(target, []) + self._memo[target] = edges + return edges + + def pack( bundle: Path, focus: str, *, hops: int = 2, max_nodes: int = 20, + use_rg: bool | None = None, + use_index: bool | None = None, ) -> dict: if not focus.startswith("/"): candidate = focus if focus.endswith(".md") else focus + ".md" @@ -96,64 +248,83 @@ def pack( else: focus = "/" + candidate.lstrip("/") - graph = build_graph(bundle) - concepts = load_map(bundle) - undirected: dict[str, list[str]] = {} - for s, tgts in graph.items(): - undirected.setdefault(s, []) - for t in tgts: - undirected.setdefault(s, []).append(t) - undirected.setdefault(t, []).append(s) - if focus in concepts: - for link in concepts[focus][0].get("links") or []: - if isinstance(link, dict) and link.get("target"): - undirected.setdefault(focus, []).append(link["target"]) - undirected.setdefault(link["target"], []).append(focus) - - seen = {focus} - order = [focus] - q = deque([(focus, 0)]) - while q and len(order) < max_nodes: - node, d = q.popleft() - if d >= hops: - continue - for nxt in undirected.get(node, []): - if nxt not in seen: - seen.add(nxt) - order.append(nxt) - q.append((nxt, d + 1)) - if len(order) >= max_nodes: - break - - nodes = [] - for rel in order: - fm, body = concepts.get(rel, ({}, "")) - title = fm.get("title") or Path(rel).stem - typ = fm.get("type") or "?" - layer = fm.get("layer") - is_root = rel == focus - nodes.append( - { - "path": rel, - "title": title, - "type": typ, - "layer": layer, - "description": fm.get("description") or "", - "body": body if is_root else "", - } - ) + inbound = ReverseIndex(bundle, use_rg=use_rg, use_index=use_index) + try: + focus_fm, focus_body = _parse_rel(bundle, focus) + extra: list[str] = [] + for tgt, _rel in iter_typed_edges(focus_fm): + if tgt and tgt != focus: + extra.append(tgt if tgt.startswith("/") else "/" + tgt.lstrip("/")) + + seen = {focus} + order = [focus] + q = deque([(focus, 0)]) + while q and len(order) < max_nodes: + node, d = q.popleft() + if d >= hops: + continue + # Outbound lineage is authored on this file (typed flow + SQL) and + # is invisible to `rg -lF ` because the file rarely mentions + # its own path. Union with inbound (index / rg / scan). + fm, body = _parse_rel(bundle, node) + outbound = _lineage_neighbors_from_file(bundle, node, fm, body) + neighbors = list(dict.fromkeys(list(outbound) + list(inbound.get(node)))) + if node == focus: + neighbors.extend(extra) + for nxt in neighbors: + if nxt not in seen: + seen.add(nxt) + order.append(nxt) + q.append((nxt, d + 1)) + if len(order) >= max_nodes: + break + + nodes = [] + for rel in order: + if inbound.engine == "index" and inbound._graph is not None: + row = inbound._graph.node(rel) + if row is not None: + fm = row.frontmatter() + body = row.body + title = row.title or Path(rel).stem + typ = row.type + layer = row.layer or fm.get("layer") + else: + fm, body = _parse_rel(bundle, rel) + title = fm.get("title") or Path(rel).stem + typ = fm.get("type") or "?" + layer = fm.get("layer") + else: + fm, body = _parse_rel(bundle, rel) + title = fm.get("title") or Path(rel).stem + typ = fm.get("type") or "?" + layer = fm.get("layer") + is_root = rel == focus + nodes.append( + { + "path": rel, + "title": title, + "type": typ, + "layer": layer, + "description": (fm.get("description") if isinstance(fm, dict) else "") or "", + "body": body if is_root else "", + } + ) - return { - "focus": focus, - "hops": hops, - "max_nodes": max_nodes, - "node_count": len(nodes), - "nodes": nodes, - "excluded_note": ( - "Nodes beyond hops/max_nodes omitted for progressive disclosure. " - "Node clip is not a token budget." - ), - } + return { + "focus": focus, + "hops": hops, + "max_nodes": max_nodes, + "node_count": len(nodes), + "nodes": nodes, + "reverse_index": inbound.engine, + "excluded_note": ( + "Nodes beyond hops/max_nodes omitted for progressive disclosure. " + "Node clip is not a token budget." + ), + } + finally: + inbound.close() def render_markdown( @@ -234,6 +405,9 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--mermaid", action="store_true") parser.add_argument("--write", action="store_true", help="Write pack under packs/") parser.add_argument("--author", default="") + parser.add_argument("--rg", action="store_true", help="Use ripgrep for inbound discovery") + parser.add_argument("--no-rg", action="store_true", help="Disable ripgrep") + parser.add_argument("--no-index", action="store_true", help="Disable the SQLite index") args = parser.parse_args(argv) from dekc_common import resolve_author @@ -245,7 +419,16 @@ def main(argv: list[str] | None = None) -> int: args.max_nodes = 8 bundle = resolve_knowledge_root(Path(args.repo).resolve(), args.bundle) - result = pack(bundle, args.focus, hops=args.hops, max_nodes=args.max_nodes) + use_index: bool | None = False if args.no_index else None + use_rg: bool | None = False if args.no_rg else (True if args.rg else None) + result = pack( + bundle, + args.focus, + hops=args.hops, + max_nodes=args.max_nodes, + use_rg=use_rg, + use_index=use_index, + ) try: if args.mermaid: diff --git a/scripts/dekc_search.py b/scripts/dekc_search.py index a12935d..dc08e76 100755 --- a/scripts/dekc_search.py +++ b/scripts/dekc_search.py @@ -1,56 +1,318 @@ #!/usr/bin/env python3 -"""Full-text + indexed search over DEKC concepts.""" +"""Full-text search over a DEKC knowledge bundle (Git stays source of truth). + +Ladder: SQLite index → ripgrep prefilter → linear scan. Ranking is always +computed in Python over the candidate files, so scores stay identical to a +full scan. `--no-index` / `--no-rg` force a lower rung. + +Usage: + python3 scripts/dekc_search.py revenue --bundle sample-knowledge + python3 scripts/dekc_search.py "gold orders" --json + python3 scripts/dekc_search.py revenue --no-rg --no-index +""" from __future__ import annotations import argparse import json +import re import sys from pathlib import Path +from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parent)) -from dekc_common import list_concepts, resolve_knowledge_root # noqa: E402 -from dekc_index import search_index, build_index # noqa: E402 +from dekc_common import ( # noqa: E402 + find_rg, + is_concept_path, + iter_concept_paths, + parse_frontmatter, + resolve_knowledge_root, + rg_list_files, +) +from dekc_index import open_graph # noqa: E402 + + +def tokenize(q: str) -> list[str]: + return [t for t in re.split(r"\s+", q.strip().lower()) if t] + + +def candidate_files( + bundle: Path, + terms: list[str], + *, + path_prefix: str | None = None, + use_rg: bool | None = None, + use_index: bool | None = None, + index_engine: str = "index", +) -> tuple[list[Path], str]: + """Return (files, engine) where engine is 'index', 'rg', or 'scan'.""" + universe = iter_concept_paths(bundle) + if path_prefix: + prefix = path_prefix.lstrip("/") + universe = [ + p + for p in universe + if ("/" + p.relative_to(bundle).as_posix()).lstrip("/").startswith(prefix) + ] + + def _filter(paths: list[Path]) -> list[Path]: + allowed = {p.resolve() for p in paths} + return [ + p + for p in universe + if p.resolve() in allowed and is_concept_path(bundle, p) + ] + + if use_index is not False: + graph = open_graph(bundle) + if graph is not None: + try: + hits = graph.candidates(terms, engine=index_engine) + return _filter(hits), "index" + finally: + graph.close() + if use_index is True: + pass + + if use_rg is False: + return universe, "scan" + if use_rg is None and not find_rg(): + return universe, "scan" + hits = rg_list_files(bundle, terms, ignore_case=True, fixed_string=False) + if hits is None: + return universe, "scan" + return _filter(hits), "rg" + + +def search( + bundle: Path, + query: str, + *, + types: list[str] | None = None, + limit: int = 20, + path_prefix: str | None = None, + use_rg: bool | None = None, + use_index: bool | None = None, + index_engine: str = "index", +) -> tuple[list[dict[str, Any]], str]: + terms = tokenize(query) + if not terms: + return [], "scan" + + type_filter = {t.lower() for t in (types or []) if t} + results: list[dict[str, Any]] = [] + files, engine = candidate_files( + bundle, + terms, + path_prefix=path_prefix, + use_rg=use_rg, + use_index=use_index, + index_engine=index_engine, + ) + + for path in files: + rel = "/" + path.relative_to(bundle).as_posix() + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + fm, body = parse_frontmatter(text) + ctype = str(fm.get("type") or "Unknown") + if type_filter and ctype.lower() not in type_filter: + continue + + hay_title = str(fm.get("title") or path.stem).lower() + hay_desc = str(fm.get("description") or "").lower() + hay_tags = " ".join(str(t) for t in (fm.get("tags") or [])).lower() + hay_body = body.lower() + full = f"{hay_title}\n{hay_desc}\n{hay_tags}\n{hay_body}" + + score = 0 + hits: list[str] = [] + missing = False + for term in terms: + if term not in full: + missing = True + break + c_title = hay_title.count(term) + c_desc = hay_desc.count(term) + c_tags = hay_tags.count(term) + c_body = hay_body.count(term) + score += c_title * 10 + c_desc * 5 + c_tags * 4 + min(c_body, 8) + if c_title: + hits.append("title") + if c_desc: + hits.append("description") + if c_tags: + hits.append("tags") + if c_body: + hits.append("body") + if missing or score <= 0: + continue + + snippet = _snippet(body or hay_desc, terms) + results.append( + { + "path": rel, + "type": ctype, + "title": fm.get("title") or path.stem, + "description": fm.get("description") or "", + "status": fm.get("status"), + "layer": fm.get("layer"), + "score": score, + "hits": sorted(set(hits)), + "snippet": snippet, + } + ) + + results.sort(key=lambda r: (-r["score"], r["title"])) + return results[:limit], engine + + +def _snippet(body: str, terms: list[str], width: int = 160) -> str: + low = body.lower() + pos = -1 + for t in terms: + i = low.find(t) + if i >= 0: + pos = i + break + if pos < 0: + text = re.sub(r"\s+", " ", body).strip() + return text[:width] + ("…" if len(text) > width else "") + start = max(0, pos - 40) + end = min(len(body), pos + width) + frag = re.sub(r"\s+", " ", body[start:end]).strip() + if start > 0: + frag = "…" + frag + if end < len(body): + frag = frag + "…" + return frag + + +def render(results: list[dict[str, Any]], query: str) -> str: + lines = [f"# Search: {query}", "", f"{len(results)} hit(s)", ""] + if not results: + lines.append("_No matches._") + return "\n".join(lines) + "\n" + for r in results: + lines.append(f"## [{r['title']}]({r['path']}) · `{r['type']}` · score {r['score']}") + if r.get("description"): + lines.append("") + lines.append(str(r["description"])) + if r.get("snippet"): + lines.append("") + lines.append(f"> {r['snippet']}") + lines.append("") + lines.append(f"_hits: {', '.join(r['hits'])}_") + lines.append("") + return "\n".join(lines) def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("query") + parser = argparse.ArgumentParser(description="DEKC full-text search") + parser.add_argument("query", help="Search terms (AND)") parser.add_argument("--repo", default=".") parser.add_argument("--bundle", default=None) + parser.add_argument("--type", dest="types", default=None, help="Comma-separated types") parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--prefix", default=None, help="Path prefix filter e.g. tables/") parser.add_argument("--json", action="store_true") - parser.add_argument("--rebuild", action="store_true") + parser.add_argument("--rebuild", action="store_true", help="Force-rebuild the SQLite index first") + parser.add_argument( + "--rg", + action="store_true", + help="Use ripgrep to prefilter candidate files (default when rg is on PATH)", + ) + parser.add_argument( + "--no-rg", + action="store_true", + help="Disable ripgrep; fall through to a full scan (unless the index is used)", + ) + parser.add_argument( + "--no-index", + action="store_true", + help="Disable the SQLite index; fall through to rg then scan", + ) + parser.add_argument( + "--engine", + choices=("auto", "index", "fts", "rg", "scan"), + default="auto", + help="Force a retrieval rung. 'fts' uses FTS5 MATCH (not score-identical).", + ) args = parser.parse_args(argv) bundle = resolve_knowledge_root(Path(args.repo).resolve(), args.bundle) - if args.rebuild or not (bundle / ".index" / "search.json").is_file(): - build_index(bundle) - hits = search_index(bundle, args.query, args.limit) - - # fallback scan if empty - if not hits: - q = args.query.lower() - for path, fm, body in list_concepts(bundle): - blob = f"{fm.get('title','')} {fm.get('description','')} {body}".lower() - if q in blob: - hits.append( - { - "path": path.relative_to(bundle).as_posix(), - "title": fm.get("title"), - "type": fm.get("type"), - "score": 1, - } + if not bundle.is_dir(): + print(f"error: bundle not found: {bundle}", file=sys.stderr) + return 1 + + if args.rg and args.no_rg: + print("error: --rg and --no-rg are mutually exclusive", file=sys.stderr) + return 2 + + if args.rebuild: + from dekc_index import refresh + + refresh(bundle, force=True) + + use_index: bool | None = None + use_rg: bool | None = None + index_engine = "index" + if args.engine == "scan": + use_index = False + use_rg = False + elif args.engine == "rg": + use_index = False + use_rg = True + elif args.engine == "index": + use_index = True + use_rg = False + elif args.engine == "fts": + use_index = True + use_rg = False + index_engine = "fts" + else: + if args.no_index: + use_index = False + if args.no_rg: + use_rg = False + elif args.rg: + use_rg = True + if not find_rg(): + print( + "dekc_search: rg not found; falling back. " + "Install ripgrep or pass --no-rg.", + file=sys.stderr, ) - if len(hits) >= args.limit: - break + + types = [t.strip() for t in (args.types or "").split(",") if t.strip()] or None + results, engine = search( + bundle, + args.query, + types=types, + limit=args.limit, + path_prefix=args.prefix, + use_rg=use_rg, + use_index=use_index, + index_engine=index_engine, + ) if args.json: - print(json.dumps({"query": args.query, "count": len(hits), "hits": hits}, indent=2)) + print( + json.dumps( + { + "query": args.query, + "count": len(results), + "engine": engine, + "hits": results, + "results": results, + }, + indent=2, + ) + ) else: - print(f"{len(hits)} hits for {args.query!r}") - for h in hits: - print(f" [{h.get('type')}] {h.get('title')} ({h.get('path')}) score={h.get('score')}") + print(render(results, args.query)) return 0 diff --git a/skills/dekc-doctor/SKILL.md b/skills/dekc-doctor/SKILL.md index 1265ffd..a12f863 100644 --- a/skills/dekc-doctor/SKILL.md +++ b/skills/dekc-doctor/SKILL.md @@ -1,10 +1,15 @@ --- name: dekc-doctor -description: Health check: validation, business coverage, orphans, index. +description: Health check: validation, business coverage, orphans, toolchain (rg / FTS5 / sqlite index). --- # Doctor +One-screen health: validation, business-object coverage, orphan tables, plus +toolchain (Python, ripgrep, SQLite FTS5) and whether +`knowledge/.dekc/index.sqlite` is present. Warms the index status on each run. +If rg is missing, offer installing ripgrep yourself — never from a hook. + ```bash python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_doctor.py" --repo . --bundle knowledge python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_validate.py" --repo . --bundle knowledge diff --git a/skills/dekc-grade/SKILL.md b/skills/dekc-grade/SKILL.md index 9e85138..3a7109e 100644 --- a/skills/dekc-grade/SKILL.md +++ b/skills/dekc-grade/SKILL.md @@ -31,7 +31,7 @@ Dict-valued `description` fields (bad YAML / mixed bundles) are coerced; grade d 1. Spawn **lineage-skeptic**, **business-skeptic**, **stream-job-skeptic**, **coverage-skeptic** 2. Aggregate with **re-adversary-judge** against `evaluation/reverse-engineering-rubric.md` (threshold 0.75) 3. On fail → orchestrator revises (capture evidence or **retract** claims) → re-grade -4. On pass → `dekc_index.py build` +4. On pass → `dekc_doctor.py` (search/pack refresh the SQLite index themselves) Orchestrators: **data-lake-walker**, **reverse-engineering-orchestrator**. diff --git a/skills/dekc-index/SKILL.md b/skills/dekc-index/SKILL.md index 2986540..4fef7d0 100644 --- a/skills/dekc-index/SKILL.md +++ b/skills/dekc-index/SKILL.md @@ -1,12 +1,27 @@ --- name: dekc-index -description: Rebuild the local second-brain index (inventory, inverted search, graph, embeddings). +description: Refresh, inspect, or drop the disposable SQLite/FTS5 knowledge index. Use when search/pack is slow, after a large git checkout, or when the user asks about the DEKC index. --- -# Index +# DEKC index + +Git + Markdown is the source of truth. `knowledge/.dekc/index.sqlite` is a +disposable accelerator. Deleting it is always valid recovery. ```bash -python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" --repo . --bundle knowledge build +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" status --repo . +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" refresh --repo . +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" refresh --force --repo . +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" drop --repo . ``` -Writes `knowledge/.index/{inventory,search,graph,embeddings,manifest}.*` for LLM retrieval. +`build` is an alias for `refresh --force` (kept for CI and older skills). + +Search and pack refresh the index themselves on each call (mtime+size, not +the curate hook). You do not need to run `refresh` first. + +Ladder: index → ripgrep → full scan. `--no-index` on search/pack skips this +rung. `DEKC_NO_INDEX=1` disables it for the process. + +Never commit `*.sqlite`. Never install packages. Missing FTS5 is not a +failure — the lower rungs still run. See `docs/designs/retrieval-ladder.md`. diff --git a/skills/dekc-search/SKILL.md b/skills/dekc-search/SKILL.md index a8b58c2..ce95054 100644 --- a/skills/dekc-search/SKILL.md +++ b/skills/dekc-search/SKILL.md @@ -1,11 +1,24 @@ --- name: dekc-search -description: Search the indexed DEKC second brain. +description: Full-text search over the DEKC knowledge bundle (AND terms, type filters). Use when finding tables, lineage, or metrics by keyword. --- # Search ```bash -python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_index.py" --repo . --bundle knowledge build -python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" "revenue" --repo . --bundle knowledge +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" "revenue" --repo . +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" revenue --type Table,Metric --json +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" revenue --engine index +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" revenue --no-index --rg +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" revenue --engine scan ``` + +AND semantics across terms. Scores title > description > tags > body +(`title×10 / description×5 / tags×4 / min(body,8)`). + +Ladder: SQLite index → ripgrep → full scan. Ranking stays in Python, so +`--engine scan` and the index path return the same scores. `--engine fts` +uses FTS5 MATCH (prefix tokens; not score-identical). Missing index or rg +is not an error. See `/dekc-index`. + +After hits on a table, offer `/dekc-context` (2-hop pack). diff --git a/skills/dekc-second-brain/SKILL.md b/skills/dekc-second-brain/SKILL.md index bdf9717..d7c2b2c 100644 --- a/skills/dekc-second-brain/SKILL.md +++ b/skills/dekc-second-brain/SKILL.md @@ -40,7 +40,7 @@ python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_brain.py" "gold order_daily" \ 1. **Retrieve** with `dekc_brain.py` (intent + query) → checklist + ranked concepts + 2-hop pack 2. **Design** using only evidence from the pack (do not invent lineage) 3. **Capture** new assets with `dekc_capture.py` matching `schemas/okf-concepts/` -4. **Index** `dekc_index.py build` +4. **Index** is optional — search/pack refresh `knowledge/.dekc/index.sqlite` themselves. `dekc_index.py build` is `refresh --force`. 5. Optional **grade** if reverse-engineering See `schemas/README.md` and `docs/user_guide/user-guide.md`. diff --git a/templates/gitignore-fragment b/templates/gitignore-fragment index 03b5281..6c76701 100644 --- a/templates/gitignore-fragment +++ b/templates/gitignore-fragment @@ -4,6 +4,15 @@ # for every bundle rather than one hardcoded name. **/.index/ +# Disposable SQLite/FTS5 retrieval index (DEKC 0.5.0). Git + Markdown stays +# source of truth; deleting the sqlite file is always valid recovery. +# Do not ignore the whole `.dekc/` directory — config.yml lives there. +**/.dekc/index.sqlite +**/.dekc/index.sqlite-journal +**/.dekc/index.sqlite-wal +**/.dekc/index.sqlite-shm + + # Per-object WriteEvents — opt in with DEKC_WRITE_EVENTS=1 or --write-event. # Keep the catalog index if you generate one; ignore the timestamp flood (#37). **/write-events/* diff --git a/tests/fixtures/fake_rg.py b/tests/fixtures/fake_rg.py new file mode 100755 index 0000000..df4080c --- /dev/null +++ b/tests/fixtures/fake_rg.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Minimal `rg -l` stand-in for tests. Not a real ripgrep. + +Understands: + -l / --files-with-matches + -i / --ignore-case + -F / --fixed-strings + --glob GLOB (including !negations) + --no-messages --color never + pattern PATH +""" + +from __future__ import annotations + +import argparse +import fnmatch +import re +import sys +from pathlib import Path + + +def match_globs(rel: str, globs: list[str]) -> bool: + include = [g for g in globs if not g.startswith("!")] + exclude = [g[1:] for g in globs if g.startswith("!")] + ok = True + if include: + ok = any(fnmatch.fnmatch(rel, g) or fnmatch.fnmatch(Path(rel).name, g) for g in include) + for g in exclude: + if fnmatch.fnmatch(rel, g) or fnmatch.fnmatch(Path(rel).name, g): + return False + return ok + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(add_help=False) + p.add_argument("-l", "--files-with-matches", action="store_true") + p.add_argument("-i", "--ignore-case", action="store_true") + p.add_argument("-F", "--fixed-strings", action="store_true") + p.add_argument("--glob", action="append", default=[]) + p.add_argument("--no-messages", action="store_true") + p.add_argument("--color", default="never") + p.add_argument("pattern") + p.add_argument("path", nargs="?", default=".") + args = p.parse_args(argv) + + root = Path(args.path).resolve() + flags = re.I if args.ignore_case else 0 + if args.fixed_strings: + needle = args.pattern.lower() if args.ignore_case else args.pattern + pred = lambda text: needle in (text.lower() if args.ignore_case else text) + else: + try: + rx = re.compile(args.pattern, flags) + except re.error: + return 2 + pred = lambda text: rx.search(text) is not None + + hits = 0 + if root.is_file(): + files = [root] + base = root.parent + else: + files = sorted(root.rglob("*")) + base = root + for path in files: + if not path.is_file(): + continue + try: + rel = path.relative_to(base).as_posix() + except ValueError: + rel = path.name + if args.glob and not match_globs(rel, args.glob): + continue + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + if pred(text): + print(path) + hits += 1 + return 0 if hits else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_dekc.py b/tests/test_dekc.py index e30d1fa..ba101d3 100644 --- a/tests/test_dekc.py +++ b/tests/test_dekc.py @@ -480,6 +480,7 @@ class TestVersionConsistency(unittest.TestCase): three others but left 8 more manifests stale (issue #8 / PR #12).""" MANIFESTS = [ + ("plugin.json", ("version",)), ("package.json", ("version",)), ("marketplace.json", ("plugins", 0, "version")), (".cursor-plugin/plugin.json", ("version",)), diff --git a/tests/test_retrieval_ladder.py b/tests/test_retrieval_ladder.py new file mode 100644 index 0000000..c5200aa --- /dev/null +++ b/tests/test_retrieval_ladder.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Retrieval ladder: index → rg → scan. Scores and pack graphs stay identical.""" + +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from dekc_common import find_rg # noqa: E402 +from dekc_doctor import doctor # noqa: E402 +from dekc_index import drop_index, refresh, status # noqa: E402 +from dekc_pack import pack # noqa: E402 +from dekc_search import candidate_files, search # noqa: E402 + +FAKE_RG = ROOT / "tests/fixtures/fake_rg.py" +SAMPLE = ROOT / "sample-knowledge" + + +class TestRipgrepAccelerator(unittest.TestCase): + def setUp(self): + self._saved = os.environ.get("DEKC_RG_PATH") + FAKE_RG.chmod(0o755) + os.environ["DEKC_RG_PATH"] = str(FAKE_RG) + + def tearDown(self): + if self._saved is None: + os.environ.pop("DEKC_RG_PATH", None) + else: + os.environ["DEKC_RG_PATH"] = self._saved + + def test_find_rg_honors_env(self): + self.assertEqual(Path(find_rg()).resolve(), FAKE_RG.resolve()) + + def test_search_rg_matches_scan_ranking(self): + scan, scan_engine = search(SAMPLE, "revenue", limit=10, use_rg=False, use_index=False) + accel, accel_engine = search(SAMPLE, "revenue", limit=10, use_rg=True, use_index=False) + self.assertGreaterEqual(len(scan), 1) + self.assertEqual([h["path"] for h in scan], [h["path"] for h in accel]) + self.assertEqual([h["score"] for h in scan], [h["score"] for h in accel]) + self.assertEqual(accel_engine, "rg") + self.assertEqual(scan_engine, "scan") + + def test_search_and_terms_intersect(self): + files, engine = candidate_files( + SAMPLE, ["revenue", "zzzz-no-such-term"], use_rg=True, use_index=False + ) + self.assertEqual(engine, "rg") + self.assertEqual(files, []) + + def test_pack_rg_matches_scan_graph(self): + # Sample 2-hop subgraph is >20 nodes; cap would make BFS order a + # false identity failure. Identity is the set, not the clip. + scan = pack(SAMPLE, "tables/gold-order-daily.md", hops=2, max_nodes=80, use_rg=False, use_index=False) + accel = pack(SAMPLE, "tables/gold-order-daily.md", hops=2, max_nodes=80, use_rg=True, use_index=False) + self.assertEqual(scan["node_count"], accel["node_count"]) + self.assertEqual( + sorted(n["path"] for n in scan["nodes"]), + sorted(n["path"] for n in accel["nodes"]), + ) + self.assertEqual(accel["reverse_index"], "rg") + self.assertEqual(scan["reverse_index"], "scan") + self.assertGreaterEqual(scan["node_count"], 5) + + +class TestSqliteIndex(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + (self.tmp / "index.md").write_text( + '---\nokf_version: "0.2"\ntitle: t\n---\n', encoding="utf-8" + ) + (self.tmp / "log.md").write_text("# log\n", encoding="utf-8") + tables = self.tmp / "tables" + tables.mkdir() + (tables / "root.md").write_text( + "---\n" + "type: Table\n" + "title: Lumenfield Orders\n" + "description: gold revenue fact\n" + "layer: gold\n" + "tags: [orders, revenue]\n" + "links:\n" + " - target: /tables/neighbor.md\n" + " rel: feeds\n" + "---\n" + "# Lumenfield Orders\n\nDaily GMV.\n", + encoding="utf-8", + ) + (tables / "neighbor.md").write_text( + "---\n" + "type: Table\n" + "title: Neighbor\n" + "description: upstream bronze\n" + "layer: bronze\n" + "---\n" + "# Neighbor\n", + encoding="utf-8", + ) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_refresh_then_noop(self): + first = refresh(self.tmp, force=True) + self.assertIsNotNone(first) + self.assertTrue(first.ok) + self.assertEqual(first.parsed, 2) + second = refresh(self.tmp) + self.assertEqual(second.parsed, 0) + self.assertEqual(second.unchanged, 2) + + def test_search_index_matches_scan(self): + scan, scan_engine = search(self.tmp, "revenue", use_index=False, use_rg=False) + idx, idx_engine = search(self.tmp, "revenue", use_index=True, use_rg=False) + self.assertEqual(idx_engine, "index") + self.assertEqual(scan_engine, "scan") + self.assertGreaterEqual(len(scan), 1) + self.assertEqual([h["path"] for h in scan], [h["path"] for h in idx]) + self.assertEqual([h["score"] for h in scan], [h["score"] for h in idx]) + + def test_pack_index_matches_scan(self): + scan = pack(self.tmp, "tables/root.md", hops=1, max_nodes=8, use_rg=False, use_index=False) + idx = pack(self.tmp, "tables/root.md", hops=1, max_nodes=8, use_rg=False, use_index=True) + self.assertEqual(idx["reverse_index"], "index") + self.assertEqual(scan["reverse_index"], "scan") + self.assertEqual( + sorted(n["path"] for n in scan["nodes"]), + sorted(n["path"] for n in idx["nodes"]), + ) + self.assertIn("/tables/neighbor.md", [n["path"] for n in idx["nodes"]]) + + def test_touch_reparses_one_file(self): + refresh(self.tmp, force=True) + target = self.tmp / "tables/root.md" + target.write_text(target.read_text(encoding="utf-8") + "\nmore\n", encoding="utf-8") + stats = refresh(self.tmp) + self.assertEqual(stats.parsed, 1) + self.assertEqual(stats.unchanged, 1) + + def test_delete_drops_row(self): + refresh(self.tmp, force=True) + (self.tmp / "tables/neighbor.md").unlink() + stats = refresh(self.tmp) + self.assertEqual(stats.deleted, 1) + self.assertEqual(status(self.tmp)["nodes"], 1) + + def test_drop_removes_sqlite(self): + refresh(self.tmp, force=True) + self.assertTrue(status(self.tmp)["present"]) + self.assertTrue(drop_index(self.tmp)) + self.assertFalse(status(self.tmp)["present"]) + + def test_no_json_index_written(self): + refresh(self.tmp, force=True) + self.assertFalse((self.tmp / ".index").exists()) + self.assertTrue((self.tmp / ".dekc" / "index.sqlite").is_file()) + + +class TestDoctorToolchain(unittest.TestCase): + def test_doctor_reports_sqlite_index(self): + report = doctor(SAMPLE) + self.assertIn("toolchain", report) + self.assertIn("fts5", report["toolchain"]["sqlite"]) + self.assertIn("index", report) + self.assertTrue(str(report["index"]["path"]).endswith("index.sqlite")) + + +class TestSampleSearchCompat(unittest.TestCase): + def test_revenue_still_hits(self): + hits, engine = search(SAMPLE, "revenue", limit=5) + self.assertTrue(hits) + self.assertIn(engine, {"index", "rg", "scan"}) + + +if __name__ == "__main__": + unittest.main()