diff --git a/README.md b/README.md index b678359e..a560f00e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ![Platforms](https://img.shields.io/badge/platforms-macOS%20arm64%20%7C%20Linux%20x64%20%7C%20Windows%20x64-lightgrey) [Get started](#get-started) · [See it in action](#see-it-in-action) · [Features](#features) · -[Benchmarks](#benchmarks) · [How it works](#how-it-works) · [Docs](docs/README.md) +[Benchmarks](docs/benchmarks.md) · [How it works](#how-it-works) · [Docs](docs/README.md) @@ -82,19 +82,6 @@ zerograph explain FR-PAY-22 # one node and everything attached to | Optional spend | `ask --rerank`, a model picking seeds from a 200-deep pool per question ([rerank](docs/rerank.md)) | | Bench | `zerograph bench` fails the process when a recall floor is missed ([benchmarks](docs/benchmarks.md)) | -## Benchmarks - -On the recorded 82 cases with the default embedder, at 244 p90 tokens an answer: - -| Suite | Cases | Default | `--no-dense` | -| --- | --- | --- | --- | -| Keyword | 40 | 40/40 | 39/40 | -| Paraphrase | 30 | 19/30 | 7/30 | -| Code | 12 | 12/12 | 12/12 | - -The graph itself costs nothing to build. Floors, exit codes and the full tables: -[benchmarks](docs/benchmarks.md). - ## Configuration `zerograph.toml` at the repository root, plus an optional machine file at diff --git a/bench/history/run-zerograph.sh b/bench/history/run-zerograph.sh index 5ff6d7be..f15cb473 100755 --- a/bench/history/run-zerograph.sh +++ b/bench/history/run-zerograph.sh @@ -30,7 +30,8 @@ trap 'rm -rf "$OUT"' EXIT # A run recorded against a fixture that moved is a run whose history rows cannot be compared, # and the comparison would silently look like a regression in zerograph. -PINNED=$(git -C "$FIXTURE" rev-parse --short HEAD) +# A fixed length: the default abbreviation grows with the object count and stops matching. +PINNED=$(git -C "$FIXTURE" rev-parse --short=8 HEAD) [ "$PINNED" = "502e8a6d" ] || { echo "fixture is at $PINNED, expected 502e8a6d" >&2; exit 1; } # `graphify-out/` is tracked in the corpus and rebuilt in place by graphify, so it is dirty by # design and says nothing about the sources. Everything else being dirty does. diff --git a/bench/history/runs.jsonl b/bench/history/runs.jsonl index b5866292..89cb2cd7 100644 --- a/bench/history/runs.jsonl +++ b/bench/history/runs.jsonl @@ -185,3 +185,5 @@ {"anchors": {"code": [12, 12], "keyword": [40, 40], "paraphrase": [9, 30]}, "arm": "bench:dense+raw", "cases": {"code/apps/api/scripts/emit-openapi.ts": 1.0, "code/apps/api/src/modules/availability/availability-snapshot.service.ts": 1.0, "code/packages/config/vitest.base.ts": 1.0, "code/packages/contracts/src/errors.ts": 1.0, "code/packages/contracts/src/money.ts": 1.0, "code/packages/db/src/connection.ts": 1.0, "code/packages/domain/src/availability/segments.ts": 1.0, "code/packages/domain/src/sync/merge.ts": 1.0, "code/packages/domain/src/time/index.ts": 1.0, "code/packages/ports/src/fiscal.ts": 1.0, "code/packages/ui/src/components/ui/badge.tsx": 1.0, "code/tools/tasks/src/cli.ts": 1.0, "keyword/ADR-024": 1.0, "keyword/BE-M11": 1.0, "keyword/FR-AI-138": 1.0, "keyword/FR-AI-145": 1.0, "keyword/FR-AI-97": 1.0, "keyword/FR-APP-02": 1.0, "keyword/FR-APP-64": 1.0, "keyword/FR-APP-67": 1.0, "keyword/FR-CAL-43": 1.0, "keyword/FR-CRM-57": 1.0, "keyword/FR-DM-128": 1.0, "keyword/FR-LIFE-29": 1.0, "keyword/FR-MIG-49": 1.0, "keyword/FR-MKT-25": 1.0, "keyword/FR-OPS-81": 1.0, "keyword/FR-PAY-110": 1.0, "keyword/FR-PAY-133": 1.0, "keyword/FR-PAY-143": 1.0, "keyword/FR-PAY-26": 1.0, "keyword/FR-PAY-28": 1.0, "keyword/FR-PAY-54": 1.0, "keyword/FR-PAY-82": 1.0, "keyword/FR-PH-43": 1.0, "keyword/FR-RPT-04": 1.0, "keyword/FR-SEC-44": 1.0, "keyword/FR-SHELL-21": 1.0, "keyword/FR-STAFF-63": 1.0, "keyword/FR-SVC-04": 1.0, "keyword/FR-SVC-50": 1.0, "keyword/FR-TOOL-18": 1.0, "keyword/FR-VIS-35": 1.0, "keyword/FR-VIS-55": 1.0, "keyword/FR-VIS-80": 1.0, "keyword/FR-VIS-85": 1.0, "keyword/FR-WEB-22": 1.0, "keyword/FR-WH-05": 1.0, "keyword/FR-WH-53": 1.0, "keyword/INV-19": 1.0, "keyword/NFR-STAFF-04": 1.0, "keyword/W-206": 1.0, "paraphrase/ADR-004": 0.0, "paraphrase/ADR-005": 0.0, "paraphrase/FR-AI-102": 0.0, "paraphrase/FR-AI-21": 0.0, "paraphrase/FR-AI-66": 1.0, "paraphrase/FR-CAL-06": 1.0, "paraphrase/FR-CAL-101": 0.0, "paraphrase/FR-CRM-11": 0.0, "paraphrase/FR-DM-66": 0.0, "paraphrase/FR-LIFE-17": 1.0, "paraphrase/FR-MIG-17": 0.0, "paraphrase/FR-MKT-35": 0.0, "paraphrase/FR-OPS-24": 1.0, "paraphrase/FR-PAY-03": 1.0, "paraphrase/FR-PAY-104": 1.0, "paraphrase/FR-PAY-49": 0.0, "paraphrase/FR-RPT-13": 0.0, "paraphrase/FR-SEC-21": 0.0, "paraphrase/FR-SHELL-52": 1.0, "paraphrase/FR-STAFF-45": 0.0, "paraphrase/FR-SVC-15": 0.0, "paraphrase/FR-SVC-39": 0.0, "paraphrase/FR-TOOL-18": 0.0, "paraphrase/FR-TOOL-22": 1.0, "paraphrase/FR-VIS-01": 0.0, "paraphrase/FR-VIS-76": 1.0, "paraphrase/FR-WEB-34": 0.0, "paraphrase/FR-WH-18": 0.0, "paraphrase/INV-16": 0.0, "paraphrase/N-109": 0.0}, "corpus": "beauty-crm", "corpus_commit": "502e8a6d2", "coverage": null, "floors": {"code": 12, "keyword": 40, "p90_tokens": 250, "paraphrase": 9}, "gated": true, "green": true, "headroom": {"code": 0, "keyword": 0, "p90_tokens": 13, "paraphrase": 0}, "metrics": {"code": [12, 12], "keyword": [40, 40], "p90_tokens": 237, "paraphrase": [9, 30]}, "note": "generic engine step 24 (defaults + OCaml on), dense-dense, e5-small, store built by 360a4c2 after reset", "source": "bench", "suite": "built-in", "tool": "repograph", "tool_commit": "360a4c2", "tool_dirty": false, "when": "2026-10-13T21:25:49+00:00", "worst_tokens": [["keyword/BE-M11", 350], ["paraphrase/FR-CAL-06", 324], ["keyword/W-206", 284]]} {"anchors": {"code": [12, 12], "keyword": [40, 40], "paraphrase": [20, 30]}, "arm": "bench:dense+raw+gemma", "cases": {"code/apps/api/scripts/emit-openapi.ts": 1.0, "code/apps/api/src/modules/availability/availability-snapshot.service.ts": 1.0, "code/packages/config/vitest.base.ts": 1.0, "code/packages/contracts/src/errors.ts": 1.0, "code/packages/contracts/src/money.ts": 1.0, "code/packages/db/src/connection.ts": 1.0, "code/packages/domain/src/availability/segments.ts": 1.0, "code/packages/domain/src/sync/merge.ts": 1.0, "code/packages/domain/src/time/index.ts": 1.0, "code/packages/ports/src/fiscal.ts": 1.0, "code/packages/ui/src/components/ui/badge.tsx": 1.0, "code/tools/tasks/src/cli.ts": 1.0, "keyword/ADR-024": 1.0, "keyword/BE-M11": 1.0, "keyword/FR-AI-138": 1.0, "keyword/FR-AI-145": 1.0, "keyword/FR-AI-97": 1.0, "keyword/FR-APP-02": 1.0, "keyword/FR-APP-64": 1.0, "keyword/FR-APP-67": 1.0, "keyword/FR-CAL-43": 1.0, "keyword/FR-CRM-57": 1.0, "keyword/FR-DM-128": 1.0, "keyword/FR-LIFE-29": 1.0, "keyword/FR-MIG-49": 1.0, "keyword/FR-MKT-25": 1.0, "keyword/FR-OPS-81": 1.0, "keyword/FR-PAY-110": 1.0, "keyword/FR-PAY-133": 1.0, "keyword/FR-PAY-143": 1.0, "keyword/FR-PAY-26": 1.0, "keyword/FR-PAY-28": 1.0, "keyword/FR-PAY-54": 1.0, "keyword/FR-PAY-82": 1.0, "keyword/FR-PH-43": 1.0, "keyword/FR-RPT-04": 1.0, "keyword/FR-SEC-44": 1.0, "keyword/FR-SHELL-21": 1.0, "keyword/FR-STAFF-63": 1.0, "keyword/FR-SVC-04": 1.0, "keyword/FR-SVC-50": 1.0, "keyword/FR-TOOL-18": 1.0, "keyword/FR-VIS-35": 1.0, "keyword/FR-VIS-55": 1.0, "keyword/FR-VIS-80": 1.0, "keyword/FR-VIS-85": 1.0, "keyword/FR-WEB-22": 1.0, "keyword/FR-WH-05": 1.0, "keyword/FR-WH-53": 1.0, "keyword/INV-19": 1.0, "keyword/NFR-STAFF-04": 1.0, "keyword/W-206": 1.0, "paraphrase/ADR-004": 1.0, "paraphrase/ADR-005": 1.0, "paraphrase/FR-AI-102": 1.0, "paraphrase/FR-AI-21": 1.0, "paraphrase/FR-AI-66": 1.0, "paraphrase/FR-CAL-06": 1.0, "paraphrase/FR-CAL-101": 0.0, "paraphrase/FR-CRM-11": 1.0, "paraphrase/FR-DM-66": 0.0, "paraphrase/FR-LIFE-17": 1.0, "paraphrase/FR-MIG-17": 0.0, "paraphrase/FR-MKT-35": 1.0, "paraphrase/FR-OPS-24": 1.0, "paraphrase/FR-PAY-03": 1.0, "paraphrase/FR-PAY-104": 1.0, "paraphrase/FR-PAY-49": 0.0, "paraphrase/FR-RPT-13": 0.0, "paraphrase/FR-SEC-21": 0.0, "paraphrase/FR-SHELL-52": 1.0, "paraphrase/FR-STAFF-45": 0.0, "paraphrase/FR-SVC-15": 1.0, "paraphrase/FR-SVC-39": 0.0, "paraphrase/FR-TOOL-18": 1.0, "paraphrase/FR-TOOL-22": 1.0, "paraphrase/FR-VIS-01": 1.0, "paraphrase/FR-VIS-76": 1.0, "paraphrase/FR-WEB-34": 1.0, "paraphrase/FR-WH-18": 1.0, "paraphrase/INV-16": 0.0, "paraphrase/N-109": 0.0}, "corpus": "beauty-crm", "corpus_commit": "502e8a6d2", "coverage": null, "floors": {"code": 12, "keyword": 40, "p90_tokens": 250, "paraphrase": 18}, "gated": true, "green": true, "headroom": {"code": 0, "keyword": 0, "p90_tokens": 2, "paraphrase": 2}, "metrics": {"code": [12, 12], "keyword": [40, 40], "p90_tokens": 248, "paraphrase": [20, 30]}, "note": "0.6.0 release check on fix/release-0-6-0-hardening, gemma default, raw store, ~/bench/beauty-crm-test re-embedded by embed", "source": "bench", "suite": "built-in", "tool": "repograph", "tool_commit": "d9593d8", "tool_dirty": false, "when": "2026-10-15T10:54:04+00:00", "worst_tokens": [["paraphrase/ADR-004", 315], ["keyword/BE-M11", 296], ["paraphrase/FR-CAL-06", 296]]} {"anchors": {"code": [12, 12], "keyword": [39, 40], "paraphrase": [7, 30]}, "arm": "bench:lexical+raw", "cases": {"code/apps/api/scripts/emit-openapi.ts": 1.0, "code/apps/api/src/modules/availability/availability-snapshot.service.ts": 1.0, "code/packages/config/vitest.base.ts": 1.0, "code/packages/contracts/src/errors.ts": 1.0, "code/packages/contracts/src/money.ts": 1.0, "code/packages/db/src/connection.ts": 1.0, "code/packages/domain/src/availability/segments.ts": 1.0, "code/packages/domain/src/sync/merge.ts": 1.0, "code/packages/domain/src/time/index.ts": 1.0, "code/packages/ports/src/fiscal.ts": 1.0, "code/packages/ui/src/components/ui/badge.tsx": 1.0, "code/tools/tasks/src/cli.ts": 1.0, "keyword/ADR-024": 1.0, "keyword/BE-M11": 1.0, "keyword/FR-AI-138": 1.0, "keyword/FR-AI-145": 1.0, "keyword/FR-AI-97": 1.0, "keyword/FR-APP-02": 1.0, "keyword/FR-APP-64": 1.0, "keyword/FR-APP-67": 1.0, "keyword/FR-CAL-43": 1.0, "keyword/FR-CRM-57": 1.0, "keyword/FR-DM-128": 1.0, "keyword/FR-LIFE-29": 1.0, "keyword/FR-MIG-49": 1.0, "keyword/FR-MKT-25": 1.0, "keyword/FR-OPS-81": 1.0, "keyword/FR-PAY-110": 1.0, "keyword/FR-PAY-133": 1.0, "keyword/FR-PAY-143": 1.0, "keyword/FR-PAY-26": 1.0, "keyword/FR-PAY-28": 1.0, "keyword/FR-PAY-54": 1.0, "keyword/FR-PAY-82": 1.0, "keyword/FR-PH-43": 0.0, "keyword/FR-RPT-04": 1.0, "keyword/FR-SEC-44": 1.0, "keyword/FR-SHELL-21": 1.0, "keyword/FR-STAFF-63": 1.0, "keyword/FR-SVC-04": 1.0, "keyword/FR-SVC-50": 1.0, "keyword/FR-TOOL-18": 1.0, "keyword/FR-VIS-35": 1.0, "keyword/FR-VIS-55": 1.0, "keyword/FR-VIS-80": 1.0, "keyword/FR-VIS-85": 1.0, "keyword/FR-WEB-22": 1.0, "keyword/FR-WH-05": 1.0, "keyword/FR-WH-53": 1.0, "keyword/INV-19": 1.0, "keyword/NFR-STAFF-04": 1.0, "keyword/W-206": 1.0, "paraphrase/ADR-004": 0.0, "paraphrase/ADR-005": 0.0, "paraphrase/FR-AI-102": 0.0, "paraphrase/FR-AI-21": 0.0, "paraphrase/FR-AI-66": 1.0, "paraphrase/FR-CAL-06": 0.0, "paraphrase/FR-CAL-101": 0.0, "paraphrase/FR-CRM-11": 0.0, "paraphrase/FR-DM-66": 0.0, "paraphrase/FR-LIFE-17": 1.0, "paraphrase/FR-MIG-17": 0.0, "paraphrase/FR-MKT-35": 0.0, "paraphrase/FR-OPS-24": 1.0, "paraphrase/FR-PAY-03": 0.0, "paraphrase/FR-PAY-104": 0.0, "paraphrase/FR-PAY-49": 0.0, "paraphrase/FR-RPT-13": 0.0, "paraphrase/FR-SEC-21": 0.0, "paraphrase/FR-SHELL-52": 1.0, "paraphrase/FR-STAFF-45": 0.0, "paraphrase/FR-SVC-15": 0.0, "paraphrase/FR-SVC-39": 0.0, "paraphrase/FR-TOOL-18": 0.0, "paraphrase/FR-TOOL-22": 0.0, "paraphrase/FR-VIS-01": 0.0, "paraphrase/FR-VIS-76": 1.0, "paraphrase/FR-WEB-34": 0.0, "paraphrase/FR-WH-18": 0.0, "paraphrase/INV-16": 1.0, "paraphrase/N-109": 1.0}, "corpus": "beauty-crm", "corpus_commit": "502e8a6d2", "coverage": null, "floors": {"code": 12, "keyword": 39, "p90_tokens": 250, "paraphrase": 7}, "gated": true, "green": true, "headroom": {"code": 0, "keyword": 0, "p90_tokens": 12, "paraphrase": 0}, "metrics": {"code": [12, 12], "keyword": [39, 40], "p90_tokens": 238, "paraphrase": [7, 30]}, "note": "0.6.0 release check on fix/release-0-6-0-hardening, gemma default, raw store, ~/bench/beauty-crm-test re-embedded by embed", "source": "bench", "suite": "built-in", "tool": "repograph", "tool_commit": "d9593d8", "tool_dirty": false, "when": "2026-10-15T10:54:04+00:00", "worst_tokens": [["keyword/BE-M11", 306], ["keyword/W-206", 283], ["paraphrase/FR-VIS-76", 266]]} +{"anchors": {"code": [12, 12], "keyword": [40, 40], "paraphrase": [20, 30]}, "arm": "bench:dense+raw+gemma", "cases": {"code/apps/api/scripts/emit-openapi.ts": 1.0, "code/apps/api/src/modules/availability/availability-snapshot.service.ts": 1.0, "code/packages/config/vitest.base.ts": 1.0, "code/packages/contracts/src/errors.ts": 1.0, "code/packages/contracts/src/money.ts": 1.0, "code/packages/db/src/connection.ts": 1.0, "code/packages/domain/src/availability/segments.ts": 1.0, "code/packages/domain/src/sync/merge.ts": 1.0, "code/packages/domain/src/time/index.ts": 1.0, "code/packages/ports/src/fiscal.ts": 1.0, "code/packages/ui/src/components/ui/badge.tsx": 1.0, "code/tools/tasks/src/cli.ts": 1.0, "keyword/ADR-024": 1.0, "keyword/BE-M11": 1.0, "keyword/FR-AI-138": 1.0, "keyword/FR-AI-145": 1.0, "keyword/FR-AI-97": 1.0, "keyword/FR-APP-02": 1.0, "keyword/FR-APP-64": 1.0, "keyword/FR-APP-67": 1.0, "keyword/FR-CAL-43": 1.0, "keyword/FR-CRM-57": 1.0, "keyword/FR-DM-128": 1.0, "keyword/FR-LIFE-29": 1.0, "keyword/FR-MIG-49": 1.0, "keyword/FR-MKT-25": 1.0, "keyword/FR-OPS-81": 1.0, "keyword/FR-PAY-110": 1.0, "keyword/FR-PAY-133": 1.0, "keyword/FR-PAY-143": 1.0, "keyword/FR-PAY-26": 1.0, "keyword/FR-PAY-28": 1.0, "keyword/FR-PAY-54": 1.0, "keyword/FR-PAY-82": 1.0, "keyword/FR-PH-43": 1.0, "keyword/FR-RPT-04": 1.0, "keyword/FR-SEC-44": 1.0, "keyword/FR-SHELL-21": 1.0, "keyword/FR-STAFF-63": 1.0, "keyword/FR-SVC-04": 1.0, "keyword/FR-SVC-50": 1.0, "keyword/FR-TOOL-18": 1.0, "keyword/FR-VIS-35": 1.0, "keyword/FR-VIS-55": 1.0, "keyword/FR-VIS-80": 1.0, "keyword/FR-VIS-85": 1.0, "keyword/FR-WEB-22": 1.0, "keyword/FR-WH-05": 1.0, "keyword/FR-WH-53": 1.0, "keyword/INV-19": 1.0, "keyword/NFR-STAFF-04": 1.0, "keyword/W-206": 1.0, "paraphrase/ADR-004": 1.0, "paraphrase/ADR-005": 1.0, "paraphrase/FR-AI-102": 1.0, "paraphrase/FR-AI-21": 1.0, "paraphrase/FR-AI-66": 1.0, "paraphrase/FR-CAL-06": 1.0, "paraphrase/FR-CAL-101": 0.0, "paraphrase/FR-CRM-11": 1.0, "paraphrase/FR-DM-66": 0.0, "paraphrase/FR-LIFE-17": 1.0, "paraphrase/FR-MIG-17": 0.0, "paraphrase/FR-MKT-35": 1.0, "paraphrase/FR-OPS-24": 1.0, "paraphrase/FR-PAY-03": 1.0, "paraphrase/FR-PAY-104": 1.0, "paraphrase/FR-PAY-49": 0.0, "paraphrase/FR-RPT-13": 0.0, "paraphrase/FR-SEC-21": 0.0, "paraphrase/FR-SHELL-52": 1.0, "paraphrase/FR-STAFF-45": 0.0, "paraphrase/FR-SVC-15": 1.0, "paraphrase/FR-SVC-39": 0.0, "paraphrase/FR-TOOL-18": 1.0, "paraphrase/FR-TOOL-22": 1.0, "paraphrase/FR-VIS-01": 1.0, "paraphrase/FR-VIS-76": 1.0, "paraphrase/FR-WEB-34": 1.0, "paraphrase/FR-WH-18": 1.0, "paraphrase/INV-16": 0.0, "paraphrase/N-109": 0.0}, "corpus": "beauty-crm", "corpus_commit": "502e8a6d2", "coverage": null, "floors": {"code": 12, "keyword": 40, "p90_tokens": 250, "paraphrase": 18}, "gated": true, "green": true, "headroom": {"code": 0, "keyword": 0, "p90_tokens": 28, "paraphrase": 2}, "metrics": {"code": [12, 12], "keyword": [40, 40], "p90_tokens": 222, "paraphrase": [20, 30]}, "note": "", "source": "bench", "suite": "built-in", "tool": "zerograph", "tool_commit": "4bd5675", "tool_dirty": false, "when": "2026-10-17T17:58:54+00:00", "worst_tokens": [["keyword/BE-M11", 294], ["paraphrase/ADR-004", 267], ["keyword/W-206", 259]]} +{"anchors": {"code": [12, 12], "keyword": [39, 40], "paraphrase": [7, 30]}, "arm": "bench:lexical+raw", "cases": {"code/apps/api/scripts/emit-openapi.ts": 1.0, "code/apps/api/src/modules/availability/availability-snapshot.service.ts": 1.0, "code/packages/config/vitest.base.ts": 1.0, "code/packages/contracts/src/errors.ts": 1.0, "code/packages/contracts/src/money.ts": 1.0, "code/packages/db/src/connection.ts": 1.0, "code/packages/domain/src/availability/segments.ts": 1.0, "code/packages/domain/src/sync/merge.ts": 1.0, "code/packages/domain/src/time/index.ts": 1.0, "code/packages/ports/src/fiscal.ts": 1.0, "code/packages/ui/src/components/ui/badge.tsx": 1.0, "code/tools/tasks/src/cli.ts": 1.0, "keyword/ADR-024": 1.0, "keyword/BE-M11": 1.0, "keyword/FR-AI-138": 1.0, "keyword/FR-AI-145": 1.0, "keyword/FR-AI-97": 1.0, "keyword/FR-APP-02": 1.0, "keyword/FR-APP-64": 1.0, "keyword/FR-APP-67": 1.0, "keyword/FR-CAL-43": 1.0, "keyword/FR-CRM-57": 1.0, "keyword/FR-DM-128": 1.0, "keyword/FR-LIFE-29": 1.0, "keyword/FR-MIG-49": 1.0, "keyword/FR-MKT-25": 1.0, "keyword/FR-OPS-81": 1.0, "keyword/FR-PAY-110": 1.0, "keyword/FR-PAY-133": 1.0, "keyword/FR-PAY-143": 1.0, "keyword/FR-PAY-26": 1.0, "keyword/FR-PAY-28": 1.0, "keyword/FR-PAY-54": 1.0, "keyword/FR-PAY-82": 1.0, "keyword/FR-PH-43": 0.0, "keyword/FR-RPT-04": 1.0, "keyword/FR-SEC-44": 1.0, "keyword/FR-SHELL-21": 1.0, "keyword/FR-STAFF-63": 1.0, "keyword/FR-SVC-04": 1.0, "keyword/FR-SVC-50": 1.0, "keyword/FR-TOOL-18": 1.0, "keyword/FR-VIS-35": 1.0, "keyword/FR-VIS-55": 1.0, "keyword/FR-VIS-80": 1.0, "keyword/FR-VIS-85": 1.0, "keyword/FR-WEB-22": 1.0, "keyword/FR-WH-05": 1.0, "keyword/FR-WH-53": 1.0, "keyword/INV-19": 1.0, "keyword/NFR-STAFF-04": 1.0, "keyword/W-206": 1.0, "paraphrase/ADR-004": 0.0, "paraphrase/ADR-005": 0.0, "paraphrase/FR-AI-102": 0.0, "paraphrase/FR-AI-21": 0.0, "paraphrase/FR-AI-66": 1.0, "paraphrase/FR-CAL-06": 0.0, "paraphrase/FR-CAL-101": 0.0, "paraphrase/FR-CRM-11": 0.0, "paraphrase/FR-DM-66": 0.0, "paraphrase/FR-LIFE-17": 1.0, "paraphrase/FR-MIG-17": 0.0, "paraphrase/FR-MKT-35": 0.0, "paraphrase/FR-OPS-24": 1.0, "paraphrase/FR-PAY-03": 0.0, "paraphrase/FR-PAY-104": 0.0, "paraphrase/FR-PAY-49": 0.0, "paraphrase/FR-RPT-13": 0.0, "paraphrase/FR-SEC-21": 0.0, "paraphrase/FR-SHELL-52": 1.0, "paraphrase/FR-STAFF-45": 0.0, "paraphrase/FR-SVC-15": 0.0, "paraphrase/FR-SVC-39": 0.0, "paraphrase/FR-TOOL-18": 0.0, "paraphrase/FR-TOOL-22": 0.0, "paraphrase/FR-VIS-01": 0.0, "paraphrase/FR-VIS-76": 1.0, "paraphrase/FR-WEB-34": 0.0, "paraphrase/FR-WH-18": 0.0, "paraphrase/INV-16": 1.0, "paraphrase/N-109": 1.0}, "corpus": "beauty-crm", "corpus_commit": "502e8a6d2", "coverage": null, "floors": {"code": 12, "keyword": 39, "p90_tokens": 250, "paraphrase": 7}, "gated": true, "green": true, "headroom": {"code": 0, "keyword": 0, "p90_tokens": 22, "paraphrase": 0}, "metrics": {"code": [12, 12], "keyword": [39, 40], "p90_tokens": 228, "paraphrase": [7, 30]}, "note": "", "source": "bench", "suite": "built-in", "tool": "zerograph", "tool_commit": "4bd5675", "tool_dirty": false, "when": "2026-10-17T17:58:54+00:00", "worst_tokens": [["keyword/BE-M11", 304], ["keyword/W-206", 260], ["paraphrase/FR-VIS-76", 241]]} diff --git a/docs/benchmarks.md b/docs/benchmarks.md index b33d6b2a..6711a434 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -65,9 +65,9 @@ are consulted, are in [the measurements](history.md#the-bench-floors-case-by-cas ## Measured -The bench fixture is 908 files and about 8.3k nodes. At 0.6.0, under the default embedder, the store -reads **keyword 40/40, paraphrase 19/30, code 12/12 at 244 p90 tokens**. Under the small model that -preceded the default it reads 40/40, 9/30, 12/12 at 237 with embeddings and 39/40, 7/30, 12/12 at +The bench fixture is 908 files and about 8.3k nodes. At 0.6.3, under the default embedder, the store +reads **keyword 40/40, paraphrase 20/30, code 12/12 at 222 p90 tokens** (19/30 at 244 at 0.6.0). At 0.6.0, under the +small model that preceded the default, it read 40/40, 9/30, 12/12 at 237 with embeddings and 39/40, 7/30, 12/12 at 238 with `--no-dense`, per case identical to the binary before the removal on the same store. Zero model tokens throughout. What `enrich`'s questions bought before 0.6.0 removed them — one paraphrase case under the default embedder, five under the small one — is in [the measurements](history.md).