From ad0977ac27fb9c15a7625c9a759450e184be5b6f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 18 Aug 2026 14:24:27 -0600 Subject: [PATCH 1/3] =?UTF-8?q?feat(experiment):=20add=20the=20evidence=20?= =?UTF-8?q?registry=20=E2=80=94=20one=20home=20for=20measured=20claims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A measured claim now has one canonical, machine-readable home. evidence/records/*.json hold typed records: claim, instrument, exact command, arms, denominator, result, evidence state (CERTIFIED, MEASURED-ONCE, RESOLVED-NULL, UNVERIFIED, KILLED), artifacts, cost, confounds, and the governing experiment seal digest. The schema (evidenceRegistryRecordSchema) is exported from ./experiment. scripts/render-evidence-index.ts generates evidence/INDEX.md from the records, and evidence:check inside verify:package fails on an invalid record or a stale index, so the human index cannot drift from the data. Initial migration: seven records — gated-stop confirm (+0.0596, CI [-0.0061,+0.1210], n=151), free-lunch (3/64 rollouts), multishot golden oracle v1, the GEPA-certified analyst prompt, prime-vs-dspy (38 rows), CAD GEPA (+9.5pp), and the VB cell-engine parity no-flip that lived only in a PR body. --- CHANGELOG.md | 5 + README.md | 2 + docs/experiment.md | 2 + evidence/INDEX.md | 175 +++++++++++++++ evidence/README.md | 37 ++++ .../records/cad-gepa-openscad-directive.json | 26 +++ .../ctb-analyst-gepa-certified-prompt.json | 29 +++ .../records/multishot-golden-oracle-v1.json | 28 +++ .../prime-vs-dspy-analyst-38-rows.json | 28 +++ .../records/tb-gated-stop-ab-confirm.json | 31 +++ evidence/records/tb-repair-free-lunch.json | 29 +++ .../vb-cell-engine-parity-no-flip.json | 29 +++ scripts/render-evidence-index.ts | 79 +++++++ src/experiment/evidence-record.test.ts | 137 ++++++++++++ src/experiment/evidence-record.ts | 205 ++++++++++++++++++ src/experiment/index.ts | 13 ++ tsconfig.script.json | 12 +- 17 files changed, 865 insertions(+), 2 deletions(-) create mode 100644 evidence/INDEX.md create mode 100644 evidence/README.md create mode 100644 evidence/records/cad-gepa-openscad-directive.json create mode 100644 evidence/records/ctb-analyst-gepa-certified-prompt.json create mode 100644 evidence/records/multishot-golden-oracle-v1.json create mode 100644 evidence/records/prime-vs-dspy-analyst-38-rows.json create mode 100644 evidence/records/tb-gated-stop-ab-confirm.json create mode 100644 evidence/records/tb-repair-free-lunch.json create mode 100644 evidence/records/vb-cell-engine-parity-no-flip.json create mode 100644 scripts/render-evidence-index.ts create mode 100644 src/experiment/evidence-record.test.ts create mode 100644 src/experiment/evidence-record.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b0b6ed..89998e77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ One repository still names three of them. `starter-foundry` imports `runAssertio ### Added +<<<<<<< HEAD - `@tangle-network/agent-eval/experiment` publishes the evidence receipt: `createEvidenceReceipt`, `verifyEvidenceReceipt`, `isIndependentEvidence`, `EVIDENCE_RECEIPT_VERSION`, `EVIDENCE_AUTHORITY_KINDS`, `INDEPENDENT_EVIDENCE_AUTHORITY_KINDS`, and the `EvidenceReceipt` / `EvidenceBinding` / `EvidenceAuthority` / `EvidenceAuthorityKind` / `EvidenceReceiptVerification` / `CreateEvidenceReceiptInput` types. A receipt binds one Runtime execution to the measurement that judged it, without either package importing the other. It carries stable pursuit and run identity, the exact candidate, evaluator, environment, input-set and output content identities, the result digest, and the authority class that made the observation. The payload is attested with the existing canonical report attestation, so mutating any bound field invalidates the receipt. @@ -49,6 +50,10 @@ One repository still names three of them. `starter-foundry` imports `runAssertio ### Changed - No existing export changed. +======= +- The evidence registry: one canonical home for measured claims. `evidence/records/*.json` hold typed records (claim, instrument, exact command, arms, denominator, result, evidence state `CERTIFIED`/`MEASURED-ONCE`/`RESOLVED-NULL`/`UNVERIFIED`/`KILLED`, artifacts, cost, confounds, optional experiment seal digest), validated by `evidenceRecordSchema` from `./experiment`. `pnpm run evidence:render` generates `evidence/INDEX.md` from the records; `pnpm run evidence:check` (inside `verify:package`) fails on an invalid record or a stale index, so the human index can never drift from the data. Initial migration: seven records spanning trace-repair (gated-stop confirm, free-lunch), multishot (golden oracle v1), trace-analysis (GEPA-certified prompt, prime-vs-dspy), creative-cad, and vertical-bench (parity no-flip). +- Add a bounded `SearchHistoryReceipt` over the canonical `SearchLedger`, plus search-history coverage and `searchHistoryPolicy: 'require-complete'` on `compareOptimizationMethods()`. Strict mode refuses missing or denominator-incomplete optimization evidence before untouched-final-test scoring. Rich events remain only in the ledger. +>>>>>>> 14322f5d (feat(experiment): add the evidence registry — one home for measured claims) --- diff --git a/README.md b/README.md index 0a7944a7..11b1071c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Model calls happen only through the clients and agents you configure. New to the package? Read [concepts](./docs/concepts.md) first — it takes five minutes and defines every word used here. +Looking for a measured result (a lift, a null, a parity verdict)? The canonical registry is [`evidence/`](./evidence/README.md) — machine-readable records, a generated index, and a freshness gate. + ## Install ```sh diff --git a/docs/experiment.md b/docs/experiment.md index fa5c2112..7c4b5400 100644 --- a/docs/experiment.md +++ b/docs/experiment.md @@ -2,6 +2,8 @@ `@tangle-network/agent-eval/experiment` turns an experiment's registration into the object that runs it. +The registry of measured claims those experiments produce lives in [`evidence/`](../evidence/README.md); a sealed experiment's digest is the `experimentDigest` its registry record carries. + ## The covenant 1. **The registered rule is the executed rule.** diff --git a/evidence/INDEX.md b/evidence/INDEX.md new file mode 100644 index 00000000..e3922373 --- /dev/null +++ b/evidence/INDEX.md @@ -0,0 +1,175 @@ +# Evidence index + + + +7 records. States: CERTIFIED > MEASURED-ONCE > RESOLVED-NULL > UNVERIFIED > KILLED. + +| state | date | id | claim | result | +| --- | --- | --- | --- | --- | +| CERTIFIED | 2026-08-16 | [`multishot-golden-oracle-v1`](#multishot-golden-oracle-v1) | The multishot loop's behaviour is frozen as deterministic golden records that any engine can be verified against, and the records are load-bearing: they detect every field-level and engine-level deviation. | Consumer parity proven before freezing: tax 4/4 deep-equal, gtm 4/4 shot-level and 8/8 matrix-level. Oracle load-bearing: every single leaf of {outcome, requests} (and {matrix, requests, judgeRequests, files}) mutated one at a time is reported for all 14 records; 8/8 engine mutants detected, each naming the deviating field. | +| CERTIFIED | 2026-08-02 | [`ctb-analyst-gepa-certified-prompt`](#ctb-analyst-gepa-certified-prompt) | The GEPA-optimized DSPy-RLM analyst prompt (sha d3829fb8) is the certified shipping prompt on CodeTraceBench: challengers lose on pooled sealed splits under the pre-registered rule. | G2 REJECTED by rule 1: pooled-cert micro-F1 stock 0.2489 vs G2 0.1928 (-5.6pp). Per-family stock OH 0.2896 / T2 0.2162. Paired CIs [-0.126, 0.058] OH, [-0.143, 0.023] T2. G2's selection-split edge (+0.064 weighted) did not transfer - third consecutive tuning-split-vs-fresh-split reversal in the campaign. The shipping prompt stands, defended twice (round-1 and round-2 challengers). | +| MEASURED-ONCE | 2026-08-15 | [`tb-gated-stop-ab-confirm`](#tb-gated-stop-ab-confirm) | At one matched total token budget on TB-Repair, an agent that may stop only after an executable held-out check passes finishes more rows than the same agent on unconditional continuation. | +0.0596 pass-rate delta (best intermediate state in both arms), 95% cluster-bootstrap CI [-0.0061, +0.1210]. The interval crosses zero: direction positive, not significant at the registered level. | +| MEASURED-ONCE | 2026-08-10 | [`tb-repair-free-lunch`](#tb-repair-free-lunch) | Unconditional continuation from the recorded end state with a real model budget (no analyst, no hint, no gate) rescues failed TB-Repair rows, so admission condition 3 has a free lunch to subtract. | 3 of 64 rollouts rescued (4.7%); 2 of 32 rows rescued at least once (6.2%). Zero-call control: 0 passes in 96 rollouts. Exit statuses: 15 submitted, 46 step-budget-exhausted, 3 model-error. | +| MEASURED-ONCE | 2026-08-08 | [`prime-vs-dspy-analyst-38-rows`](#prime-vs-dspy-analyst-38-rows) | Prime-agent as a one-shot trace analyst beats the GEPA-certified DSPy-RLM analyst on stored-baseline CodeTraceBench rows. | Prime wins the dev-pool comparison: mean per-case scored F1 0.3218 vs 0.1833; 18W/4L/16T (all ties 0-0); two-sided sign test p=0.0043. Direction consistent across both baseline sources including the derivation-verified stock-smoke slice (OH 0.377 vs 0.231, T2 0.416 vs 0.163). | +| MEASURED-ONCE | 2026-06-08 | [`cad-gepa-openscad-directive`](#cad-gepa-openscad-directive) | GEPA optimization of the OpenSCAD system directive lifts the CAD agent's held-out build quality over the hand-written directive. | +9.5pp held-out over the baseline directive; the winning directive shipped with automatic video capture of the built models. | +| RESOLVED-NULL | 2026-08-15 | [`vb-cell-engine-parity-no-flip`](#vb-cell-engine-parity-no-flip) | The VerticalBench graph cell-engine reaches behavioural parity with the loop cell-engine on a live cell, so the default engine can flip to the graph. | NO FLIP. Parity holds on dispatched shot-1 prompt (byte-identical), realness-firewall activation, verification score on the same graded state (blended 0.31 both arms), and toolchain detection. Parity fails on coder turn depth (graph settles recon-only in ~3-4m vs the loop working the full 30m wall; opencode SSE errored 2/4 cells), fixed-budget adherence (structural in the loop vs probabilistic in the graph: 1/3 post-contract cells dispatched shot 2), and board classification (loop 2/2 fair attempts, graph 0/2 - quarantined, never silently scored). | + +## multishot-golden-oracle-v1 + +**CERTIFIED** · 2026-08-16 · multishot · source repo `agent-eval` + +**Claim.** The multishot loop's behaviour is frozen as deterministic golden records that any engine can be verified against, and the records are load-bearing: they detect every field-level and engine-level deviation. + +**Result.** Consumer parity proven before freezing: tax 4/4 deep-equal, gtm 4/4 shot-level and 8/8 matrix-level. Oracle load-bearing: every single leaf of {outcome, requests} (and {matrix, requests, judgeRequests, files}) mutated one at a time is reported for all 14 records; 8/8 engine mutants detected, each naming the deviating field. + +- **Instrument**: golden-record oracle: scripted transports and executors, request ledger + outcome per scenario, double capture with mandatory agreement, exhaustive single-field mutation and engine-mutant detection in the suite +- **Command**: `pnpm vitest run src/multishot/golden/golden.test.ts` +- **Arms**: recorded-loop-v1 vs candidate-engine +- **n**: 14 records (13 shot scenarios + 1 matrix scenario, each captured twice with agreement required; v1 captured from the loop at 0.145.21) +- **Cost**: $0.00 +- **Artifacts**: + - src/multishot/golden/records/v1.json + - src/multishot/golden/golden.test.ts + - docs/multishot-golden-records.md + - https://github.com/tangle-network/agent-eval/pull/627 +- **Confounds**: + - deterministic-only coverage: scripted transports, fixed personas and budgets; live-provider behaviour (latency, substitution, SSE faults) is out of the oracle's scope by design +- **Notes**: CERTIFIED because the check re-runs on every suite run and records are frozen: the recorder refuses to overwrite a version, so a behaviour change mints a new version file and the diff is the reviewable evidence. Later consumer parity counts quoted as 14/14 and 15/15 live in the loop-retention PR chain (#617 and its consumers) - PR-comment-only until migrated here. + +## ctb-analyst-gepa-certified-prompt + +**CERTIFIED** · 2026-08-02 · trace-analysis · source repo `agent-eval` + +**Claim.** The GEPA-optimized DSPy-RLM analyst prompt (sha d3829fb8) is the certified shipping prompt on CodeTraceBench: challengers lose on pooled sealed splits under the pre-registered rule. + +**Result.** G2 REJECTED by rule 1: pooled-cert micro-F1 stock 0.2489 vs G2 0.1928 (-5.6pp). Per-family stock OH 0.2896 / T2 0.2162. Paired CIs [-0.126, 0.058] OH, [-0.143, 0.023] T2. G2's selection-split edge (+0.064 weighted) did not transfer - third consecutive tuning-split-vs-fresh-split reversal in the campaign. The shipping prompt stands, defended twice (round-1 and round-2 challengers). + +- **Instrument**: sealed certification runs: pre-registration written before the winner was known, pooled OH-cert32 + T2-cert32 sealed splits, scored micro-F1 against human step labels, paired CIs +- **Command**: `gepa-analyst-campaign certification phase: 4 serial runs, 2 reps, mutex-held seat (see .evolve/certification-2026-08-02-preregistration.md for the registered rule)` +- **Arms**: stock-d3829fb8 vs g2-challenger-1bb303e7 +- **n**: 64 sealed cases (OH-cert32 + T2-cert32, 2 reps (128 case-runs; G2 failed 3/128, stock 0/128); both cert32 splits spent for this config family) +- **Cost**: $31.07 +- **Artifacts**: + - ~/bench-cache/ctb-20260801/cert2/ (4 result.json + logs + winner-instructions.txt) + - .evolve/certification-2026-08-02-preregistration.md + - .evolve/experiments.jsonl (arm cert2-stock-vs-g2) +- **Confounds**: + - cross-family gap is the dominant residual: shipping analyst OH 0.29 / T2 0.22 vs mini-SWE wide 0.56-0.63 (~2x), later diagnosed as localization (far-class mass), not prompt quality + - gold-mass-weighted selection metric recorded as overfit-prone at n=12 + - CodeTraceBench frozen as an optimization target on 2026-08-04: ~60% of residual misses are missing labels (H1, n=20 adjudication), so absolute F1 under-reads the analyst +- **Notes**: Companion baseline: dev-pool scored-F1 baseline for the dspy-rlm analyst stack is 32.73%/30.99% (retired direct runner excluded). Prime-agent comparison against these stored baselines is recorded separately (prime-vs-dspy-analyst-38-rows). + +## tb-gated-stop-ab-confirm + +**MEASURED-ONCE** · 2026-08-15 · trace-repair · source repo `agent-eval` + +**Claim.** At one matched total token budget on TB-Repair, an agent that may stop only after an executable held-out check passes finishes more rows than the same agent on unconditional continuation. + +**Result.** +0.0596 pass-rate delta (best intermediate state in both arms), 95% cluster-bootstrap CI [-0.0061, +0.1210]. The interval crosses zero: direction positive, not significant at the registered level. + +- **Instrument**: tb-gated-stop-ab confirmatory draw: sealed registered experiment, servedModel abort gate, paired cluster-bootstrap contrast over best intermediate state +- **Command**: `pnpm tsx scripts/tb-gated-stop-ab.ts (design/screen/pilot/confirm phases; seal e13a4f8d re-executed by the confirm runner)` +- **Arms**: blind-continue vs gated-continue +- **n**: 151 rows (22 clusters; settlingDraw at power 0.8009 (search sim) / 0.8013 (registered sim) for effect 0.10; admitted ceiling 216 rows at power 0.9189) +- **Cost**: not captured +- **Experiment seal**: `e13a4f8d9ff437dbf7d24898d64cba55c90c682c3902ad30bb528ca241939626` +- **Artifacts**: + - docs/trace-repair-gated-stop.md + - benchmarks/trace-repair/gated-stop-ab/design.json + - ~/bench-cache/gated-stop-ab/ (design.json, pilot.json, confirm-runs.json, confirm-refusal.json, finalize.out) + - https://github.com/tangle-network/agent-eval/pull/601 +- **Confounds**: + - 95% CI crosses zero; the draw was powered for effect 0.10 and the observed effect is 0.06 + - provider retired the pinned glm-5.2 mid-campaign; the servedModel gate aborted the stale seal at $0 and the pin moved to glm-5.3 (seal ad82d342 -> e13a4f8d), so screen-era and confirm-era rows saw different provider states + - confirm run absorbed HTTP 429 rate-limit interruptions on several rows (retried; logged in confirm-run.out) +- **Notes**: Aggregate router spend was not captured as one number for the confirm draw; per-row token and grade costs are in confirm-runs.json. + +## tb-repair-free-lunch + +**MEASURED-ONCE** · 2026-08-10 · trace-repair · source repo `agent-eval` + +**Claim.** Unconditional continuation from the recorded end state with a real model budget (no analyst, no hint, no gate) rescues failed TB-Repair rows, so admission condition 3 has a free lunch to subtract. + +**Result.** 3 of 64 rollouts rescued (4.7%); 2 of 32 rows rescued at least once (6.2%). Zero-call control: 0 passes in 96 rollouts. Exit statuses: 15 submitted, 46 step-budget-exhausted, 3 model-error. + +- **Instrument**: tb-repair-freelunch runner: pinned mini-swe-agent continuation policy, network none inside the continuation container, deterministic oracle grading, servedModel check per pass +- **Command**: `bash ~/bench-cache/freelunch-20260810/run3.sh -> tsx scripts/tb-repair-freelunch.ts (policy tb-repair-continuation-v1, glm-5.2 pinned, seed 20260810, $10 ceiling)` +- **Arms**: zero-call-control vs budgeted-continuation +- **n**: 64 rollouts (32 prefix-faithful rows x uniform n=2; denominator chain 48 evaluated -> 43 deterministic oracle -> 35 clean exit -> 32 prefix-fidelity ok) +- **Cost**: $9.64 +- **Artifacts**: + - docs/trace-repair-free-lunch.md + - ~/bench-cache/freelunch-20260810/ (freelunch-pass1.json, freelunch-pass2.json, rollouts.jsonl, PREREG.md, status.txt) + - https://github.com/tangle-network/agent-eval/pull/558 +- **Confounds**: + - one full pass was served a substituted model (deepseek-v3.2 request answered as deepseek-v4-flash) and was aborted pre-count; only glm-5.2-served rollouts are in the denominator + - uniform n stopped at 2 because a third pass would cross the $10 ceiling + - 3 prefix-divergent rows quarantined to a secondary set and excluded from the primary denominator +- **Notes**: The free-lunch rate is the floor any repair intervention must beat; it is subtracted, not celebrated. + +## prime-vs-dspy-analyst-38-rows + +**MEASURED-ONCE** · 2026-08-08 · trace-analysis · source repo `traces` + +**Claim.** Prime-agent as a one-shot trace analyst beats the GEPA-certified DSPy-RLM analyst on stored-baseline CodeTraceBench rows. + +**Result.** Prime wins the dev-pool comparison: mean per-case scored F1 0.3218 vs 0.1833; 18W/4L/16T (all ties 0-0); two-sided sign test p=0.0043. Direction consistent across both baseline sources including the derivation-verified stock-smoke slice (OH 0.377 vs 0.231, T2 0.416 vs 0.163). + +- **Instrument**: batch comparison over stored dspy baselines: prime-RLM one-shot with a bounded structured-output repair turn, scored per-case F1 against human step labels +- **Command**: `prime-vs-dspy batch over 38 positive-labeled OH+T2 dev rows via cli-bridge prime backend (systemd-run scoped spawn, nix-built prime-agent 0.7.0)` +- **Arms**: prime-agent-0.7.0 vs dspy-rlm-stored-baseline +- **n**: 38 rows (positive-labeled OH+T2 dev rows; 35/38 completed ok, 5 needed the repair turn) +- **Cost**: $5.30 +- **Artifacts**: + - https://gist.github.com/drewstone/cd6ee0f0ca659334215001d792d0dacf + - .evolve/experiments.jsonl (arm prime-38-row-comparison) +- **Confounds**: + - compared against STORED dspy baselines, not a same-day paired rerun; provider state differed between the arms' measurement dates + - cost asymmetry ~4.7x dspy (220K input tokens per case mean), so the win is not cost-normalized + - dev-pool rows, not the sealed cert splits; 3/38 prime rows failed structured output +- **Notes**: Until this registry, the durable copy of this result was a secret gist plus an uncommitted .evolve line in a working tree. + +## cad-gepa-openscad-directive + +**MEASURED-ONCE** · 2026-06-08 · creative-cad · source repo `creative-agent` + +**Claim.** GEPA optimization of the OpenSCAD system directive lifts the CAD agent's held-out build quality over the hand-written directive. + +**Result.** +9.5pp held-out over the baseline directive; the winning directive shipped with automatic video capture of the built models. + +- **Instrument**: creative-agent CAD self-improvement loop: GEPA over the OpenSCAD directive, held-out scenario split, produced-state scoring of compiled models +- **Command**: not preserved +- **Arms**: hand-written-directive vs gepa-optimized-directive +- **n**: 1 held-out split (single held-out evaluation of the winning directive; split size recorded in the creative-agent run artifacts) +- **Cost**: not captured +- **Artifacts**: + - creative-agent: .evolve/evals/ and the CAD self-improve loop run artifacts (repo tangle-network/creative-agent) +- **Confounds**: + - single held-out evaluation, no replication or challenger round + - exact invocation and split size were not preserved in a durable record - this registry entry is the pointer that stops the number from degrading into folklore; the next CAD run should re-derive and upgrade or kill this record +- **Notes**: Weakest record in the initial migration by design: it demonstrates the UNVERIFIED->MEASURED-ONCE boundary. The +9.5pp number traces to the CAD loop session record, but the run directory is not pinned here yet. + +## vb-cell-engine-parity-no-flip + +**RESOLVED-NULL** · 2026-08-15 · vertical-bench · source repo `blueprint-agent` + +**Claim.** The VerticalBench graph cell-engine reaches behavioural parity with the loop cell-engine on a live cell, so the default engine can flip to the graph. + +**Result.** NO FLIP. Parity holds on dispatched shot-1 prompt (byte-identical), realness-firewall activation, verification score on the same graded state (blended 0.31 both arms), and toolchain detection. Parity fails on coder turn depth (graph settles recon-only in ~3-4m vs the loop working the full 30m wall; opencode SSE errored 2/4 cells), fixed-budget adherence (structural in the loop vs probabilistic in the graph: 1/3 post-contract cells dispatched shot 2), and board classification (loop 2/2 fair attempts, graph 0/2 - quarantined, never silently scored). + +- **Instrument**: instrumented live parity cell: tempo-payments tempo-tip20-balance-viewer, opencode glm-5.3 (router-served id verified), shots=2, verify=full, judges deepseek-v4-flash, identical env across arms +- **Command**: `blueprint-agent experiments harness, parity phase 3 of tangle-network/agent-runtime#694 (loop arm + four instrumented graph attempts)` +- **Arms**: loop-cell-engine vs graph-cell-engine +- **n**: 1 leaf (1 rep per arm; 4 instrumented graph cycles, each archived with its defect) +- **Cost**: not captured +- **Artifacts**: + - blueprint-agent: scripts/experiments/results/p3-parity-20260815/ (loop + four graph attempts, each archived with its defect) + - https://github.com/tangle-network/blueprint-agent/pull/2330 +- **Confounds**: + - n=1 leaf, 1 rep per arm; stochastic model trajectories + - arms ran hours apart on a live provider + - the loop's cells were wall-capped at 30m, so its 0.31 grades a half-build + - glm-5.3 had no rate in SANDBOX_MODEL_COSTS/cost-estimator at run time, so spend did not accumulate against caps (left loud, not guessed) +- **Notes**: Two structural gaps block the flip and are named in the engine header: TODO(graph-cell/bridge-turn-depth) and TODO(graph-cell/fixed-budget-adherence). The flip is one line once both close. Until this registry, these numbers lived only in the PR body. diff --git a/evidence/README.md b/evidence/README.md new file mode 100644 index 00000000..20bd60a2 --- /dev/null +++ b/evidence/README.md @@ -0,0 +1,37 @@ +# Where evidence lives + +One home, one shape, one index. +This directory is the canonical registry for measured claims about our agents, prompts, policies, and instruments — the numbers that used to rot in per-repo docs, `.evolve/` diaries, results directories, PR comments, and gists. + +## The rule + +- **A measured claim that anyone will cite later gets a record here.** + One JSON file per claim in [`records/`](./records/), validated by the exported schema (`evidenceRecordSchema` from `@tangle-network/agent-eval/experiment`). +- **The registry lives in agent-eval** because the measurement substrate owns evidence legitimacy. + Every other repo keeps at most a pointer file (`docs/EVIDENCE.md`, three lines: "measured results for this repo live in agent-eval `evidence/`; do not restate numbers here"). +- **Humans read [`INDEX.md`](./INDEX.md); machines read `records/*.json`.** + The index is generated (`pnpm run evidence:render`) and drift-checked (`pnpm run evidence:check`, inside `verify:package`), so prose can never drift from data. +- **Session diaries stay diaries.** `.evolve/experiments.jsonl` remains the append-only lab notebook. + A registry record is the distilled, addressable claim a notebook line earned. + +## The record + +Each record states, in typed fields: the one-sentence **claim** (written so it can fail), the **instrument**, the exact **command** (`null` = not preserved, a named gap), the **arms**, the denominator **n**, the **result** with uncertainty, **artifacts** (run dirs, PRs, gists — never empty), **cost** (`null` = not captured, never a silent zero), **confounds** (stated before anyone reads the verdict), and an **evidence state**: + +| state | meaning | +| --- | --- | +| `CERTIFIED` | pre-registered rule + sealed/held-out data, defended or re-run in the suite | +| `MEASURED-ONCE` | one honest measurement on the real path, not replicated | +| `RESOLVED-NULL` | adequate instrument, effect did not appear under the registered rule | +| `UNVERIFIED` | stated somewhere load-bearing, no independent check yet | +| `KILLED` | refuted, invalidated, or superseded | + +When a run was governed by a sealed experiment (`@tangle-network/agent-eval/experiment`), the record carries the seal as `experimentDigest` — the registered rule is the addressing scheme. + +## How to add a record + +1. Write `records/.json` (`id` = kebab-case filename). +2. `pnpm run evidence:render` — validates and regenerates `INDEX.md`. +3. Commit both. A stale index or invalid record fails `verify:package`. + +A result that changes state (replicated, refuted, superseded) gets its state moved or a new record with `supersedes` — never a silent edit of the numbers. diff --git a/evidence/records/cad-gepa-openscad-directive.json b/evidence/records/cad-gepa-openscad-directive.json new file mode 100644 index 00000000..186dda83 --- /dev/null +++ b/evidence/records/cad-gepa-openscad-directive.json @@ -0,0 +1,26 @@ +{ + "id": "cad-gepa-openscad-directive", + "date": "2026-06-08", + "claim": "GEPA optimization of the OpenSCAD system directive lifts the CAD agent's held-out build quality over the hand-written directive.", + "domain": "creative-cad", + "instrument": "creative-agent CAD self-improvement loop: GEPA over the OpenSCAD directive, held-out scenario split, produced-state scoring of compiled models", + "command": null, + "arms": ["hand-written-directive", "gepa-optimized-directive"], + "n": { + "value": 1, + "unit": "held-out split", + "detail": "single held-out evaluation of the winning directive; split size recorded in the creative-agent run artifacts" + }, + "result": "+9.5pp held-out over the baseline directive; the winning directive shipped with automatic video capture of the built models.", + "evidenceState": "MEASURED-ONCE", + "artifacts": [ + "creative-agent: .evolve/evals/ and the CAD self-improve loop run artifacts (repo tangle-network/creative-agent)" + ], + "costUsd": null, + "confounds": [ + "single held-out evaluation, no replication or challenger round", + "exact invocation and split size were not preserved in a durable record - this registry entry is the pointer that stops the number from degrading into folklore; the next CAD run should re-derive and upgrade or kill this record" + ], + "sourceRepo": "creative-agent", + "notes": "Weakest record in the initial migration by design: it demonstrates the UNVERIFIED->MEASURED-ONCE boundary. The +9.5pp number traces to the CAD loop session record, but the run directory is not pinned here yet." +} diff --git a/evidence/records/ctb-analyst-gepa-certified-prompt.json b/evidence/records/ctb-analyst-gepa-certified-prompt.json new file mode 100644 index 00000000..33600256 --- /dev/null +++ b/evidence/records/ctb-analyst-gepa-certified-prompt.json @@ -0,0 +1,29 @@ +{ + "id": "ctb-analyst-gepa-certified-prompt", + "date": "2026-08-02", + "claim": "The GEPA-optimized DSPy-RLM analyst prompt (sha d3829fb8) is the certified shipping prompt on CodeTraceBench: challengers lose on pooled sealed splits under the pre-registered rule.", + "domain": "trace-analysis", + "instrument": "sealed certification runs: pre-registration written before the winner was known, pooled OH-cert32 + T2-cert32 sealed splits, scored micro-F1 against human step labels, paired CIs", + "command": "gepa-analyst-campaign certification phase: 4 serial runs, 2 reps, mutex-held seat (see .evolve/certification-2026-08-02-preregistration.md for the registered rule)", + "arms": ["stock-d3829fb8", "g2-challenger-1bb303e7"], + "n": { + "value": 64, + "unit": "sealed cases", + "detail": "OH-cert32 + T2-cert32, 2 reps (128 case-runs; G2 failed 3/128, stock 0/128); both cert32 splits spent for this config family" + }, + "result": "G2 REJECTED by rule 1: pooled-cert micro-F1 stock 0.2489 vs G2 0.1928 (-5.6pp). Per-family stock OH 0.2896 / T2 0.2162. Paired CIs [-0.126, 0.058] OH, [-0.143, 0.023] T2. G2's selection-split edge (+0.064 weighted) did not transfer - third consecutive tuning-split-vs-fresh-split reversal in the campaign. The shipping prompt stands, defended twice (round-1 and round-2 challengers).", + "evidenceState": "CERTIFIED", + "artifacts": [ + "~/bench-cache/ctb-20260801/cert2/ (4 result.json + logs + winner-instructions.txt)", + ".evolve/certification-2026-08-02-preregistration.md", + ".evolve/experiments.jsonl (arm cert2-stock-vs-g2)" + ], + "costUsd": 31.07, + "confounds": [ + "cross-family gap is the dominant residual: shipping analyst OH 0.29 / T2 0.22 vs mini-SWE wide 0.56-0.63 (~2x), later diagnosed as localization (far-class mass), not prompt quality", + "gold-mass-weighted selection metric recorded as overfit-prone at n=12", + "CodeTraceBench frozen as an optimization target on 2026-08-04: ~60% of residual misses are missing labels (H1, n=20 adjudication), so absolute F1 under-reads the analyst" + ], + "sourceRepo": "agent-eval", + "notes": "Companion baseline: dev-pool scored-F1 baseline for the dspy-rlm analyst stack is 32.73%/30.99% (retired direct runner excluded). Prime-agent comparison against these stored baselines is recorded separately (prime-vs-dspy-analyst-38-rows)." +} diff --git a/evidence/records/multishot-golden-oracle-v1.json b/evidence/records/multishot-golden-oracle-v1.json new file mode 100644 index 00000000..7d01d674 --- /dev/null +++ b/evidence/records/multishot-golden-oracle-v1.json @@ -0,0 +1,28 @@ +{ + "id": "multishot-golden-oracle-v1", + "date": "2026-08-16", + "claim": "The multishot loop's behaviour is frozen as deterministic golden records that any engine can be verified against, and the records are load-bearing: they detect every field-level and engine-level deviation.", + "domain": "multishot", + "instrument": "golden-record oracle: scripted transports and executors, request ledger + outcome per scenario, double capture with mandatory agreement, exhaustive single-field mutation and engine-mutant detection in the suite", + "command": "pnpm vitest run src/multishot/golden/golden.test.ts", + "arms": ["recorded-loop-v1", "candidate-engine"], + "n": { + "value": 14, + "unit": "records", + "detail": "13 shot scenarios + 1 matrix scenario, each captured twice with agreement required; v1 captured from the loop at 0.145.21" + }, + "result": "Consumer parity proven before freezing: tax 4/4 deep-equal, gtm 4/4 shot-level and 8/8 matrix-level. Oracle load-bearing: every single leaf of {outcome, requests} (and {matrix, requests, judgeRequests, files}) mutated one at a time is reported for all 14 records; 8/8 engine mutants detected, each naming the deviating field.", + "evidenceState": "CERTIFIED", + "artifacts": [ + "src/multishot/golden/records/v1.json", + "src/multishot/golden/golden.test.ts", + "docs/multishot-golden-records.md", + "https://github.com/tangle-network/agent-eval/pull/627" + ], + "costUsd": 0, + "confounds": [ + "deterministic-only coverage: scripted transports, fixed personas and budgets; live-provider behaviour (latency, substitution, SSE faults) is out of the oracle's scope by design" + ], + "sourceRepo": "agent-eval", + "notes": "CERTIFIED because the check re-runs on every suite run and records are frozen: the recorder refuses to overwrite a version, so a behaviour change mints a new version file and the diff is the reviewable evidence. Later consumer parity counts quoted as 14/14 and 15/15 live in the loop-retention PR chain (#617 and its consumers) - PR-comment-only until migrated here." +} diff --git a/evidence/records/prime-vs-dspy-analyst-38-rows.json b/evidence/records/prime-vs-dspy-analyst-38-rows.json new file mode 100644 index 00000000..f3413379 --- /dev/null +++ b/evidence/records/prime-vs-dspy-analyst-38-rows.json @@ -0,0 +1,28 @@ +{ + "id": "prime-vs-dspy-analyst-38-rows", + "date": "2026-08-08", + "claim": "Prime-agent as a one-shot trace analyst beats the GEPA-certified DSPy-RLM analyst on stored-baseline CodeTraceBench rows.", + "domain": "trace-analysis", + "instrument": "batch comparison over stored dspy baselines: prime-RLM one-shot with a bounded structured-output repair turn, scored per-case F1 against human step labels", + "command": "prime-vs-dspy batch over 38 positive-labeled OH+T2 dev rows via cli-bridge prime backend (systemd-run scoped spawn, nix-built prime-agent 0.7.0)", + "arms": ["prime-agent-0.7.0", "dspy-rlm-stored-baseline"], + "n": { + "value": 38, + "unit": "rows", + "detail": "positive-labeled OH+T2 dev rows; 35/38 completed ok, 5 needed the repair turn" + }, + "result": "Prime wins the dev-pool comparison: mean per-case scored F1 0.3218 vs 0.1833; 18W/4L/16T (all ties 0-0); two-sided sign test p=0.0043. Direction consistent across both baseline sources including the derivation-verified stock-smoke slice (OH 0.377 vs 0.231, T2 0.416 vs 0.163).", + "evidenceState": "MEASURED-ONCE", + "artifacts": [ + "https://gist.github.com/drewstone/cd6ee0f0ca659334215001d792d0dacf", + ".evolve/experiments.jsonl (arm prime-38-row-comparison)" + ], + "costUsd": 5.3, + "confounds": [ + "compared against STORED dspy baselines, not a same-day paired rerun; provider state differed between the arms' measurement dates", + "cost asymmetry ~4.7x dspy (220K input tokens per case mean), so the win is not cost-normalized", + "dev-pool rows, not the sealed cert splits; 3/38 prime rows failed structured output" + ], + "sourceRepo": "traces", + "notes": "Until this registry, the durable copy of this result was a secret gist plus an uncommitted .evolve line in a working tree." +} diff --git a/evidence/records/tb-gated-stop-ab-confirm.json b/evidence/records/tb-gated-stop-ab-confirm.json new file mode 100644 index 00000000..9f69a260 --- /dev/null +++ b/evidence/records/tb-gated-stop-ab-confirm.json @@ -0,0 +1,31 @@ +{ + "id": "tb-gated-stop-ab-confirm", + "date": "2026-08-15", + "claim": "At one matched total token budget on TB-Repair, an agent that may stop only after an executable held-out check passes finishes more rows than the same agent on unconditional continuation.", + "domain": "trace-repair", + "instrument": "tb-gated-stop-ab confirmatory draw: sealed registered experiment, servedModel abort gate, paired cluster-bootstrap contrast over best intermediate state", + "command": "pnpm tsx scripts/tb-gated-stop-ab.ts (design/screen/pilot/confirm phases; seal e13a4f8d re-executed by the confirm runner)", + "arms": ["blind-continue", "gated-continue"], + "n": { + "value": 151, + "unit": "rows", + "detail": "22 clusters; settlingDraw at power 0.8009 (search sim) / 0.8013 (registered sim) for effect 0.10; admitted ceiling 216 rows at power 0.9189" + }, + "result": "+0.0596 pass-rate delta (best intermediate state in both arms), 95% cluster-bootstrap CI [-0.0061, +0.1210]. The interval crosses zero: direction positive, not significant at the registered level.", + "evidenceState": "MEASURED-ONCE", + "artifacts": [ + "docs/trace-repair-gated-stop.md", + "benchmarks/trace-repair/gated-stop-ab/design.json", + "~/bench-cache/gated-stop-ab/ (design.json, pilot.json, confirm-runs.json, confirm-refusal.json, finalize.out)", + "https://github.com/tangle-network/agent-eval/pull/601" + ], + "costUsd": null, + "confounds": [ + "95% CI crosses zero; the draw was powered for effect 0.10 and the observed effect is 0.06", + "provider retired the pinned glm-5.2 mid-campaign; the servedModel gate aborted the stale seal at $0 and the pin moved to glm-5.3 (seal ad82d342 -> e13a4f8d), so screen-era and confirm-era rows saw different provider states", + "confirm run absorbed HTTP 429 rate-limit interruptions on several rows (retried; logged in confirm-run.out)" + ], + "sourceRepo": "agent-eval", + "experimentDigest": "e13a4f8d9ff437dbf7d24898d64cba55c90c682c3902ad30bb528ca241939626", + "notes": "Aggregate router spend was not captured as one number for the confirm draw; per-row token and grade costs are in confirm-runs.json." +} diff --git a/evidence/records/tb-repair-free-lunch.json b/evidence/records/tb-repair-free-lunch.json new file mode 100644 index 00000000..a9a1ed08 --- /dev/null +++ b/evidence/records/tb-repair-free-lunch.json @@ -0,0 +1,29 @@ +{ + "id": "tb-repair-free-lunch", + "date": "2026-08-10", + "claim": "Unconditional continuation from the recorded end state with a real model budget (no analyst, no hint, no gate) rescues failed TB-Repair rows, so admission condition 3 has a free lunch to subtract.", + "domain": "trace-repair", + "instrument": "tb-repair-freelunch runner: pinned mini-swe-agent continuation policy, network none inside the continuation container, deterministic oracle grading, servedModel check per pass", + "command": "bash ~/bench-cache/freelunch-20260810/run3.sh -> tsx scripts/tb-repair-freelunch.ts (policy tb-repair-continuation-v1, glm-5.2 pinned, seed 20260810, $10 ceiling)", + "arms": ["zero-call-control", "budgeted-continuation"], + "n": { + "value": 64, + "unit": "rollouts", + "detail": "32 prefix-faithful rows x uniform n=2; denominator chain 48 evaluated -> 43 deterministic oracle -> 35 clean exit -> 32 prefix-fidelity ok" + }, + "result": "3 of 64 rollouts rescued (4.7%); 2 of 32 rows rescued at least once (6.2%). Zero-call control: 0 passes in 96 rollouts. Exit statuses: 15 submitted, 46 step-budget-exhausted, 3 model-error.", + "evidenceState": "MEASURED-ONCE", + "artifacts": [ + "docs/trace-repair-free-lunch.md", + "~/bench-cache/freelunch-20260810/ (freelunch-pass1.json, freelunch-pass2.json, rollouts.jsonl, PREREG.md, status.txt)", + "https://github.com/tangle-network/agent-eval/pull/558" + ], + "costUsd": 9.64, + "confounds": [ + "one full pass was served a substituted model (deepseek-v3.2 request answered as deepseek-v4-flash) and was aborted pre-count; only glm-5.2-served rollouts are in the denominator", + "uniform n stopped at 2 because a third pass would cross the $10 ceiling", + "3 prefix-divergent rows quarantined to a secondary set and excluded from the primary denominator" + ], + "sourceRepo": "agent-eval", + "notes": "The free-lunch rate is the floor any repair intervention must beat; it is subtracted, not celebrated." +} diff --git a/evidence/records/vb-cell-engine-parity-no-flip.json b/evidence/records/vb-cell-engine-parity-no-flip.json new file mode 100644 index 00000000..1f8ef20f --- /dev/null +++ b/evidence/records/vb-cell-engine-parity-no-flip.json @@ -0,0 +1,29 @@ +{ + "id": "vb-cell-engine-parity-no-flip", + "date": "2026-08-15", + "claim": "The VerticalBench graph cell-engine reaches behavioural parity with the loop cell-engine on a live cell, so the default engine can flip to the graph.", + "domain": "vertical-bench", + "instrument": "instrumented live parity cell: tempo-payments tempo-tip20-balance-viewer, opencode glm-5.3 (router-served id verified), shots=2, verify=full, judges deepseek-v4-flash, identical env across arms", + "command": "blueprint-agent experiments harness, parity phase 3 of tangle-network/agent-runtime#694 (loop arm + four instrumented graph attempts)", + "arms": ["loop-cell-engine", "graph-cell-engine"], + "n": { + "value": 1, + "unit": "leaf", + "detail": "1 rep per arm; 4 instrumented graph cycles, each archived with its defect" + }, + "result": "NO FLIP. Parity holds on dispatched shot-1 prompt (byte-identical), realness-firewall activation, verification score on the same graded state (blended 0.31 both arms), and toolchain detection. Parity fails on coder turn depth (graph settles recon-only in ~3-4m vs the loop working the full 30m wall; opencode SSE errored 2/4 cells), fixed-budget adherence (structural in the loop vs probabilistic in the graph: 1/3 post-contract cells dispatched shot 2), and board classification (loop 2/2 fair attempts, graph 0/2 - quarantined, never silently scored).", + "evidenceState": "RESOLVED-NULL", + "artifacts": [ + "blueprint-agent: scripts/experiments/results/p3-parity-20260815/ (loop + four graph attempts, each archived with its defect)", + "https://github.com/tangle-network/blueprint-agent/pull/2330" + ], + "costUsd": null, + "confounds": [ + "n=1 leaf, 1 rep per arm; stochastic model trajectories", + "arms ran hours apart on a live provider", + "the loop's cells were wall-capped at 30m, so its 0.31 grades a half-build", + "glm-5.3 had no rate in SANDBOX_MODEL_COSTS/cost-estimator at run time, so spend did not accumulate against caps (left loud, not guessed)" + ], + "sourceRepo": "blueprint-agent", + "notes": "Two structural gaps block the flip and are named in the engine header: TODO(graph-cell/bridge-turn-depth) and TODO(graph-cell/fixed-budget-adherence). The flip is one line once both close. Until this registry, these numbers lived only in the PR body." +} diff --git a/scripts/render-evidence-index.ts b/scripts/render-evidence-index.ts new file mode 100644 index 00000000..1ea157cb --- /dev/null +++ b/scripts/render-evidence-index.ts @@ -0,0 +1,79 @@ +/** + * Render `evidence/INDEX.md` from `evidence/records/*.json`, or verify it. + * + * pnpm run evidence:render — validate every record and rewrite the index + * pnpm run evidence:check — validate + fail when the committed index + * differs from what the records render to + * + * The check runs inside `verify:package`, so a record that breaks the schema + * or an index edited by hand fails the same local gate that guards releases. + */ + +import { readdirSync, readFileSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { renderEvidenceIndex } from '../src/experiment/evidence-record' + +const repoRoot = fileURLToPath(new URL('..', import.meta.url)) +const recordsDir = resolve(repoRoot, 'evidence/records') +const indexPath = resolve(repoRoot, 'evidence/INDEX.md') +const checkMode = process.argv.includes('--check') + +const files = readdirSync(recordsDir) + .filter((name) => name.endsWith('.json')) + .sort() + +if (files.length === 0) { + console.error(`no evidence records found in ${recordsDir}`) + process.exit(1) +} + +const raws = files.map((name) => { + const path = resolve(recordsDir, name) + let parsed: unknown + try { + parsed = JSON.parse(readFileSync(path, 'utf8')) + } catch (error) { + console.error(`evidence record is not valid JSON: ${path}`) + console.error(String(error)) + process.exit(1) + } + const id = (parsed as { id?: unknown }).id + if (id !== name.replace(/\.json$/, '')) { + console.error( + `evidence record filename must equal its id: ${name} declares id ${JSON.stringify(id)}`, + ) + process.exit(1) + } + return parsed +}) + +let rendered: string +try { + rendered = renderEvidenceIndex(raws) +} catch (error) { + console.error('evidence registry validation failed:') + console.error(String(error)) + process.exit(1) +} + +if (checkMode) { + let committed: string + try { + committed = readFileSync(indexPath, 'utf8') + } catch { + console.error(`evidence index is missing: ${indexPath} — run pnpm run evidence:render`) + process.exit(1) + } + if (committed !== rendered) { + console.error( + 'evidence/INDEX.md is stale: it does not match evidence/records/*.json.\n' + + 'Run pnpm run evidence:render and commit the result.', + ) + process.exit(1) + } + console.log(`evidence index is fresh: ${files.length} records, index matches`) +} else { + writeFileSync(indexPath, rendered) + console.log(`rendered ${indexPath} from ${files.length} records`) +} diff --git a/src/experiment/evidence-record.test.ts b/src/experiment/evidence-record.test.ts new file mode 100644 index 00000000..2a351341 --- /dev/null +++ b/src/experiment/evidence-record.test.ts @@ -0,0 +1,137 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + EVIDENCE_STATES, + type EvidenceRegistryRecord, + parseEvidenceRegistryRecord, + renderEvidenceIndex, + validateEvidenceRegistry, +} from './evidence-record' + +const valid: EvidenceRegistryRecord = { + id: 'sample-claim', + date: '2026-08-15', + claim: 'The sample instrument separates the arms.', + domain: 'sample', + instrument: 'paired A/B over scripted rows', + command: 'pnpm tsx scripts/sample.ts', + arms: ['control', 'treatment'], + n: { value: 151, unit: 'rows', detail: '22 clusters' }, + result: '+0.0596, 95% CI [-0.0061, +0.1210]', + evidenceState: 'MEASURED-ONCE', + artifacts: ['~/bench-cache/sample/'], + costUsd: 9.64, + confounds: ['CI crosses zero'], + sourceRepo: 'agent-eval', +} + +describe('evidenceRegistryRecordSchema', () => { + it('accepts a complete record', () => { + expect(parseEvidenceRegistryRecord(valid)).toEqual(valid) + }) + + it('accepts null command and null cost as named gaps', () => { + const record = parseEvidenceRegistryRecord({ ...valid, command: null, costUsd: null }) + expect(record.command).toBeNull() + expect(record.costUsd).toBeNull() + }) + + it('rejects an unknown evidence state', () => { + expect(() => + parseEvidenceRegistryRecord({ ...valid, evidenceState: 'PROBABLY-FINE' }), + ).toThrow() + }) + + it('rejects empty artifacts — a claim with no artifact is prose', () => { + expect(() => parseEvidenceRegistryRecord({ ...valid, artifacts: [] })).toThrow() + }) + + it('rejects unknown keys, a non-kebab id, and a malformed date', () => { + expect(() => parseEvidenceRegistryRecord({ ...valid, vibe: 'good' })).toThrow() + expect(() => parseEvidenceRegistryRecord({ ...valid, id: 'Sample_Claim' })).toThrow() + expect(() => parseEvidenceRegistryRecord({ ...valid, date: '08/15/2026' })).toThrow() + }) + + it('rejects a negative cost', () => { + expect(() => parseEvidenceRegistryRecord({ ...valid, costUsd: -1 })).toThrow() + }) +}) + +describe('validateEvidenceRegistry', () => { + it('rejects duplicate ids', () => { + expect(() => validateEvidenceRegistry([valid, valid])).toThrow(/duplicate/) + }) + + it('rejects supersedes pointing at an unknown record', () => { + expect(() => validateEvidenceRegistry([{ ...valid, supersedes: ['never-recorded'] }])).toThrow( + /unknown record/, + ) + }) + + it('sorts by trust ladder, then date desc, then id', () => { + const certified = { ...valid, id: 'a-certified', evidenceState: 'CERTIFIED' as const } + const newerNull = { + ...valid, + id: 'b-null', + date: '2026-08-16', + evidenceState: 'RESOLVED-NULL' as const, + } + const sorted = validateEvidenceRegistry([newerNull, valid, certified]) + expect(sorted.map((r) => r.id)).toEqual(['a-certified', 'sample-claim', 'b-null']) + }) +}) + +describe('renderEvidenceIndex', () => { + it('is deterministic and carries every load-bearing field', () => { + const first = renderEvidenceIndex([valid]) + expect(renderEvidenceIndex([valid])).toBe(first) + for (const needle of [ + valid.claim, + valid.result, + valid.instrument, + '151 rows (22 clusters)', + '$9.64', + 'CI crosses zero', + 'MEASURED-ONCE', + ]) { + expect(first).toContain(needle) + } + }) + + it('labels the named gaps instead of dropping them', () => { + const rendered = renderEvidenceIndex([{ ...valid, command: null, costUsd: null }]) + expect(rendered).toContain('not preserved') + expect(rendered).toContain('not captured') + }) +}) + +describe('committed registry', () => { + const recordsDir = resolve(__dirname, '../../evidence/records') + const files = readdirSync(recordsDir) + .filter((name) => name.endsWith('.json')) + .sort() + + it('holds at least the initial migration', () => { + expect(files.length).toBeGreaterThanOrEqual(7) + }) + + it('every committed record parses, ids match filenames, registry validates', () => { + const raws = files.map((name) => { + const parsed = JSON.parse(readFileSync(resolve(recordsDir, name), 'utf8')) + expect(parsed.id).toBe(name.replace(/\.json$/, '')) + return parsed + }) + const records = validateEvidenceRegistry(raws) + expect(records).toHaveLength(files.length) + for (const state of records.map((r) => r.evidenceState)) { + expect(EVIDENCE_STATES).toContain(state) + } + }) + + it('the committed index matches the records byte for byte', () => { + const raws = files.map((name) => JSON.parse(readFileSync(resolve(recordsDir, name), 'utf8'))) + const committed = readFileSync(resolve(recordsDir, '../INDEX.md'), 'utf8') + expect(committed).toBe(renderEvidenceIndex(raws)) + }) +}) diff --git a/src/experiment/evidence-record.ts b/src/experiment/evidence-record.ts new file mode 100644 index 00000000..ea9c76ba --- /dev/null +++ b/src/experiment/evidence-record.ts @@ -0,0 +1,205 @@ +/** + * The evidence registry record — the canonical, machine-readable unit for a + * measured claim about an agent, a prompt, a policy, or an instrument. + * + * One record answers, in typed fields, the questions a scattered results doc + * leaves to prose: what was claimed, what instrument measured it, the exact + * command that reproduces it, which arms ran, the denominator, what it cost, + * what confounds the comparison, and how much trust the evidence has earned. + * + * Records live as JSON files in `evidence/records/` at the repo root — one + * file per claim, filename `.json`. The registry lives in agent-eval + * because the measurement substrate owns evidence legitimacy; every other + * repo keeps at most a pointer file. `scripts/render-evidence-index.ts` + * generates the human index from these records, so prose can never drift + * from data, and `pnpm run evidence:check` (part of `verify:package`) fails + * on an invalid record or a stale index. + * + * The shape extends the `.evolve/experiments.jsonl` session-log vocabulary + * (ts/hypothesis/change/result/evidence) into a curated registry record. + * Session logs stay append-only diaries; a registry record is the durable, + * addressable claim distilled from them. + */ + +import { z } from 'zod' + +/** + * Trust ladder for a recorded claim, strongest first. + * + * - `CERTIFIED` — pre-registered rule, sealed or held-out data, and either a + * defended challenge or a check that re-runs in the suite. + * - `MEASURED-ONCE` — one honest measurement on the real path, not replicated. + * - `RESOLVED-NULL` — measured with an adequate instrument; the effect did not + * appear under the registered rule. A null is a result, not a failure. + * - `UNVERIFIED` — stated somewhere load-bearing, but no independent check has + * run. Admitting this state is what keeps folklore out of the other four. + * - `KILLED` — refuted, invalidated, or superseded by a stronger record. + */ +export const EVIDENCE_STATES = [ + 'CERTIFIED', + 'MEASURED-ONCE', + 'RESOLVED-NULL', + 'UNVERIFIED', + 'KILLED', +] as const + +export type EvidenceState = (typeof EVIDENCE_STATES)[number] + +const idPattern = /^[a-z0-9]+(?:-[a-z0-9]+)*$/ +const datePattern = /^\d{4}-\d{2}-\d{2}$/ +const sha256Pattern = /^[0-9a-f]{64}$/ + +/** Denominator of the measurement. A claim without an `n` is prose, not evidence. */ +export const evidenceDenominatorSchema = z.strictObject({ + value: z.number().finite(), + /** What one unit is: rows, cases, rollouts, records, leaves… */ + unit: z.string().min(1), + /** Clustering, reps, split names — whatever the bare number hides. */ + detail: z.string().min(1).optional(), +}) + +export type EvidenceDenominator = z.infer + +export const evidenceRegistryRecordSchema = z.strictObject({ + /** Stable kebab-case identity; the registry filename is `.json`. */ + id: z.string().regex(idPattern), + /** Date the decisive measurement completed, `YYYY-MM-DD`. */ + date: z.string().regex(datePattern), + /** The one-sentence load-bearing claim, stated so it can fail. */ + claim: z.string().min(1), + /** Domain the claim lives in: trace-repair, multishot, analyst, vertical-bench… */ + domain: z.string().min(1), + /** The instrument that produced the number — script, harness, oracle, judge. */ + instrument: z.string().min(1), + /** + * The exact invocation that reproduces the measurement. + * `null` means the invocation was not preserved — a named gap, never a guess. + */ + command: z.string().min(1).nullable(), + /** Compared arms. Empty for a single-arm measurement. */ + arms: z.array(z.string().min(1)), + n: evidenceDenominatorSchema, + /** The measured numbers, with uncertainty when it exists. Numbers, not adjectives. */ + result: z.string().min(1), + evidenceState: z.enum(EVIDENCE_STATES), + /** Where the raw data lives: repo paths, PR/gist URLs, run directories. Never empty. */ + artifacts: z.array(z.string().min(1)).min(1), + /** + * Measured spend in USD. `0` is a true zero (deterministic, no paid calls); + * `null` means spend was not captured — a named gap, never a silent zero. + */ + costUsd: z.number().nonnegative().nullable(), + /** Every known asymmetry or threat to validity. Empty only when none is known. */ + confounds: z.array(z.string().min(1)), + /** Repo whose work produced the evidence (the registry itself lives here). */ + sourceRepo: z.string().min(1), + /** Seal digest of the registered experiment (`./experiment`), when one governed the run. */ + experimentDigest: z.string().regex(sha256Pattern).optional(), + /** Ids of records this one supersedes. */ + supersedes: z.array(z.string().regex(idPattern)).optional(), + notes: z.string().min(1).optional(), +}) + +export type EvidenceRegistryRecord = z.infer + +/** Parse one record; throws with the zod issue list on any violation. */ +export function parseEvidenceRegistryRecord(raw: unknown): EvidenceRegistryRecord { + return evidenceRegistryRecordSchema.parse(raw) +} + +export class EvidenceRegistryError extends Error {} + +/** + * Validate a set of records as one registry: every record parses, ids are + * unique, and every `supersedes` target exists. Returns records sorted in + * the registry's canonical order (trust ladder, then date desc, then id). + */ +export function validateEvidenceRegistry(raws: readonly unknown[]): EvidenceRegistryRecord[] { + const records = raws.map((raw) => parseEvidenceRegistryRecord(raw)) + const seen = new Set() + for (const record of records) { + if (seen.has(record.id)) { + throw new EvidenceRegistryError(`duplicate evidence record id: ${record.id}`) + } + seen.add(record.id) + } + for (const record of records) { + for (const target of record.supersedes ?? []) { + if (!seen.has(target)) { + throw new EvidenceRegistryError(`record ${record.id} supersedes unknown record: ${target}`) + } + } + } + const stateRank = new Map(EVIDENCE_STATES.map((state, index) => [state, index])) + return [...records].sort((a, b) => { + const byState = (stateRank.get(a.evidenceState) ?? 0) - (stateRank.get(b.evidenceState) ?? 0) + if (byState !== 0) return byState + if (a.date !== b.date) return a.date < b.date ? 1 : -1 + return a.id < b.id ? -1 : 1 + }) +} + +function mdEscape(text: string): string { + return text.replace(/\|/g, '\\|').replace(/\n/g, ' ') +} + +function formatCost(costUsd: number | null): string { + if (costUsd === null) return 'not captured' + return `$${costUsd.toFixed(2)}` +} + +function formatN(n: EvidenceDenominator): string { + const base = `${n.value} ${n.unit}` + return n.detail ? `${base} (${n.detail})` : base +} + +/** + * Render the registry's human index from records. Deterministic: identical + * records always produce identical output, so a byte diff against the + * committed index is the freshness check. No clocks, no environment. + */ +export function renderEvidenceIndex(raws: readonly unknown[]): string { + const records = validateEvidenceRegistry(raws) + const lines: string[] = [ + '# Evidence index', + '', + '', + '', + `${records.length} records. States: ${EVIDENCE_STATES.join(' > ')}.`, + '', + '| state | date | id | claim | result |', + '| --- | --- | --- | --- | --- |', + ] + for (const r of records) { + lines.push( + `| ${r.evidenceState} | ${r.date} | [\`${r.id}\`](#${r.id}) | ${mdEscape(r.claim)} | ${mdEscape(r.result)} |`, + ) + } + for (const r of records) { + lines.push('', `## ${r.id}`, '') + lines.push(`**${r.evidenceState}** · ${r.date} · ${r.domain} · source repo \`${r.sourceRepo}\``) + lines.push('', `**Claim.** ${r.claim}`, '', `**Result.** ${r.result}`, '') + lines.push(`- **Instrument**: ${r.instrument}`) + lines.push(`- **Command**: ${r.command === null ? 'not preserved' : `\`${r.command}\``}`) + lines.push(`- **Arms**: ${r.arms.length === 0 ? 'single-arm' : r.arms.join(' vs ')}`) + lines.push(`- **n**: ${formatN(r.n)}`) + lines.push(`- **Cost**: ${formatCost(r.costUsd)}`) + if (r.experimentDigest) lines.push(`- **Experiment seal**: \`${r.experimentDigest}\``) + lines.push(`- **Artifacts**:`) + for (const artifact of r.artifacts) lines.push(` - ${artifact}`) + if (r.confounds.length === 0) { + lines.push('- **Confounds**: none recorded') + } else { + lines.push('- **Confounds**:') + for (const confound of r.confounds) lines.push(` - ${confound}`) + } + if (r.supersedes && r.supersedes.length > 0) { + lines.push(`- **Supersedes**: ${r.supersedes.map((s) => `\`${s}\``).join(', ')}`) + } + if (r.notes) lines.push(`- **Notes**: ${r.notes}`) + } + lines.push('') + return lines.join('\n') +} diff --git a/src/experiment/index.ts b/src/experiment/index.ts index f43c2bb9..860c31b4 100644 --- a/src/experiment/index.ts +++ b/src/experiment/index.ts @@ -199,6 +199,19 @@ export { isIndependentEvidence, verifyEvidenceReceipt, } from './evidence-receipt' +// ── Evidence registry records (evidence/records/*.json) ────────────── +export { + EVIDENCE_STATES, + type EvidenceDenominator, + EvidenceRegistryError, + type EvidenceRegistryRecord, + type EvidenceState, + evidenceDenominatorSchema, + evidenceRegistryRecordSchema, + parseEvidenceRegistryRecord, + renderEvidenceIndex, + validateEvidenceRegistry, +} from './evidence-record' // ── Denominator chain ──────────────────────────────────────────────── export { type AdmissionExecution, diff --git a/tsconfig.script.json b/tsconfig.script.json index 5a9e61ed..cedd517c 100644 --- a/tsconfig.script.json +++ b/tsconfig.script.json @@ -1,7 +1,11 @@ { "extends": "./tsconfig.json", - "compilerOptions": { "rootDir": ".", "noEmit": true }, + "compilerOptions": { + "rootDir": ".", + "noEmit": true + }, "include": [ + "scripts/render-evidence-index.ts", "scripts/record-multishot-golden.ts", "scripts/tb-corpus-rows.ts", "scripts/tb-gated-stop-ab.ts", @@ -11,5 +15,9 @@ "scripts/tb-repair-milestone1.ts", "src" ], - "exclude": ["node_modules", "dist", "tests"] + "exclude": [ + "node_modules", + "dist", + "tests" + ] } From 5baa04dcfd9f460faf81870e91e7bc3d03929cf8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 18 Aug 2026 14:37:07 -0600 Subject: [PATCH 2/3] fix(experiment): require a positive denominator and name the schema export correctly The denominator schema refused NaN and infinity but admitted zero and negative values; n.value is now positive. evidence/README.md names the exported schema evidenceRegistryRecordSchema. --- CHANGELOG.md | 6 +----- evidence/README.md | 2 +- src/experiment/evidence-record.test.ts | 7 +++++++ src/experiment/evidence-record.ts | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89998e77..9822e4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ One repository still names three of them. `starter-foundry` imports `runAssertio ### Added -<<<<<<< HEAD +- The evidence registry: one canonical home for measured claims. `evidence/records/*.json` hold typed records (claim, instrument, exact command, arms, denominator, result, evidence state `CERTIFIED`/`MEASURED-ONCE`/`RESOLVED-NULL`/`UNVERIFIED`/`KILLED`, artifacts, cost, confounds, optional experiment seal digest), validated by `evidenceRecordSchema` from `./experiment`. `pnpm run evidence:render` generates `evidence/INDEX.md` from the records; `pnpm run evidence:check` (inside `verify:package`) fails on an invalid record or a stale index, so the human index can never drift from the data. Initial migration: seven records spanning trace-repair (gated-stop confirm, free-lunch), multishot (golden oracle v1), trace-analysis (GEPA-certified prompt, prime-vs-dspy), creative-cad, and vertical-bench (parity no-flip). - `@tangle-network/agent-eval/experiment` publishes the evidence receipt: `createEvidenceReceipt`, `verifyEvidenceReceipt`, `isIndependentEvidence`, `EVIDENCE_RECEIPT_VERSION`, `EVIDENCE_AUTHORITY_KINDS`, `INDEPENDENT_EVIDENCE_AUTHORITY_KINDS`, and the `EvidenceReceipt` / `EvidenceBinding` / `EvidenceAuthority` / `EvidenceAuthorityKind` / `EvidenceReceiptVerification` / `CreateEvidenceReceiptInput` types. A receipt binds one Runtime execution to the measurement that judged it, without either package importing the other. It carries stable pursuit and run identity, the exact candidate, evaluator, environment, input-set and output content identities, the result digest, and the authority class that made the observation. The payload is attested with the existing canonical report attestation, so mutating any bound field invalidates the receipt. @@ -50,10 +50,6 @@ One repository still names three of them. `starter-foundry` imports `runAssertio ### Changed - No existing export changed. -======= -- The evidence registry: one canonical home for measured claims. `evidence/records/*.json` hold typed records (claim, instrument, exact command, arms, denominator, result, evidence state `CERTIFIED`/`MEASURED-ONCE`/`RESOLVED-NULL`/`UNVERIFIED`/`KILLED`, artifacts, cost, confounds, optional experiment seal digest), validated by `evidenceRecordSchema` from `./experiment`. `pnpm run evidence:render` generates `evidence/INDEX.md` from the records; `pnpm run evidence:check` (inside `verify:package`) fails on an invalid record or a stale index, so the human index can never drift from the data. Initial migration: seven records spanning trace-repair (gated-stop confirm, free-lunch), multishot (golden oracle v1), trace-analysis (GEPA-certified prompt, prime-vs-dspy), creative-cad, and vertical-bench (parity no-flip). -- Add a bounded `SearchHistoryReceipt` over the canonical `SearchLedger`, plus search-history coverage and `searchHistoryPolicy: 'require-complete'` on `compareOptimizationMethods()`. Strict mode refuses missing or denominator-incomplete optimization evidence before untouched-final-test scoring. Rich events remain only in the ledger. ->>>>>>> 14322f5d (feat(experiment): add the evidence registry — one home for measured claims) --- diff --git a/evidence/README.md b/evidence/README.md index 20bd60a2..2932b27f 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -6,7 +6,7 @@ This directory is the canonical registry for measured claims about our agents, p ## The rule - **A measured claim that anyone will cite later gets a record here.** - One JSON file per claim in [`records/`](./records/), validated by the exported schema (`evidenceRecordSchema` from `@tangle-network/agent-eval/experiment`). + One JSON file per claim in [`records/`](./records/), validated by the exported schema (`evidenceRegistryRecordSchema` from `@tangle-network/agent-eval/experiment`). - **The registry lives in agent-eval** because the measurement substrate owns evidence legitimacy. Every other repo keeps at most a pointer file (`docs/EVIDENCE.md`, three lines: "measured results for this repo live in agent-eval `evidence/`; do not restate numbers here"). - **Humans read [`INDEX.md`](./INDEX.md); machines read `records/*.json`.** diff --git a/src/experiment/evidence-record.test.ts b/src/experiment/evidence-record.test.ts index 2a351341..9f95a2fe 100644 --- a/src/experiment/evidence-record.test.ts +++ b/src/experiment/evidence-record.test.ts @@ -56,6 +56,13 @@ describe('evidenceRegistryRecordSchema', () => { it('rejects a negative cost', () => { expect(() => parseEvidenceRegistryRecord({ ...valid, costUsd: -1 })).toThrow() }) + + it('rejects a zero or negative denominator', () => { + expect(() => parseEvidenceRegistryRecord({ ...valid, n: { value: 0, unit: 'rows' } })).toThrow() + expect(() => + parseEvidenceRegistryRecord({ ...valid, n: { value: -5, unit: 'rows' } }), + ).toThrow() + }) }) describe('validateEvidenceRegistry', () => { diff --git a/src/experiment/evidence-record.ts b/src/experiment/evidence-record.ts index ea9c76ba..74823db5 100644 --- a/src/experiment/evidence-record.ts +++ b/src/experiment/evidence-record.ts @@ -51,7 +51,7 @@ const sha256Pattern = /^[0-9a-f]{64}$/ /** Denominator of the measurement. A claim without an `n` is prose, not evidence. */ export const evidenceDenominatorSchema = z.strictObject({ - value: z.number().finite(), + value: z.number().positive().finite(), /** What one unit is: rows, cases, rollouts, records, leaves… */ unit: z.string().min(1), /** Clustering, reps, split names — whatever the bare number hides. */ From 3e0c1a33cf4e8030ef05b1a765893f3749a6a5ad Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 19 Aug 2026 01:01:59 -0600 Subject: [PATCH 3/3] fix(analyst): repin both digests for the rebased tree The implementation pin absorbs the 522/524 retry change on main; the dependency-lock pin absorbs the evidence-check script additions. --- CHANGELOG.md | 5 ++++- package.json | 4 +++- src/analyst/benchmark-implementation.ts | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9822e4bc..77b89a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ## [Unreleased] +### Added + +- The evidence registry: one canonical home for measured claims. `evidence/records/*.json` hold typed records (claim, instrument, exact command, arms, denominator, result, evidence state `CERTIFIED`/`MEASURED-ONCE`/`RESOLVED-NULL`/`UNVERIFIED`/`KILLED`, artifacts, cost, confounds, optional experiment seal digest), validated by `evidenceRecordSchema` from `./experiment`. `pnpm run evidence:render` generates `evidence/INDEX.md` from the records; `pnpm run evidence:check` (inside `verify:package`) fails on an invalid record or a stale index, so the human index can never drift from the data. Initial migration: seven records spanning trace-repair (gated-stop confirm, free-lunch), multishot (golden oracle v1), trace-analysis (GEPA-certified prompt, prime-vs-dspy), creative-cad, and vertical-bench (parity no-flip). + --- ## [0.149.0] — 2026-08-18 @@ -34,7 +38,6 @@ One repository still names three of them. `starter-foundry` imports `runAssertio ### Added -- The evidence registry: one canonical home for measured claims. `evidence/records/*.json` hold typed records (claim, instrument, exact command, arms, denominator, result, evidence state `CERTIFIED`/`MEASURED-ONCE`/`RESOLVED-NULL`/`UNVERIFIED`/`KILLED`, artifacts, cost, confounds, optional experiment seal digest), validated by `evidenceRecordSchema` from `./experiment`. `pnpm run evidence:render` generates `evidence/INDEX.md` from the records; `pnpm run evidence:check` (inside `verify:package`) fails on an invalid record or a stale index, so the human index can never drift from the data. Initial migration: seven records spanning trace-repair (gated-stop confirm, free-lunch), multishot (golden oracle v1), trace-analysis (GEPA-certified prompt, prime-vs-dspy), creative-cad, and vertical-bench (parity no-flip). - `@tangle-network/agent-eval/experiment` publishes the evidence receipt: `createEvidenceReceipt`, `verifyEvidenceReceipt`, `isIndependentEvidence`, `EVIDENCE_RECEIPT_VERSION`, `EVIDENCE_AUTHORITY_KINDS`, `INDEPENDENT_EVIDENCE_AUTHORITY_KINDS`, and the `EvidenceReceipt` / `EvidenceBinding` / `EvidenceAuthority` / `EvidenceAuthorityKind` / `EvidenceReceiptVerification` / `CreateEvidenceReceiptInput` types. A receipt binds one Runtime execution to the measurement that judged it, without either package importing the other. It carries stable pursuit and run identity, the exact candidate, evaluator, environment, input-set and output content identities, the result digest, and the authority class that made the observation. The payload is attested with the existing canonical report attestation, so mutating any bound field invalidates the receipt. diff --git a/package.json b/package.json index caf98014..7e86183a 100644 --- a/package.json +++ b/package.json @@ -180,7 +180,9 @@ "check:model-ids": "node scripts/check-model-id-requests.mjs", "check:analyst-benchmark": "node scripts/check-analyst-benchmark-implementation.mjs", "openapi": "node dist/cli.js openapi --out dist/openapi.json", - "verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs" + "verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run evidence:check", + "evidence:render": "tsx scripts/render-evidence-index.ts", + "evidence:check": "tsx scripts/render-evidence-index.ts --check" }, "dependencies": { "@asteasolutions/zod-to-openapi": "^9.1.0", diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index c8e5dac3..47b3fae9 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '3e17b679d34215f636931e6d39cb5215d878cdaa501c38fd6cef513ce844677f' + 'd486ebde3bc6205e6486cd74547e3890c8136a8ad0873111bd5974ce4204737a' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the @@ -136,7 +136,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '6fe839500ca54d707265f11d6cd0bf8cbca55b9087d1b7d6ae751dc6960bfb1b' + 'ac95f22b1942262ff5090ad142b7593c43f598c942ae217f0d115159e4f33a67' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256