diff --git a/packages/loopover-engine/package.json b/packages/loopover-engine/package.json index 16523a8f4f..fbdaed805d 100644 --- a/packages/loopover-engine/package.json +++ b/packages/loopover-engine/package.json @@ -81,6 +81,10 @@ "./calibration/backtest-corpus": { "types": "./dist/calibration/backtest-corpus.d.ts", "default": "./dist/calibration/backtest-corpus.js" + }, + "./calibration/backtest-checksum": { + "types": "./dist/calibration/backtest-checksum.d.ts", + "default": "./dist/calibration/backtest-checksum.js" } }, "files": [ diff --git a/packages/loopover-engine/src/calibration/backtest-checksum.ts b/packages/loopover-engine/src/calibration/backtest-checksum.ts new file mode 100644 index 0000000000..cad3dff04f --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-checksum.ts @@ -0,0 +1,40 @@ +// The backtest corpus freeze point (#8136's reproducibility posture, moved here by #9639). +// +// This lived in scripts/backtest-corpus-export-core.ts, which made it reachable only from the CI-side CLIs. +// The in-Worker threshold backtest (src/services/threshold-backtest-run.ts) needs the SAME function to stamp +// its own runs: a deployment whose only backtest history is in-Worker was publishing no freeze point at all, +// so /v1/public/eval-scores served zero EvalScoreRecords. Two checksum implementations would have been worse +// than none -- a manifest frozen by one and validated by the other would disagree for no visible reason -- +// so the function moved rather than being copied. +// +// It is byte-stable by contract, not by accident: existing corpus manifests on disk carry checksums produced +// by the pre-move implementation, and scripts/backtest-logic-check.ts and scripts/attested-backtest-run.ts +// both re-validate a manifest against it before trusting the cases. Any change to the canonicalization or the +// hash input invalidates every manifest ever exported. backtest-checksum.test.ts pins the exact digest of a +// fixed fixture for that reason. +// +// node:crypto rather than Web Crypto: this must be SYNCHRONOUS (checksumCases is called inside pure, +// non-async manifest builders), and the Worker runs with nodejs_compat -- the same basis on which +// attester.ts, attestation-envelope.ts, backtest-split.ts and counterfactual-fixtures.ts in this same +// directory already import createHash. +import { createHash } from "node:crypto"; +import type { BacktestCase } from "./backtest-corpus.js"; + +/** Canonicalize one case (sort keys) so property-order differences don't change the checksum -- same technique + * as scripts/export-d1-core.ts's canonicalizeRow. */ +function canonicalizeCase(backtestCase: BacktestCase): Record { + // Sorted with the DEFAULT comparator over the keys, not a hand-written `a < b ? -1 : a > b ? 1 : 0`. For + // strings the two give the identical total order -- the pinned digest in this module's two test suites is + // what proves the output did not move -- but the hand-written form carries an equality arm that + // Object.keys can never trigger, since it yields each key exactly once. An unreachable branch cannot be + // tested, so it either sits uncovered or needs an ignore pragma; not writing it beats both. (The pragma + // was also flag-dependent: vitest's v8 provider honoured it, the engine package's own coverage did not.) + const source = backtestCase as unknown as Record; + return Object.fromEntries(Object.keys(source).sort().map((key) => [key, source[key]])); +} + +/** Deterministic SHA-256 over the canonicalized cases -- mirrors scripts/export-d1-core.ts's checksumRows + * exactly (canonicalize each entry, JSON-stringify the array, hash). */ +export function checksumCases(cases: readonly BacktestCase[]): string { + return createHash("sha256").update(JSON.stringify(cases.map(canonicalizeCase))).digest("hex"); +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 9eab8e2a6f..fc84cd7c7f 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -164,6 +164,9 @@ export * from "./governor/action-mode.js"; export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; +// #9639: the corpus freeze point, moved out of scripts/ so the in-Worker threshold backtest can stamp its +// own runs with the same checksum the CI-side manifests are frozen and validated by. +export * from "./calibration/backtest-checksum.js"; export * from "./calibration/repo-corpus-slice.js"; export * from "./calibration/ams-prediction-corpus.js"; export * from "./calibration/ams-rank-corpus.js"; diff --git a/packages/loopover-engine/test/backtest-checksum.test.ts b/packages/loopover-engine/test/backtest-checksum.test.ts new file mode 100644 index 0000000000..d92a655ba1 --- /dev/null +++ b/packages/loopover-engine/test/backtest-checksum.test.ts @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { checksumCases } from "../dist/index.js"; +import type { BacktestCase } from "../dist/calibration/backtest-corpus.js"; + +// #9639: checksumCases moved here from scripts/backtest-corpus-export-core.ts so the in-Worker threshold +// backtest can stamp its runs with the same freeze point the CI-side manifests are frozen and validated by. +// +// The root test/unit/backtest-checksum.test.ts covers this from the host side; this is its coverage in its +// OWN package, where the `engine` flag measures it. More than a flag exercise, though: this suite runs +// against dist/, so it is the only place the moved function is checked through the exact artifact published +// consumers import. + +// A fixed fixture with deliberately unsorted keys, so canonicalization is exercised and not just the hash. +const FIXTURE: BacktestCase[] = [ + { targetKey: "acme/widgets#2", ruleId: "ai_consensus_defect", outcome: "close", label: "reversed", decidedAt: "2026-01-03T00:00:00.000Z", firedAt: "2026-01-02T00:00:00.000Z", metadata: { confidence: 0.91 } }, + { ruleId: "ai_consensus_defect", firedAt: "2026-01-01T00:00:00.000Z", decidedAt: "2026-01-02T00:00:00.000Z", targetKey: "acme/widgets#1", label: "confirmed", outcome: "merge", metadata: { confidence: 0.42 } }, +]; + +// Produced by the PRE-MOVE implementation. Corpus manifests already on disk carry checksums from it, and +// scripts/backtest-logic-check.ts and scripts/attested-backtest-run.ts both re-validate a manifest against +// this function before trusting its cases -- so drift here does not fail loudly, it silently rejects every +// corpus ever exported. If this constant has to be edited to make the test pass, that has happened. +const FIXTURE_DIGEST = "bba3c7db2e1ff0b6943802416ebf94c789f37ac5ed962cc7167c2dd33a33a861"; + +test("checksumCases produces the pinned pre-move digest", () => { + assert.equal(checksumCases(FIXTURE), FIXTURE_DIGEST); +}); + +test("checksumCases ignores property order", () => { + const reordered = FIXTURE.map((c) => Object.fromEntries(Object.entries(c).reverse()) as unknown as BacktestCase); + assert.equal(checksumCases(reordered), FIXTURE_DIGEST); +}); + +test("checksumCases does NOT ignore case order -- the corpus is a sequence", () => { + assert.notEqual(checksumCases([...FIXTURE].reverse()), FIXTURE_DIGEST); +}); + +test("checksumCases distinguishes a changed value, so it genuinely commits to the cases", () => { + const mutated = FIXTURE.map((c, i) => (i === 0 ? { ...c, reversed: false } : c)); + assert.notEqual(checksumCases(mutated), FIXTURE_DIGEST); +}); + +test("checksumCases is deterministic and 64 hex chars for the empty corpus", () => { + // The value EMPTY_CORPUS_CHECKSUM recognises: identical for every rule, window and deployment, which is + // why eval-score-records.ts refuses to publish a record against it. + assert.equal(checksumCases([]), checksumCases([])); + assert.match(checksumCases([]), /^[0-9a-f]{64}$/); +}); diff --git a/scripts/attested-backtest-run.ts b/scripts/attested-backtest-run.ts index 0c020a56d2..2c86b8a979 100644 --- a/scripts/attested-backtest-run.ts +++ b/scripts/attested-backtest-run.ts @@ -30,7 +30,7 @@ import { readFileSync } from "node:fs"; import { assembleAttestationEnvelope, createSampleAttester, type Attester } from "@loopover/engine/calibration/attester"; import type { BacktestCase } from "@loopover/engine/calibration/backtest-corpus"; -import { checksumCases } from "./backtest-corpus-export-core"; +import { checksumCases } from "@loopover/engine/calibration/backtest-checksum"; import { attestedRunExitCode, buildAttestedRunAuditInsertSql, diff --git a/scripts/backtest-corpus-export-core.ts b/scripts/backtest-corpus-export-core.ts index 328e4e1f3d..4e30179f73 100644 --- a/scripts/backtest-corpus-export-core.ts +++ b/scripts/backtest-corpus-export-core.ts @@ -2,9 +2,15 @@ // built BacktestCase[] (from buildBacktestCorpus) into a versioned, checksummed manifest a scorer can reload // without re-querying D1. No IO here — the CLI (backtest-corpus-export.ts) does the wrangler/D1 reads and the // file write — so this stays unit-testable. Mirrors scripts/export-d1-core.ts's pure-core / thin-IO split. -import { createHash } from "node:crypto"; +import { checksumCases } from "@loopover/engine"; import type { BacktestCase } from "@loopover/engine/calibration/backtest-corpus"; +// #9639: checksumCases moved into the engine so the in-Worker threshold backtest can use it too. Re-exported +// here because this module's own consumers (and its test) have always imported it from this path, and the +// checksum must stay ONE function -- a manifest frozen by one copy and validated by another would disagree +// for no visible reason. +export { checksumCases }; + export type BacktestCorpusManifest = { ruleId: string; caseCount: number; @@ -12,18 +18,6 @@ export type BacktestCorpusManifest = { cases: BacktestCase[]; }; -/** Canonicalize one case (sort keys) so property-order differences don't change the checksum — same technique - * as export-d1-core.ts's canonicalizeRow. */ -function canonicalizeCase(backtestCase: BacktestCase): Record { - return Object.fromEntries(Object.entries(backtestCase).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); -} - -/** Deterministic SHA-256 over the canonicalized cases — mirrors export-d1-core.ts's checksumRows exactly - * (canonicalize each entry, JSON-stringify the array, hash). */ -export function checksumCases(cases: readonly BacktestCase[]): string { - return createHash("sha256").update(JSON.stringify(cases.map(canonicalizeCase))).digest("hex"); -} - /** * Build the export manifest for one rule's labeled corpus. Spreads an optional `meta` bag into the result * (the CLI attaches `generatedAt`); this core never reads the clock. Mirrors export-d1-core.ts's diff --git a/scripts/backtest-logic-check.ts b/scripts/backtest-logic-check.ts index 34f9574d1c..bf53760202 100644 --- a/scripts/backtest-logic-check.ts +++ b/scripts/backtest-logic-check.ts @@ -17,7 +17,7 @@ import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { spawnSync } from "node:child_process"; import type { BacktestCase } from "@loopover/engine"; -import { checksumCases } from "./backtest-corpus-export-core"; +import { checksumCases } from "@loopover/engine"; import { buildLogicBacktestAuditInsertSql, filterReplayableCases, diff --git a/scripts/replay-runner-image-manifest.json b/scripts/replay-runner-image-manifest.json index 889e035274..118a0ddcdf 100644 --- a/scripts/replay-runner-image-manifest.json +++ b/scripts/replay-runner-image-manifest.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, "baseImageRef": "node:22-slim@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3", - "dockerfileSha256": "7e4d3c18fbc3a4153d7ba25e3cff8329b07fceb6a52a62350ba0f3c65f54f2ff", - "packageLockSha256": "050ad66cb6a9ba6cfceb6e379f59341fea11eac0bdf5ffc2cfe17febdb0d5038", + "dockerfileSha256": "18b434af0c6359105293a9bd6bdd890f4b5662ce72308df4bcfc3bb30ae7c1ee", + "packageLockSha256": "35c61915f0b21d5980f5f5aa1bff532da3d9d51c71257c8b28904f7b798e0d71", "sourceFiles": { "scripts/attested-backtest-run-core.ts": "e5f5659dc9204f9406c9874317d71cb02fd7f4c993c9f5caa92545595f522167", - "scripts/attested-backtest-run.ts": "0db6b462afdf80086eb68da793a6dccbe3d5751eceeb19e338d2746a1c319fa5", - "scripts/backtest-corpus-export-core.ts": "6a268410ea1d8041e95af345149aeb4848376c4c72127b801ee9da377b81e85e", + "scripts/attested-backtest-run.ts": "49e0e046e3c4882c0e4afd422a464957b6c4789bad28f5b6682afa9d534424f7", + "scripts/backtest-corpus-export-core.ts": "060d48a2a6073a7d859efc64c87c18ae1d3d25e06391ce94d6ed40874050b2e8", "scripts/snp-attester.ts": "b917e8541908150bd0c885fbe57e178247e5cd5f78f80eda7dc9e8b77da6deb3" }, - "digest": "cdf7c3aaf83f814cf5b0ce37887abb264c9b5ee2d903ddb47a3d9dc60c64fa0c" + "digest": "5b8665f586317a0f3d77c5ac76891b4cf2ba8f1784977d268b093ce6644b2210" } diff --git a/scripts/replay-runner/Dockerfile b/scripts/replay-runner/Dockerfile index 59de929a12..d4f1855769 100644 --- a/scripts/replay-runner/Dockerfile +++ b/scripts/replay-runner/Dockerfile @@ -54,8 +54,11 @@ COPY --from=runtime-deps /app/node_modules ./node_modules # @loopover/engine is constructed here directly from its own compiled output -- NOT `npm ci`'d -- because npm # would install every dependency the FULL package declares (@anthropic-ai/claude-agent-sdk, tree-sitter-wasms, # ...), regardless of which files this image actually imports. attester.js/attestation-envelope.js/ -# backtest-corpus.js's entire runtime closure is these three files plus node:crypto (verified: `grep -n -# "^import" packages/loopover-engine/dist/calibration/*.js` -- attested-backtest-run.ts's own subpath imports, +# backtest-corpus.js/backtest-checksum.js's entire runtime closure is these four files plus node:crypto +# (verified: `grep -n "^import" packages/loopover-engine/dist/calibration/*.js` -- backtest-checksum.js, added +# by #9639, compiles to a single `node:crypto` import because its only other import is type-only and erased, +# so it widens this image's file list without widening its trusted computing base at all) +# -- attested-backtest-run.ts's own subpath imports, # #9214, are what make this closure reachable without ever loading the barrel's web-tree-sitter/ # claude-agent-sdk-carrying re-export graph in the first place). This measured image is meant to run inside # an eventual TEE (#8534); keeping its dependency footprint to exactly what replay needs is a real security @@ -64,6 +67,7 @@ COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-eng COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/attester.js ./node_modules/@loopover/engine/dist/calibration/attester.js COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/attestation-envelope.js ./node_modules/@loopover/engine/dist/calibration/attestation-envelope.js COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/backtest-corpus.js ./node_modules/@loopover/engine/dist/calibration/backtest-corpus.js +COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/backtest-checksum.js ./node_modules/@loopover/engine/dist/calibration/backtest-checksum.js # The runner script's own sources -- listed individually (rather than `COPY scripts/`) so this image's content # is EXACTLY the set replay-runner-image-manifest.json enumerates under sourceFiles; adding a new script # dependency here without a matching manifest entry is precisely the drift `replay-runner-manifest:check` catches. diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 9b99ea7c68..bbbb27d9c2 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -8595,9 +8595,12 @@ export async function resolveThresholdBacktestAdvisory( if (mode === "off") return ""; if (!files.some((file) => THRESHOLD_BACKTEST_WATCHED_PATHS.has(file.path))) return ""; try { - const { changed, comparisons } = await runThresholdBacktestAdvisory(env, buildSecretScanDiff(files)); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, buildSecretScanDiff(files)); if (comparisons.length === 0) return ""; - await persistThresholdBacktestRuns(env, repoFullName, pr.number, changed, comparisons); + // #9639: the freeze point travels with the run. Without it the persisted event is invisible to + // loadPublicRulePrecision's `corpusChecksum IS NOT NULL` filter, so /v1/public/eval-scores publishes + // nothing no matter how many backtests this deployment has actually run. + await persistThresholdBacktestRuns(env, repoFullName, pr.number, changed, comparisons, corpusChecksumByRuleId); // #8105 block mode: a REGRESSED verdict becomes a configured gate blocker. The finding only exists in // block mode (advisory keeps today's comment-only behavior byte-identically), and // isConfiguredGateBlocker's own `backtest_regression` branch is the defense-in-depth mirror of this diff --git a/src/services/threshold-backtest-run.ts b/src/services/threshold-backtest-run.ts index c90c6bb3e5..93a50ccb7f 100644 --- a/src/services/threshold-backtest-run.ts +++ b/src/services/threshold-backtest-run.ts @@ -9,7 +9,7 @@ // • A SignalStore read failure for one ruleId degrades that ruleId to an empty corpus (a real, if uninformative, // backtest result -- null precision/recall, per #8085's own null-when-no-data discipline) rather than // aborting the whole advisory. This never blocks the PR either way -- see the epic's own Boundaries. -import { buildBacktestCorpus, type BacktestCase, type BacktestComparison } from "@loopover/engine"; +import { buildBacktestCorpus, checksumCases, type BacktestCase, type BacktestComparison } from "@loopover/engine"; import { createSignalStore } from "../review/signal-tracking-wire"; import { recordAuditEvent } from "../db/repositories"; import { backtestChangedThreshold, detectChangedThresholds, type ChangedThreshold } from "./threshold-backtest"; @@ -32,6 +32,13 @@ async function fetchCorpus(env: Env, ruleId: string, nowMs: number): Promise; }; /** @@ -43,14 +50,21 @@ export type ThresholdBacktestRunResult = { */ export async function runThresholdBacktestAdvisory(env: Env, diff: string, nowMs: number = Date.now()): Promise { const changed = detectChangedThresholds(diff); - if (changed.length === 0) return { changed: [], comparisons: [] }; + if (changed.length === 0) return { changed: [], comparisons: [], corpusChecksumByRuleId: new Map() }; const ruleIds = [...new Set(changed.flatMap((change) => change.ruleIds))]; const corpusByRuleId = new Map(); for (const ruleId of ruleIds) corpusByRuleId.set(ruleId, await fetchCorpus(env, ruleId, nowMs)); + // Checksummed HERE, over the same map backtestChangedThreshold is about to score -- not re-read from the + // store afterwards. A second read could return a different corpus (a new override landing mid-run), and a + // freeze point that does not match the cases actually scored is worse than none: it invites a reader to + // re-derive different numbers and conclude the published ones are wrong. + const corpusChecksumByRuleId = new Map(); + for (const [ruleId, cases] of corpusByRuleId) corpusChecksumByRuleId.set(ruleId, checksumCases(cases)); + const comparisons = changed.flatMap((change) => backtestChangedThreshold(change, corpusByRuleId)); - return { changed, comparisons }; + return { changed, comparisons, corpusChecksumByRuleId }; } /** Persist each comparison for #8140's future track-record tool -- structured data via the shared @@ -63,12 +77,17 @@ export async function persistThresholdBacktestRuns( prNumber: number, changed: readonly ChangedThreshold[], comparisons: readonly BacktestComparison[], + // #9639: optional so existing call sites that only have `changed`/`comparisons` keep compiling, but every + // in-tree caller passes it -- without a checksum the event is exactly the one loadPublicRulePrecision + // filters out with `corpusChecksum IS NOT NULL`, which is the bug this closes. + corpusChecksumByRuleId: ReadonlyMap = new Map(), ): Promise { const comparisonsByRuleId = new Map(comparisons.map((comparison) => [comparison.ruleId, comparison])); for (const change of changed) { for (const ruleId of change.ruleIds) { const comparison = comparisonsByRuleId.get(ruleId); if (!comparison) continue; + const corpusChecksum = corpusChecksumByRuleId.get(ruleId); await recordAuditEvent(env, { eventType: THRESHOLD_BACKTEST_EVENT_TYPE, actor: "loopover", @@ -79,7 +98,17 @@ export async function persistThresholdBacktestRuns( // how every other telemetry-shaped audit event in this codebase uses "completed" for "happened, see detail". outcome: "completed", detail: `${change.constantName} threshold backtest for ${ruleId}: ${comparison.verdict}`, - metadata: { comparison, constantName: change.constantName }, + // corpusChecksum is a SIBLING of the existing keys, in the same position and under the same name + // buildLogicBacktestAuditInsertSql writes it (scripts/backtest-logic-check-core.ts) -- the two writers + // are siblings by design and loadPublicRulePrecision reads them through one query. `comparison` and + // `constantName` are untouched: rule-calibration-trend.ts reads `$.comparison.verdict` off these rows. + // Omitted rather than written as null when absent, so the reader's `IS NOT NULL` filter stays the + // single place that decides whether a run is a usable freeze point. + metadata: { + comparison, + constantName: change.constantName, + ...(corpusChecksum === undefined ? {} : { corpusChecksum }), + }, }).catch(() => undefined); } } diff --git a/test/unit/backtest-checksum.test.ts b/test/unit/backtest-checksum.test.ts new file mode 100644 index 0000000000..efe7b2077f --- /dev/null +++ b/test/unit/backtest-checksum.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +// Imported by RELATIVE SOURCE path, not by the "@loopover/engine" specifier: that specifier resolves through +// the package's exports map to the compiled dist/, and v8 coverage would attribute every hit to the gitignored +// built file instead of this source -- the same misattribution vitest.config.ts's contractSourceAliases() +// exists to prevent for @loopover/contract, and the shape every other engine unit test here already uses. +import { checksumCases } from "../../packages/loopover-engine/src/calibration/backtest-checksum"; +import { checksumCases as checksumCasesViaBarrel } from "@loopover/engine"; +import type { BacktestCase } from "@loopover/engine/calibration/backtest-corpus"; +import { checksumCases as checksumCasesViaExportCore } from "../../scripts/backtest-corpus-export-core"; + +// #9639: checksumCases moved from scripts/backtest-corpus-export-core.ts into the engine so the in-Worker +// threshold backtest can stamp its runs with the SAME freeze point the CI-side manifests use. +// +// The move is only safe if the output is byte-identical. Corpus manifests already on disk carry checksums +// produced by the pre-move implementation, and scripts/backtest-logic-check.ts and +// scripts/attested-backtest-run.ts both re-validate a manifest against this function before trusting its +// cases -- so a drifted implementation does not fail loudly, it rejects every previously-exported corpus. + +// A fixed fixture, written out literally rather than generated: the digest below is only meaningful if the +// input can never change. Keys are deliberately NOT in sorted order, so the test also exercises the +// canonicalization rather than just the hash. +const FIXTURE: BacktestCase[] = [ + { targetKey: "acme/widgets#2", ruleId: "ai_consensus_defect", outcome: "close", label: "reversed", decidedAt: "2026-01-03T00:00:00.000Z", firedAt: "2026-01-02T00:00:00.000Z", metadata: { confidence: 0.91 } }, + { ruleId: "ai_consensus_defect", firedAt: "2026-01-01T00:00:00.000Z", decidedAt: "2026-01-02T00:00:00.000Z", targetKey: "acme/widgets#1", label: "confirmed", outcome: "merge", metadata: { confidence: 0.42 } }, +]; + +// Produced by the pre-move implementation in scripts/backtest-corpus-export-core.ts. If this constant has to +// be edited to make the test pass, the freeze point has drifted and every exported manifest is invalidated. +const FIXTURE_DIGEST = "bba3c7db2e1ff0b6943802416ebf94c789f37ac5ed962cc7167c2dd33a33a861"; + +describe("checksumCases byte-stability across the #9639 move", () => { + it("produces the pinned digest for the fixed fixture", () => { + expect(checksumCases(FIXTURE)).toBe(FIXTURE_DIGEST); + }); + + it("agrees byte-for-byte through the engine barrel and through scripts/backtest-corpus-export-core", () => { + // Behavioral sameness, not reference identity: the three paths resolve through different module + // instances (source here, dist via the package specifier), and it is the DIGEST that must not fork -- + // a manifest frozen through one path and validated through another has to agree. + expect(checksumCasesViaBarrel(FIXTURE)).toBe(FIXTURE_DIGEST); + expect(checksumCasesViaExportCore(FIXTURE)).toBe(FIXTURE_DIGEST); + }); + + it("ignores property order, so a reordered case set freezes to the same digest", () => { + const reordered = FIXTURE.map((c) => Object.fromEntries(Object.entries(c).reverse()) as unknown as BacktestCase); + expect(checksumCases(reordered)).toBe(FIXTURE_DIGEST); + }); + + it("does NOT ignore case order -- the corpus is a sequence, and a reordered export is a different freeze point", () => { + expect(checksumCases([...FIXTURE].reverse())).not.toBe(FIXTURE_DIGEST); + }); + + it("distinguishes a changed value, so the checksum actually commits to the cases", () => { + const mutated = FIXTURE.map((c, i) => (i === 0 ? { ...c, reversed: false } : c)); + expect(checksumCases(mutated)).not.toBe(FIXTURE_DIGEST); + }); + + it("hashes the empty corpus to a stable, rule-independent digest", () => { + // This is the value EMPTY_CORPUS_CHECKSUM exists to recognise: identical for every rule and every + // deployment, which is why eval-score-records.ts refuses to publish a record against it. + expect(checksumCases([])).toBe(checksumCases([])); + expect(checksumCases([])).toHaveLength(64); + }); +}); diff --git a/test/unit/eval-score-records.test.ts b/test/unit/eval-score-records.test.ts index ff47c9875b..f657092e44 100644 --- a/test/unit/eval-score-records.test.ts +++ b/test/unit/eval-score-records.test.ts @@ -10,7 +10,10 @@ import { type EvalScoreRecord, } from "../../src/review/eval-score-records"; import { contentDigest, sha256Hex } from "../../src/review/decision-record"; -import type { PublicRulePrecision } from "../../src/review/public-rule-precision"; +import { loadPublicRulePrecision, PUBLIC_PRECISION_MIN_DECIDED, type PublicRulePrecision } from "../../src/review/public-rule-precision"; +import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import { persistThresholdBacktestRuns, runThresholdBacktestAdvisory } from "../../src/services/threshold-backtest-run"; +import { createTestEnv } from "../helpers/d1"; const ISSUED_AT = "2026-07-27T12:00:00.000Z"; @@ -237,3 +240,69 @@ describe("per-rule corpus commitments when no backtest run is persisted (#9805)" expect(await buildEvalScoreRecordsFromRulePrecision(precisionOf([rule("ai_consensus_defect")]), ISSUED_AT)).toEqual([]); }); }); + +// #9639 Deliverable 4: the /v1/public/eval-scores regression, pinned end to end over a real D1. The unit +// tests above feed buildEvalScoreRecordsFromRulePrecision a hand-built PublicRulePrecision; this one starts +// from an in-Worker threshold backtest and goes all the way to records, because the defect lived in the seam +// between the writer and the reader -- both halves were individually correct and produced [] together. +describe("in-Worker backtest -> /v1/public/eval-scores records (#9639)", () => { + const diff = [ + "diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts", + "@@ -980,7 +980,7 @@", + "-export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.5;", + "+export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.4;", + ].join("\n"); + + async function seedRunAndLoad(env: Env, now: number) { + const store = createSignalStore(env); + // Enough decided cases to clear PUBLIC_PRECISION_MIN_DECIDED, so a rule actually reaches the block. + for (let i = 0; i < PUBLIC_PRECISION_MIN_DECIDED + 2; i += 1) { + await store.recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + outcome: "unaddressed", + occurredAt: new Date(now - (i + 2) * 1000).toISOString(), + metadata: { confidence: 0.3 + (i % 4) * 0.15 }, + }); + await store.recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + verdict: i % 2 === 0 ? "reversed" : "confirmed", + occurredAt: new Date(now - (i + 1) * 1000).toISOString(), + }); + } + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + return { run, precision: await loadPublicRulePrecision(env, now) }; + } + + it("REGRESSION: emits one record per rule instead of [] -- the whole surface was empty before the writer stamped a checksum", async () => { + const env = createTestEnv(); + const now = Date.now(); + const { run, precision } = await seedRunAndLoad(env, now); + expect(precision.rules.length).toBeGreaterThan(0); + + const records = await buildEvalScoreRecordsFromRulePrecision(precision, ISSUED_AT); + + expect(records).toHaveLength(precision.rules.length); + for (const record of records) { + expect(record.commitments.corpusChecksum).toBe(run.corpusChecksumByRuleId.get("linked_issue_scope_mismatch")); + expect(record.commitments.corpusChecksum).not.toBe(EMPTY_CORPUS_CHECKSUM); + expect(record.trust.tier).toBe("reproducible"); + // Each record still commits to its own content, so the freeze point cannot be swapped undetected. + await expect(verifyEvalScoreRecordDigest(record)).resolves.toBe(true); + } + }); + + it("goes back to [] when the same pipeline runs against an empty corpus", async () => { + // The counter-case for the one above: the run is persisted either way, so an assertion that records exist + // is only meaningful alongside one showing they still do not when the commitment is worthless. + const env = createTestEnv(); + const now = Date.now(); + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + + const precision = await loadPublicRulePrecision(env, now); + expect(await buildEvalScoreRecordsFromRulePrecision(precision, ISSUED_AT)).toEqual([]); + }); +}); diff --git a/test/unit/public-rule-precision.test.ts b/test/unit/public-rule-precision.test.ts index 4e1746b978..bc2cbc39b5 100644 --- a/test/unit/public-rule-precision.test.ts +++ b/test/unit/public-rule-precision.test.ts @@ -8,6 +8,7 @@ import { import { sha256Hex } from "../../src/review/decision-record"; import { recordAuditEvent } from "../../src/db/repositories"; import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import { persistThresholdBacktestRuns, runThresholdBacktestAdvisory } from "../../src/services/threshold-backtest-run"; import { createTestEnv } from "../helpers/d1"; // #8230 (epic #8211 track G): the public measured-accuracy block. Load-bearing properties: the public @@ -192,3 +193,61 @@ describe("loadPublicRulePrecision (#8230)", () => { expect(serialized).not.toMatch(/acme|#\d|targetKey|confidence|wallet|hotkey|trust|reward|payout/i); }); }); + +// #9639: the reader's `IN ('calibration.threshold_backtest_run', 'calibration.logic_backtest_run')` listed two +// writers but only the CI-side logic one ever satisfied the companion `corpusChecksum IS NOT NULL` filter. +// These pin the in-Worker writer's row as a first-class freeze point, end to end through the real service. +describe("latestBacktestRun from an in-Worker threshold run (#9639)", () => { + const diff = [ + "diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts", + "@@ -980,7 +980,7 @@", + "-export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.5;", + "+export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.4;", + ].join("\n"); + + const seedAndRun = async (env: Env, now: number) => { + const store = createSignalStore(env); + // A labeled case needs BOTH halves: the firing (with its confidence) and the human verdict that scores it. + // Firings alone build an empty corpus, whose checksum is the rule-independent EMPTY_CORPUS_CHECKSUM. + for (let i = 0; i < 4; i += 1) { + await store.recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + outcome: "unaddressed", + occurredAt: new Date(now - (i + 2) * 1000).toISOString(), + metadata: { confidence: 0.3 + i * 0.2 }, + }); + await store.recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + verdict: i % 2 === 0 ? "reversed" : "confirmed", + occurredAt: new Date(now - (i + 1) * 1000).toISOString(), + }); + } + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + return run; + }; + + it("REGRESSION: a threshold_backtest_run is now a usable freeze point -- this returned null before the writer emitted corpusChecksum", async () => { + const env = createTestEnv(); + const now = Date.now(); + const run = await seedAndRun(env, now); + + const block = await loadPublicRulePrecision(env, now); + expect(block.latestBacktestRun).not.toBeNull(); + expect(block.latestBacktestRun?.corpusChecksum).toBe(run.corpusChecksumByRuleId.get("linked_issue_scope_mismatch")); + expect(block.latestBacktestRun?.corpusChecksum).not.toBe(EMPTY_CORPUS_CHECKSUM); + }); + + it("still returns null when the same writer ran against an empty corpus -- a hash over zero cases commits to nothing", async () => { + // No seeded history, so the corpus is empty and its checksum is the rule-independent EMPTY_CORPUS_CHECKSUM. + // The row IS written (the run happened); the reader is what declines to treat it as a commitment. + const env = createTestEnv(); + const now = Date.now(); + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + + expect((await loadPublicRulePrecision(env, now)).latestBacktestRun).toBeNull(); + }); +}); diff --git a/test/unit/threshold-backtest-run.test.ts b/test/unit/threshold-backtest-run.test.ts index 7107d933e6..4c08eabaa3 100644 --- a/test/unit/threshold-backtest-run.test.ts +++ b/test/unit/threshold-backtest-run.test.ts @@ -4,6 +4,7 @@ import { createSignalStore } from "../../src/review/signal-tracking-wire"; import * as repositories from "../../src/db/repositories"; import { listAuditEventsByType } from "../../src/db/repositories"; import { persistThresholdBacktestRuns, runThresholdBacktestAdvisory, THRESHOLD_BACKTEST_EVENT_TYPE } from "../../src/services/threshold-backtest-run"; +import { checksumCases } from "@loopover/engine"; import { createTestEnv } from "../helpers/d1"; afterEach(() => { @@ -18,7 +19,7 @@ describe("runThresholdBacktestAdvisory (#8138) — real D1 round-trip", () => { it("returns empty changed/comparisons when the diff touches no known threshold, without any D1 read", async () => { const env = createTestEnv(); const result = await runThresholdBacktestAdvisory(env, "diff --git a/README.md b/README.md\n-old\n+new"); - expect(result).toEqual({ changed: [], comparisons: [] }); + expect(result).toEqual({ changed: [], comparisons: [], corpusChecksumByRuleId: new Map() }); }); it("backtests a changed LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR against real recorded history", async () => { @@ -111,3 +112,121 @@ describe("persistThresholdBacktestRuns (#8138) — real D1 round-trip", () => { await expect(persistThresholdBacktestRuns(env, "acme/widgets", 8, changed, comparisons)).resolves.toBeUndefined(); }); }); + +// #9639: loadPublicRulePrecision selects the latest backtest run with +// `json_extract(metadata_json, '$.corpusChecksum') IS NOT NULL`. This writer never emitted the field, so a +// deployment whose only backtest history is in-Worker had `latestBacktestRun: null` forever -- and +// buildEvalScoreRecordsFromRulePrecision early-returns [] on that, which is why /v1/public/eval-scores +// served zero records with no diagnostic. +describe("persistThresholdBacktestRuns writes the corpus freeze point (#9639)", () => { + const seedHistory = async (env: Env, now: number) => { + const store = createSignalStore(env); + // A labeled case needs BOTH halves: the firing (with its confidence) and the human verdict that scores it. + // Firings alone build an empty corpus, whose checksum is the rule-independent EMPTY_CORPUS_CHECKSUM. + for (let i = 0; i < 4; i += 1) { + await store.recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + outcome: "unaddressed", + occurredAt: new Date(now - (i + 2) * 1000).toISOString(), + metadata: { confidence: 0.3 + i * 0.2 }, + }); + await store.recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + verdict: i % 2 === 0 ? "reversed" : "confirmed", + occurredAt: new Date(now - (i + 1) * 1000).toISOString(), + }); + } + }; + + it("REGRESSION: writes metadata.corpusChecksum, the field the public reader filters on", async () => { + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, changed, comparisons, corpusChecksumByRuleId); + + const rows = await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()); + expect(rows).toHaveLength(1); + const checksum = rows[0]!.metadata.corpusChecksum; + expect(checksum).toMatch(/^[0-9a-f]{64}$/); + // Not the rule-independent empty-corpus digest -- that would commit to nothing re-derivable. + expect(checksum).not.toBe(checksumCases([])); + // Not just "a hash": the SAME hash the shared freeze-point function produces over the scored corpus. + expect(checksum).toBe(corpusChecksumByRuleId.get("linked_issue_scope_mismatch")); + }); + + it("leaves metadata.comparison and metadata.constantName untouched -- rule-calibration-trend.ts reads $.comparison.verdict off these rows", async () => { + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + await persistThresholdBacktestRuns(env, "acme/widgets", 9, changed, comparisons, corpusChecksumByRuleId); + + const row = (await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()))[0]!; + expect(row.metadata.constantName).toBe("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR"); + const comparison = row.metadata.comparison as { ruleId: string; verdict: string }; + expect(comparison.ruleId).toBe("linked_issue_scope_mismatch"); + expect(typeof comparison.verdict).toBe("string"); + }); + + it("checksums the corpus that was SCORED, not a fresh read -- a later override does not move the freeze point", async () => { + // The freeze point has to describe the cases the verdict came from. If it were re-read at persist time, + // a reader re-deriving from it would get numbers that disagree with the published ones and conclude the + // publication was wrong. + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + const frozen = corpusChecksumByRuleId.get("linked_issue_scope_mismatch"); + + await createSignalStore(env).recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#999", + outcome: "unaddressed", + occurredAt: new Date(now - 2000).toISOString(), + metadata: { confidence: 0.99 }, + }); + await createSignalStore(env).recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#999", + verdict: "reversed", + occurredAt: new Date(now - 1500).toISOString(), + }); + await persistThresholdBacktestRuns(env, "acme/widgets", 11, changed, comparisons, corpusChecksumByRuleId); + + const row = (await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()))[0]!; + expect(row.metadata.corpusChecksum).toBe(frozen); + }); + + it("omits corpusChecksum entirely (never null) when a direct caller supplies no map, so the reader's IS NOT NULL filter stays the single gate", async () => { + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + await persistThresholdBacktestRuns(env, "acme/widgets", 13, changed, comparisons); + + const row = (await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()))[0]!; + expect("corpusChecksum" in row.metadata).toBe(false); + }); + + it("still records a freeze point for a ruleId whose corpus read FAILED -- the empty corpus is a true statement about what was backtested", async () => { + // fetchCorpus degrades a failed read to [], and that is genuinely what got scored. The reader is what + // refuses to publish against EMPTY_CORPUS_CHECKSUM; the writer must not silently omit the field and make + // a failed read indistinguishable from an old deployment that never wrote one. + const env = createTestEnv(); + const now = Date.now(); + vi.spyOn(signalTrackingWire, "createSignalStore").mockReturnValue({ + queryRuleHistory: () => Promise.reject(new Error("simulated read failure")), + } as unknown as ReturnType); + + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + vi.restoreAllMocks(); + await persistThresholdBacktestRuns(env, "acme/widgets", 15, changed, comparisons, corpusChecksumByRuleId); + + const rows = await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()); + if (rows.length > 0) expect(rows[0]!.metadata.corpusChecksum).toBe(checksumCases([])); + expect(corpusChecksumByRuleId.get("linked_issue_scope_mismatch")).toBe(checksumCases([])); + }); +});