From 5c957ce2916a08ebfc47cd659dc8460abf491b2f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 21:46:55 -0700 Subject: [PATCH 01/16] ci(oracles): run the e2e oracle's random and determinism units on every PR CI-1 (random seeds run on every PR) and P12 (every job under 10 minutes). pr-shards.mjs gets two partitions: oracle-gaps-random, in two shards, and oracle-gaps-determinism, in four. Each shard is a longest-first packing of one Arclet run of main at 35422e3: the random units take 624 s and the determinism units 1,337 s, and the largest shard is about 340 s of unit time. pr.yml runs them as blocking Linux jobs, and the preview waits for them. The e2e-shards comment that said CI runs only the fixed-seed job is updated. Co-authored-by: Isaac --- .github/workflows/pr.yml | 46 +++++++++++- packages/oracles/scripts/e2e-shards.mjs | 5 +- packages/oracles/scripts/pr-shards.mjs | 95 +++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8cbab1b3..52cd5129 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -120,11 +120,53 @@ jobs: - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The end-to-end oracle's fixed-seed units, in three shards. Its random - # units, calibrations, and determinism units run locally. + # units and determinism units run in their own jobs, and its + # calibrations run locally. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} + test-oracle-gaps-random: + name: Gap report end-to-end oracle (random seed, ${{ matrix.shard }}/2) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + # The end-to-end oracle's random units, in two shards. Each run draws a + # fresh campaign seed (the ORACLE_SEED line in the log), and a failing + # unit prints its Replay line. A counterexample is filed as an issue. + - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-random "$SHARD")" + working-directory: packages/oracles + env: + SHARD: ${{ matrix.shard }} + test-oracle-gaps-determinism: + name: Gap report end-to-end oracle (determinism, ${{ matrix.shard }}/4) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + # The end-to-end oracle's determinism units, in four shards. Each runs a + # history twice and compares the two traces. + - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism "$SHARD")" + working-directory: packages/oracles + env: + SHARD: ${{ matrix.shard }} test-vitest-5: name: Test on Vitest 5 (${{ matrix.suite }}) # The package's vitest peer range includes 5; this job is the evidence. @@ -167,6 +209,8 @@ jobs: - test-oracle-framework - test-oracle-gaps-fast - test-oracle-gaps-end-to-end + - test-oracle-gaps-random + - test-oracle-gaps-determinism - test-vitest-5 # The blocking Windows jobs (the non-blocking static and e2e jobs are not # a dependency until they are green). diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index ee1e2456..09c94038 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -36,8 +36,9 @@ const suite = `oracle-gaps.report` // cost follows its seed, so a shard's time varies from run to run. Option M // of the e2e speed plan bounds the random units' tail: a generated history // with the witness module takes no steps, and the one edit sequence over it -// is a constructed history. CI keeps its one fixed-seed job until the -// maintainer decides which units run on a pull request. +// is a constructed history. A pull request runs the fixed-seed, random, and +// determinism units as the partitions of pr-shards.mjs; these shards are +// the full-suite plan, which this file's check keeps partitioned. export const shards = { // 522 s in the measured run, about 221 s idle. 'e2e-1': [ diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 6f93b39c..441de344 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -313,6 +313,101 @@ export const partitions = { 3: rest, }, }, + // The end-to-end gap oracle's random units, which every pull request runs + // (maintainer decision CI-1): each run draws a fresh campaign seed, and a + // counterexample prints its replay line. 38 units, measured on main at + // 35422e3 on one Arclet (8 vCPU Linux, one Vitest worker): 624 s in all. + // The witness module's edit sequence alone takes 178 s, so two shards of + // about 312 s each are the fewest that keep a job near 5 minutes. + 'oracle-gaps-random': { + args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], + suite: `oracle-gaps.report`, + select: `random seed: `, + shards: { + // 312 s: the witness module's edit sequence (178 s), brokenOwner + // (74 s), and the small units. + 1: anyOf([ + 'an edit sequence over the witness module', + 'a declaration fault: brokenOwner', + 'a declaration fault: brokenVitest', + 'a declaration fault: noTypescript', + 'an edit that changed the cold result after a hang replaced the only context', + 'generated', + 'an edit to a module that only mutated runs load', + 'a failing owner under a config that --config names', + 'a load failure beside a hang', + 'an ambiguous authored mutant', + 'a package whose parent directory has its config', + 'a .ts config in a package with no module type that calls require', + 'a config that --config names', + 'ordinary tests left out by default', + 'a declaration fault: configNull', + 'a declaration fault: configThrows', + 'a declaration fault: missingSource', + 'a declaration fault: nonLiteral', + 'a config fault in a run that owns nothing', + 'a declaration fault: outsideSource', + ]), + // The rest: every outcome class (104 s), apiLessTypescript (75 s), the + // daemon's edit sequence (65 s), and the second daemon check (44 s), + // about 312 s, and any unit added later. + 2: rest, + }, + }, + // The end-to-end gap oracle's determinism units, which every pull request + // runs: each runs one constructed or generated history twice and compares + // the two traces. 40 units, measured in the same run on main at 35422e3: + // 1,337 s in all. Six of them fail on main (their full time is counted): + // the traces differ in the heartbeat list, which records which busy mutants + // beat, so the check is wall-clock dependent. The longest unit, the witness + // module's edit sequence, takes 339 s, so four shards of 333 to 339 s are + // the fewest that keep each job near 6 minutes. + 'oracle-gaps-determinism': { + args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], + suite: `oracle-gaps.report`, + select: `determinism: `, + shards: { + // 339 s: the witness module's edit sequence, alone. + 1: 'constructed history: an edit sequence over the witness module', + // 333 s: every outcome class (223 s) and the small units. + 2: anyOf([ + 'constructed history: every outcome class', + 'constructed history: a declaration fault: brokenVitest', + 'constructed history: a stale authored mutant and a reformat', + 'constructed history: an edit that changed the cold result after a hang replaced the only context', + 'constructed history: two authored mutants', + 'constructed history: an ambiguous authored mutant', + 'constructed history: a run with eleven contexts', + 'constructed history: a package whose parent directory has its config', + 'constructed history: a declaration fault: configNoDefault', + 'constructed history: a declaration fault: configThrows', + 'constructed history: a declaration fault: nonLiteral', + 'constructed history: a declaration fault: projects', + 'constructed history: a config fault in a run that owns nothing', + ]), + // 333 s: generated 2 (192 s), the second daemon check (88 s), and the + // small units. + 3: anyOf([ + 'generated 2', + 'constructed history: a second daemon check of the budget witness', + 'constructed history: a lazy module loaded before a hang in the same batch', + 'constructed history: an edit to a module that only mutated runs load', + 'constructed history: a failing owner under a config that --config names', + 'constructed history: a load failure beside a hang', + 'constructed history: a config that --config names', + 'constructed history: a .ts config in a package with no module type that calls require', + 'constructed history: ordinary tests left out by default', + 'constructed history: a declaration fault: configSyntax', + 'constructed history: a declaration fault: missingSource', + 'generated 0', + 'constructed history: a declaration fault: outsideSource', + ]), + // The rest: the daemon's edit sequence (134 s), noTypescript (89 s), + // brokenOwner (76 s), and the small units, about 333 s, and any unit + // added later. + 4: rest, + }, + }, 'oracle-gaps-static-windows': { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-static`], suite: `oracle-gaps.report-static`, From 10f69371a1e7977a1682f8617eb8a468701ef05b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 22:19:19 -0700 Subject: [PATCH 02/16] ci(oracles): exclude the six determinism units that fail on main, each with an issue The determinism job gates every unit except six that fail on main (#91 heartbeat list, #92 daemon note): every outcome class, brokenOwner, noTypescript, the witness module's edit sequence, generated 2, and the daemon's edit sequence. Each excluded unit has its own issue (#93 to #98). pr-shards.mjs takes an excluded list: the rest shard skips it, the partition check fails if a shard holds an excluded name or an excluded name is not a test, and an excluded test is in no shard on purpose. CI ran the random shards about 1.7 times slower than the Arclet, so random is three shards and determinism is two. Co-authored-by: Isaac --- .github/workflows/pr.yml | 15 +-- packages/oracles/scripts/pr-shards.mjs | 168 ++++++++++++++----------- 2 files changed, 100 insertions(+), 83 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 52cd5129..0dfc6cc8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -127,12 +127,12 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-random: - name: Gap report end-to-end oracle (random seed, ${{ matrix.shard }}/2) + name: Gap report end-to-end oracle (random seed, ${{ matrix.shard }}/3) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -140,7 +140,7 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's random units, in two shards. Each run draws a + # The end-to-end oracle's random units, in three shards. Each run draws a # fresh campaign seed (the ORACLE_SEED line in the log), and a failing # unit prints its Replay line. A counterexample is filed as an issue. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-random "$SHARD")" @@ -148,12 +148,12 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-determinism: - name: Gap report end-to-end oracle (determinism, ${{ matrix.shard }}/4) + name: Gap report end-to-end oracle (determinism, ${{ matrix.shard }}/2) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4] + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -161,8 +161,9 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's determinism units, in four shards. Each runs a - # history twice and compares the two traces. + # The end-to-end oracle's determinism units, in two shards, except the + # six that fail on main and are excluded in pr-shards.mjs, each with an + # issue. Each unit runs a history twice and compares the two traces. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism "$SHARD")" working-directory: packages/oracles env: diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 441de344..a6472f95 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -317,95 +317,96 @@ export const partitions = { // (maintainer decision CI-1): each run draws a fresh campaign seed, and a // counterexample prints its replay line. 38 units, measured on main at // 35422e3 on one Arclet (8 vCPU Linux, one Vitest worker): 624 s in all. - // The witness module's edit sequence alone takes 178 s, so two shards of - // about 312 s each are the fewest that keep a job near 5 minutes. + // CI ran the random shards 1.7 times slower than the Arclet (9 min for a + // 312 s shard, in #90's first run), and a random unit's cost follows its + // seed. So the shards are three of about 208 s here, which is about 6 min + // in CI. The witness module's edit sequence alone is 178 s. 'oracle-gaps-random': { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], suite: `oracle-gaps.report`, select: `random seed: `, shards: { - // 312 s: the witness module's edit sequence (178 s), brokenOwner - // (74 s), and the small units. + // 208 s here. 1: anyOf([ - 'an edit sequence over the witness module', - 'a declaration fault: brokenOwner', - 'a declaration fault: brokenVitest', - 'a declaration fault: noTypescript', - 'an edit that changed the cold result after a hang replaced the only context', - 'generated', - 'an edit to a module that only mutated runs load', - 'a failing owner under a config that --config names', - 'a load failure beside a hang', - 'an ambiguous authored mutant', - 'a package whose parent directory has its config', - 'a .ts config in a package with no module type that calls require', - 'a config that --config names', - 'ordinary tests left out by default', - 'a declaration fault: configNull', - 'a declaration fault: configThrows', - 'a declaration fault: missingSource', - 'a declaration fault: nonLiteral', - 'a config fault in a run that owns nothing', - 'a declaration fault: outsideSource', + `an edit sequence over the witness module`, + `a declaration fault: noTypescript`, + `a lazy module loaded before a hang in the same batch`, + `a load failure beside a hang`, + `an ambiguous authored mutant`, + `a .ts config in a package with no module type that calls require`, + `an awaiting config in a package with no module type`, + `a declaration fault: configNull`, + `a declaration fault: configSyntax`, + `a declaration fault: missingSource`, + `a config fault in a run that owns nothing`, ]), - // The rest: every outcome class (104 s), apiLessTypescript (75 s), the - // daemon's edit sequence (65 s), and the second daemon check (44 s), - // about 312 s, and any unit added later. - 2: rest, + // 208 s here. + 2: anyOf([ + `every outcome class`, + `an edit sequence through the daemon`, + `a declaration fault: brokenVitest`, + `an edit that changed the cold result after a hang replaced the only context`, + `an edit to a module that only mutated runs load`, + `a failing owner under a config that --config names`, + `a failing owner under its parent's config`, + `a package whose parent directory has its config`, + `a package with no config`, + `ordinary tests left out by default`, + `a declaration fault: configNoDefault`, + `a declaration fault: badEntry`, + `a declaration fault: projects`, + `a declaration fault: outsideSource`, + ]), + // The rest: 208 s here, and any unit added later. + 3: rest, }, }, - // The end-to-end gap oracle's determinism units, which every pull request - // runs: each runs one constructed or generated history twice and compares - // the two traces. 40 units, measured in the same run on main at 35422e3: - // 1,337 s in all. Six of them fail on main (their full time is counted): - // the traces differ in the heartbeat list, which records which busy mutants - // beat, so the check is wall-clock dependent. The longest unit, the witness - // module's edit sequence, takes 339 s, so four shards of 333 to 339 s are - // the fewest that keep each job near 6 minutes. + // The end-to-end gap oracle's determinism units, which pull requests run + // except the excluded ones below. Each unit runs one history twice and + // compares the two traces. 40 units, measured on main at 35422e3 on one + // Arclet: 1,337 s in all. Six fail on main, and their time is what they + // take to fail: the traces differ in the heartbeat list (wall-clock + // timing, #91) or in a daemon note (#92). Those six are excluded, one + // issue each (#93 to #98), so this job gates every other unit. A test that + // a pull request adds runs here, since only the names below are excluded; + // a change to an excluded unit's history does not run it until its issue + // closes. The other 34 units take 284 s here, in two shards. 'oracle-gaps-determinism': { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], suite: `oracle-gaps.report`, select: `determinism: `, + excluded: [ + // Heartbeat list (#91): #93 to #97. + `constructed history: every outcome class`, + `constructed history: a declaration fault: brokenOwner`, + `constructed history: a declaration fault: noTypescript`, + `constructed history: an edit sequence over the witness module`, + `generated 2`, + // Daemon test-output note (#92): #98. + `constructed history: an edit sequence through the daemon`, + ], shards: { - // 339 s: the witness module's edit sequence, alone. - 1: 'constructed history: an edit sequence over the witness module', - // 333 s: every outcome class (223 s) and the small units. - 2: anyOf([ - 'constructed history: every outcome class', - 'constructed history: a declaration fault: brokenVitest', - 'constructed history: a stale authored mutant and a reformat', - 'constructed history: an edit that changed the cold result after a hang replaced the only context', - 'constructed history: two authored mutants', - 'constructed history: an ambiguous authored mutant', - 'constructed history: a run with eleven contexts', - 'constructed history: a package whose parent directory has its config', - 'constructed history: a declaration fault: configNoDefault', - 'constructed history: a declaration fault: configThrows', - 'constructed history: a declaration fault: nonLiteral', - 'constructed history: a declaration fault: projects', - 'constructed history: a config fault in a run that owns nothing', - ]), - // 333 s: generated 2 (192 s), the second daemon check (88 s), and the - // small units. - 3: anyOf([ - 'generated 2', - 'constructed history: a second daemon check of the budget witness', - 'constructed history: a lazy module loaded before a hang in the same batch', - 'constructed history: an edit to a module that only mutated runs load', - 'constructed history: a failing owner under a config that --config names', - 'constructed history: a load failure beside a hang', - 'constructed history: a config that --config names', - 'constructed history: a .ts config in a package with no module type that calls require', - 'constructed history: ordinary tests left out by default', - 'constructed history: a declaration fault: configSyntax', - 'constructed history: a declaration fault: missingSource', - 'generated 0', - 'constructed history: a declaration fault: outsideSource', + // 142 s here. + 1: anyOf([ + `constructed history: a second daemon check of the budget witness`, + `constructed history: an edit that changed the cold result after a hang replaced the only context`, + `constructed history: a declaration fault: apiLessTypescript`, + `constructed history: two authored mutants`, + `constructed history: an ambiguous authored mutant`, + `constructed history: a failing owner under its parent's config`, + `constructed history: a config that --config names`, + `constructed history: a .ts config in a package with no module type that calls require`, + `constructed history: a package whose parent directory has its config`, + `constructed history: a declaration fault: configNoDefault`, + `constructed history: a declaration fault: configThrows`, + `constructed history: a declaration fault: configSyntax`, + `constructed history: a declaration fault: nonLiteral`, + `constructed history: a declaration fault: projects`, + `constructed history: test.projects in a parent's config`, + `constructed history: a declaration fault: outsideSource`, ]), - // The rest: the daemon's edit sequence (134 s), noTypescript (89 s), - // brokenOwner (76 s), and the small units, about 333 s, and any unit - // added later. - 4: rest, + // The rest: 142 s here, and any unit added later. + 2: rest, }, }, 'oracle-gaps-static-windows': { @@ -674,11 +675,18 @@ export const partitions = { }, } +/** A partition's excluded names, relative to its select: tests in no shard. */ +const excludedOf = (partition) => new Set(partition.excluded ?? []) + /** A shard's pattern over the name that follows the suite's. */ function bodyOf(partition, shard) { const own = partition.shards[shard] if (own !== rest) return `${escape(partition.select)}(?:${own})` - const others = Object.values(partition.shards).filter((body) => body !== rest) + // A rest shard takes what no other shard and no exclusion takes. + const others = [ + ...Object.values(partition.shards).filter((body) => body !== rest), + ...[...excludedOf(partition)].map(escape), + ] return `(?!${escape(partition.select)}(?:${others.join(`|`)})$)${escape(partition.select)}.*` } @@ -726,7 +734,12 @@ export function partitionErrors(name, listed) { joined.every((text) => owners[0].pattern.test(text)) const own = full.startsWith(prefix) ? full.slice(prefix.length) : undefined const selected = own !== undefined && own.startsWith(partition.select) - if (selected && !whole) + // An excluded test is in no shard on purpose; its issue is the record. + const excluded = + selected && excludedOf(partition).has(own.slice(partition.select.length)) + if (excluded && owners.length > 0) + errors.push(`${JSON.stringify(full)} is excluded, but is in a shard`) + if (selected && !whole && !excluded) errors.push( `${JSON.stringify(full)} is in ${owners.length === 0 ? `no shard` : owners.length === 1 ? `shard ${owners[0].shard} under only one of Vitest's name joins` : `shards ${owners.map(({ shard }) => shard).join(`, `)}`}`, ) @@ -744,6 +757,9 @@ export function partitionErrors(name, listed) { for (const exception of partition.exceptions ?? []) if (!names.has(`${prefix}${exception}`)) errors.push(`exception ${JSON.stringify(exception)} names no test`) + for (const exclusion of excludedOf(partition)) + if (!names.has(`${prefix}${partition.select}${exclusion}`)) + errors.push(`excluded ${JSON.stringify(exclusion)} names no test`) for (const [shard, body] of Object.entries(partition.shards)) if ( body !== rest && From dee496cdb2fd3eabab73d72d49144045094c82dd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:23:14 -0700 Subject: [PATCH 03/16] fix(oracles): judge heartbeats for shape only, never by their set Maintainer decision, 2026-11-14: heartbeats are progress, not outcome. A heartbeat's set and count depend on wall-clock timing, so the e2e oracle's determinism check compared a timing-dependent trace and failed five units on main (#91). The trace now keeps no heartbeat. The driver observes the breaches of the shape law instead: a beat names the baseline or a mutant of its run, in a file that runs it, and a beat that follows its mutant's own streamed gap line names a mutant that was not running. The check reports a breach as a streaming failure. The presence obligation (a slow owner must be named in a heartbeat) is retired with the set, and so is the model's owed-heartbeat flag. Calibrations feed wrong beat observations to the same check. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 22 --- .../gap-report-oracle.property.test.mjs | 144 ++++++++++++------ 2 files changed, 95 insertions(+), 71 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 36a093a0..04e4aa3a 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4099,18 +4099,6 @@ function filesRun(runs) { return ran } -/** - * The owner files in which a slow test waits under a mutant, among the - * files its run reaches: each copy of such a file is busy for SLOW_MS, so - * the CLI owes a heartbeat for it. - */ -function slowedFiles(plan, tests, sites, active) { - const slowed = new Set() - for (const [file, own] of filesRun(runsOf(plan, tests, sites, active))) - if (own.some((r) => r.slowed)) slowed.add(file) - return slowed -} - /** * A mutant's kind from its owner tests' runs. An operator mutant's * confirming runs share its budget (`budgeted`); an authored mutant's runs @@ -4506,15 +4494,6 @@ function model(plan) { ordinal: site.ordinal, probe: site.probe, kind, - // A mutant whose run reaches a slow owner test keeps a copy busy for - // SLOW_MS, so the CLI must name it in a heartbeat while it runs. - heartbeat: - kind !== undefined && - ![`unowned`, `unreached`, `setup-error`, `indeterminate`].includes( - kind, - ) && - slowedFiles(plan, ownerTests(site.fnId), sites, { site: site.id }) - .size > 0, ordinaryOutcome: plan.ordinary && survivorKinds.has(kind) && ordinaryReaches(site), } @@ -5894,7 +5873,6 @@ export { ownersOf, interpret, judge, - slowedFiles, MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 547f4e36..10a2b4d6 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1023,6 +1023,34 @@ function streamOf(lines, name) { } } +/** + * Law 11, shape only (maintainer decision, 2026-11-14). A heartbeat names the + * baseline or a mutant of this run, in a file that runs it, and a CLI prints + * it only while its copy runs, so a beat after its mutant's own streamed gap + * line names a mutant that was not running. Returns the breaches as text, + * none in a correct run. The beats' set and count depend on wall-clock timing, + * so they are progress, not outcome: no trace keeps them. + */ +function heartbeatBreaches(cli) { + const byId = new Map((cli.mutants ?? []).map((m) => [m.id, m])) + return (cli.beats ?? []).flatMap(({ label, file, index, finishedAt }) => { + if (label === `baseline`) + return /^tests\/[abc]\.test\.ts$/.test(file) + ? [] + : [`baseline in ${file}: not a baseline file`] + const m = byId.get(label) + if (m === undefined) return [`${label} in ${file}: no mutant of this run`] + if (finishedAt !== undefined && finishedAt < index) + return [ + `${label} in ${file}: its gap line came first, so it was not running`, + ] + const runs = m.authored + ? /^tests\/\w+\.test\.ts$/.test(file) && file !== `tests/c.test.ts` + : [...(m.oracleFiles ?? []), ...(m.ordinaryFiles ?? [])].includes(file) + return runs ? [] : [`${label} in ${file}: the file does not run it`] + }) +} + function readRun(dir, out, status, stderr, cli, stdout) { // The output names the CLI directory that ran and the package, as scratch // directories do, before it is cut to its last characters. @@ -1068,6 +1096,27 @@ function readRun(dir, out, status, stderr, cli, stdout) { ] .sort() .map((beat) => JSON.parse(beat)) + // Each beat in stream order, with the stream index of its mutant's own + // streamed gap line when that came first (law 11's shape). The driver + // judges these with heartbeatBreaches; the trace observes only the breaches. + const streamedAt = new Map() + lines.forEach((line, index) => { + const id = line.match(streamedGap)?.[1] + if (id !== undefined && !streamedAt.has(id)) streamedAt.set(id, index) + }) + const beats = lines.flatMap((line, index) => { + const beat = line.match(/^oracle-gaps: busy (\S+) in (\S+) for \d+s$/) + return beat + ? [ + { + label: beat[1], + file: beat[2], + index, + finishedAt: streamedAt.get(beat[1]), + }, + ] + : [] + }) // The stdout summary's lead, up to its first blank line (law 14), its // last line, and the files the baseline lines on stderr name. Since the // stream has no blank line (law 15), the lead is the whole stdout, whose @@ -1092,6 +1141,7 @@ function readRun(dir, out, status, stderr, cli, stdout) { return file === undefined ? [] : [file] }), heartbeats, + beats, // The config the run says it used (law 3), and the warnings Node printed // for the process. configLines: lines.filter((line) => @@ -1777,7 +1827,16 @@ async function driver(plan, observe) { const tornRead = reached > 0 // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. - observe(`cli`, { ...cli, evidence: undefined }) + // Heartbeats are progress, not outcome (maintainer decision, 2026-11-14): + // their set depends on wall-clock timing, so the trace keeps none of them. + // Their shape breaches are observed as `beats`, which is empty in a correct run. + observe(`cli`, { + ...cli, + evidence: undefined, + beats: undefined, + heartbeats: undefined, + }) + observe(`beats`, { breaches: heartbeatBreaches(cli) }) // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) observe(`replay`, { command: replay, ran: replayed(dir, replay) }) @@ -2566,39 +2625,16 @@ function check(plan, observations, expected) { `streaming`, { actual: cli.streamed, expected: cli.gaps }, ) - // A heartbeat names what its busy copy runs: a mutant, and a test file - // that runs it (an owner file whose tests reached it, an ordinary file for - // the ordinary tier, any oracle file for an authored mutant), or the - // baseline and a test file. Each mutant whose run reaches a slow owner - // test keeps a copy busy past the interval, so a heartbeat names it. - const byId = new Map(cli.mutants.map((m) => [m.id, m])) - for (const beat of cli.heartbeats) { - const m = byId.get(beat.label) - const runsIt = - beat.label === `baseline` - ? /^tests\/[abc]\.test\.ts$/.test(beat.file) - : m?.authored - ? /^tests\/\w+\.test\.ts$/.test(beat.file) && - beat.file !== `tests/c.test.ts` - : [...(m?.oracleFiles ?? []), ...(m?.ordinaryFiles ?? [])].includes( - beat.file, - ) - if (!runsIt) - return violation( - `cli`, - `a heartbeat names ${beat.label} in ${beat.file}, which does not run it`, - `streaming`, - { actual: beat }, - ) - } - for (const [owed, got] of pairs) - if (owed.heartbeat && !cli.heartbeats.some((beat) => beat.label === got.id)) - return violation( - `cli`, - `${site(got)}: no heartbeat named it while its slow owner test ran`, - `streaming`, - { actual: cli.heartbeats, expected: got.id }, - ) + // Law 11, shape only (maintainer decision, 2026-11-14): the driver observes + // the heartbeats that break the shape, and a breach is a streaming failure. + const beats = observations.find((o) => o.checkpoint === `beats`)?.value + if (beats?.breaches.length > 0) + return violation( + `cli`, + `a heartbeat breaks its shape: ${beats.breaches[0]}`, + `streaming`, + { actual: beats.breaches }, + ) for (const [owed, got] of pairs) if (!owed.probe && got.ordinaryOutcome !== owed.ordinaryOutcome) return violation( @@ -3024,6 +3060,16 @@ const corruptCli = (change) => (trace) => { if (value === undefined) return undefined return trace.map((o, i) => (i === index ? { ...o, value } : o)) } +// A checker calibration's wrong beat observation: one breach of the shape law, +// set in a run that completed (a run that did not complete has no beats). +const corruptBeats = (breach) => (trace) => { + const cli = trace.find((o) => o.checkpoint === `cli`)?.value + const index = trace.findIndex((o) => o.checkpoint === `beats`) + if (cli?.exit !== 0 || index < 0) return undefined + return trace.map((o, i) => + i === index ? { ...o, value: { breaches: [breach] } } : o, + ) +} const firstWhere = (predicate, change) => corruptCli((cli) => { if (cli.exit !== 0) return undefined @@ -3253,25 +3299,25 @@ calibrations.push( })), }, { - name: `a busy mutant is named in no heartbeat`, + name: `a heartbeat names a file that does not run its mutant`, law: `streaming`, - corrupt: corruptCli((cli) => - cli.exit !== 0 || cli.heartbeats.length === 0 - ? undefined - : { ...cli, heartbeats: [] }, + corrupt: corruptBeats( + `sut.ts:M1 in tests/elsewhere.test.ts: the file does not run it`, ), }, { - name: `a heartbeat names a file that does not run its mutant`, + name: `a heartbeat names a mutant whose gap line came first`, law: `streaming`, - corrupt: corruptCli((cli) => { - const beat = cli.heartbeats.find((b) => b.label !== `baseline`) - if (cli.exit !== 0 || !beat) return undefined - return { - ...cli, - heartbeats: [{ ...beat, file: `tests/elsewhere.test.ts` }], - } - }), + corrupt: corruptBeats( + `sut.ts:M1 in tests/a.test.ts: its gap line came first, so it was not running`, + ), + }, + { + name: `a heartbeat names no mutant of its run`, + law: `streaming`, + corrupt: corruptBeats( + `sut.ts:M999 in tests/a.test.ts: no mutant of this run`, + ), }, // Law 15: a line without the stream's prefix, the decisive baseline line // after a mutant's line, a completed run that ends with the old @@ -3680,7 +3726,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats; beats are checked for shape only (maintainer decision, 2026-11-14): each names a mutant of the run and a file that runs it, and a beat that follows its mutant's gap line is a breach, while their set and count, which depend on timing, are not compared and no beat's presence is required; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs From 8e1e820329bbbd0464a70e3ac7ebcb071f38af5c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:34:11 -0700 Subject: [PATCH 04/16] fix(oracles): keep heartbeats out of every observed run, and judge every run's beats The stability rerun and the partial run observed their CLI runs with their heartbeat lists still in them, so the determinism check compared timing in two more observations. Every observed run now drops its beats, and each run that is observed also observes its shape breaches. The check judges every beats observation. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 38 +++++++++++-------- packages/oracles/scripts/pr-shards.mjs | 13 ++++++- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 10a2b4d6..b805d3ba 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1023,6 +1023,17 @@ function streamOf(lines, name) { } } +// A run as the trace observes it: its heartbeats and their positions are +// progress (see heartbeatBreaches), and its evidence counts are timing. +function unbeaten(run) { + return { + ...run, + evidence: undefined, + beats: undefined, + heartbeats: undefined, + } +} + /** * Law 11, shape only (maintainer decision, 2026-11-14). A heartbeat names the * baseline or a mutant of this run, in a file that runs it, and a CLI prints @@ -1830,12 +1841,7 @@ async function driver(plan, observe) { // Heartbeats are progress, not outcome (maintainer decision, 2026-11-14): // their set depends on wall-clock timing, so the trace keeps none of them. // Their shape breaches are observed as `beats`, which is empty in a correct run. - observe(`cli`, { - ...cli, - evidence: undefined, - beats: undefined, - heartbeats: undefined, - }) + observe(`cli`, unbeaten(cli)) observe(`beats`, { breaches: heartbeatBreaches(cli) }) // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) @@ -2284,7 +2290,9 @@ async function driver(plan, observe) { const moved = { ...files, [SOURCE]: edited(plan) } // The evidence counts are not observed, as for `cli`: they depend on // timing (oracles-followups.md G1). Verdicts and keys are. - observe(`stability`, { ...runCli(dir, moved, how), evidence: undefined }) + const again = runCli(dir, moved, how) + observe(`beats`, { breaches: heartbeatBreaches(again) }) + observe(`stability`, unbeaten(again)) states.add(`stability rerun`) } if (plan.steps.length > 0) @@ -2627,14 +2635,14 @@ function check(plan, observations, expected) { ) // Law 11, shape only (maintainer decision, 2026-11-14): the driver observes // the heartbeats that break the shape, and a breach is a streaming failure. - const beats = observations.find((o) => o.checkpoint === `beats`)?.value - if (beats?.breaches.length > 0) - return violation( - `cli`, - `a heartbeat breaks its shape: ${beats.breaches[0]}`, - `streaming`, - { actual: beats.breaches }, - ) + for (const { value } of observations.filter((o) => o.checkpoint === `beats`)) + if (value.breaches.length > 0) + return violation( + `cli`, + `a heartbeat breaks its shape: ${value.breaches[0]}`, + `streaming`, + { actual: value.breaches }, + ) for (const [owed, got] of pairs) if (!owed.probe && got.ordinaryOutcome !== owed.ordinaryOutcome) return violation( diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index a6472f95..c7e33762 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -25,7 +25,12 @@ import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`) -const anyOf = (names) => `(?:${names.map(escape).join(`|`)})` +// The names a shard lists are kept with its pattern, so the check can find a +// name that matches no test. +const anyOf = (names) => ({ + names, + toString: () => `(?:${names.map(escape).join(`|`)})`, +}) const rest = Symbol(`rest`) // The self-oracle's test names, split by the start of the name after its @@ -760,6 +765,12 @@ export function partitionErrors(name, listed) { for (const exclusion of excludedOf(partition)) if (!names.has(`${prefix}${partition.select}${exclusion}`)) errors.push(`excluded ${JSON.stringify(exclusion)} names no test`) + for (const body of Object.values(partition.shards)) + for (const name of body.names ?? []) + if (!names.has(`${prefix}${partition.select}${name}`)) + errors.push( + `a shard names ${JSON.stringify(name)}, which matches no test`, + ) for (const [shard, body] of Object.entries(partition.shards)) if ( body !== rest && From 78bddab62b03ae6edb3d4e769e03163c851f172a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:46:08 -0700 Subject: [PATCH 05/16] fix(oracles): stop observing the heartbeat-derived reach state The reach state "a heartbeat named a busy mutant" is derived from a beat's timing, so determinism compared a timing-dependent observation. Under the maintainer's 2026-11-14 decision that heartbeats are progress, the state is no longer observed or required. #84 owns restoring its coverage through a deterministic pause point. Recorded in the Limits. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index b805d3ba..3708f891 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -2154,8 +2154,6 @@ async function driver(plan, observe) { states.add(`a hang in one owner and a survivor in another timed out`) if (m.function === `w_slowpair` && m.outcome === `timeout`) states.add(`confirmations past a mutant's budget timed out`) - if (cli.heartbeats.some((beat) => beat.label === m.id)) - states.add(`a heartbeat named a busy mutant`) const named = { killed: `killed`, 'random-only-kill': `random-only kill`, @@ -3564,7 +3562,6 @@ const required = [ `a hang in one owner and a kill in another killed`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, - `a heartbeat named a busy mutant`, `a run stopped by SIGTERM while a worker spun`, `a run stopped by SIGINT while a worker spun`, `a stop signal after the completed line`, @@ -3734,7 +3731,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats; beats are checked for shape only (maintainer decision, 2026-11-14): each names a mutant of the run and a file that runs it, and a beat that follows its mutant's gap line is a breach, while their set and count, which depend on timing, are not compared and no beat's presence is required; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats; beats are checked for shape only (maintainer decision, 2026-11-14): each names a mutant of the run and a file that runs it, and a beat that follows its mutant's gap line is a breach, while their set and count, which depend on timing, are not compared and no beat's presence is required; so the reach state "a heartbeat named a busy mutant" is no longer observed or required (it was progress, derived from a beat's timing; #84 owns restoring its coverage through a deterministic pause point); a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs From 3cdb8a23246b81208ef39697eceb3717d3a9a185 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:57:59 -0700 Subject: [PATCH 06/16] fix(oracles): stop observing the timing-derived timed-out reuse state The reach state "a timed-out run reused after an edit" compares the cut counts of a daemon check and a cold run. Those counts are timing, so the state depended on wall-clock timeouts, and the determinism check failed on the witness module's edit sequence. It is no longer observed or required, and the witness history no longer enters it. Recorded in the Limits. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 3708f891..a5aa7b3b 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1521,18 +1521,6 @@ async function incremental(dir, plan, cli, copy, observe, states) { `an unchanged check through the daemon computed nothing while a test read a source through fs`, ) } - // The witness module's hangs are cut off in every cold run. A daemon - // check after an edit that cut off fewer runs than the cold run of the - // same state reused a timed-out run, so law 9 judged a timeout that came - // from the store. - if ( - plan.witness !== false && - step !== `start` && - !daemonEvents.has(step) && - reused > 0 && - (daemon.evidence?.cut ?? 0) < (coldRun.evidence?.cut ?? 0) - ) - states.add(`a timed-out run reused after an edit`) // Undoing an edit to a file returns to a state the daemon has seen, so // it reruns nothing. if ( @@ -3618,7 +3606,6 @@ const required = [ `a config fault in a run that owns nothing`, ...incrementalStates, ...ownedIncrementalStates, - `a timed-out run reused after an edit`, `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, `a lazy edit changed the cold result after a batch was cut off`, @@ -3701,7 +3688,6 @@ const separate = [ `a failing owner replayed under a config that --config names`, ...incrementalStates, ...ownedIncrementalStates, - `a timed-out run reused after an edit`, `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, `a lazy edit changed the cold result after a batch was cut off`, @@ -3731,7 +3717,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats; beats are checked for shape only (maintainer decision, 2026-11-14): each names a mutant of the run and a file that runs it, and a beat that follows its mutant's gap line is a breach, while their set and count, which depend on timing, are not compared and no beat's presence is required; so the reach state "a heartbeat named a busy mutant" is no longer observed or required (it was progress, derived from a beat's timing; #84 owns restoring its coverage through a deterministic pause point); a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats; beats are checked for shape only (maintainer decision, 2026-11-14): each names a mutant of the run and a file that runs it, and a beat that follows its mutant's gap line is a breach, while their set and count, which depend on timing, are not compared and no beat's presence is required; so the reach state "a heartbeat named a busy mutant" is no longer observed or required (it was progress, derived from a beat's timing; #84 owns restoring its coverage through a deterministic pause point); and \"a timed-out run reused after an edit\" is no longer observed or required, because its predicate compares evidence cut counts, which depend on timing (the evidence counts are not observed, as for cli), so the witness module's edit sequence no longer enters it; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4254,7 +4240,6 @@ oracleProperty({ // cheapest legal setting, so that the history costs about two cold // runs and four daemon checks of the witness module. name: `an edit sequence over the witness module`, - enters: [`a timed-out run reused after an edit`], build: (generated) => ({ ...generated, functions: [ From d77b63e9364f5396108d1e76cf84868ed776f8bc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:59:08 -0700 Subject: [PATCH 07/16] fix(oracles): give the witness history an empty enters list The removed reach state was the witness history's only enters entry, so the file failed to load. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index a5aa7b3b..33b3f67e 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -4240,6 +4240,7 @@ oracleProperty({ // cheapest legal setting, so that the history costs about two cold // runs and four daemon checks of the witness module. name: `an edit sequence over the witness module`, + enters: [], build: (generated) => ({ ...generated, functions: [ From 082991ddef24631baa6b77ab2d3f8a976a5023c9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 00:09:45 -0700 Subject: [PATCH 08/16] ci(oracles): run the determinism units, excluding only the daemon note (#92) With the heartbeat fix, the five units that failed on main pass, so the determinism partition takes them back: the witness module's edit sequence in its own shard, and the other 49 units in three packed shards and a rest shard, five jobs in all. Only #92 stays excluded. The partition check now fails on a shard name that matches no test, which the explicit names had never been checked for. Co-authored-by: Isaac --- .github/workflows/pr.yml | 10 +-- packages/oracles/scripts/pr-shards.mjs | 93 ++++++++++++++++---------- 2 files changed, 64 insertions(+), 39 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0dfc6cc8..5dfd032a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -148,12 +148,12 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-determinism: - name: Gap report end-to-end oracle (determinism, ${{ matrix.shard }}/2) + name: Gap report end-to-end oracle (determinism, ${{ matrix.shard }}/5) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3, 4, 5] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -161,9 +161,9 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's determinism units, in two shards, except the - # six that fail on main and are excluded in pr-shards.mjs, each with an - # issue. Each unit runs a history twice and compares the two traces. + # The end-to-end oracle's determinism units, in five shards, except the + # one excluded in pr-shards.mjs (#92). Each unit runs a history twice and + # compares the results and fixed lines; heartbeats are checked for shape. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism "$SHARD")" working-directory: packages/oracles env: diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index c7e33762..5f1e873f 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -5,10 +5,12 @@ // shard of its partition takes. The check lists the suite's tests with // `vitest list` and fails unless every selected test is in exactly one // shard, no other test is in any shard, and every test that a shard names -// exists. So the jobs together run exactly the tests that one job ran -// before: a test in no shard would leave its law out of CI, and a test in -// two would run twice. A new test lands in the `rest` shard, so the jobs' -// balance can drift, but no test is left out. +// exists. A partition's `excluded` names are the exception: each is in no +// shard on purpose, with its own issue, and must exist. So the jobs together +// run exactly the tests that one job ran before: a test in no shard would +// leave its law out of CI, and a test in two would run twice. A new test +// lands in the `rest` shard, so the jobs' balance can drift, but no test is +// left out. // // node scripts/pr-shards.mjs check checks every // partition @@ -367,51 +369,74 @@ export const partitions = { }, }, // The end-to-end gap oracle's determinism units, which pull requests run - // except the excluded ones below. Each unit runs one history twice and - // compares the two traces. 40 units, measured on main at 35422e3 on one - // Arclet: 1,337 s in all. Six fail on main, and their time is what they - // take to fail: the traces differ in the heartbeat list (wall-clock - // timing, #91) or in a daemon note (#92). Those six are excluded, one - // issue each (#93 to #98), so this job gates every other unit. A test that - // a pull request adds runs here, since only the names below are excluded; - // a change to an excluded unit's history does not run it until its issue - // closes. The other 34 units take 284 s here, in two shards. + // except the one excluded below. Each unit runs one history twice and + // compares the two traces: results and fixed lines, not the heartbeats, + // which are progress (maintainer decision, 2026-11-14). 51 units; measured + // on Arclets at d77b63e, one Vitest worker: 1,305 s for the 50 units that + // are not excluded. The witness module's edit sequence is 344 s alone, so + // it is a shard by itself; the other 49 units are packed longest first into + // three shards of 240 s, and the rest shard takes the fourth bin and any + // unit added later. The excluded unit, the daemon's test-output note (#92), + // has a cause that is not heartbeats, so it stays out until #92 is fixed. 'oracle-gaps-determinism': { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], suite: `oracle-gaps.report`, select: `determinism: `, excluded: [ - // Heartbeat list (#91): #93 to #97. - `constructed history: every outcome class`, - `constructed history: a declaration fault: brokenOwner`, - `constructed history: a declaration fault: noTypescript`, - `constructed history: an edit sequence over the witness module`, - `generated 2`, - // Daemon test-output note (#92): #98. + // The daemon's test-output note (#92). `constructed history: an edit sequence through the daemon`, ], shards: { - // 142 s here. + // 344 s: the witness module's edit sequence, alone. 1: anyOf([ - `constructed history: a second daemon check of the budget witness`, - `constructed history: an edit that changed the cold result after a hang replaced the only context`, + `constructed history: an edit sequence over the witness module`, + ]), + // 240 s here. + 2: anyOf([ + `constructed history: every outcome class`, + `constructed history: an edit to data a test reads through fs (statDir)`, `constructed history: a declaration fault: apiLessTypescript`, - `constructed history: two authored mutants`, - `constructed history: an ambiguous authored mutant`, `constructed history: a failing owner under its parent's config`, - `constructed history: a config that --config names`, + `constructed history: a package with no config`, `constructed history: a .ts config in a package with no module type that calls require`, - `constructed history: a package whose parent directory has its config`, `constructed history: a declaration fault: configNoDefault`, - `constructed history: a declaration fault: configThrows`, - `constructed history: a declaration fault: configSyntax`, - `constructed history: a declaration fault: nonLiteral`, - `constructed history: a declaration fault: projects`, - `constructed history: test.projects in a parent's config`, + `constructed history: a declaration fault: badEntry`, `constructed history: a declaration fault: outsideSource`, ]), - // The rest: 142 s here, and any unit added later. - 2: rest, + // 240 s here. + 3: anyOf([ + `generated 2`, + `constructed history: an edit during a check through the daemon`, + `constructed history: an edit to data a test reads through fs (sync)`, + `constructed history: an unchanged check through the daemon while a test reads a source through fs`, + `constructed history: two authored mutants`, + `constructed history: a capture row read while the worker wrote it`, + `constructed history: two batches each read a capture row while its worker wrote it`, + `constructed history: a package whose parent directory has its config`, + `constructed history: a declaration fault: configNull`, + `constructed history: a declaration fault: missingSource`, + `constructed history: a config fault in a run that owns nothing`, + ]), + // 240 s here. + 4: anyOf([ + `constructed history: a declaration fault: noTypescript`, + `constructed history: a declaration fault: brokenVitest`, + `constructed history: a stale authored mutant and a reformat`, + `constructed history: an edit that changed the cold result after a hang replaced the only context`, + `constructed history: a test edits the helper it read, during a check through the daemon`, + `generated 1`, + `constructed history: an edit to data a test reads through fs (callback)`, + `constructed history: an unchanged check through the daemon under a config's pre plugin`, + `constructed history: an awaiting config in a package with no module type`, + `constructed history: a run with eleven contexts`, + `constructed history: a .cjs config that calls require`, + `constructed history: ordinary tests left out by default`, + `constructed history: a declaration fault: setupThrows`, + `constructed history: a declaration fault: nonLiteral`, + `generated 0`, + ]), + // The rest: 240 s here, and any unit added later. + 5: rest, }, }, 'oracle-gaps-static-windows': { From 2c5e0e7608175f24f561fe16f08b5bf1236f26a1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 00:32:00 -0700 Subject: [PATCH 09/16] test(oracles): canary for the constructed-history order The e2e gap oracle's constructed histories are built from seed + index, so a history inserted mid-list changes every later one (generator rule, register 2026-11-14: new constructed histories are appended at the end). The canary records the registered names in oracle-gaps/constructed-order.json and fails unless the current list begins with them. Co-authored-by: Isaac --- .../oracle-gaps/constructed-order.json | 52 +++++++++++++++++++ .../gap-report-history-fingerprint.test.ts | 48 +++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 packages/oracles/oracle-gaps/constructed-order.json diff --git a/packages/oracles/oracle-gaps/constructed-order.json b/packages/oracles/oracle-gaps/constructed-order.json new file mode 100644 index 00000000..b7bf32de --- /dev/null +++ b/packages/oracles/oracle-gaps/constructed-order.json @@ -0,0 +1,52 @@ +{ + "constructed": [ + "every outcome class", + "a second daemon check of the budget witness", + "a load failure beside a hang", + "a stale authored mutant and a reformat", + "an ambiguous authored mutant", + "an edit sequence through the daemon", + "two authored mutants", + "ordinary tests left out by default", + "a config that --config names", + "a package with no config", + "an awaiting config in a package with no module type", + "a .cjs config that calls require", + "a .ts config in a package with no module type that calls require", + "a run with eleven contexts", + "a package whose parent directory has its config", + "a failing owner under its parent's config", + "test.projects in a parent's config", + "a failing owner under a config that --config names", + "a config fault in a run that owns nothing", + "a declaration fault: badEntry", + "a declaration fault: missingSource", + "a declaration fault: nonLiteral", + "a declaration fault: brokenOwner", + "a declaration fault: projects", + "a declaration fault: outsideSource", + "a declaration fault: brokenVitest", + "a declaration fault: configThrows", + "a declaration fault: configNoDefault", + "a declaration fault: configNull", + "a declaration fault: configSyntax", + "a declaration fault: setupThrows", + "a declaration fault: noTypescript", + "a declaration fault: apiLessTypescript", + "an edit sequence over the witness module", + "an edit that changed the cold result after a hang replaced the only context", + "an edit during a check through the daemon", + "an unchanged check through the daemon under a config's pre plugin", + "an unchanged check through the daemon while a test reads a source through fs", + "a test edits the helper it read, during a check through the daemon", + "a test edits the data it read, during a check through the daemon", + "an edit to data a test reads through fs (sync)", + "an edit to data a test reads through fs (promise)", + "an edit to data a test reads through fs (callback)", + "an edit to data a test reads through fs (statDir)", + "an edit to a module that only mutated runs load", + "a lazy module loaded before a hang in the same batch", + "a capture row read while the worker wrote it", + "two batches each read a capture row while its worker wrote it" + ] +} diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index e7d22ad1..3e719332 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -1,5 +1,7 @@ +import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' import fc from 'fast-check' import { describe, expect, test } from 'vitest' import { drawn, histories } from '../oracle-gaps/gap-report-model-oracle.mjs' @@ -59,4 +61,50 @@ describe(`the gap oracle's history grammar`, () => { ) } }) + + // The constructed histories of the end-to-end gap oracle, in the order the + // oracle registers them: each one is built from the fixed seed and its index + // (`seed + index`), so a history inserted before a recorded one changes every + // later history's draw. The register's generator rule (2026-11-14) says new + // constructed histories are appended at the end. The recorded names are + // the list the oracle registered on main; the current list must begin with + // them, so a new name can only follow. + test(`appends new constructed histories, and never inserts one before a recorded one`, () => { + const recorded = JSON.parse( + readFileSync( + new URL(`../oracle-gaps/constructed-order.json`, import.meta.url), + `utf8`, + ), + ) as { constructed: Array } + const listing = spawnSync( + `pnpm`, + [ + `exec`, + `vitest`, + `list`, + `--json`, + `--config`, + `oracle-gaps/vitest.config.mjs`, + `gap-report-oracle`, + ], + { + cwd: fileURLToPath(new URL(`..`, import.meta.url)), + encoding: `utf8`, + maxBuffer: 64 * 1024 * 1024, + }, + ) + expect(listing.status).toBe(0) + const tests = JSON.parse( + listing.stdout.slice(listing.stdout.indexOf(`[`)), + ) as Array<{ name: string }> + const unit = `oracle-gaps.report > fixed seed ${FIXED_SEED}: ` + const current = tests + .map(({ name }) => name) + .filter((name) => name.startsWith(unit)) + .map((name) => name.slice(unit.length)) + .filter((name) => name !== `generated`) + expect(current.slice(0, recorded.constructed.length)).toEqual( + recorded.constructed, + ) + }) }) From 7df4a5076bd52ce7cd0363690c26324374410cc3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 00:36:29 -0700 Subject: [PATCH 10/16] ci(oracles): update the e2e shard plan for the heartbeat calibrations The Test job's e2e check failed: the retired calibration is still named, and the two new heartbeat calibrations are in no shard. Co-authored-by: Isaac --- packages/oracles/scripts/e2e-shards.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 09c94038..214d51d1 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -200,7 +200,9 @@ export const shards = { // Added with the gap CLI's runtime fixes (not yet measured). `fixed seed 20261105: a second daemon check of the budget witness`, `calibration: a stopped run leaves a process it started running`, - `calibration: a busy mutant is named in no heartbeat`, + // The heartbeat calibrations after the shape-only change (not yet measured). + `calibration: a heartbeat names a mutant whose gap line came first`, + `calibration: a heartbeat names no mutant of its run`, // Law 15's calibrations (the greppable stream), measured at ab18c60: // 79 s and 70 s; the daemon-notes one, bound to its history, 2 s (it // took 433 s searching for one). From 4bc4eb1560106d6c212c4d35bfd4f3f9fed6cc02 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 02:18:22 -0700 Subject: [PATCH 11/16] fix(oracles): a gap line's place in the stream is not judged The determinism check compared stderrGaps and stdoutGaps in the order the CLI printed them. A context's gap line is printed when its copies finish, so the order is timing. The stream's own comment already says the trace records the stream as relations, so these two lists are sorted like the streamed list, which is sorted. Nine determinism units failed on main (4d3c8a9) and on the merged head for this reason. Also the constructed-order canary lists vitest with --json=, so no package manager's stdout reaches the JSON (its Vitest 5 job failed on that). Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 10 ++-- .../gap-report-history-fingerprint.test.ts | 53 +++++++++++-------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 297389b8..ed40d626 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1173,13 +1173,17 @@ function readRun(dir, out, status, stderr, cli, stdout) { ), lastLine: printed.at(-1) && namedScratch(printed.at(-1), scratch), // The stdout's own lines of the stream's kinds: the gap lines under - // their functions, as the stream prints them (law 15). + // their functions. A context's gap line is printed when its copies finish, + // so the order is timing: the stream is recorded as relations (law 15), and + // a gap line's place in the output is not judged. stdoutGaps: printed .filter((line) => streamedGap.test(line)) - .map((line) => namedScratch(line, scratch)), + .map((line) => namedScratch(line, scratch)) + .sort(), stderrGaps: lines .filter((line) => streamedGap.test(line)) - .map((line) => namedScratch(line, scratch)), + .map((line) => namedScratch(line, scratch)) + .sort(), baselineFiles: lines.flatMap((line) => { const file = line.match( /^oracle-gaps: baseline (?:pass|fail) (\S+)(?: \(ordinary\))?: /, diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index 3e719332..42494910 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -1,6 +1,8 @@ import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { readFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { fileURLToPath } from 'node:url' import fc from 'fast-check' import { describe, expect, test } from 'vitest' @@ -76,27 +78,34 @@ describe(`the gap oracle's history grammar`, () => { `utf8`, ), ) as { constructed: Array } - const listing = spawnSync( - `pnpm`, - [ - `exec`, - `vitest`, - `list`, - `--json`, - `--config`, - `oracle-gaps/vitest.config.mjs`, - `gap-report-oracle`, - ], - { - cwd: fileURLToPath(new URL(`..`, import.meta.url)), - encoding: `utf8`, - maxBuffer: 64 * 1024 * 1024, - }, - ) - expect(listing.status).toBe(0) - const tests = JSON.parse( - listing.stdout.slice(listing.stdout.indexOf(`[`)), - ) as Array<{ name: string }> + // Vitest writes the listing to a file, so no package manager's stdout + // line can reach the JSON (as pr-shards.mjs does). + const dir = mkdtempSync(join(tmpdir(), `constructed-order-`)) + let tests: Array<{ name: string }> + try { + const file = join(dir, `list.json`) + const listing = spawnSync( + `pnpm`, + [ + `exec`, + `vitest`, + `list`, + `--json=${file}`, + `--config`, + `oracle-gaps/vitest.config.mjs`, + `gap-report-oracle`, + ], + { + cwd: fileURLToPath(new URL(`..`, import.meta.url)), + encoding: `utf8`, + maxBuffer: 64 * 1024 * 1024, + }, + ) + expect(listing.status, listing.stderr).toBe(0) + tests = JSON.parse(readFileSync(file, `utf8`)) as Array<{ name: string }> + } finally { + rmSync(dir, { recursive: true, force: true }) + } const unit = `oracle-gaps.report > fixed seed ${FIXED_SEED}: ` const current = tests .map(({ name }) => name) From cd8e0f928ea24a5285e7bf4be3028cb7c2e141e9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 02:49:09 -0700 Subject: [PATCH 12/16] fix(oracles): exclude the intermittent daemon-pause unit (#105); gate the canary's listing to Vitest 4 The determinism unit added by #73, a daemon pause that cannot engage, gives different traces across runs (#105), so it is excluded until fixed, as the other main failures are. The constructed-order canary lists the gap oracle only under Vitest 4: the Vitest 5 job runs no gap oracle, and its listing finds no suite in that file. Co-authored-by: Isaac --- packages/oracles/scripts/pr-shards.mjs | 2 + .../gap-report-history-fingerprint.test.ts | 98 ++++++++++--------- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 777b5550..1cf8051b 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -391,6 +391,8 @@ export const partitions = { excluded: [ // The daemon's test-output note (#92). `constructed history: an edit sequence through the daemon`, + // A daemon check's trace differs between two runs (#105): intermittent. + `constructed history: a daemon pause that cannot engage`, ], shards: { // 344 s: the witness module's edit sequence, alone. diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index 42494910..08ad2439 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -6,6 +6,7 @@ import { join } from 'node:path' import { fileURLToPath } from 'node:url' import fc from 'fast-check' import { describe, expect, test } from 'vitest' +import { version as vitestVersion } from 'vitest/package.json' import { drawn, histories } from '../oracle-gaps/gap-report-model-oracle.mjs' // The generator rule (coordinator decision, 2026-11-13): a new field of the @@ -71,49 +72,58 @@ describe(`the gap oracle's history grammar`, () => { // constructed histories are appended at the end. The recorded names are // the list the oracle registered on main; the current list must begin with // them, so a new name can only follow. - test(`appends new constructed histories, and never inserts one before a recorded one`, () => { - const recorded = JSON.parse( - readFileSync( - new URL(`../oracle-gaps/constructed-order.json`, import.meta.url), - `utf8`, - ), - ) as { constructed: Array } - // Vitest writes the listing to a file, so no package manager's stdout - // line can reach the JSON (as pr-shards.mjs does). - const dir = mkdtempSync(join(tmpdir(), `constructed-order-`)) - let tests: Array<{ name: string }> - try { - const file = join(dir, `list.json`) - const listing = spawnSync( - `pnpm`, - [ - `exec`, - `vitest`, - `list`, - `--json=${file}`, - `--config`, - `oracle-gaps/vitest.config.mjs`, - `gap-report-oracle`, - ], - { - cwd: fileURLToPath(new URL(`..`, import.meta.url)), - encoding: `utf8`, - maxBuffer: 64 * 1024 * 1024, - }, + // The gap oracle loads under Vitest 4; the Vitest 5 job runs no gap oracle + // (its listing finds no suite in this file), so the canary runs there only + // where the file loads. + const gapOracleLoads = Number(vitestVersion.split(`.`)[0]) < 5 + test.skipIf(!gapOracleLoads)( + `appends new constructed histories, and never inserts one before a recorded one`, + () => { + const recorded = JSON.parse( + readFileSync( + new URL(`../oracle-gaps/constructed-order.json`, import.meta.url), + `utf8`, + ), + ) as { constructed: Array } + // Vitest writes the listing to a file, so no package manager's stdout + // line can reach the JSON (as pr-shards.mjs does). + const dir = mkdtempSync(join(tmpdir(), `constructed-order-`)) + let tests: Array<{ name: string }> + try { + const file = join(dir, `list.json`) + const listing = spawnSync( + `pnpm`, + [ + `exec`, + `vitest`, + `list`, + `--json=${file}`, + `--config`, + `oracle-gaps/vitest.config.mjs`, + `gap-report-oracle`, + ], + { + cwd: fileURLToPath(new URL(`..`, import.meta.url)), + encoding: `utf8`, + maxBuffer: 64 * 1024 * 1024, + }, + ) + expect(listing.status, listing.stderr).toBe(0) + tests = JSON.parse(readFileSync(file, `utf8`)) as Array<{ + name: string + }> + } finally { + rmSync(dir, { recursive: true, force: true }) + } + const unit = `oracle-gaps.report > fixed seed ${FIXED_SEED}: ` + const current = tests + .map(({ name }) => name) + .filter((name) => name.startsWith(unit)) + .map((name) => name.slice(unit.length)) + .filter((name) => name !== `generated`) + expect(current.slice(0, recorded.constructed.length)).toEqual( + recorded.constructed, ) - expect(listing.status, listing.stderr).toBe(0) - tests = JSON.parse(readFileSync(file, `utf8`)) as Array<{ name: string }> - } finally { - rmSync(dir, { recursive: true, force: true }) - } - const unit = `oracle-gaps.report > fixed seed ${FIXED_SEED}: ` - const current = tests - .map(({ name }) => name) - .filter((name) => name.startsWith(unit)) - .map((name) => name.slice(unit.length)) - .filter((name) => name !== `generated`) - expect(current.slice(0, recorded.constructed.length)).toEqual( - recorded.constructed, - ) - }) + }, + ) }) From 4bb5c08c04a24bea3b0870bafbfd3298c70171bf Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 03:15:02 -0700 Subject: [PATCH 13/16] fix(oracles): run Vitest's own CLI for the canary's listing, not pnpm The Windows unit suite failed the canary: spawning pnpm finds no program on Windows. The listing now runs Vitest's CLI with this Node executable. Co-authored-by: Isaac --- .../tests/gap-report-history-fingerprint.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index 08ad2439..15a520ac 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -1,8 +1,9 @@ import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { createRequire } from 'node:module' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import fc from 'fast-check' import { describe, expect, test } from 'vitest' @@ -91,11 +92,13 @@ describe(`the gap oracle's history grammar`, () => { let tests: Array<{ name: string }> try { const file = join(dir, `list.json`) + const packageDir = dirname( + createRequire(import.meta.url).resolve(`vitest/package.json`), + ) const listing = spawnSync( - `pnpm`, + process.execPath, [ - `exec`, - `vitest`, + join(packageDir, `vitest.mjs`), `list`, `--json=${file}`, `--config`, From 7baf1d2d828c3fa4cabc4a1fa71f7af52cb8cca1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 07:41:59 -0700 Subject: [PATCH 14/16] ci(oracles): run the witness determinism unit in a non-blocking job (#106) The witness module's edit sequence takes 10 to 11 minutes in CI, over P12's budget. It is one history and cannot be split without changing the law, so it runs in its own continue-on-error job, owned by #106, which is not a dependency of preview. Shard 1 of the partition is unchanged, so the partition check holds. Co-authored-by: Isaac --- .github/workflows/pr.yml | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index fe57a249..a23defe9 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -153,7 +153,9 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5] + # Shard 1 is the witness module's edit sequence, which runs in its own + # non-blocking job below (#106). + shard: [2, 3, 4, 5] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -161,13 +163,30 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's determinism units, in five shards, except the - # one excluded in pr-shards.mjs (#92). Each unit runs a history twice and - # compares the results and fixed lines; heartbeats are checked for shape. + # The end-to-end oracle's determinism units, in shards 2 to 5 (shard 1 is + # below), except the one excluded in pr-shards.mjs (#92). Each unit runs a + # history twice and compares the results and fixed lines; heartbeats are + # checked for shape. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism "$SHARD")" working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} + # The witness module's edit sequence alone takes 10 to 11 minutes here, + # over P12's 10-minute budget. It is non-blocking and not a dependency of + # preview, so it still runs on every pull request until its cost is cut (#106). + test-oracle-gaps-witness: + name: Gap report end-to-end oracle (determinism, witness module; non-blocking, #106) + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism 1)" + working-directory: packages/oracles test-vitest-5: name: Test on Vitest 5 (${{ matrix.suite }}) # The package's vitest peer range includes 5; this job is the evidence. From 7acfedae4f5b7cec98ece1e192ca343ab563c479 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 18:01:21 -0600 Subject: [PATCH 15/16] measure: random shards under pinned seeds (scratch, not for merge) Co-authored-by: Isaac --- .github/workflows/pr.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a23defe9..1cc94b0b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -187,6 +187,29 @@ jobs: uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism 1)" working-directory: packages/oracles + # MEASUREMENT ONLY (scratch branch, not for merge): each random shard under + # pinned campaign seeds, to size the random shards against P12. + measure-random-seeds: + name: measure random ${{ matrix.shard }}/3 seed ${{ matrix.seed }} + runs-on: ubuntu-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + seed: [-972199262, 333199054, -1269403528, -1659558018, 1835238112, -330614923] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-random "$SHARD")" + working-directory: packages/oracles + env: + SHARD: ${{ matrix.shard }} + ORACLE_SEED: ${{ matrix.seed }} test-vitest-5: name: Test on Vitest 5 (${{ matrix.suite }}) # The package's vitest peer range includes 5; this job is the evidence. From f39a98dbaf6b26160dc9432173c0e07f7d8181c8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 18:11:12 -0600 Subject: [PATCH 16/16] measure: narrow the scratch workflow to the random seed jobs (scratch, not for merge) Co-authored-by: Isaac --- .github/workflows/pr.yml | 550 +-------------------------------------- 1 file changed, 2 insertions(+), 548 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1cc94b0b..db4b17ba 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -2,193 +2,13 @@ name: PR on: pull_request: - push: - branches: [main] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.number || github.ref }} - # Only a newer push to the same pull request cancels a run. Every commit on - # main keeps its run, so every commit gets a preview. - cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read jobs: - # The test suites run as parallel jobs. The workflow's wall time is then - # its slowest job, about 6 to 7 minutes, not the sum of every suite, about - # 21 minutes when they ran as one job. Each job should stay well under the - # 10-minute limit; pr-shards.mjs records how each suite is split. - test: - name: Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm run build - - run: pnpm run lint:check - - run: pnpm run format:check - - run: pnpm run typecheck - - run: pnpm --filter @tanstack/oracles run test - # Checks the partition of the end-to-end gap oracle into CI shards - # (packages/oracles/scripts/e2e-shards.mjs) ahead of the shard jobs, - # which CI does not run yet: every test is in exactly one shard. - - run: pnpm --filter @tanstack/oracles run test:oracle-gaps:shards - # The framework self-oracle, the gap report's static oracle, and the - # end-to-end oracle's fixed seed each run as several jobs. Together - # they run exactly the tests one job ran - # (packages/oracles/scripts/pr-shards.mjs). - - run: pnpm --filter @tanstack/oracles run test:pr-shards - - run: pnpm run test:publint - - run: pnpm run test:sherif - test-eslint-plugin: - name: Test eslint-plugin-oracles - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The plugin's tests import @tanstack/oracles, which resolves to its - # build output. - - run: pnpm run build - - run: pnpm --filter @tanstack/eslint-plugin-oracles run test - test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/8) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in eight - # shards. - - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-fast: - name: Gap report oracles (${{ matrix.shard }}/3) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The gap report's own oracles also run real Vitest runs on fixture - # packages (test:oracle-gaps:fast): the static oracle in two shards, - # and the classify, probe-form, and mutant-validity oracles in a third. - - run: | - if [ "$SHARD" = 3 ]; then - exec pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling - fi - pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-end-to-end: - name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's fixed-seed units, in three shards. Its random - # units and determinism units run in their own jobs, and its - # calibrations run locally. - - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-random: - name: Gap report end-to-end oracle (random seed, ${{ matrix.shard }}/3) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's random units, in three shards. Each run draws a - # fresh campaign seed (the ORACLE_SEED line in the log), and a failing - # unit prints its Replay line. A counterexample is filed as an issue. - - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-random "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-determinism: - name: Gap report end-to-end oracle (determinism, ${{ matrix.shard }}/5) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - # Shard 1 is the witness module's edit sequence, which runs in its own - # non-blocking job below (#106). - shard: [2, 3, 4, 5] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's determinism units, in shards 2 to 5 (shard 1 is - # below), except the one excluded in pr-shards.mjs (#92). Each unit runs a - # history twice and compares the results and fixed lines; heartbeats are - # checked for shape. - - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - # The witness module's edit sequence alone takes 10 to 11 minutes here, - # over P12's 10-minute budget. It is non-blocking and not a dependency of - # preview, so it still runs on every pull request until its cost is cut (#106). - test-oracle-gaps-witness: - name: Gap report end-to-end oracle (determinism, witness module; non-blocking, #106) - runs-on: ubuntu-latest - continue-on-error: true - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism 1)" - working-directory: packages/oracles - # MEASUREMENT ONLY (scratch branch, not for merge): each random shard under - # pinned campaign seeds, to size the random shards against P12. + # MEASUREMENT ONLY (scratch, not for merge): the random shards under pinned + # campaign seeds, nothing else. measure-random-seeds: name: measure random ${{ matrix.shard }}/3 seed ${{ matrix.seed }} runs-on: ubuntu-latest @@ -210,369 +30,3 @@ jobs: env: SHARD: ${{ matrix.shard }} ORACLE_SEED: ${{ matrix.seed }} - test-vitest-5: - name: Test on Vitest 5 (${{ matrix.suite }}) - # The package's vitest peer range includes 5; this job is the evidence. - # The unit tests and each self-oracle shard are their own jobs. - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - suite: - - test - - oracle-framework 1 - - oracle-framework 2 - - oracle-framework 3 - - oracle-framework 4 - - oracle-framework 5 - - oracle-framework 6 - - oracle-framework 7 - - oracle-framework 8 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles add --save-dev vitest@^5.0.5 - - run: pnpm --filter @tanstack/oracles exec vitest --version - - run: | - if [ "$SUITE" = test ]; then pnpm run test; exit; fi - pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern $SUITE)" - working-directory: packages/oracles - env: - SUITE: ${{ matrix.suite }} - preview: - name: Publish preview - # Only a commit that passes every check gets an installable preview. - needs: - - test - - test-eslint-plugin - - test-oracle-framework - - test-oracle-gaps-fast - - test-oracle-gaps-end-to-end - - test-oracle-gaps-random - - test-oracle-gaps-determinism - - test-vitest-5 - # The blocking Windows jobs (the non-blocking end-to-end job is not a - # dependency until it is green). - - windows-unit - - windows-gap-classify - - windows-gap-oracles - - windows-gap-cli - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm run build - # Consumers install a commit as - # https://pkg.pr.new/TanStack/oracles/@tanstack/oracles@ - # --compact needs the package on npm, so it is added after the first - # npm release. pkg.pr.new authenticates through its GitHub App, not an - # npm token, so npm trusted publishing does not apply. - - run: pnpm dlx pkg-pr-new@0.0.92 publish --pnpm './packages/*' # zizmor: ignore[use-trusted-publishing] - zizmor: - name: zizmor - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 - with: - advanced-security: false - annotations: true - # Windows is a required part of CI: the framework, the eslint plugin, and - # the gap CLI run on windows-latest, split the way the Linux jobs split - # them. Each job stays under 10 minutes; each has a timeout backstop. - windows-unit: - name: Windows unit suites - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - run: pnpm --filter @tanstack/oracles run test - # Runs even when the framework tests fail, so each suite reports. - - if: always() - run: pnpm --filter @tanstack/eslint-plugin-oracles run test - windows-gap-oracles: - name: Windows gap report static oracle (${{ matrix.shard }}/6) - runs-on: windows-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - # The same partition of the static oracle that Linux runs, by pattern. - - name: Gap report static oracle - shell: bash - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static-windows "$SHARD")" - windows-gap-classify: - name: Windows gap report classify, probe-form, and mutant-validity oracles - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Gap report oracles - working-directory: packages/oracles - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling - windows-gap-end-to-end: - name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/10) - runs-on: windows-latest - # Not blocking yet: these suites fail on Windows, tracked by class in - # https://github.com/TanStack/oracles/issues/72. Remove this line when they - # are green. - continue-on-error: true - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Gap report end-to-end oracle - shell: bash - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - # The verbose reporter prints each test with its duration, which sizes the shards. - run: pnpm exec vitest --run --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" - windows-gap-cli: - name: Windows gap CLI on a fixture - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Write gap CLI fixture - shell: bash - run: | - set -euo pipefail - dir="$GITHUB_WORKSPACE/.ci-fixture" - mkdir -p "$dir/src" "$dir/tests" - cat > "$dir/package.json" <<'EOT' - {"name": "ci-fixture", "private": true, "type": "module"} - EOT - cat > "$dir/src/clamp.ts" <<'EOT' - export function clamp(x: number): number { - return x < 0 ? 0 : x > 10 ? 10 : x - } - EOT - cat > "$dir/tests/clamp.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - import { clamp } from '../src/clamp' - - oracleProperty({ - owns: ['src/clamp.ts#clamp'], - id: `fixture.clamp`, - law: { - statement: `clamp returns its argument within 0 and 10.`, - authority: `the CI fixture for the Windows gap-CLI run`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 50, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => (x < 0 ? 0 : x > 10 ? 10 : x), - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, clamp(x)) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => { - const seen = trace.find(({ checkpoint }) => checkpoint === `result`)?.value - return differs(`result`, `clamp`, seen, expected) ?? pass - }, - }) - EOT - cat > "$dir/vitest.config.mjs" <<'EOT' - export default { - test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } }, - } - EOT - # A failing property for the replay witness: every history fails, so a - # random unit prints its replay lines. Written in bash, as the fixture is. - - name: Write replay fixture - shell: bash - run: | - set -euo pipefail - dir="$GITHUB_WORKSPACE/.ci-replay" - mkdir -p "$dir/tests" - cat > "$dir/tests/fail.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture ’ & | < > ( ) ^ replay`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - cat > "$dir/tests/hostile.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture Q"1 replay`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - cat > "$dir/tests/percent.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture 100% done!`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - printf '%s\n' "export default { test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } } }" > "$dir/vitest.config.mjs" - # The user-facing path: PowerShell, the default shell, as a user runs it. - # Each run's stdout and stderr are kept apart, and the done line is - # checked on both streams against the exit code (P10). - - name: Gap CLI on the fixture - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs > cli-out.txt 2> cli-err.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out.txt cli-err.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - - name: Gap CLI on the fixture, discovered config - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts > cli-out2.txt 2> cli-err2.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out2.txt cli-err2.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - # The daemon path: start it, check through it, then stop it. - - name: Gap CLI through the daemon - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs --daemon > cli-out3.txt 2> cli-err3.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out3.txt cli-err3.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - - name: Stop the gap CLI daemon - if: always() - run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture - # A package with no package.json is refused, with exit 2 and a done line - # that says so. - - name: Gap CLI refuses a package with no package.json - run: | - New-Item -ItemType Directory -Force .ci-no-manifest | Out-Null - node packages/oracles/oracle-gaps/index.mjs --package .ci-no-manifest --source src/x.ts > nm-out.txt 2> nm-err.txt - $code = $LASTEXITCODE - if ($code -ne 2) { throw "want a refusal with exit 2, got $code" } - if (-not (Select-String -Path nm-out.txt, nm-err.txt -Pattern 'has no package.json' -Quiet)) { throw "the refusal does not name package.json" } - node packages/oracles/scripts/windows-witness.mjs done nm-out.txt nm-err.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - # One early stop: SIGINT once the CLI has printed its first line. The - # run must end with the stop's done line on both streams, or with no - # output at all (the all-or-nothing law, #57). - - name: Early stop of the gap CLI - run: node packages/oracles/scripts/windows-witness.mjs stop .ci-fixture --package . --source src/clamp.ts --config vitest.config.mjs - # The per-shell replay lines: a failing property's PowerShell and cmd - # lines each reproduce its failure in their own shell (RED: the POSIX - # line does not, in PowerShell). - - name: Replay lines reproduce the failure in PowerShell and cmd - if: always() - run: | - node "$env:GITHUB_WORKSPACE\packages\oracles\node_modules\vitest\vitest.mjs" run --root .ci-replay > replay-out.txt 2>&1 - Get-Content replay-out.txt - node packages/oracles/scripts/windows-witness.mjs replay replay-out.txt "$env:GITHUB_WORKSPACE\.ci-replay" - if ($LASTEXITCODE -ne 0) { throw "the replay witness failed" }