From 28915ee9aa6951ac3946bc28a388f44430c35df1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 16:37:30 -0700 Subject: [PATCH 01/32] test(oracle-gaps): a slow owner's kill reports as it does when it does not wait Determinism under load (2026-11-13): a run's outcome must not depend on how long a run took. A constructed history, "a slow owner's kill and another owner's kill", has one mutant that file a's random-route test and file b's fixed-seed test both kill. Its driver also runs the slow twin: the same package, where a's test waits 3 s past its first budget wherever the output differs and then judges as before. The law compares the two reports, `randomAlso` included. On main, a's slow copy is cut at its first budget and queued for a confirmation, b's kill decides the mutant first, and a's random failure is never recorded: the twin reports `randomAlso: false` where the package reports `true`. `slowTwin` is constructed-only (no generated history draws it; `histories` takes it back without it). Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 7 +- .../gap-report-oracle.property.test.mjs | 90 +++++++++++++++++++ packages/oracles/scripts/e2e-shards.mjs | 3 + packages/oracles/scripts/pr-shards.mjs | 1 + 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6ebe9f35..ef6dae86 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -805,6 +805,8 @@ const keepsSteps = (history) => { // P1-a: the budget witness only in its own histories). The map is its own // inverse on the histories it keeps, so fast-check takes every one of them // back and shrinks it. +// A constructed history may set `slowTwin`, which no generated history +// draws, and which is taken back without it. const histories = drawn.map( (history) => (keepsSteps(history) ? history : { ...history, steps: null }), (value) => { @@ -812,7 +814,10 @@ const histories = drawn.map( /** @type {{ witness: unknown, steps: unknown[] | null }} */ (value) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) - return value + const { slowTwin, ...drawnValue } = /** @type {Record} */ ( + value + ) + return slowTwin === undefined ? value : drawnValue }, ) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 9f0fe4c7..03127452 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1604,11 +1604,38 @@ async function driver(plan, observe) { // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) + // The slow twin (`slowTwin`): the same package, where file a's first + // test waits 3 s past its first budget wherever the output differs, and + // then judges as before. Its verdicts are the same, so its report must + // be too ("Determinism under load", 2026-11-13: an outcome does not + // depend on how long a run took). + let twinSame = false + if (plan.history.slowTwin) { + const twin = resolveHistory({ + ...plan.history, + slowTwin: undefined, + tests: plan.history.tests.map((t, i) => + i === 0 ? { ...t, slow: true } : t, + ), + }) + const slow = runCli(dir, fixtureFiles(twin), how) + twinSame = + slow.exit === cli.exit && + JSON.stringify(slow.report?.results) === + JSON.stringify(cli.report?.results) + observe(`twin`, { + fast: cli.report?.results, + slow: slow.report?.results, + exit: slow.exit, + }) + } // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if (twinSame) + states.add(`a slow owner's kill reported as when it did not wait`) if (cli.exit === 0 && !plan.consoleOutput) states.add(`a completed run whose tests wrote nothing`) // Law 14: a partial run of one owned function's mutants, whose baseline @@ -2143,6 +2170,23 @@ function daemonCheck(observations) { function check(plan, observations, expected) { const cli = observations.find((o) => o.checkpoint === `cli`)?.value if (!cli) return violation(`cli`, `no CLI run was observed`, `completion`) + // A run's outcome does not depend on how long its runs took: the slow + // twin reports each mutant as the package does (determinism under load). + const twin = observations.find((o) => o.checkpoint === `twin`)?.value + if (twin) { + const at = (twin.fast ?? []).findIndex( + (r, i) => JSON.stringify(r) !== JSON.stringify(twin.slow?.[i]), + ) + if (twin.exit !== cli.exit || at >= 0) + return violation( + `twin`, + `the slow twin reports ${at >= 0 ? twin.fast[at].key : `exit ${twin.exit}`} otherwise than the package whose test does not wait`, + `determinism under load`, + at >= 0 + ? { actual: twin.slow?.[at], expected: twin.fast[at] } + : undefined, + ) + } if (cli.strayWrites?.length > 0) return violation( `cli`, @@ -3165,6 +3209,7 @@ const required = [ `each owner's value change killed its own mutant`, `a hang alone timed out`, `a hang in one owner and a kill in another killed`, + `a slow owner's kill reported as when it did not wait`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, `a heartbeat named a busy mutant`, @@ -3511,6 +3556,51 @@ oracleProperty({ authored: { stmt: 0, kind: `ambiguous` }, }), }, + { + // File a's random-route test and file b's fixed-seed test both kill + // one mutant (as w_crosskill does). In the slow twin, a's test + // waits 3 s past its first budget before it judges, so its copy is + // cut and queued for a confirmation; b's kill decides the mutant + // first. The report must still be the one the package without the + // wait gives, its `randomAlso` included. + name: `a slow owner's kill and another owner's kill`, + enters: [`a slow owner's kill reported as when it did not wait`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `random seed`, 7), fn: 0 }, + { ...judging(`b`, `fixed seed`, 7), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + slowTwin: true, + }), + }, { // Two small owned functions, so that each check is quick, and every // edit in an order where each one changes the cold result: file b's diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 986c4047..b4c4ed12 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -60,6 +60,9 @@ export const shards = { `determinism: constructed history: a declaration fault: projects`, `determinism: constructed history: a package with no config`, `determinism: constructed history: an edit sequence through the daemon`, + `fixed seed 20261105: a slow owner's kill and another owner's kill`, + `random seed: a slow owner's kill and another owner's kill`, + `determinism: constructed history: a slow owner's kill and another owner's kill`, // The witness module's edit sequence (option M): 189 s here, measured // with the daemon fix of fix/daemon-current-inputs. `determinism: constructed history: an edit sequence over the witness module`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 9e162886..c65f3191 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -546,6 +546,7 @@ export const partitions = { 'a declaration fault: outsideSource', 'a load failure beside a hang', 'an ambiguous authored mutant', + "a slow owner's kill and another owner's kill", 'every outcome class', ]), 5: anyOf([ From 9412eac1433ae8c182814b16bced84fabb5c733e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 16:55:06 -0700 Subject: [PATCH 02/32] test(oracle-gaps): RED for the work budget, a kill past Vitest's limit, and a timer under load Deterministic cuts, option A (register M, 2026-11-14): no wall-clock cut decides an outcome, so a mutant's copies are judged by count, not by time. Three laws are added to the end-to-end oracle, each RED on the parent commit before any production change. - The budget witness w_slowpair becomes a work-budget witness: its two owners' tests call the function PAIR_LOOPS times where their output differs. Each copy stays inside its call budget, the two together pass the mutant's work budget (WORK_FLOOR), so the mutant is a timeout by count, whatever the clock says. The model judges the budget in owner order, a kill before the budget runs out still deciding. - A slow kill whose test waits 16 s, past Vitest's 15 s per-test limit, is killed: no limit that the CLI leaves to Vitest decides the outcome. - A copy whose test starts a timer, spins under load, then waits for the timer, is killed and never cut as idle (the idle law is state, not time). Constructed-only test fields (slowMs, loops, busy) are held by the drawn test record at neutral constants and taken back to them by the histories' map, so no generated history draws them (generator rule, 2026-11-14). The constructed histories are appended at the end of the list, and the slow owner's history moved there from the middle of it. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 149 +++++++++++++--- .../gap-report-oracle.property.test.mjs | 165 +++++++++++++----- packages/oracles/scripts/e2e-shards.mjs | 6 + packages/oracles/scripts/pr-shards.mjs | 7 +- 4 files changed, 252 insertions(+), 75 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index ef6dae86..6520bb09 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -135,13 +135,17 @@ const FULL_LIMIT_MS = 20000 // How long a slow test waits, in real milliseconds: past the 1 s first // budget that the driver passes (`--first-limit`), within the full limit. const SLOW_MS = 3000 -// How long each slow test of `w_slowpair` waits: past its file's first -// budget (the 1 s `--first-limit` plus three times its one-test file's -// unmutated time t), so its copy is always cut and confirmed; past half the -// mutant's budget, so two such confirmations never fit it; and under the -// full limit, so each would fit alone. The window holds while t < 3 s and -// ten times the two files' time stays under 2 × 10 s (their sum under 2 s). -const PAIR_SLOW_MS = 10000 +// The site evaluations that each copy of `w_slowpair` makes: its test calls +// the function this many times where its output differs. Each copy stays +// inside its call budget (CALL_FLOOR in check.mjs, 100,000 site evaluations), +// and the two copies together pass the mutant's work budget (WORK_FLOOR). +// Work is counted, not timed, so the outcome is the same on any machine. +const PAIR_LOOPS = 37_500 +// A mutant's work budget (maintainer disposition G1, under the deterministic +// cuts decision of 2026-11-14): the larger of this floor and ten times its +// owner files' unmutated work, in site evaluations. The floor is the copies' +// call budget, so two copies that each stay inside theirs can pass it. +const WORK_FLOOR = 100_000 // A mutant's budget for the runs that confirm its cut-off copies, which the // driver passes (`--mutant-limit`): one slow owner's confirmation fits it // (SLOW_MS), and two of `w_slowpair`'s do not. Ten times the fixture's @@ -414,6 +418,12 @@ const testCase = fc.record({ // Only a witness waits on a timer, sleeps, reads the clock, or waits a // turn: its work could follow the clock. timed: fc.constant(false), + // The constructed-only fields (see `histories`): a witness's wait where its + // output differs, its repeated calls, and its load. A generated test has the + // neutral values, which change nothing. + slowMs: fc.constant(SLOW_MS), + loops: fc.constant(0), + busy: fc.constant(null), }) // Law 9's edits. Each content edit changes what some evidence depends on: // a test file, a source statement, the source's line numbers, an owns list, @@ -806,7 +816,18 @@ const keepsSteps = (history) => { // inverse on the histories it keeps, so fast-check takes every one of them // back and shrinks it. // A constructed history may set `slowTwin`, which no generated history -// draws, and which is taken back without it. +// draws, and which is taken back without it. Its tests may also set the +// constructed-only fields `slowMs`, `loops` and `busy`, which the drawn +// test record holds as constants at their neutral values (the generator rule, +// 2026-11-14: a field that changes what a history runs is set only by a +// constructed history, and taken back to its neutral value here). +const takenBack = (t) => ({ + ...t, + slow: false, + slowMs: SLOW_MS, + loops: 0, + busy: null, +}) const histories = drawn.map( (history) => (keepsSteps(history) ? history : { ...history, steps: null }), (value) => { @@ -817,7 +838,8 @@ const histories = drawn.map( const { slowTwin, ...drawnValue } = /** @type {Record} */ ( value ) - return slowTwin === undefined ? value : drawnValue + const tests = /** @type {Record[]} */ (drawnValue.tests) + return { ...drawnValue, tests: tests.map(takenBack) } }, ) @@ -1617,16 +1639,8 @@ const witness = [ owners: PAIR_FILES, body: [kill], tests: [ - judging(`p`, `fixed seed`, 7, { - judges: false, - slow: true, - slowMs: PAIR_SLOW_MS, - }), - judging(`q`, `fixed seed`, 7, { - judges: false, - slow: true, - slowMs: PAIR_SLOW_MS, - }), + judging(`p`, `fixed seed`, 7, { judges: false, loops: PAIR_LOOPS }), + judging(`q`, `fixed seed`, 7, { judges: false, loops: PAIR_LOOPS }), ], }, // A kill as usual. In a run that the driver stops (`GAP_STOP_DIR` set), @@ -3907,16 +3921,26 @@ function runsOf(plan, tests, sites, active) { for (const part of partsOf(t)) { const base = interpret(plan, t.fn, part.input, sites) const mutated = interpret(plan, t.fn, part.input, sites, active) + const route = routeOf(t, part) + const changed = + !mutated.threw && mutated.trace.join() !== base.trace.join() + // Where the output differs, a `loops` witness calls the function that + // many more times, so its work is the call's work once per call. if (!t.judgesShared) { before += base.work - after += mutated.work + after += mutated.work * (1 + (changed ? (t.loops ?? 0) : 0)) } - const route = routeOf(t, part) // A loop that never ends cuts the file's run off, as a hang does. if (mutated.spun) - return { t, route, hang: true, verdicts: found, changed: true } - const changed = - !mutated.threw && mutated.trace.join() !== base.trace.join() + return { + t, + route, + hang: true, + verdicts: found, + changed: true, + work: after, + before, + } changedAny ||= changed slowed ||= changed && (t.slow || t.ioWait) if (mutated.threw) @@ -3927,17 +3951,35 @@ function runsOf(plan, tests, sites, active) { verdicts: found, changed: changedAny, slowed, + work: after, + before, } // A test that hangs where another would judge never ends. if (changed && t.hangs) - return { t, route, hang: true, verdicts: found, changed: true } + return { + t, + route, + hang: true, + verdicts: found, + changed: true, + work: after, + before, + } if (changed && t.judges) { found.push({ route, law: t.judgeBy === `verdict` ? t.law : `(unnamed)`, }) if (t.judgeBy !== `soft`) - return { t, route, verdicts: found, changed: true, slowed } + return { + t, + route, + verdicts: found, + changed: true, + slowed, + work: after, + before, + } } } // A test that passed: its route is the last one it registered, and @@ -3966,6 +4008,8 @@ function runsOf(plan, tests, sites, active) { changed: changedAny, slowed, rose: judgesWork && after > before, + work: after, + before, } }) } @@ -4023,20 +4067,51 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { // a timeout); failing that, the mutant is a timeout. const hung = new Set() const slowed = new Map() - for (const file of new Set(runs.map((r) => r.t.file))) + // The mutant's work budget (G1): its owner files' copies are counted in + // their order, each copy's work being the site evaluations of its runs up + // to a hang or a kill. A copy past the budget is a timeout, and so is every + // copy after it, so a kill among them does not count. The budget is the + // larger of WORK_FLOOR and ten times the owner files' unmutated work, where + // a file's unmutated work is its most-working test's. + const files = [...new Set(runs.map((r) => r.t.file))] + const unmutated = files.reduce( + (sum, file) => + sum + + Math.max( + 0, + ...runs.filter((r) => r.t.file === file).map((r) => r.before), + ), + 0, + ) + const workBudget = Math.max(WORK_FLOOR, 10 * unmutated) + const counted = new Set() + let spent = 0 + let exhausted = false + for (const file of files) { + let work = 0 for (const r of runs.filter((x) => x.t.file === file)) { if (r.slowed) slowed.set(file, r.t.slowMs ?? SLOW_MS) if (r.hang) hung.add(file) + work += r.work if (r.hang || killsIn(r)) break } - const live = runs.filter((r) => !hung.has(r.t.file)) + if (budgeted && spent + work > workBudget) { + exhausted = true + break + } + spent += work + counted.add(file) + } + const live = runs.filter( + (r) => counted.has(r.t.file) && !hung.has(r.t.file), + ) const verdicts = (route) => live.flatMap((r) => r.verdicts.filter((v) => v.route === route).map((v) => v.law), ) const reproducible = verdicts(`reproducible`) if (reproducible.some((law) => law !== `reach agreement`)) return `killed` - if (hung.size > 0) return `timeout` + if (hung.size > 0 || exhausted) return `timeout` // The runs that confirm a mutant's cut-off copies share its budget // (`--mutant-limit`, MUTANT_LIMIT_MS here; maintainer disposition G1, // 2026-11-11). Under two budgets, a slow owner file is cut at its first @@ -4708,6 +4783,22 @@ function oracleTestFile(plan, file) { ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.slowMs ?? SLOW_MS}))`, ] : []), + // Where its output differs, a test that calls the function `loops` + // more times: each call is work the copy does, and the work budget + // counts it. + ...(t.loops + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) for (let k${i} = 0; k${i} < ${t.loops}; k${i} += 1) ${callFor(t.fn)}(${part.input}, [])`, + ] + : []), + // Where its output differs, a test that starts a timer, spins for + // `busyMs` (the load), and then waits for the timer. The timer is + // pending throughout, so the copy is never idle. + ...(t.busy + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { const waiting${i} = new Promise((r) => setTimeout(r, ${t.busy.waitMs})); const until${i} = performance.now() + ${t.busy.busyMs}; while (performance.now() < until${i}) {}; await waiting${i} }`, + ] + : []), // A test that waits on I/O (a child process, about 3 s) where the // output differs, then judges as usual. ...(t.ioWait diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 03127452..9adb1d67 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1636,6 +1636,13 @@ async function driver(plan, observe) { const states = new Set() if (twinSame) states.add(`a slow owner's kill reported as when it did not wait`) + // A kill that waits past Vitest's 15 s test limit, and one whose copy + // waits on a timer under load, each killed as the report says. + const killed = cli.report?.results?.some((r) => r.outcome === `killed`) + if (killed && plan.history.tests.some((t) => t.slowMs > 15000)) + states.add(`a slow kill past fifteen seconds killed`) + if (killed && plan.history.tests.some((t) => t.busy)) + states.add(`a copy waiting on a timer under load killed`) if (cli.exit === 0 && !plan.consoleOutput) states.add(`a completed run whose tests wrote nothing`) // Law 14: a partial run of one owned function's mutants, whose baseline @@ -3210,6 +3217,8 @@ const required = [ `a hang alone timed out`, `a hang in one owner and a kill in another killed`, `a slow owner's kill reported as when it did not wait`, + `a slow kill past fifteen seconds killed`, + `a copy waiting on a timer under load killed`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, `a heartbeat named a busy mutant`, @@ -3375,6 +3384,40 @@ const withDeclaration = (constructed) => }), })) +// A constructed history whose only owned function is a kill, owned by file a +// alone, with the tests given (the rest of the generated base is left as the +// grammar draws it, as the histories above do). +const soloKill = (generated, tests) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests, + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, +}) oracleProperty({ id: `oracle-gaps.report`, law: { @@ -3556,51 +3599,6 @@ oracleProperty({ authored: { stmt: 0, kind: `ambiguous` }, }), }, - { - // File a's random-route test and file b's fixed-seed test both kill - // one mutant (as w_crosskill does). In the slow twin, a's test - // waits 3 s past its first budget before it judges, so its copy is - // cut and queued for a confirmation; b's kill decides the mutant - // first. The report must still be the one the package without the - // wait gives, its `randomAlso` included. - name: `a slow owner's kill and another owner's kill`, - enters: [`a slow owner's kill reported as when it did not wait`], - build: (generated) => ({ - ...generated, - functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], - containers: [], - overloads: [], - owners: [`both`], - library: [], - libraryOwners: [], - runes: [], - derived: null, - dotGen: false, - dataOwners: [], - exportOwners: [], - importer: `none`, - wholeFile: `none`, - template: false, - tests: [ - { ...judging(`a`, `random seed`, 7), fn: 0 }, - { ...judging(`b`, `fixed seed`, 7), fn: 0 }, - ], - witness: false, - authored: null, - secondAuthored: false, - fault: `none`, - stability: `none`, - ordinary: false, - native: `none`, - boot: false, - budget: `two`, - cliPath: `plain`, - concurrentStart: false, - contexts: 1, - steps: null, - slowTwin: true, - }), - }, { // Two small owned functions, so that each check is quick, and every // edit in an order where each one changes the cold result: file b's @@ -4133,6 +4131,83 @@ oracleProperty({ steps: [`lazy`], }), }, + { + // File a's random-route test and file b's fixed-seed test both kill + // one mutant (as w_crosskill does). In the slow twin, a's test + // waits 3 s past its first budget before it judges, so its copy is + // cut and queued for a confirmation; b's kill decides the mutant + // first. The report must still be the one the package without the + // wait gives, its `randomAlso` included. + name: `a slow owner's kill and another owner's kill`, + enters: [`a slow owner's kill reported as when it did not wait`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `random seed`, 7), fn: 0 }, + { ...judging(`b`, `fixed seed`, 7), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + slowTwin: true, + }), + }, + // Constructed histories added by the deterministic-cuts change follow. + // A slow kill whose test waits 16 s where its output differs, past + // Vitest's 15 s per-test limit: no wall-clock limit decides an outcome, + // so the mutant is still killed. + { + name: `a slow kill past Vitest's 15 s limit`, + enters: [`a slow kill past fifteen seconds killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + slow: true, + slowMs: 16000, + }, + ]), + }, + // A copy whose test starts a timer due inside its horizon, then spins + // under load, then waits for it: the copy is busy, never idle, and the + // mutant is killed. + { + name: `a copy waiting on a timer under load`, + enters: [`a copy waiting on a timer under load killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + busy: { busyMs: 600, waitMs: 2000 }, + }, + ]), + }, ]), }, resolve: resolveHistory, diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index b4c4ed12..36f76fbf 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -63,6 +63,12 @@ export const shards = { `fixed seed 20261105: a slow owner's kill and another owner's kill`, `random seed: a slow owner's kill and another owner's kill`, `determinism: constructed history: a slow owner's kill and another owner's kill`, + `fixed seed 20261105: a slow kill past Vitest's 15 s limit`, + `random seed: a slow kill past Vitest's 15 s limit`, + `determinism: constructed history: a slow kill past Vitest's 15 s limit`, + `fixed seed 20261105: a copy waiting on a timer under load`, + `random seed: a copy waiting on a timer under load`, + `determinism: constructed history: a copy waiting on a timer under load`, // The witness module's edit sequence (option M): 189 s here, measured // with the daemon fix of fix/daemon-current-inputs. `determinism: constructed history: an edit sequence over the witness module`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index c65f3191..dce1d222 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -301,12 +301,15 @@ export const partitions = { 'a stale authored mutant and a reformat', 'a failing owner under a config that --config names', ]), - // 134 s, 101 s, 27 s, and 15 s. + // 134 s, 101 s, 27 s, and 15 s; the two constructed histories of the + // deterministic cuts (a 16 s wait, and a timer under load), about 45 s. 2: anyOf([ 'every outcome class', 'a second daemon check of the budget witness', 'a declaration fault: brokenVitest', 'an edit that changed the cold result after a hang replaced the only context', + "a slow kill past Vitest's 15 s limit", + 'a copy waiting on a timer under load', ]), // The rest: the edit sequence through the daemon (95 s), brokenOwner // (83 s), generated (23 s), the lazy module (12 s), and small units. @@ -547,6 +550,8 @@ export const partitions = { 'a load failure beside a hang', 'an ambiguous authored mutant', "a slow owner's kill and another owner's kill", + "a slow kill past Vitest's 15 s limit", + 'a copy waiting on a timer under load', 'every outcome class', ]), 5: anyOf([ From b71be91c63b94a7a02011defefd6ca699a0d7f35 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 17:00:41 -0700 Subject: [PATCH 03/32] test(oracle-gaps): size the work witness's loops so two copies pass the work budget The first RED (9412eac) lets each copy of w_slowpair make 37,500 calls beyond its first. At three site evaluations a call that is 112,500 per copy, past the 100,000 call budget, so main cuts each copy by count and the mutant is a timeout on either side; the work law is not reached. At 30,000 calls a copy makes 60,000 to 90,000 site evaluations, inside its budget, and the two copies together pass the work budget, so main must report survived. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-model-oracle.mjs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6520bb09..71a12d7d 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -135,12 +135,13 @@ const FULL_LIMIT_MS = 20000 // How long a slow test waits, in real milliseconds: past the 1 s first // budget that the driver passes (`--first-limit`), within the full limit. const SLOW_MS = 3000 -// The site evaluations that each copy of `w_slowpair` makes: its test calls -// the function this many times where its output differs. Each copy stays -// inside its call budget (CALL_FLOOR in check.mjs, 100,000 site evaluations), -// and the two copies together pass the mutant's work budget (WORK_FLOOR). +// The calls that each copy of `w_slowpair` makes beyond its first: its test +// calls the function this many more times where its output differs. A call +// evaluates two or three instrumented sites, so a copy makes 60,000 to 90,000 +// site evaluations: inside its call budget (CALL_FLOOR in check.mjs, 100,000), +// and the two copies together past the mutant's work budget (WORK_FLOOR). // Work is counted, not timed, so the outcome is the same on any machine. -const PAIR_LOOPS = 37_500 +const PAIR_LOOPS = 30_000 // A mutant's work budget (maintainer disposition G1, under the deterministic // cuts decision of 2026-11-14): the larger of this floor and ten times its // owner files' unmutated work, in site evaluations. The floor is the copies' From 54c1645303bb7ca3af6f5e88f772df6e96be3439 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 17:07:34 -0700 Subject: [PATCH 04/32] test(oracle-gaps): the model no longer predicts a timeout from a wall-clock budget Register M entry "Deterministic cuts, option A" (2026-11-14): no wall-clock cut decides an outcome; a mutant's budget is a work count, and the wall-clock budget is only a backstop that reports a gap. The model's wall-clock clause (two budgets: a slow owner file's confirmations past --mutant-limit predict a timeout) is removed as the repair the M entry requires. This is not a model change to match production: the clause's authority is the retired confirmation path, and the M entry retires it. Consequence, a RED on main: the constructed history "a slow kill past Vitest's 15 s limit" (a 16 s wait where its output differs) now predicts `killed`, and main reports `timeout` for it, because main's confirmation is cut by the mutant budget at 15 s (diagnostic on a scratch branch: the copy's outcome is "run time limit", not Vitest's "Test timed out"). Also: a test that awaits its busy witness is async (the emitter marks it so), so the baseline loads. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 71a12d7d..a93a6b09 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4067,7 +4067,6 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { // earlier verdicts. A kill in another file still decides (a kill outranks // a timeout); failing that, the mutant is a timeout. const hung = new Set() - const slowed = new Map() // The mutant's work budget (G1): its owner files' copies are counted in // their order, each copy's work being the site evaluations of its runs up // to a hang or a kill. A copy past the budget is a timeout, and so is every @@ -4091,7 +4090,6 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { for (const file of files) { let work = 0 for (const r of runs.filter((x) => x.t.file === file)) { - if (r.slowed) slowed.set(file, r.t.slowMs ?? SLOW_MS) if (r.hang) hung.add(file) work += r.work if (r.hang || killsIn(r)) break @@ -4103,9 +4101,7 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { spent += work counted.add(file) } - const live = runs.filter( - (r) => counted.has(r.t.file) && !hung.has(r.t.file), - ) + const live = runs.filter((r) => counted.has(r.t.file) && !hung.has(r.t.file)) const verdicts = (route) => live.flatMap((r) => r.verdicts.filter((v) => v.route === route).map((v) => v.law), @@ -4113,20 +4109,10 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { const reproducible = verdicts(`reproducible`) if (reproducible.some((law) => law !== `reach agreement`)) return `killed` if (hung.size > 0 || exhausted) return `timeout` - // The runs that confirm a mutant's cut-off copies share its budget - // (`--mutant-limit`, MUTANT_LIMIT_MS here; maintainer disposition G1, - // 2026-11-11). Under two budgets, a slow owner file is cut at its first - // budget and confirmed alone, and its confirmation takes longer than the - // time its slow test waits. When the confirmations of every slow owner - // file do not fit the budget together, one is cut, and the mutant is a - // timeout; a kill still decides first. Under one budget no copy is - // confirmed, so the budget never binds. - if ( - budgeted && - plan.budget === `two` && - [...slowed.values()].reduce((sum, ms) => sum + ms, 0) >= MUTANT_LIMIT_MS - ) - return `timeout` + // No wall-clock budget decides here (maintainer decision, deterministic + // cuts, option A, 2026-11-14): a slow copy runs to its end, and the mutant + // budget is a work count (above). The wall-clock backstops report a gap no + // history here reaches, so a slow owner's wait never makes a timeout. if (reproducible.length > 0) return `agreement-only-kill` if (verdicts(`random`).length > 0) return `random-only-kill` if (verdicts(`unknown`).length > 0) return `unregistered-kill` @@ -4862,7 +4848,7 @@ function oracleTestFile(plan, file) { : []), t.asProp ? ` fcTest.prop([fcv.integer({ min: 0, max: 10 })], { numRuns: 20, seed: 1 })('${testName(t, index)}', async (n) => {` - : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timed || lazyGated(plan, file, t) ? `async ` : ``}() => {`, + : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.busy || t.timed || lazyGated(plan, file, t) ? `async ` : ``}() => {`, ...partsOf(t).flatMap(campaign), ` })`, `})`, From 4b063c8f5dfaf8c096075d6650fabde34d3055a2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 17:10:12 -0700 Subject: [PATCH 05/32] fix(oracle-gaps): Vitest's per-test limit is a backstop gap, and the states are listed apart Register M entry "Deterministic cuts, option A" (2026-11-14), with the coordinator's ruling 3 (fallback): a test that Vitest's own limit ends ("Test timed out") is reported as a backstop gap, like a timed-out run. It is `timeout` after any kill, and it never keeps a kill from deciding. The previous rule read a Vitest timeout only when every failing test had one, so a timed-out test beside a verdict-less failure was `crashed`, and a timed-out test beside an unregistered kill was not a timeout at all. The `every` branch and the `!timeout` filter are now redundant and removed. Law: a run's outcome does not depend on how long a test took (determinism under load); a clock limit may only report a gap. RED: the classify oracle on the parent classify.mjs (HEAD before this commit, built from the same fixtures) fails with "mutant m0: unregistered-kill, owed timeout", and the parent's crashed/timeout split fails the same law. Mutant: timeout checked before the kills (scratch copy): the classify oracle fails, "mutant m0: timeout, owed killed"; the mutant is rejected. GREEN: the classify oracle passes (276 tests). The model counts a timeout message only in a file that loaded, as production does (a file that fails to load gives no tests). The e2e oracle's states "a slow owner's kill reported as when it did not wait", "a slow kill past fifteen seconds killed" and "a copy waiting on a timer under load killed" are listed as separate: they have their own histories, so "every outcome class" must not enter them. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/classify.mjs | 17 ++++---- ...p-report-classify-oracle.property.test.mjs | 42 +++++++++++-------- .../gap-report-oracle.property.test.mjs | 5 +++ 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/packages/oracles/oracle-gaps/classify.mjs b/packages/oracles/oracle-gaps/classify.mjs index 72da6178..9eb9d3a6 100644 --- a/packages/oracles/oracle-gaps/classify.mjs +++ b/packages/oracles/oracle-gaps/classify.mjs @@ -47,9 +47,15 @@ export function outcomeOf( assertions = false, pastBound = () => false, ) { - // A run that timed out recorded nothing. A kill in another run still - // decides; failing that, the mutant is `timeout`. - const timeout = units.some((unit) => unit.timedOut) + // A run that timed out recorded nothing, and a test that Vitest's own limit + // ended (`Test timed out`) is the same kind of backstop (deterministic cuts, + // 2026-11-14): a gap that never keeps a kill from deciding. A kill in another + // run still decides; failing that, the mutant is `timeout`. + const timedOutTest = (test) => + test.state === `failed` && /Test timed out/.test(test.message) + const timeout = units.some( + (unit) => unit.timedOut || (unit.tests ?? []).some(timedOutTest), + ) ? { outcome: `timeout`, detail: `run time limit` } : undefined const failures = [] @@ -87,7 +93,6 @@ export function outcomeOf( // fast-check's wrapper text ("Property failed …") is not an // assertion; the wrapped error, which its message quotes, decides. /AssertionError|expected .+ to /.test(test.message), - timeout: /Test timed out/.test(test.message), // fast-check quotes the error its predicate threw after its own // header; that error is the cause the report shows. message: ( @@ -153,9 +158,7 @@ export function outcomeOf( laws: lawsOf(unregistered), } const tests = failures.filter((f) => f.test) - if (tests.length > 0 && tests.every((f) => f.timeout)) - return { outcome: `timeout`, by: tests.map((f) => f.test) } - const crashed = tests.filter((f) => f.route !== `framework` && !f.timeout) + const crashed = tests.filter((f) => f.route !== `framework`) if (crashed.length > 0) return { outcome: `crashed`, diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 5765025d..554b6bd0 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -36,8 +36,8 @@ * so a kill names a non-empty part of them. An operator mutant's owner * files run one after another in report order (law 7), and the runner * stops at the first that kills, so its kill names laws of that file - * alone. Failing that, a run that - * timed out makes it `timeout`. Failing that, reproducible verdicts under + * alone. Failing that, a run that timed out, or a test that Vitest's limit + * ended, makes it `timeout`. Failing that, reproducible verdicts under * reach agreement alone make it an `agreement-only-kill`, which names * them all. Failing that, a verdict on the random campaign * makes it a `random-only-kill`, and one on an unknown or unregistered @@ -62,7 +62,10 @@ * and the design decided with it that day: a speed shortcut is owed a * cost law, not an exit, so a regression that deletes it must be * catchable, by a deterministic count and never by wall-clock time. - * 4. Failing that, if every failing test timed out, it is `timeout`; a + * 4. Failing that, a test that Vitest's own limit ended (its message is + * `Test timed out`) is a backstop, with a run that timed out: the mutant + * is `timeout`, a gap that never keeps a kill from deciding (deterministic + * cuts, option A, 2026-11-14). Otherwise a * failing test outside the framework's own checks makes it `crashed`; a * failing framework check alone (the `calibration` route, which covers * calibrations, probe calibrations, and the determinism test) makes it a @@ -720,7 +723,14 @@ function boundBelow(before, after) { // at a kill reads no work. function failureOf(h, slot, tests, work = false) { const under = ({ t }) => t.under[slot].kind - const timedOut = tests.some((x) => under(x) === `runTimeout`) + // A test that Vitest's own limit ended is a backstop, as a cut run is, + // in a file that loaded (a file that failed to load has no tests to end). + const timedOut = tests.some( + (x) => + under(x) === `runTimeout` || + (under(x) === `timeoutMessage` && + !h.files[x.fi].tests.some((t) => t.under[slot].kind === `setupError`)), + ) const setup = tests.some((x) => under(x) === `setupError`) const ran = tests.filter( (x) => @@ -771,11 +781,7 @@ function failureOf(h, slot, tests, work = false) { return naming(`random-only-kill`, verdictsIn(`random`)) if (verdictsIn(`unknown`).length > 0) return naming(`unregistered-kill`, verdictsIn(`unknown`)) - if (failing.length > 0 && failing.every((x) => under(x) === `timeoutMessage`)) - return decided({ outcome: `timeout` }) - const crashed = failing.filter( - (x) => under(x) !== `timeoutMessage` && kindOf(x.t.route) !== `framework`, - ) + const crashed = failing.filter((x) => kindOf(x.t.route) !== `framework`) if (crashed.length > 0) return naming(`crashed`, crashed) if (failing.length > 0) return naming(`framework-error`, failing) const unhandled = ran.some((x) => under(x) === `unhandled`) @@ -1102,7 +1108,14 @@ function operandsSeen(h, mi, owners) { function ordinaryOutcome(h, mi, files) { const tests = files.flatMap((fi) => h.files[fi].tests.map((t) => ({ t, fi }))) const under = ({ t }) => t.under[mi].kind - const timedOut = tests.some((x) => under(x) === `runTimeout`) + // A test that Vitest's own limit ended is a backstop, as a cut run is, + // in a file that loaded (a file that failed to load has no tests to end). + const timedOut = tests.some( + (x) => + under(x) === `runTimeout` || + (under(x) === `timeoutMessage` && + !h.files[x.fi].tests.some((t) => t.under[mi].kind === `setupError`)), + ) const ran = tests.filter( (x) => !h.files[x.fi].tests.some((t) => @@ -1130,14 +1143,7 @@ function ordinaryOutcome(h, mi, files) { if (kills.length > 0) return `agreement-only-kill` if (judgedOn(`random`).length > 0) return `random-only-kill` if (judgedOn(`unknown`).length > 0) return `unregistered-kill` - if (failing.length > 0 && failing.every((x) => under(x) === `timeoutMessage`)) - return `timeout` - if ( - failing.some( - (x) => under(x) !== `timeoutMessage` && kindOf(x.t.route) !== `framework`, - ) - ) - return `crashed` + if (failing.some((x) => kindOf(x.t.route) !== `framework`)) return `crashed` if (failing.length > 0) return `framework-error` if (tests.some((x) => under(x) === `setupError` || under(x) === `unhandled`)) return `setup-error` diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 9adb1d67..b751f876 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -3345,6 +3345,11 @@ const separate = [ `a session failure larger than a socket buffer`, `a config fault in a run that owns nothing`, `a refused run whose package lies inside the CLI's directory`, + // The deterministic-cuts states have constructed histories of their own, so + // no history that names every state (`every outcome class`) must enter them. + `a slow owner's kill reported as when it did not wait`, + `a slow kill past fifteen seconds killed`, + `a copy waiting on a timer under load killed`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, `a package with no config ran`, From 72db011a2943676a3ca91d655f2578ad80f4df81 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 17:16:49 -0700 Subject: [PATCH 06/32] fix(oracle-gaps): no wall-clock cut decides an outcome Register M entry "Deterministic cuts, option A" (2026-11-14). A copy runs until it finishes, goes idle, or spins; the only wall-clock limit is a backstop that reports a gap and never decides a kill. - No first budget and no confirmation: a copy runs once, to its end, under the backstop limit (hang-limit plus three seconds per second of the file's unmutated time). The cut marking and the confirming run are removed. - G1 becomes a work budget: a mutant's copies, summed in their owner files' report order, may do at most max(WORK_FLOOR, 10 x the owners' unmutated calls) site evaluations (WORK_FLOOR = CALL_FLOOR, 100,000). Past it, the copy that crosses and every later copy is a timeout, so a kill among them does not count. This is deterministic because the work is a count; the owner order is the report's, not the completion order. - The wall-clock mutant budget (--mutant-limit) is a backstop checked after all copies: no kill and elapsed over budget is a timeout; it never decides a kill. - Defaults (coordinator rulings, 2026-11-14): --hang-limit 60 s, --mutant-limit 180 s, --test-timeout 65 s (hang-limit plus 5 s). Cost to state: an unmutated hang that no CLI limit cuts now fails after 65 s, not 15 s; under option A an idle hang is cut by `no progress` in about 500 ms, so only a busy hang with no instrumented site waits for the backstop. - --first-limit is validated and retired: it prints a note and changes nothing. - A test that Vitest's own limit ends is a backstop gap (classify.mjs; its RED and mutant are in the prior commit, 4b063c8). Correction to 4b063c8's message: that message said the parent's crashed/timeout split fails the classify oracle. I did not run that case. The verified RED is the one in 4b063c8's body: the parent gives "unregistered-kill" where the model owes "timeout". Co-authored-by: Isaac --- packages/oracles/oracle-gaps/args.mjs | 25 ++- packages/oracles/oracle-gaps/check.mjs | 191 +++++++----------- packages/oracles/oracle-gaps/classify.mjs | 4 +- .../oracle-gaps/gap-report-model-oracle.mjs | 3 +- .../gap-report-oracle.property.test.mjs | 11 +- 5 files changed, 93 insertions(+), 141 deletions(-) diff --git a/packages/oracles/oracle-gaps/args.mjs b/packages/oracles/oracle-gaps/args.mjs index d8f7d02f..34978ae7 100644 --- a/packages/oracles/oracle-gaps/args.mjs +++ b/packages/oracles/oracle-gaps/args.mjs @@ -210,17 +210,20 @@ function optionsOf(args, cwd) { sample: args.sample ? Number(args.sample) : undefined, // A mutant can make a test hang. Oracles that run in milliseconds do not // need Vitest's default limits, which would make each hang cost seconds. - testTimeout: Number(args[`test-timeout`] ?? 15000), - // The base of each mutant run's time limit, in milliseconds; the limit - // adds three times the file's unmutated duration (see limitFor). - hangLimit: Number(args[`hang-limit`] ?? 10000), - // The base of a mutant run's first, short budget; a copy past it runs - // again alone with the full limit before it can be a timeout. - firstLimit: Number(args[`first-limit`] ?? 2000), - // The floor of a mutant's budget, which the runs that confirm its - // cut-off copies share (check.mjs, `budgetOf`): 30 s by default, room - // for three confirmations at the default hang limit. - mutantLimit: Number(args[`mutant-limit`] ?? 30000), + // Vitest's per-test limit, above the copies' backstop (`hangLimit` plus + // five seconds, so a copy that waits for its backstop is cut by it first). + testTimeout: Number(args[`test-timeout`] ?? 65000), + // The base of each mutant run's backstop time limit, in milliseconds; the + // limit adds three times the file's unmutated duration (see limitFor). A + // backstop reports a gap and never decides a kill (deterministic cuts, + // option A, 2026-11-14). + hangLimit: Number(args[`hang-limit`] ?? 60000), + // The `--first-limit` option is retired (option A): it is validated and + // noted, and changes nothing. + retiredFirstLimit: args[`first-limit`] !== undefined, + // The floor of a mutant's wall-clock backstop (check.mjs, `budgetOf`): 180 s + // by default, three full copies at the default backstop. + mutantLimit: Number(args[`mutant-limit`] ?? 180000), // Vitest's worker pool. Threads start much faster than child processes; // a package with a native addon in its tree runs with `forks`. pool: args.pool ?? defaultPool(pkg), diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 9ca8b5b0..e9f747a6 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -95,6 +95,11 @@ const IDLE_MS = 500 // in one test before it counts as a loop that never ends. A file's budget is // a hundred times the most calls any of its unmutated tests made, if more. const CALL_FLOOR = 100_000 +// The least work (site evaluations) a mutant's copies may do together: the +// larger of this and ten times their owner files' unmutated work (`workBudgetOf`). +// It is the call floor: two copies that each stay inside their own call budget +// can pass it together. +const WORK_FLOOR = CALL_FLOOR const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) /** The host that a session directory names, if it names one. */ function sessionHostOf(dir) { @@ -1282,9 +1287,13 @@ export default async (env) => { async function check(options, store, log = console.error) { const authored = options.mutants ? loadAuthored(options.mutants) : [] seedVariable = `${options.envPrefix}_SEED` + if (options.retiredFirstLimit) + log( + `oracle-gaps: note --first-limit is retired: a copy has no first budget, so it changes nothing (deterministic cuts, option A)`, + ) if (options.hangLimit > options.mutantLimit) log( - `oracle-gaps: note --hang-limit ${options.hangLimit} ms is above --mutant-limit ${options.mutantLimit} ms; a mutant's confirmations together get at most the larger of --mutant-limit and ten times its owner files' unmutated time`, + `oracle-gaps: note --hang-limit ${options.hangLimit} ms is above --mutant-limit ${options.mutantLimit} ms; a mutant's copies together get at most the larger of --mutant-limit and ten times its owner files' unmutated time`, ) pulse = { log, @@ -1410,7 +1419,7 @@ export default async (env) => { spin: 0, rerun: 0, unchanged: 0, - // Confirmations that a mutant's budget cut short or left no time. + // Copies past their mutant's work budget (each a timeout). budget: 0, } // Each file's imports in its unmutated run. A run that timed out reports @@ -1492,8 +1501,7 @@ export default async (env) => { ) return { ...batch, inputs } } - // A unit that depends on more than its inputs (a confirmation that its - // mutant's budget cut short) is not kept (`keep`). + // A unit that depends on more than its inputs is not kept (`keep`). async function unit(context, file, spec, keep, runOptions = {}) { const index = indexOf.get(context) const id = idOf(file, spec, runOptions.select, runOptions.ignore) @@ -1527,9 +1535,9 @@ export default async (env) => { return result } // Units for many requests at once. Each is looked up first; the rest run - // as one batch (runBatch). A batch cut off by a hang queues that item's - // confirming run, which any free slot takes, and reruns the rest in a - // fresh context; a batch with + // as one batch (runBatch). A batch cut off (a copy's backstop, idleness, + // or a spin) settles that copy as a timeout and reruns the rest in a fresh + // context; a batch with // errors outside any test runs its items alone. `needed` says, from the // units found so far, whether a request must still run; a request that // need not run, gets no unit; a copy that skipped its tests gets a @@ -1576,25 +1584,18 @@ export default async (env) => { evidence.computed += 1 }) if (batch.timedOutAt === undefined) break - // A copy past its first budget may only be slow: a loaded machine - // slows a run that ends. It runs again alone, in a fresh context, - // with the full limit, and only a run cut off there too is a timeout. + // A copy cut off by its backstop, for idleness, or for spinning is a + // timeout as it stands: no clock decides a kill (deterministic cuts). const hung = rest[batch.timedOutAt] - const full = limitFor(hung.file) evidence.cut += 1 if (batch.idle) evidence.idle += 1 if (batch.spin) evidence.spin += 1 await replace(slot) - // A copy that already had the full limit, or that sat idle, is - // final; any other comes back marked `cut`, for its mutant's own - // confirmation (`runOwners`). - if (hung.limitMs >= full || batch.idle || batch.spin) - settle( - hung, - { file: hung.file, timedOut: true, seconds: full / 1000 }, - batch.inputs[batch.timedOutAt], - ) - else got[hung.i] = { file: hung.file, cut: true } + settle( + hung, + { file: hung.file, timedOut: true, seconds: hung.limitMs / 1000 }, + batch.inputs[batch.timedOutAt], + ) rest = [ ...rest.slice(0, batch.timedOutAt).filter((_, k) => !batch.units[k]), ...rest.slice(batch.timedOutAt + 1), @@ -1619,53 +1620,6 @@ export default async (env) => { if (u.timedOut) await replace(slot) return u } - // A hung copy's confirming run: alone, in a clean context, with the full - // limit, or with what is left of its mutant's budget when that is less - // (`left`, in milliseconds). Only a run cut off there too is a timeout. - // An idle copy in a file with no I/O is final at once (see unitsBatch): - // it waits on nothing that can come. The unit is marked `confirmed`, so - // its seconds count against the budget whenever it is used, reused ones - // too. One that the budget cut short, or left no time, depends on the - // mutant's other units, so it is not kept. - async function confirmAlone(slot, request, left) { - // The confirmation is kept under the request's own identity, so a - // later check reuses it in place of the request's first run. - const hung = { - ...request, - id: idOf(request.file, request.spec, request.select, request.ignore), - } - const full = limitFor(hung.file) - const limit = Math.min(full, left) - const confirmed = (u) => ({ ...u, confirmed: true }) - if (limit <= 0) { - evidence.budget += 1 - return confirmed({ file: hung.file, timedOut: true, seconds: 0 }) - } - const alone = await batchIn(slot, [{ ...hung, limitMs: limit }]) - if (alone.unattributed) - return confirmed( - await runAlone(slot, { ...hung, limitMs: limit }, limit >= full), - ) - const [inputs] = alone.inputs - if (alone.timedOutAt === 0) { - evidence.cut += 1 - if (alone.idle) evidence.idle += 1 - if (limit < full) evidence.budget += 1 - const u = confirmed({ - file: hung.file, - timedOut: true, - seconds: limit / 1000, - }) - if (limit >= full) remember(hung.id, u, inputs) - await replace(slot) - return u - } - const u = confirmed(alone.units[0]) - remember(hung.id, u, inputs) - evidence.computed += 1 - return u - } - const vitest = contexts[0] const fileSpecs = new Map(contextSpecs.get(vitest)) const relative = (file) => relativeWithin(pkg, file) ?? posix(file) @@ -1683,41 +1637,34 @@ export default async (env) => { // detect anything, and it may read source text that instrumentation // changes. Each file's unmutated duration sets a mutant run's time limit. const fileSeconds = new Map() - // A mutant can slow a run that still ends, for example by lengthening a - // retry backoff, and a loaded machine slows every run. A mutant run - // first gets a short budget (`--first-limit`); a copy cut off there runs - // again alone, with the full limit (`--hang-limit`, 10 s by default: - // a 5 s base once turned a slow kill into a timeout), and only a run - // cut off there too is a timeout (see unitsBatch). + // A copy's time limit is a backstop: the full limit, which reports a gap and + // never decides a kill (deterministic cuts, option A, 2026-11-14). There is + // no first budget, so a copy runs once, to its end. const limitFor = (file) => options.hangLimit + 3000 * (fileSeconds.get(file) ?? 10) - const firstLimitFor = (file) => - Math.min( - limitFor(file), - options.firstLimit + 3000 * (fileSeconds.get(file) ?? 10), - ) - // Each mutant's budget (maintainer disposition G1, 2026-11-11), for the - // runs that confirm its cut-off copies: the larger of `--mutant-limit` - // (30 s by default) and ten times its owner files' unmutated time. Its - // confirmations run in its owner files' order, and each gets at most the - // smaller of its full limit and what the budget has left after the - // seconds of the mutant's earlier confirmed units, reused ones included - // (`runOwners`), so the outcome is a function of the units, whatever ran - // first or was reused. With the default limits the budget cuts a - // confirmation below its full limit (10 s plus three times its file's - // time t) only when the mutant's confirmations together pass it: for - // one owner file that is never, since 10t >= 10 + 3t once t >= 1.43 s - // and the 30 s floor covers a smaller t; it binds when `--hang-limit` is - // raised past the budget, or when several owner files need long - // confirmations. It does not bound a mutant's first runs (each has its - // first budget) or the context replacements after cuts (each up to a - // 10 s close and a warm start). + // A mutant's wall-clock backstop (`--mutant-limit`, or ten times its owner + // files' unmutated time, whichever is larger): past it, a mutant with no kill + // is a timeout (runOwners). It never decides a kill. const budgetOf = (files) => Math.max( options.mutantLimit, 10_000 * files.reduce((sum, file) => sum + (fileSeconds.get(file) ?? 0), 0), ) + // A mutant's work budget (maintainer disposition G1, option A): the larger of + // WORK_FLOOR and ten times its owner files' unmutated work, in site + // evaluations, each file counted by its most-working unmutated test. + const workBudgetOf = (files) => + Math.max( + WORK_FLOOR, + 10 * files.reduce((sum, file) => sum + (mostCalls.get(file) ?? 0), 0), + ) + // A copy's work: the site evaluations its tests made. A copy that timed out + // recorded none, and counts none. + const workOf = (unit) => + unit.timedOut + ? 0 + : (unit.rows ?? []).reduce((sum, [, row]) => sum + (row.calls ?? 0), 0) // A run cut off is still running, so its context is replaced; closing // it stops the worker. const replace = async (slot) => { @@ -2029,8 +1976,12 @@ export default async (env) => { // verdicts of its own yet. const kills = (u) => !u.skipped && - !u.cut && outcomeOf([u], baseline.nondeterministic).outcome === `killed` + // A mutant's copies run in their owner files' order and are summed in that + // order (the work budget, below). The order is the report's, so it does not + // depend on the timing of the copies. A later owner's kill does not count + // once the budget is passed; that is deterministic because the work is a + // count, and it is the rule the design note (option A) states. const runOwners = async (slot, states, filesOf, keysOf) => { const active = states.filter((st) => !st.done) const requests = [] @@ -2047,7 +1998,7 @@ export default async (env) => { spec: `mutant:${st.mutant.number}`, mutant: st.mutant.number, select: await selectionFor(slot, file, keysOf(st)), - limitMs: firstLimitFor(file), + limitMs: limitFor(file), // Idle detection only where no unmutated test of the file did I/O. idleMs: ioFiles.has(file) ? null : IDLE_MS, // A loop that spins on instrumented code is cut by count. @@ -2073,46 +2024,42 @@ export default async (env) => { (request, found) => !killedBefore(request, found), ) for (const st of active) { - // The mutant's budget, spent in its files' order by the seconds of - // each confirmed unit, reused or run now, so its outcome is a - // function of its units, whatever ran first or was reused. - let left = budgetOf(filesOf(st)) + const files = filesOf(st) + // The mutant's copies in their files' order, judged by count: each copy's + // work is added, and once the sum passes the work budget, that copy and + // every later one is a timeout, so a kill among them does not count. + const budget = workBudgetOf(files) + let work = 0 for (const [i, request] of requests.entries()) { if (request.st !== st) continue - // A copy its batch skipped (an earlier copy killed the mutant) - // runs again through unitsBatch, so it is confirmed like any copy - // while no kill is in hand. + // A copy its batch skipped (an earlier copy killed the mutant) runs + // again through unitsBatch, while no kill is in hand. if (got[i]?.skipped) { if (st.units.some(kills)) break got[i] = (await unitsBatch(slot, [request], () => true))[0] } - // A kill earlier in the mutant's files leaves a file without a - // unit. Any other missing unit stops the run. + // A kill earlier in the mutant's files leaves a file without a unit. + // Any other missing unit stops the run. if (got[i] === undefined || got[i].skipped) { if (st.units.some(kills)) break throw new Error( `${st.mutant.id}: ${relative(request.file)} has no unit and no earlier kill`, ) } - // A cut copy is confirmed only while no later owner's unit already - // killed the mutant: a kill decides, so its confirmation would - // spend time for nothing. - if ( - got[i].cut && - requests.some( - (other, j) => - other.st === st && - j > i && - got[j] !== undefined && - kills(got[j]), - ) - ) - continue - if (got[i].cut) got[i] = await confirmAlone(slot, request, left) - if (got[i].confirmed) left -= got[i].seconds * 1000 + work += workOf(got[i]) + if (work > budget) { + evidence.budget += 1 + st.units.push({ file: request.file, timedOut: true, seconds: 0 }) + break + } st.units.push(got[i]) if (kills(got[i])) break } + // The wall-clock backstop: no kill, and the copies' measured time past + // the mutant's wall budget, makes a timeout. A kill decides before it. + const elapsed = st.units.reduce((sum, u) => sum + (u.seconds ?? 0), 0) + if (!st.units.some(kills) && elapsed * 1000 > budgetOf(files)) + st.units.push({ file: files[0], timedOut: true, seconds: 0 }) st.done = true } } diff --git a/packages/oracles/oracle-gaps/classify.mjs b/packages/oracles/oracle-gaps/classify.mjs index 9eb9d3a6..6c2a0ac5 100644 --- a/packages/oracles/oracle-gaps/classify.mjs +++ b/packages/oracles/oracle-gaps/classify.mjs @@ -281,7 +281,9 @@ export function targetsOf(mutant, baseline, owners, oracleFiles) { return { oracleKeys, testKeys, - oracleTargets: [...new Set(oracleKeys.map(fileOf))], + // Report order (byPath), so a mutant's owners are summed in one order + // however its copies ran. + oracleTargets: [...new Set(oracleKeys.map(fileOf))].sort(byPath), testTargets: [...new Set(testKeys.map(fileOf))], } } diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index a93a6b09..8a73700e 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4091,7 +4091,8 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { let work = 0 for (const r of runs.filter((x) => x.t.file === file)) { if (r.hang) hung.add(file) - work += r.work + // A copy cut off by its backstop, or by spinning, records no work. + work += r.hang ? 0 : r.work if (r.hang || killsIn(r)) break } if (budgeted && spent + work > workBudget) { diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index b751f876..d05706fc 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -608,12 +608,11 @@ function cliArgs( // in a fixed order; results do not depend on the count. `--contexts`, String(contexts), - // A fixture's tests run in milliseconds. A 1 s first budget cuts off - // the slow witnesses, and the full limit lets them end; a hang is cut - // for idleness. With one budget, a run past the full limit is a timeout - // at once. - `--first-limit`, - budget === `one` ? String(FULL_LIMIT_MS) : `1000`, + // A fixture's tests run in milliseconds. A run's copies have no first + // budget (option A): the full limit is the only time limit, and a hang + // is cut for idleness. `--first-limit` is retired; the budget `one` + // history passes it, and the CLI notes that it changes nothing. + ...(budget === `one` ? [`--first-limit`, `1000`] : []), `--hang-limit`, String(FULL_LIMIT_MS), // The budget that a mutant's confirming runs share, the full limit of From b377f9ebafb3cad4136abeba13f467747f8c0c9a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 17:25:40 -0700 Subject: [PATCH 07/32] test(oracle-gaps): the timer witness waits on one timer, with no load in the test The counter-witness (register, 2026-11-13 "Determinism under load"; the 2026-11-14 entry "Deterministic cuts, option A") was a copy whose test spun for 600 ms and then waited 2 s on a timer. Its spin kept event-loop utilization above 5%, which alone keeps a copy from looking idle, so the timer check never mattered: mutant 2 (elapsed-only idle, the timer checks disabled) passed the witness. The witness now waits on one timer due 3 s out, inside the horizon, with nothing else in the test. A copy with its timer pending is never idle; only that check stands between it and an idle cut. A loaded variant, if wanted, is a CPU hog outside the worker. The state is "a copy waiting on a timer killed"; the constructed field is timerWait (a constructed-only test field, taken back to null; no generated test draws it). Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 17 +++++++------- .../gap-report-oracle.property.test.mjs | 22 +++++++++---------- packages/oracles/scripts/e2e-shards.mjs | 6 ++--- packages/oracles/scripts/pr-shards.mjs | 6 ++--- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 8a73700e..03041505 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -424,7 +424,7 @@ const testCase = fc.record({ // neutral values, which change nothing. slowMs: fc.constant(SLOW_MS), loops: fc.constant(0), - busy: fc.constant(null), + timerWait: fc.constant(null), }) // Law 9's edits. Each content edit changes what some evidence depends on: // a test file, a source statement, the source's line numbers, an owns list, @@ -827,7 +827,7 @@ const takenBack = (t) => ({ slow: false, slowMs: SLOW_MS, loops: 0, - busy: null, + timerWait: null, }) const histories = drawn.map( (history) => (keepsSteps(history) ? history : { ...history, steps: null }), @@ -4779,12 +4779,13 @@ function oracleTestFile(plan, file) { ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) for (let k${i} = 0; k${i} < ${t.loops}; k${i} += 1) ${callFor(t.fn)}(${part.input}, [])`, ] : []), - // Where its output differs, a test that starts a timer, spins for - // `busyMs` (the load), and then waits for the timer. The timer is - // pending throughout, so the copy is never idle. - ...(t.busy + // Where its output differs, a test that waits on one timer due well + // inside its horizon. Nothing else keeps it busy: no load is in the test + // (a loaded variant would be a CPU hog outside the worker), so the timer + // check alone stands between this copy and an idle cut. + ...(t.timerWait ? [ - ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { const waiting${i} = new Promise((r) => setTimeout(r, ${t.busy.waitMs})); const until${i} = performance.now() + ${t.busy.busyMs}; while (performance.now() < until${i}) {}; await waiting${i} }`, + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.timerWait.waitMs}))`, ] : []), // A test that waits on I/O (a child process, about 3 s) where the @@ -4849,7 +4850,7 @@ function oracleTestFile(plan, file) { : []), t.asProp ? ` fcTest.prop([fcv.integer({ min: 0, max: 10 })], { numRuns: 20, seed: 1 })('${testName(t, index)}', async (n) => {` - : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.busy || t.timed || lazyGated(plan, file, t) ? `async ` : ``}() => {`, + : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timerWait || t.timed || lazyGated(plan, file, t) ? `async ` : ``}() => {`, ...partsOf(t).flatMap(campaign), ` })`, `})`, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index d05706fc..3abdc7c4 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -385,7 +385,7 @@ * * Limits. Sources are one file of exported functions and classes over one * number. A hang is generated only as an owner test that waits on a promise - * that never settles, under a 1 s first budget and a 20 s full limit + * that never settles, under a 20 s full limit * (`--first-limit`, `--hang-limit`, and a 15 s `--mutant-limit`); a slow run * that ends is only a witness, a test that waits 3 s where its output * changed and then judges, or 10 s in two one-test owners without judging. Not generated: a busy @@ -1636,12 +1636,12 @@ async function driver(plan, observe) { if (twinSame) states.add(`a slow owner's kill reported as when it did not wait`) // A kill that waits past Vitest's 15 s test limit, and one whose copy - // waits on a timer under load, each killed as the report says. + // waits on a timer, each killed as the report says. const killed = cli.report?.results?.some((r) => r.outcome === `killed`) if (killed && plan.history.tests.some((t) => t.slowMs > 15000)) states.add(`a slow kill past fifteen seconds killed`) - if (killed && plan.history.tests.some((t) => t.busy)) - states.add(`a copy waiting on a timer under load killed`) + if (killed && plan.history.tests.some((t) => t.timerWait)) + states.add(`a copy waiting on a timer killed`) if (cli.exit === 0 && !plan.consoleOutput) states.add(`a completed run whose tests wrote nothing`) // Law 14: a partial run of one owned function's mutants, whose baseline @@ -3217,7 +3217,7 @@ const required = [ `a hang in one owner and a kill in another killed`, `a slow owner's kill reported as when it did not wait`, `a slow kill past fifteen seconds killed`, - `a copy waiting on a timer under load killed`, + `a copy waiting on a timer killed`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, `a heartbeat named a busy mutant`, @@ -3348,7 +3348,7 @@ const separate = [ // no history that names every state (`every outcome class`) must enter them. `a slow owner's kill reported as when it did not wait`, `a slow kill past fifteen seconds killed`, - `a copy waiting on a timer under load killed`, + `a copy waiting on a timer killed`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, `a package with no config ran`, @@ -3425,9 +3425,9 @@ const soloKill = (generated, tests) => ({ oracleProperty({ id: `oracle-gaps.report`, law: { - statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, + statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds a mutant's copies by one work budget per mutant (site evaluations, never time) and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; the retired --first-limit judged only as a note; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with the full limit, since no copy has a first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4201,14 +4201,14 @@ oracleProperty({ // under load, then waits for it: the copy is busy, never idle, and the // mutant is killed. { - name: `a copy waiting on a timer under load`, - enters: [`a copy waiting on a timer under load killed`], + name: `a copy waiting on a timer`, + enters: [`a copy waiting on a timer killed`], build: (generated) => soloKill(generated, [ { ...judging(`a`, `fixed seed`, 7), fn: 0, - busy: { busyMs: 600, waitMs: 2000 }, + timerWait: { waitMs: 3000 }, }, ]), }, diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 36f76fbf..bdd42c97 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -66,9 +66,9 @@ export const shards = { `fixed seed 20261105: a slow kill past Vitest's 15 s limit`, `random seed: a slow kill past Vitest's 15 s limit`, `determinism: constructed history: a slow kill past Vitest's 15 s limit`, - `fixed seed 20261105: a copy waiting on a timer under load`, - `random seed: a copy waiting on a timer under load`, - `determinism: constructed history: a copy waiting on a timer under load`, + `fixed seed 20261105: a copy waiting on a timer`, + `random seed: a copy waiting on a timer`, + `determinism: constructed history: a copy waiting on a timer`, // The witness module's edit sequence (option M): 189 s here, measured // with the daemon fix of fix/daemon-current-inputs. `determinism: constructed history: an edit sequence over the witness module`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index dce1d222..2bcb21c3 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -302,14 +302,14 @@ export const partitions = { 'a failing owner under a config that --config names', ]), // 134 s, 101 s, 27 s, and 15 s; the two constructed histories of the - // deterministic cuts (a 16 s wait, and a timer under load), about 45 s. + // deterministic cuts (a 16 s wait, and a timer), about 45 s. 2: anyOf([ 'every outcome class', 'a second daemon check of the budget witness', 'a declaration fault: brokenVitest', 'an edit that changed the cold result after a hang replaced the only context', "a slow kill past Vitest's 15 s limit", - 'a copy waiting on a timer under load', + 'a copy waiting on a timer', ]), // The rest: the edit sequence through the daemon (95 s), brokenOwner // (83 s), generated (23 s), the lazy module (12 s), and small units. @@ -551,7 +551,7 @@ export const partitions = { 'an ambiguous authored mutant', "a slow owner's kill and another owner's kill", "a slow kill past Vitest's 15 s limit", - 'a copy waiting on a timer under load', + 'a copy waiting on a timer', 'every outcome class', ]), 5: anyOf([ From e78f30f0436e9688053fd0a08230006990899fdc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:08:14 -0700 Subject: [PATCH 08/32] test(oracle-gaps): RED for the outcome rule: a spin ends the mutant's copies Register C entry "Deterministic cuts, outcome rule (under M option A)", 2026-11-14: a copy cut by a clock or by its call count ends the mutant's copies; no later copy runs. The model now predicts that: a copy that spins (a count) stops the owner files after it, so a later owner's kill does not count and the outcome is a timeout. Witness: a constructed history whose one function spins in the mutant (the until loop of w_spin) with two owners, a and b. Owner a's copy spins first; owner b's judging test would kill. Production (main at this commit) runs b's copy after a's spin and reports killed. The model owes timeout. RED: on this commit, the new witness fails (the CLI reports killed where the model owes timeout; pool run dc-rule-red-*). Limits: the model's rule covers copies cut by a count; a wall-clock cut (the backstop and Vitest's limit) is a backstop gap under the same rule, and its own witness is pending (finding 4, the H1 witness). Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 11 +++-- .../gap-report-oracle.property.test.mjs | 48 +++++++++++++++++++ packages/oracles/scripts/e2e-shards.mjs | 3 ++ packages/oracles/scripts/pr-shards.mjs | 1 + 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index f6024ad6..3737f8aa 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -913,9 +913,8 @@ const histories = drawn.map( ) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) - const { slowTwin, ...drawnValue } = /** @type {Record} */ ( - value - ) + const { slowTwin, spinFirst, ...drawnValue } = + /** @type {Record} */ (value) const tests = /** @type {Record[]} */ (drawnValue.tests) const plain = /** @type {{ witness: unknown, steps: unknown[] | null, tornRows: unknown }} */ ({ @@ -4204,6 +4203,12 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { work += r.hang ? 0 : r.work if (r.hang || killsIn(r)) break } + // A copy cut by a clock or by its call count ends the mutant's copies + // here: no later copy runs (register, 2026-11-14, the outcome rule). + if (hung.has(file)) { + exhausted = true + break + } if (budgeted && spent + work > workBudget) { exhausted = true break diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index d6afe03a..ffe78335 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1816,6 +1816,11 @@ async function driver(plan, observe) { states.add(`a slow kill past fifteen seconds killed`) if (killed && plan.history.tests.some((t) => t.timerWait)) states.add(`a copy waiting on a timer killed`) + if ( + plan.history.spinFirst && + cli.report?.results?.some((r) => r.outcome === `timeout`) + ) + states.add(`a spin in an owner ahead of a kill in another timed out`) if (tornRead && cli.reported) states.add(`a capture row read while the worker wrote it`) if (cli.exit === 0 && !plan.consoleOutput) @@ -3561,6 +3566,7 @@ const required = [ `a slow owner's kill reported as when it did not wait`, `a slow kill past fifteen seconds killed`, `a copy waiting on a timer killed`, + `a spin in an owner ahead of a kill in another timed out`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, `a heartbeat named a busy mutant`, @@ -3693,6 +3699,7 @@ const separate = [ `a slow owner's kill reported as when it did not wait`, `a slow kill past fifteen seconds killed`, `a copy waiting on a timer killed`, + `a spin in an owner ahead of a kill in another timed out`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, `a package with no config ran`, @@ -5088,6 +5095,47 @@ oracleProperty({ }), }, // Constructed histories added by the deterministic-cuts change follow. + // An owner whose copy spins (a count) ends the mutant's copies before a + // later owner's test that would kill it: the outcome is a timeout. + { + name: `a spin in an owner ahead of a kill in another`, + enters: [`a spin in an owner ahead of a kill in another timed out`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `until`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `fixed seed`, 1), fn: 0 }, + { ...judging(`b`, `fixed seed`, 1), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + spinFirst: true, + }), + }, // A slow kill whose test waits 16 s where its output differs, past // Vitest's 15 s per-test limit: no wall-clock limit decides an outcome, // so the mutant is still killed. diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 8c54733c..3353ef65 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -69,6 +69,9 @@ export const shards = { `fixed seed 20261105: a copy waiting on a timer`, `random seed: a copy waiting on a timer`, `determinism: constructed history: a copy waiting on a timer`, + `fixed seed 20261105: a spin in an owner ahead of a kill in another`, + `random seed: a spin in an owner ahead of a kill in another`, + `determinism: constructed history: a spin in an owner ahead of a kill in another`, // The witness module's edit sequence (option M): 189 s here, measured // with the daemon fix of fix/daemon-current-inputs. `determinism: constructed history: an edit sequence over the witness module`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 2b3bfc26..b4f6731d 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -558,6 +558,7 @@ export const partitions = { "a slow owner's kill and another owner's kill", "a slow kill past Vitest's 15 s limit", 'a copy waiting on a timer', + 'a spin in an owner ahead of a kill in another', 'a capture row read while the worker wrote it', 'two batches each read a capture row while its worker wrote it', 'every outcome class', From a3ee642ce9bc79295d0c2f34ca56a00f67634dd5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:10:40 -0700 Subject: [PATCH 09/32] fix(oracle-gaps): the outcome rule: a clock or count cut ends the mutant, uncharged Register C entry "Deterministic cuts, outcome rule (under M option A)", 2026-11-14: only counts and states decide a mutant's outcome. - A copy cut by a clock or by its call count (its own timeout, or Vitest's "Test timed out") is never charged, and it ends the mutant's copies: no later copy runs, and a kill among later copies does not count (review B2). - A copy's work is the calls of its deterministic, unclocked tests (`baseline.work`, classify.mjs), and the owner files' most-working tests set the budget and the call budget (review B3). - The mutant-level elapsed-seconds backstop is removed: it summed cached units' stored seconds, so a daemon reuse could differ from a cold run (review finding 3). Each copy's own wall cut remains the backstop. Whether --mutant-limit still has a role is open (it now only affects identity). - Model: a copy cut by a count or a clock ends the owner files (no later file runs), so the model owes the same outcome. Witness: a constructed history with two owners, each making 30,000 extra calls; owner a's copy is under the budget, owner b's would kill but passes the budget, so the mutant is a timeout by count. The reversed-sum mutant (review H2, 968bfd7) is the mutant it must fail. Not witnessed here: a spin in an earlier owner ahead of a kill in a later one. A spin witness was tried first and removed: the later owner runs the same spinning function, so it spins too and never kills, and the witness passed on the parent. Finding 4's backstop witness (H1) is pending. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/check.mjs | 55 ++++++++++--------- .../oracle-gaps/gap-report-model-oracle.mjs | 5 +- .../gap-report-oracle.property.test.mjs | 32 +++++++---- packages/oracles/scripts/e2e-shards.mjs | 6 +- packages/oracles/scripts/pr-shards.mjs | 2 +- 5 files changed, 57 insertions(+), 43 deletions(-) diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 1a8c2c98..88dca8a7 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -1744,15 +1744,6 @@ export default async (env) => { // no first budget, so a copy runs once, to its end. const limitFor = (file) => options.hangLimit + 3000 * (fileSeconds.get(file) ?? 10) - // A mutant's wall-clock backstop (`--mutant-limit`, or ten times its owner - // files' unmutated time, whichever is larger): past it, a mutant with no kill - // is a timeout (runOwners). It never decides a kill. - const budgetOf = (files) => - Math.max( - options.mutantLimit, - 10_000 * - files.reduce((sum, file) => sum + (fileSeconds.get(file) ?? 0), 0), - ) // A mutant's work budget (maintainer disposition G1, option A): the larger of // WORK_FLOOR and ten times its owner files' unmutated work, in site // evaluations, each file counted by its most-working unmutated test. @@ -1761,12 +1752,27 @@ export default async (env) => { WORK_FLOOR, 10 * files.reduce((sum, file) => sum + (mostCalls.get(file) ?? 0), 0), ) - // A copy's work: the site evaluations its tests made. A copy that timed out - // recorded none, and counts none. + // A copy cut by a clock or by its call count: its own run's timeout, or + // Vitest's own limit ("Test timed out"), which the copy outlived. Such a + // copy is never charged, and it ends the mutant's copies (the outcome rule, + // register 2026-11-14, `Deterministic cuts, outcome rule`). + const cutByTime = (unit) => + unit.timedOut || + (unit.tests ?? []).some( + (test) => + test.state === `failed` && /Test timed out/.test(test.message), + ) + // A copy's work: the calls of its deterministic, unclocked tests + // (`baseline.work`, classify.mjs), so a row that a clock could move counts + // nothing. const workOf = (unit) => - unit.timedOut + cutByTime(unit) ? 0 - : (unit.rows ?? []).reduce((sum, [, row]) => sum + (row.calls ?? 0), 0) + : (unit.rows ?? []).reduce( + (sum, [key, row]) => + sum + (baseline.work.has(key) ? (row.calls ?? 0) : 0), + 0, + ) // A run cut off is still running, so its context is replaced; closing // it stops the worker. const replace = async (slot) => { @@ -2014,13 +2020,11 @@ export default async (env) => { // any of its unmutated tests made, and at least CALL_FLOOR. A copy past it // is spinning (capture-setup.mjs) and is a timeout as it stands. const mostCalls = new Map() - for (const rows of [first.rows, second.rows]) - for (const [key, row] of rows) - if (!row.background) - mostCalls.set( - fileOf(key), - Math.max(mostCalls.get(fileOf(key)) ?? 0, row.calls ?? 0), - ) + for (const [key, calls] of baseline.work) + mostCalls.set( + fileOf(key), + Math.max(mostCalls.get(fileOf(key)) ?? 0, calls), + ) const callBudgetFor = (file) => Math.max(CALL_FLOOR, 100 * (mostCalls.get(file) ?? 0)) const changesNothing = (mutant, targets) => @@ -2148,6 +2152,12 @@ export default async (env) => { `${st.mutant.id}: ${relative(request.file)} has no unit and no earlier kill`, ) } + // A copy cut by a clock or its count is a timeout, uncharged, and no + // later copy runs (the outcome rule). + if (cutByTime(got[i])) { + st.units.push(got[i]) + break + } work += workOf(got[i]) if (work > budget) { evidence.budget += 1 @@ -2157,11 +2167,6 @@ export default async (env) => { st.units.push(got[i]) if (kills(got[i])) break } - // The wall-clock backstop: no kill, and the copies' measured time past - // the mutant's wall budget, makes a timeout. A kill decides before it. - const elapsed = st.units.reduce((sum, u) => sum + (u.seconds ?? 0), 0) - if (!st.units.some(kills) && elapsed * 1000 > budgetOf(files)) - st.units.push({ file: files[0], timedOut: true, seconds: 0 }) st.done = true } } diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 3737f8aa..c5e7dd63 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -913,8 +913,9 @@ const histories = drawn.map( ) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) - const { slowTwin, spinFirst, ...drawnValue } = - /** @type {Record} */ (value) + const { slowTwin, ...drawnValue } = /** @type {Record} */ ( + value + ) const tests = /** @type {Record[]} */ (drawnValue.tests) const plain = /** @type {{ witness: unknown, steps: unknown[] | null, tornRows: unknown }} */ ({ diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index ffe78335..3d752aa7 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1816,11 +1816,14 @@ async function driver(plan, observe) { states.add(`a slow kill past fifteen seconds killed`) if (killed && plan.history.tests.some((t) => t.timerWait)) states.add(`a copy waiting on a timer killed`) + // Two owners that each make 30,000 extra calls: the second owner's copy + // passes the work budget, and its kill does not count (the outcome rule). if ( - plan.history.spinFirst && + plan.history.tests.length === 2 && + plan.history.tests.every((t) => t.loops === 30_000) && cli.report?.results?.some((r) => r.outcome === `timeout`) ) - states.add(`a spin in an owner ahead of a kill in another timed out`) + states.add(`a kill past the work budget in a later owner does not count`) if (tornRead && cli.reported) states.add(`a capture row read while the worker wrote it`) if (cli.exit === 0 && !plan.consoleOutput) @@ -3566,7 +3569,7 @@ const required = [ `a slow owner's kill reported as when it did not wait`, `a slow kill past fifteen seconds killed`, `a copy waiting on a timer killed`, - `a spin in an owner ahead of a kill in another timed out`, + `a kill past the work budget in a later owner does not count`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, `a heartbeat named a busy mutant`, @@ -3699,7 +3702,7 @@ const separate = [ `a slow owner's kill reported as when it did not wait`, `a slow kill past fifteen seconds killed`, `a copy waiting on a timer killed`, - `a spin in an owner ahead of a kill in another timed out`, + `a kill past the work budget in a later owner does not count`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, `a package with no config ran`, @@ -5095,14 +5098,15 @@ oracleProperty({ }), }, // Constructed histories added by the deterministic-cuts change follow. - // An owner whose copy spins (a count) ends the mutant's copies before a - // later owner's test that would kill it: the outcome is a timeout. + // Owner a's copy makes 30,000 extra calls (under its budget), and so does + // owner b's, which would kill. Together they pass the work budget, so b's + // kill does not count: the mutant is a timeout by count. { - name: `a spin in an owner ahead of a kill in another`, - enters: [`a spin in an owner ahead of a kill in another timed out`], + name: `a kill past the work budget in a later owner`, + enters: [`a kill past the work budget in a later owner does not count`], build: (generated) => ({ ...generated, - functions: [[{ kind: `until`, quote: `'` }]], + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], containers: [], overloads: [], owners: [`both`], @@ -5117,8 +5121,13 @@ oracleProperty({ wholeFile: `none`, template: false, tests: [ - { ...judging(`a`, `fixed seed`, 1), fn: 0 }, - { ...judging(`b`, `fixed seed`, 1), fn: 0 }, + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + loops: 30_000, + }, + { ...judging(`b`, `fixed seed`, 7), fn: 0, loops: 30_000 }, ], witness: false, authored: null, @@ -5133,7 +5142,6 @@ oracleProperty({ concurrentStart: false, contexts: 1, steps: null, - spinFirst: true, }), }, // A slow kill whose test waits 16 s where its output differs, past diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 3353ef65..6b25e565 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -69,9 +69,9 @@ export const shards = { `fixed seed 20261105: a copy waiting on a timer`, `random seed: a copy waiting on a timer`, `determinism: constructed history: a copy waiting on a timer`, - `fixed seed 20261105: a spin in an owner ahead of a kill in another`, - `random seed: a spin in an owner ahead of a kill in another`, - `determinism: constructed history: a spin in an owner ahead of a kill in another`, + `fixed seed 20261105: a kill past the work budget in a later owner`, + `random seed: a kill past the work budget in a later owner`, + `determinism: constructed history: a kill past the work budget in a later owner`, // The witness module's edit sequence (option M): 189 s here, measured // with the daemon fix of fix/daemon-current-inputs. `determinism: constructed history: an edit sequence over the witness module`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index b4f6731d..6a826758 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -558,7 +558,7 @@ export const partitions = { "a slow owner's kill and another owner's kill", "a slow kill past Vitest's 15 s limit", 'a copy waiting on a timer', - 'a spin in an owner ahead of a kill in another', + 'a kill past the work budget in a later owner', 'a capture row read while the worker wrote it', 'two batches each read a capture row while its worker wrote it', 'every outcome class', From 32e60aa35133271816ee77bb17c59651fa816dd2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 23:28:15 -0700 Subject: [PATCH 10/32] fix(oracle-gaps): refined outcome rule: count and state cuts end their copy, wall cuts end the mutant Register C entry "Deterministic cuts, outcome rule (under M option A)", as refined by the coordinator: a copy ending in "no progress" (a state) or a spin past its call budget (a count) is a deterministic timeout for that copy. It is charged nothing (the rows of a cut copy are not kept, and the per-copy call budget already bounds it), and the next owner's copy still runs, so a later kill decides. Only a wall-clock cut (the CLI backstop, or Vitest's "Test timed out") ends the remaining copies, as a backstop gap that is never charged. - Model: runsOf tags each cut with its kind (wall, count, state); judge ends the owner files only on a wall cut. - Check: spin and idle copies settle with spin or idle set; cutByTime covers wall cuts only. - --mutant-limit is retired (validated, noted, changes nothing); the driver no longer passes it. - Constructed history "a backstop cut ends the copies ahead of a later kill": owner a's copy waits past a 2 s backstop; owner b would kill, and does not run. The state "a backstop cut ends the copies ahead of a later kill" is entered. Open: "a declaration fault: noTypescript" (shard 3) streams 161 gap lines where the model expects 159; a per-mutant check is pending. It is a model and CLI disagreement, not a duplicate line (the scratch diagnostic found no repeated line). Co-authored-by: Isaac --- packages/oracles/oracle-gaps/args.mjs | 6 +- packages/oracles/oracle-gaps/check.mjs | 17 +++-- .../oracle-gaps/gap-report-model-oracle.mjs | 35 ++++++++--- .../gap-report-oracle.property.test.mjs | 62 +++++++++++++++++-- packages/oracles/scripts/e2e-shards.mjs | 3 + packages/oracles/scripts/pr-shards.mjs | 1 + 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/packages/oracles/oracle-gaps/args.mjs b/packages/oracles/oracle-gaps/args.mjs index 34978ae7..e5b10015 100644 --- a/packages/oracles/oracle-gaps/args.mjs +++ b/packages/oracles/oracle-gaps/args.mjs @@ -221,9 +221,9 @@ function optionsOf(args, cwd) { // The `--first-limit` option is retired (option A): it is validated and // noted, and changes nothing. retiredFirstLimit: args[`first-limit`] !== undefined, - // The floor of a mutant's wall-clock backstop (check.mjs, `budgetOf`): 180 s - // by default, three full copies at the default backstop. - mutantLimit: Number(args[`mutant-limit`] ?? 180000), + // `--mutant-limit` is retired too (deterministic cuts, option A): the work + // budget bounds a mutant, so it is validated and noted, and changes nothing. + retiredMutantLimit: args[`mutant-limit`] !== undefined, // Vitest's worker pool. Threads start much faster than child processes; // a package with a native addon in its tree runs with `forks`. pool: args.pool ?? defaultPool(pkg), diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 88dca8a7..7578d08b 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -1373,9 +1373,9 @@ export default async (env) => { log( `oracle-gaps: note --first-limit is retired: a copy has no first budget, so it changes nothing (deterministic cuts, option A)`, ) - if (options.hangLimit > options.mutantLimit) + if (options.retiredMutantLimit) log( - `oracle-gaps: note --hang-limit ${options.hangLimit} ms is above --mutant-limit ${options.mutantLimit} ms; a mutant's copies together get at most the larger of --mutant-limit and ten times its owner files' unmutated time`, + `oracle-gaps: note --mutant-limit is retired: a mutant's work budget bounds it, so it changes nothing (deterministic cuts, option A)`, ) pulse = { log, @@ -1486,7 +1486,6 @@ export default async (env) => { settings, // A run cut off under one limit may end under a longer one. options.hangLimit, - options.mutantLimit, ]), ) // How the evidence was produced: units reused and computed, runs cut off @@ -1693,9 +1692,17 @@ export default async (env) => { if (batch.idle) evidence.idle += 1 if (batch.spin) evidence.spin += 1 await replace(slot) + // A count or state cut (a spin, or no progress) is a timeout for its + // copy alone; only a wall-clock cut ends the mutant's copies (cutByTime). settle( hung, - { file: hung.file, timedOut: true, seconds: hung.limitMs / 1000 }, + { + file: hung.file, + timedOut: true, + seconds: hung.limitMs / 1000, + ...(batch.spin ? { spin: true } : {}), + ...(batch.idle ? { idle: true } : {}), + }, batch.inputs[batch.timedOutAt], ) rest = [ @@ -1757,7 +1764,7 @@ export default async (env) => { // copy is never charged, and it ends the mutant's copies (the outcome rule, // register 2026-11-14, `Deterministic cuts, outcome rule`). const cutByTime = (unit) => - unit.timedOut || + (unit.timedOut && !unit.spin && !unit.idle) || (unit.tests ?? []).some( (test) => test.state === `failed` && /Test timed out/.test(test.message), diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index c5e7dd63..f872b157 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -913,9 +913,8 @@ const histories = drawn.map( ) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) - const { slowTwin, ...drawnValue } = /** @type {Record} */ ( - value - ) + const { slowTwin, hangMs, ...drawnValue } = + /** @type {Record} */ (value) const tests = /** @type {Record[]} */ (drawnValue.tests) const plain = /** @type {{ witness: unknown, steps: unknown[] | null, tornRows: unknown }} */ ({ @@ -4040,12 +4039,29 @@ function runsOf(plan, tests, sites, active) { before += base.work after += mutated.work * (1 + (changed ? (t.loops ?? 0) : 0)) } + // A wait past the history's backstop (its `hangMs`, else the full limit) + // and a margin of 6 s for three seconds per second of a one-test file + // (the file's time is under 2 s) is a backstop cut: the file's run is cut + // off, as a hang is, with no clock decision in the verdicts. + const backstopMs = plan.history.hangMs ?? FULL_LIMIT_MS + if (changed && t.slow && (t.slowMs ?? SLOW_MS) > backstopMs + 6000) + return { + t, + route, + hang: true, + cut: 'wall', + verdicts: found, + changed: true, + work: after, + before, + } // A loop that never ends cuts the file's run off, as a hang does. if (mutated.spun) return { t, route, hang: true, + cut: 'count', verdicts: found, changed: true, work: after, @@ -4070,6 +4086,7 @@ function runsOf(plan, tests, sites, active) { t, route, hang: true, + cut: 'state', verdicts: found, changed: true, work: after, @@ -4176,6 +4193,10 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { // earlier verdicts. A kill in another file still decides (a kill outranks // a timeout); failing that, the mutant is a timeout. const hung = new Set() + // Only a wall-clock cut (the backstop) ends the owner files after it. A count + // or a state cut (a spin, or no progress) ends its own copy only (the outcome + // rule, 2026-11-14, as refined: the next owner's copy still runs). + const wallHung = new Set() // The mutant's work budget (G1): its owner files' copies are counted in // their order, each copy's work being the site evaluations of its runs up // to a hang or a kill. A copy past the budget is a timeout, and so is every @@ -4200,13 +4221,13 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { let work = 0 for (const r of runs.filter((x) => x.t.file === file)) { if (r.hang) hung.add(file) - // A copy cut off by its backstop, or by spinning, records no work. + if (r.hang && r.cut === `wall`) wallHung.add(file) + // A cut copy records no work (its calls are not rows the CLI keeps). work += r.hang ? 0 : r.work if (r.hang || killsIn(r)) break } - // A copy cut by a clock or by its call count ends the mutant's copies - // here: no later copy runs (register, 2026-11-14, the outcome rule). - if (hung.has(file)) { + // A copy cut by a clock ends the mutant's copies here: no later copy runs. + if (wallHung.has(file)) { exhausted = true break } diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 3d752aa7..8ebd9230 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -621,11 +621,7 @@ function cliArgs( // history passes it, and the CLI notes that it changes nothing. ...(budget === `one` ? [`--first-limit`, `1000`] : []), `--hang-limit`, - String(FULL_LIMIT_MS), - // The budget that a mutant's confirming runs share, the full limit of - // one copy: one slow owner's confirmation fits it, two do not (law 4). - `--mutant-limit`, - String(MUTANT_LIMIT_MS), + String(plan.history.hangMs ?? FULL_LIMIT_MS), `--out`, path(out), ] @@ -1816,6 +1812,13 @@ async function driver(plan, observe) { states.add(`a slow kill past fifteen seconds killed`) if (killed && plan.history.tests.some((t) => t.timerWait)) states.add(`a copy waiting on a timer killed`) + // A copy that waits past the history's backstop is cut off, and no later + // owner runs, so the mutant is a timeout, not the later owner's kill. + if ( + plan.history.hangMs && + cli.report?.results?.some((r) => r.outcome === `timeout`) + ) + states.add(`a backstop cut ends the copies ahead of a later kill`) // Two owners that each make 30,000 extra calls: the second owner's copy // passes the work budget, and its kill does not count (the outcome rule). if ( @@ -3570,6 +3573,7 @@ const required = [ `a slow kill past fifteen seconds killed`, `a copy waiting on a timer killed`, `a kill past the work budget in a later owner does not count`, + `a backstop cut ends the copies ahead of a later kill`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, `a heartbeat named a busy mutant`, @@ -3703,6 +3707,7 @@ const separate = [ `a slow kill past fifteen seconds killed`, `a copy waiting on a timer killed`, `a kill past the work budget in a later owner does not count`, + `a backstop cut ends the copies ahead of a later kill`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, `a package with no config ran`, @@ -5098,6 +5103,53 @@ oracleProperty({ }), }, // Constructed histories added by the deterministic-cuts change follow. + // Owner a's copy waits past the history's 2 s backstop, so it is cut + // off; owner b would kill. The cut ends the copies, so b does not run: + // the mutant is a backstop gap (a timeout), never a kill. + { + name: `a backstop cut ends the copies ahead of a later kill`, + enters: [`a backstop cut ends the copies ahead of a later kill`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + slow: true, + slowMs: 12_000, + }, + { ...judging(`b`, `fixed seed`, 7), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + hangMs: 2_000, + }), + }, // Owner a's copy makes 30,000 extra calls (under its budget), and so does // owner b's, which would kill. Together they pass the work budget, so b's // kill does not count: the mutant is a timeout by count. diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 6b25e565..659fa622 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -72,6 +72,9 @@ export const shards = { `fixed seed 20261105: a kill past the work budget in a later owner`, `random seed: a kill past the work budget in a later owner`, `determinism: constructed history: a kill past the work budget in a later owner`, + `fixed seed 20261105: a backstop cut ends the copies ahead of a later kill`, + `random seed: a backstop cut ends the copies ahead of a later kill`, + `determinism: constructed history: a backstop cut ends the copies ahead of a later kill`, // The witness module's edit sequence (option M): 189 s here, measured // with the daemon fix of fix/daemon-current-inputs. `determinism: constructed history: an edit sequence over the witness module`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 6a826758..c3824db9 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -559,6 +559,7 @@ export const partitions = { "a slow kill past Vitest's 15 s limit", 'a copy waiting on a timer', 'a kill past the work budget in a later owner', + 'a backstop cut ends the copies ahead of a later kill', 'a capture row read while the worker wrote it', 'two batches each read a capture row while its worker wrote it', 'every outcome class', From 245a31726b2ab27da23266055cc8aa4e7e794013 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 00:33:27 -0700 Subject: [PATCH 11/32] test(oracle-gaps): pass the history's backstop to cliArgs (plan was not in scope there) Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 8ebd9230..b453978c 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -586,6 +586,7 @@ function cliArgs( ordinary = false, sources = `${SOURCE},${LIB}`, budget = `two`, + hangMs = FULL_LIMIT_MS, outName = daemon ? `out-daemon` : `out`, config, contexts = 4, @@ -621,7 +622,7 @@ function cliArgs( // history passes it, and the CLI notes that it changes nothing. ...(budget === `one` ? [`--first-limit`, `1000`] : []), `--hang-limit`, - String(plan.history.hangMs ?? FULL_LIMIT_MS), + String(hangMs), `--out`, path(out), ] @@ -1311,6 +1312,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + hangMs: plan.history.hangMs, ...configArgs(plan), // After the `config` edit, `--config` names the second config. ...(state.altConfig ? { config: altConfigPath(plan) } : {}), @@ -1733,6 +1735,7 @@ async function driver(plan, observe) { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + hangMs: plan.history.hangMs, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), } From 4c25a6e1e9bb2d2001eeeb30c2e06c91695fdd66 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 00:33:58 -0700 Subject: [PATCH 12/32] fix(oracle-gaps): a context close past the 60 s backstop is a loud gap, never a silent one Review finding 5 (register L8). The 10 s close waits in closeContexts and in replace continued silently after a close was still pending. Now each close waits up to the 60 s backstop; a close still pending is an error that names the backstop gap, and no replacement is warmed while the old context is still closing. Closes that finish are unchanged. Witness: none. A pending close needs a context whose close never settles, which costs 60 s per run; it is a limit of this change (the coverage is by inspection of closeWithin and its two callers). Co-authored-by: Isaac --- packages/oracles/oracle-gaps/check.mjs | 28 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 7578d08b..8acf4925 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -802,14 +802,25 @@ export default async (env) => { } contextSpecs.set(context, map) } + // A context's close, up to the backstop (60 s). A close still pending past it + // is a loud backstop gap (register L8), never a silent one: the caller never + // warms a replacement while the old context is still closing. + async function closeWithin(context, label) { + const closed = await Promise.race([ + closeContext(context).then( + () => true, + () => true, + ), + sleep(60_000).then(() => false), + ]) + if (!closed) + throw new Error( + `oracle-gaps: gap backstop: a context close (${label}) is still pending after 60 s; no replacement is warmed while it closes`, + ) + } async function closeContexts() { await Promise.all( - contexts.map((context) => - Promise.race([ - closeContext(context).catch(() => undefined), - sleep(10_000), - ]), - ), + contexts.map((context) => closeWithin(context, `end of run`)), ) contexts = [] lastServed.clear() @@ -1784,10 +1795,7 @@ export default async (env) => { // it stops the worker. const replace = async (slot) => { const started = Date.now() - await Promise.race([ - closeContext(slot.context).catch(() => undefined), - sleep(10_000), - ]) + await closeWithin(slot.context, `slot ${slot.index}`) slot.context = await warmContext(settings, slot.index) await globSpecs(slot.context) contexts[slot.index] = slot.context From 59c4d4f5597bf165c134d625cd9de80272a9dad2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 00:49:38 -0700 Subject: [PATCH 13/32] test(oracle-gaps): the H1 backstop witness never finishes and never goes idle Owner a's copy now awaits a promise that never resolves, while a repeating timer stays due inside its horizon, so the copy is never idle and no site spins. Only the backstop cuts it, on every machine. The witness no longer depends on whether a 12 s wait lands inside the backstop band (register 2026-11-14, outcome rule item 5). Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 26 +++++++++++++------ .../gap-report-oracle.property.test.mjs | 12 +++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index f872b157..b08d931c 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -420,11 +420,13 @@ const testCase = fc.record({ // turn: its work could follow the clock. timed: fc.constant(false), // The constructed-only fields (see `histories`): a witness's wait where its - // output differs, its repeated calls, and its load. A generated test has the - // neutral values, which change nothing. + // output differs, its repeated calls, its load, and a repeating timer kept + // due while a hang waits. A generated test has the neutral values, which + // change nothing. slowMs: fc.constant(SLOW_MS), loops: fc.constant(0), timerWait: fc.constant(null), + pulse: fc.constant(false), }) // Law 9's edits. Each content edit changes what some evidence depends on: // a test file, a source statement, the source's line numbers, an owns list, @@ -895,14 +897,16 @@ const keepsSteps = (history) => { // take-back reads it as the drawn constant `false` (as `claimScenario` does): a // shrink is torn-row-free. A constructed history may set `slowTwin`, which no // generated history draws, and which is taken back without it. Its tests may -// set the constructed-only fields `slowMs`, `loops` and `timerWait`, which the -// drawn test record holds at neutral values (the generator rule, 2026-11-14). +// set the constructed-only fields `slowMs`, `loops`, `timerWait` and `pulse`, +// which the drawn test record holds at neutral values (the generator rule, +// 2026-11-14). const takenBack = (t) => ({ ...t, slow: false, slowMs: SLOW_MS, loops: 0, timerWait: null, + pulse: false, }) const histories = drawn.map( (history) => (keepsSteps(history) ? history : { ...history, steps: null }), @@ -4080,13 +4084,15 @@ function runsOf(plan, tests, sites, active) { work: after, before, } - // A test that hangs where another would judge never ends. + // A test that hangs where another would judge never ends. Without a + // `pulse` it goes idle (no progress); with one, its timer keeps it busy, + // so only the backstop cuts it (a wall-clock cut). if (changed && t.hangs) return { t, route, hang: true, - cut: 'state', + cut: t.pulse ? 'wall' : 'state', verdicts: found, changed: true, work: after, @@ -4999,10 +5005,14 @@ function oracleTestFile(plan, file) { ` }`, ] : []), - // A flaky test never hangs: the model leaves flaky tests out. + // A flaky test never hangs: the model leaves flaky tests out. A + // `pulse` hang keeps a repeating timer due inside its horizon, so the + // copy is never idle, and it never finishes: only the backstop cuts it. ...(t.hangs && !t.flaky ? [ - ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise(() => {})`, + t.pulse + ? ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { setInterval(() => {}, 100); await new Promise(() => {}) }` + : ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise(() => {})`, ] : []), ...(!t.judges diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index b453978c..0462fd6a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -5106,9 +5106,11 @@ oracleProperty({ }), }, // Constructed histories added by the deterministic-cuts change follow. - // Owner a's copy waits past the history's 2 s backstop, so it is cut - // off; owner b would kill. The cut ends the copies, so b does not run: - // the mutant is a backstop gap (a timeout), never a kill. + // Owner a's copy never finishes and never goes idle: it awaits a promise + // that never resolves, while a repeating timer stays due inside its + // horizon (`pulse`). So it is cut only by the 2 s backstop, on every + // machine, and owner b would kill. The cut ends the copies, so b does + // not run: the mutant is a backstop gap (a timeout), never a kill. { name: `a backstop cut ends the copies ahead of a later kill`, enters: [`a backstop cut ends the copies ahead of a later kill`], @@ -5132,8 +5134,8 @@ oracleProperty({ { ...judging(`a`, `fixed seed`, 7), fn: 0, - slow: true, - slowMs: 12_000, + hangs: true, + pulse: true, }, { ...judging(`b`, `fixed seed`, 7), fn: 0 }, ], From d974b030c823e54c5605ff52954959123b18222a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 00:59:07 -0700 Subject: [PATCH 14/32] fix(oracle-gaps): each timeout names its cut: backstop, spin, no progress, or work budget A copy's cut is now recorded as its cause and carried into the mutant's timeout detail, results.json, and the gap line (cause: ...). The model predicts the cause of each timeout: a backstop cut (a wall-clock limit), a spin past the copy's call budget (a count, outcome rule item 4), no progress (a state), or the mutant's work budget. The model had no per-copy call budget, so the work-budget history's 4-sites-per-call mutant was a spin and not a work-budget timeout; the model now predicts that. ORACLE_GAPS_PROFILE rows carry each copy's cause, its file's measured time, its limit and its duration. RED: the e2e oracle, with the cause predicted and production unchanged, fails on the backstop history (its timeout is cut by run time limit). Co-authored-by: Isaac --- packages/oracles/oracle-gaps/check.mjs | 83 +++++++++++++++---- packages/oracles/oracle-gaps/classify.mjs | 11 ++- ...p-report-classify-oracle.property.test.mjs | 20 ++++- .../oracle-gaps/gap-report-model-oracle.mjs | 60 ++++++++++---- .../gap-report-oracle.property.test.mjs | 17 ++++ packages/oracles/oracle-gaps/stream.mjs | 4 +- 6 files changed, 154 insertions(+), 41 deletions(-) diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 8acf4925..eba6f6ed 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -972,7 +972,12 @@ export default async (env) => { forceCancel(context) span(`run`, started, { ...profiled, timedOut: true }) rmSync(capture, { force: true }) - return { file, timedOut: true, seconds, ...(spun ? { spin: true } : {}) } + return { + file, + timedOut: true, + seconds, + cause: spun ? `spin` : `backstop`, + } } // The result also lists test modules from earlier runs, with their old // results. Only this run's file counts. @@ -1011,7 +1016,7 @@ export default async (env) => { return { file, timedOut: true, - spin: true, + cause: `spin`, seconds, inputs: importsOf(file, modules, closing), readAt: closing.readAt, @@ -1200,6 +1205,10 @@ export default async (env) => { let current = 0 let copyStarted = Date.now() let timer + // The cut that ended the running copy, if the watchdog cut it off, and how + // long the copy had run then. + let cutCause + let cutMs const beat = heartbeat() // Each poll counts itself, after it has read the capture (the hold in // capture-setup.mjs waits for two of these). @@ -1218,12 +1227,18 @@ export default async (env) => { items[current].file, copyStarted, ) - if ( - idle.has(copies[current]) || - Date.now() - (spunAt.get(copies[current]) ?? Infinity) > IDLE_MS || - Date.now() - copyStarted > items[current].limitMs - ) + // The cut that ends the copy, in this order: a spin past the call + // budget (a count), no progress (a state), then the backstop (a clock). + const copy = copies[current] + if (Date.now() - (spunAt.get(copy) ?? Infinity) > IDLE_MS) + cutCause = `spin` + else if (idle.has(copy)) cutCause = `no progress` + else if (Date.now() - copyStarted > items[current].limitMs) + cutCause = `backstop` + if (cutCause !== undefined) { + cutMs = Date.now() - copyStarted return resolve(undefined) + } timer = setTimeout(poll, 50) } timer = setTimeout(poll, 50) @@ -1263,7 +1278,7 @@ export default async (env) => { const spunUnit = (item) => ({ file: item.file, timedOut: true, - spin: true, + cause: `spin`, seconds, }) try { @@ -1317,8 +1332,8 @@ export default async (env) => { } }), timedOutAt: current, - idle: idle.has(copies[current]), - spin: spun.has(copies[current]), + cause: cutCause, + cutMs, } } // Errors outside any test cannot be attributed to one copy, so the @@ -1654,6 +1669,22 @@ export default async (env) => { // units found so far, whether a request must still run; a request that // need not run, gets no unit; a copy that skipped its tests gets a // skipped marker, which its caller runs again (runOwners). + // A profile row for one copy (ORACLE_GAPS_PROFILE): its mutant, its file's + // measured time, its limit, the cause that ended it or `completed`, and its + // duration. + const copyRow = (request, { cause, durationMs }) => + appendFileSync( + profileFile, + `${JSON.stringify({ + kind: `copy`, + mutant: request.label, + file: baseOf(request.file), + cause, + fileMs: Math.round((fileSeconds.get(request.file) ?? 0) * 1000), + limitMs: request.limitMs, + durationMs, + })}\n`, + ) async function unitsBatch(slot, requests, needed = () => true) { const index = indexOf.get(slot.context) const got = new Array(requests.length) @@ -1691,17 +1722,29 @@ export default async (env) => { // A skipped copy keeps its marker, so the caller can run it again. got[rest[k].i] = u if (u.skipped) return - if (u.spin) evidence.spin += 1 + if (u.cause === `spin`) evidence.spin += 1 remember(rest[k].id, u, batch.inputs[k]) evidence.computed += 1 }) + // Per-copy rows for ORACLE_GAPS_PROFILE: each copy's cut or completion, + // with its file's measured time and its limit, to see what a cut was. + if (profileFile) + batch.units.forEach((u, k) => { + if (u === undefined || u.skipped) return + copyRow(rest[k], { + cause: u.cause ?? `completed`, + durationMs: Math.round(u.seconds * 1000), + }) + }) if (batch.timedOutAt === undefined) break // A copy cut off by its backstop, for idleness, or for spinning is a // timeout as it stands: no clock decides a kill (deterministic cuts). const hung = rest[batch.timedOutAt] + if (profileFile) + copyRow(hung, { cause: batch.cause, durationMs: batch.cutMs }) evidence.cut += 1 - if (batch.idle) evidence.idle += 1 - if (batch.spin) evidence.spin += 1 + if (batch.cause === `no progress`) evidence.idle += 1 + if (batch.cause === `spin`) evidence.spin += 1 await replace(slot) // A count or state cut (a spin, or no progress) is a timeout for its // copy alone; only a wall-clock cut ends the mutant's copies (cutByTime). @@ -1711,8 +1754,7 @@ export default async (env) => { file: hung.file, timedOut: true, seconds: hung.limitMs / 1000, - ...(batch.spin ? { spin: true } : {}), - ...(batch.idle ? { idle: true } : {}), + cause: batch.cause, }, batch.inputs[batch.timedOutAt], ) @@ -1736,7 +1778,7 @@ export default async (env) => { limitMs: r.limitMs, callBudget: r.callBudget, }) - if (u.spin) evidence.spin += 1 + if (u.cause === `spin`) evidence.spin += 1 if (u.timedOut) await replace(slot) return u } @@ -1775,7 +1817,7 @@ export default async (env) => { // copy is never charged, and it ends the mutant's copies (the outcome rule, // register 2026-11-14, `Deterministic cuts, outcome rule`). const cutByTime = (unit) => - (unit.timedOut && !unit.spin && !unit.idle) || + unit.cause === `backstop` || (unit.tests ?? []).some( (test) => test.state === `failed` && /Test timed out/.test(test.message), @@ -2176,7 +2218,12 @@ export default async (env) => { work += workOf(got[i]) if (work > budget) { evidence.budget += 1 - st.units.push({ file: request.file, timedOut: true, seconds: 0 }) + st.units.push({ + file: request.file, + timedOut: true, + seconds: 0, + cause: `work budget`, + }) break } st.units.push(got[i]) diff --git a/packages/oracles/oracle-gaps/classify.mjs b/packages/oracles/oracle-gaps/classify.mjs index 6c2a0ac5..ea4e84dc 100644 --- a/packages/oracles/oracle-gaps/classify.mjs +++ b/packages/oracles/oracle-gaps/classify.mjs @@ -53,10 +53,13 @@ export function outcomeOf( // run still decides; failing that, the mutant is `timeout`. const timedOutTest = (test) => test.state === `failed` && /Test timed out/.test(test.message) - const timeout = units.some( - (unit) => unit.timedOut || (unit.tests ?? []).some(timedOutTest), - ) - ? { outcome: `timeout`, detail: `run time limit` } + // The timeout's detail is the cause of the first cut in report order: the + // copy's `cause` (`backstop`, `spin`, `no progress`, or `work budget`). + const cut = [...units] + .sort((a, b) => byPath(a.file, b.file)) + .find((unit) => unit.timedOut || (unit.tests ?? []).some(timedOutTest)) + const timeout = cut + ? { outcome: `timeout`, detail: cut.timedOut ? cut.cause : `backstop` } : undefined const failures = [] // By file path (report order, `inReportOrder`), in whatever order the diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 554b6bd0..bbae942c 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -772,7 +772,8 @@ function failureOf(h, slot, tests, work = false) { laws: [...new Set(behavioral)].sort(), lawsMayBePart: true, }) - if (timedOut) return decided({ outcome: `timeout` }) + // A Vitest limit is a backstop cut: the timeout names it (classify.mjs). + if (timedOut) return decided({ outcome: `timeout`, detail: `backstop` }) if (kills.length > 0) return naming(`agreement-only-kill`, verdictsIn(`reproducible`), { laws: [`reach agreement`], @@ -1493,7 +1494,7 @@ const messages = { verdict: `Error: violation`, assertion: `AssertionError: expected [ 'a' ] to deeply equal [ 'b' ]`, crash: `TypeError: x.y is not a function`, - timeoutMessage: `Error: Test timed out in 15000ms.`, + timeoutMessage: `Error: Test timed out in 65000ms.`, } // The capture's comparison and the swap a comparison mutant states, as the // instrumented code evaluates them (capture-setup.mjs `cmp`). @@ -1639,7 +1640,12 @@ function unitOf(h, fi, slot) { const f = h.files[fi] const under = (t) => t.under[slot] if (f.tests.some((t) => under(t).kind === `runTimeout`)) - return { file: fileName(h, fi), timedOut: true, seconds: 1 } + return { + file: fileName(h, fi), + timedOut: true, + cause: `backstop`, + seconds: 1, + } const loadFailed = f.tests.some((t) => under(t).kind === `setupError`) const tests = loadFailed ? [] : f.tests return { @@ -2265,8 +2271,14 @@ function check(h, observations, expected) { return violation(`classified`, `nothing was classified`, `completion`) // A kill names a non-empty part of its behavioral laws (the runner may // stop at the first); an agreement-only kill names reach agreement. + // A timeout names the cut that ended it (its `detail`, the cause). const view = (o) => - JSON.stringify([o.outcome, o.class, o.testOutcome ?? null]) + JSON.stringify([ + o.outcome, + o.class, + o.testOutcome ?? null, + o.outcome === `timeout` ? o.detail : null, + ]) for (const [mi, owed] of expected.outcomes.entries()) { const actual = got.outcomes[mi] const lawsFit = diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index b08d931c..1eb0ed97 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -147,6 +147,11 @@ const PAIR_LOOPS = 30_000 // owner files' unmutated work, in site evaluations. The floor is the copies' // call budget, so two copies that each stay inside theirs can pass it. const WORK_FLOOR = 100_000 +// The least call budget a copy gets (check.mjs CALL_FLOOR): site evaluations in +// one copy's runs before it counts as a loop that never ends. A file's budget +// is a hundred times the most its unmutated tests did, if more. Past it, the +// copy is cut by count (a spin) and charged nothing (the outcome rule, item 4). +const CALL_FLOOR = 100_000 // A mutant's budget for the runs that confirm its cut-off copies, which the // driver passes (`--mutant-limit`): one slow owner's confirmation fits it // (SLOW_MS), and two of `w_slowpair`'s do not. Ten times the fixture's @@ -4186,12 +4191,16 @@ function slowedFiles(plan, tests, sites, active) { return slowed } +// The timeout cause that each kind of cut names (the CLI's `cause`, `backstop`, +// `spin` and `no progress`; `work budget` is the mutant's own count). +const CUT_CAUSE = { wall: `backstop`, count: `spin`, state: `no progress` } + /** * A mutant's kind from its owner tests' runs. An operator mutant's * confirming runs share its budget (`budgeted`); an authored mutant's runs * are each final at the full limit, so it has no confirmations to budget. */ -function judge(plan, tests, sites, active, { budgeted = true } = {}) { +function judge(plan, tests, sites, active, { budgeted = true, detail } = {}) { const runs = runsOf(plan, tests, sites, active) // A file's tests run in order, and a kill (a behavioral verdict on a // reproducible route) stops the file. A hang before that cuts the file's @@ -4199,6 +4208,9 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { // earlier verdicts. A kill in another file still decides (a kill outranks // a timeout); failing that, the mutant is a timeout. const hung = new Set() + // The cause of a timeout: the first cut in the order the copies run, which + // the CLI names on the mutant's timeout (`detail.cause`). + let cut // Only a wall-clock cut (the backstop) ends the owner files after it. A count // or a state cut (a spin, or no progress) ends its own copy only (the outcome // rule, 2026-11-14, as refined: the next owner's copy still runs). @@ -4210,27 +4222,29 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { // larger of WORK_FLOOR and ten times the owner files' unmutated work, where // a file's unmutated work is its most-working test's. const files = [...new Set(runs.map((r) => r.t.file))] - const unmutated = files.reduce( - (sum, file) => - sum + - Math.max( - 0, - ...runs.filter((r) => r.t.file === file).map((r) => r.before), - ), - 0, - ) + // A file's most-working unmutated test, in site evaluations. + const mostBefore = (file) => + Math.max(0, ...runs.filter((r) => r.t.file === file).map((r) => r.before)) + const unmutated = files.reduce((sum, file) => sum + mostBefore(file), 0) const workBudget = Math.max(WORK_FLOOR, 10 * unmutated) const counted = new Set() let spent = 0 let exhausted = false for (const file of files) { let work = 0 + const callBudget = Math.max(CALL_FLOOR, 100 * mostBefore(file)) for (const r of runs.filter((x) => x.t.file === file)) { - if (r.hang) hung.add(file) + // A copy whose runs pass its call budget is cut by count, where the run + // goes past it (the CLI's spin row). + const spun = !r.hang && work + r.work > callBudget + if (r.hang || spun) { + hung.add(file) + cut ??= CUT_CAUSE[spun ? `count` : r.cut] + } if (r.hang && r.cut === `wall`) wallHung.add(file) // A cut copy records no work (its calls are not rows the CLI keeps). - work += r.hang ? 0 : r.work - if (r.hang || killsIn(r)) break + work += r.hang || spun ? 0 : r.work + if (r.hang || spun || killsIn(r)) break } // A copy cut by a clock ends the mutant's copies here: no later copy runs. if (wallHung.has(file)) { @@ -4239,6 +4253,7 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { } if (budgeted && spent + work > workBudget) { exhausted = true + cut ??= `work budget` break } spent += work @@ -4251,7 +4266,10 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { ) const reproducible = verdicts(`reproducible`) if (reproducible.some((law) => law !== `reach agreement`)) return `killed` - if (hung.size > 0 || exhausted) return `timeout` + if (hung.size > 0 || exhausted) { + if (detail) detail.cause = cut + return `timeout` + } // No wall-clock budget decides here (maintainer decision, deterministic // cuts, option A, 2026-11-14): a slow copy runs to its end, and the mutant // budget is a work count (above). The wall-clock backstops report a gap no @@ -4593,6 +4611,12 @@ function model(plan) { `indeterminate`, `unreached`, ]) + // A timeout's cause: the cut that the copies ran into first (`judge`). + const causeOf = (tests, active, options) => { + const detail = {} + judge(plan, tests, sites, active, { ...options, detail }) + return detail.cause + } const mutants = sites.map((site) => { const kind = site.probe ? undefined @@ -4609,6 +4633,10 @@ function model(plan) { ordinal: site.ordinal, probe: site.probe, kind, + cause: + kind === `timeout` + ? causeOf(ownerTests(site.fnId), { site: site.id }) + : undefined, // A mutant whose run reaches a slow owner test keeps a copy busy for // SLOW_MS, so the CLI must name it in a heartbeat while it runs. heartbeat: @@ -4644,6 +4672,10 @@ function model(plan) { ), ), line: a.applies ? a.line : undefined, + cause: + a.applies && owners(a.fn).size > 0 + ? causeOf(ownerTests(a.fn), { authored: a.label }, { budgeted: false }) + : undefined, })) const authored = authoredAll[0]?.kind const isGap = (kind) => diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 0462fd6a..b7a92053 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1168,6 +1168,8 @@ function readRun(dir, out, status, stderr, cli, stdout) { outcome: r.outcome, class: r.class, laws: r.laws, + // The cut that ended a timeout (classify.mjs `outcomeOf`). + cause: r.outcome === `timeout` ? r.detail : undefined, probe: r.probeGuard === true, ordinaryOutcome: r.testOutcome !== undefined, })), @@ -2620,6 +2622,14 @@ function check(plan, observations, expected) { : `classification`, { actual: got, expected: owed }, ) + // A timeout names the cut that ended it (its `detail`). + if (kind === `timeout` && got.cause !== owed.cause) + return violation( + `cli`, + `${site(got)}: its timeout is cut by ${got.cause}, owed ${owed.cause}`, + `classification`, + { actual: got, expected: owed }, + ) } // Each gap is streamed once as its mutant is judged (stream.mjs), and the // streamed gaps are the report's gaps, each once. @@ -2689,6 +2699,13 @@ function check(plan, observations, expected) { `authored mutants`, { actual: kind, expected: owed.kind }, ) + if (kind === `timeout` && got.cause !== owed.cause) + return violation( + `cli`, + `the authored mutant ${owed.id} is cut by ${got.cause}, owed ${owed.cause}`, + `authored mutants`, + { actual: got.cause, expected: owed.cause }, + ) if (owed.line !== undefined && got.line !== owed.line) return violation( `cli`, diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index bae8a959..0f7761b4 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -52,9 +52,11 @@ export function change(result) { * and again in the summary. */ export function gapLine(result) { + // A timeout names the cut that ended it (classify.mjs `outcomeOf`). + const cut = result.outcome === `timeout` ? ` (cause: ${result.detail})` : `` return streamLine( `gap`, - `${kindOf(result)} L${result.line ?? `?`} ${result.id} ${change(result)}`, + `${kindOf(result)} L${result.line ?? `?`} ${result.id} ${change(result)}${cut}`, ) } From f640d1c1893bb37311b0ed198c14c7e3de52e7c7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 01:12:48 -0700 Subject: [PATCH 15/32] test(oracle-gaps): the fingerprint records the constructed-only fields as constants The deterministic-cuts constructed fields are constants in every drawn record, so no draw changes: the tests line hashes the records without them and is main's (e9029433a23d1f10). slowMs, loops, timerWait and pulse each get a line after the others, with its neutral value checked on every generated test, and hangMs is a drawn constant (null where generated) with its own line. The constructed histories #99 adds are appended after main's 26, by name and index. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 73 +++++++++---------- .../oracle-gaps/history-fingerprint.json | 14 +++- .../gap-report-history-fingerprint.test.ts | 54 ++++++++++++-- 3 files changed, 92 insertions(+), 49 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 1eb0ed97..7b6b3e24 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -129,11 +129,10 @@ const TOP_LOOP = `@..gen` // Whether this filesystem ignores letter case, so that `src/Sut.ts` names // `src/sut.ts`; the `cased` spelling applies only where it does. const caseless = existsSync(fileURLToPath(import.meta.url).toUpperCase()) -// The full limit of a cut-off copy's confirmation that the driver passes -// (`--hang-limit`), and, under one budget, the first budget too. +// The backstop base that the driver passes (`--hang-limit`): a copy that runs +// past it, plus three seconds per second of its file's time, is cut. const FULL_LIMIT_MS = 20000 -// How long a slow test waits, in real milliseconds: past the 1 s first -// budget that the driver passes (`--first-limit`), within the full limit. +// How long a slow test waits, in real milliseconds, where its output differs. const SLOW_MS = 3000 // The calls that each copy of `w_slowpair` makes beyond its first: its test // calls the function this many more times where its output differs. A call @@ -152,17 +151,11 @@ const WORK_FLOOR = 100_000 // is a hundred times the most its unmutated tests did, if more. Past it, the // copy is cut by count (a spin) and charged nothing (the outcome rule, item 4). const CALL_FLOOR = 100_000 -// A mutant's budget for the runs that confirm its cut-off copies, which the -// driver passes (`--mutant-limit`): one slow owner's confirmation fits it -// (SLOW_MS), and two of `w_slowpair`'s do not. Ten times the fixture's -// one-test files' time stays under it, so the floor is the budget. -const MUTANT_LIMIT_MS = 15000 // How often the CLI says which mutant a busy copy runs (its heartbeat): the // driver shortens it from 30 s (`ORACLE_GAPS_HEARTBEAT_MS`), so that a slow // test's wait of SLOW_MS passes it. A copy beats once its own run has lasted -// an interval, so the interval is twice the 1 s first budget: a copy cut at -// its first budget never lasts it, and whether a run beats is a function of -// the history, not of the machine's load, as the determinism tests require. +// an interval, and whether a run beats is a function of the history, not of +// the machine's load, as the determinism tests require. const HEARTBEAT_MS = 2000 const ORACLE_FILES = [`a`, `b`] // The owners of the budget witness (`w_slowpair`): two oracle files of one @@ -577,9 +570,10 @@ const drawn = fc.record({ tests: fc.array(testCase, { minLength: 1, maxLength: 5 }), // The witness module: all of it, or only the functions whose outcomes // need real test runs (see `captureWitness`). - // The budget witness alone (`budget`) costs two 10 s confirmations per - // mutant in every CLI run of a history, so generated histories rarely - // draw it; its constructed history (the daemon recheck) always does. + // The budget witness alone (`budget`) makes two copies of about 90,000 + // site evaluations each in every CLI run of a history, so generated + // histories rarely draw it; its constructed history (the daemon recheck) + // always does. witness: fc.oneof( { weight: 30, arbitrary: fc.constantFrom(false, true, `capture`) }, { weight: 1, arbitrary: fc.constant(`budget`) }, @@ -699,9 +693,9 @@ const drawn = fc.record({ // module loads, with 0. A mutant under which it throws then fails every // file that imports the module, before any test runs. boot: fc.constantFrom(false, false, false, true), - // The run's time budgets: a short first budget confirmed by a run alone - // with the full limit, or one budget (`--first-limit` equal to - // `--hang-limit`), where a run past it is a timeout at once. + // The run's budget mode: `one` passes the retired `--first-limit`, which + // the CLI checks and notes and which changes nothing; `two` passes no + // retired option. budget: fc.constantFrom(`two`, `two`, `two`, `one`), // Runs that the driver stops by a signal while the stop witness's test // spins (`w_stop`), each in a pool: Vitest's workers as child processes or @@ -858,6 +852,9 @@ const drawn = fc.record({ // `histories` takes a constructed one back as this constant (as // `claimScenario` does). Its neutral value is recorded by the canary. tornRows: fc.constant(false), + // A constructed history's backstop (`--hang-limit`), which no generated + // history sets: null is the full limit. Recorded by the canary as well. + hangMs: fc.constant(null), }) // The edit sequence over the witness module, the one list of steps that a @@ -866,7 +863,7 @@ const drawn = fc.record({ // only file b, so the daemon reuses file a's timed-out runs; the revert // returns to a state that the daemon has seen. The budget witness alone // (`budget`) takes its own one step instead: a second daemon check with -// nothing changed, which reuses the confirmations that the first check ran. +// nothing changed, which reuses the results that the first check ran. const witnessSteps = [`test`, `helper`, `revert`] const budgetSteps = [`recheck`] const keepsSteps = (history) => { @@ -887,8 +884,8 @@ const keepsSteps = (history) => { // witness module" takes `witnessSteps`, so every campaign still judges law // 9 over the witness module's outcome classes, along one short sequence // (maintainer decision, 2026-11-11, option M of the CI plan; a stated -// limit of the property). The budget witness's confirmations cost about -// 15 s a mutant, so its histories follow the same rule, and its constructed +// limit of the property). The budget witness's two copies make about 90,000 +// site evaluations each, so its histories follow the same rule, and its constructed // history "a second daemon check of the budget witness" takes // `budgetSteps` (the review of the gap CLI's runtime fixes, 2026-11-12, // P1-a: the budget witness only in its own histories). The map is its own @@ -922,8 +919,9 @@ const histories = drawn.map( ) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) - const { slowTwin, hangMs, ...drawnValue } = - /** @type {Record} */ (value) + const { slowTwin, ...drawnValue } = /** @type {Record} */ ( + value + ) const tests = /** @type {Record[]} */ (drawnValue.tests) const plain = /** @type {{ witness: unknown, steps: unknown[] | null, tornRows: unknown }} */ ({ @@ -1724,18 +1722,17 @@ const witness = [ body: [kill], tests: [judging(`a`, `random seed`, 7), judging(`b`, `fixed seed`, 7)], }, - // A slow kill: the run ends after the first budget, within the full - // limit, so the confirming rerun kills it (a run is a timeout only when - // it passes the full limit alone). + // A slow kill: its owner test waits 3 s where its output differs, then + // judges, so its copy runs to its end and the kill decides. { id: `w_slowkill`, owners: [`a`], body: [kill], tests: [judging(`a`, `fixed seed`, 7, { slow: true })], }, - // Slow in both owners, and judged by neither: each owner's copy is cut at - // its first budget, and the two confirmations together need more than the - // mutant's budget, so the mutant is a timeout, not a survivor. + // Slow in both owners, and judged by neither: each owner's copy makes + // 30,000 more calls, and the two copies together pass the work budget, so + // the mutant is a timeout by count, not a survivor. { id: `w_slowpair`, owners: PAIR_FILES, @@ -1835,8 +1832,8 @@ const witness = [ tests: [judging(`a`, `fixed seed`, 1)], }, // A test that waits on a child process where the output differs: an I/O - // handle is open, so its copy is busy, never cut for idleness, and the - // confirming run kills it. + // handle is open, so its copy is busy, never cut for idleness, and its + // kill decides. { id: `w_iowait`, owners: [`a`], @@ -2594,8 +2591,8 @@ function resolveHistory(history) { ? witness .filter( (w) => - // The budget witness costs two 10 s confirmations per mutant, - // so only histories of the `budget` mode carry it. + // The budget witness's two copies make about 90,000 site + // evaluations each, so only histories of the `budget` mode carry it. (history.witness === true && w.id !== `w_slowpair`) || (history.witness === `capture` && captureWitness.has(w.id)) || (history.witness === `budget` && w.id === `w_slowpair`), @@ -4196,9 +4193,9 @@ function slowedFiles(plan, tests, sites, active) { const CUT_CAUSE = { wall: `backstop`, count: `spin`, state: `no progress` } /** - * A mutant's kind from its owner tests' runs. An operator mutant's - * confirming runs share its budget (`budgeted`); an authored mutant's runs - * are each final at the full limit, so it has no confirmations to budget. + * A mutant's kind from its owner tests' runs. An operator mutant's copies + * share its work budget (`budgeted`); an authored mutant's copies are judged + * alone, with no budget. */ function judge(plan, tests, sites, active, { budgeted = true, detail } = {}) { const runs = runsOf(plan, tests, sites, active) @@ -4985,8 +4982,7 @@ function oracleTestFile(plan, file) { ` if (reads${i} !== 0) throw new Error('the payload was read')`, ] : []), - // A slow test waits past the first budget, within the full limit, - // where the output differs; it then judges as usual. + // A slow test waits where the output differs; it then judges as usual. ...(t.slow ? [ ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.slowMs ?? SLOW_MS}))`, @@ -6051,7 +6047,6 @@ export { interpret, judge, slowedFiles, - MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, refusal, diff --git a/packages/oracles/oracle-gaps/history-fingerprint.json b/packages/oracles/oracle-gaps/history-fingerprint.json index 79e58ac2..45200f5e 100644 --- a/packages/oracles/oracle-gaps/history-fingerprint.json +++ b/packages/oracles/oracle-gaps/history-fingerprint.json @@ -52,7 +52,12 @@ ["prePlugin", "7093bcf61828dea9"], ["fsRead", "265eb24ce88fdbe2"], ["laterSteps", "fb28f2575c00f223"], - ["tornRows", "ed6866229ecc4969"] + ["tornRows", "ed6866229ecc4969"], + ["hangMs", "f6e71aca47bcb3e2"], + ["tests.slowMs", "33494e17be1a5ba5"], + ["tests.loops", "4bc867dd43aad13c"], + ["tests.timerWait", "02f66613e9490d15"], + ["tests.pulse", "04378653193a64e7"] ], "histories": [ ["functions", "66c0e9c56951c377"], @@ -107,6 +112,11 @@ ["prePlugin", "7093bcf61828dea9"], ["fsRead", "265eb24ce88fdbe2"], ["laterSteps", "fb28f2575c00f223"], - ["tornRows", "ed6866229ecc4969"] + ["tornRows", "ed6866229ecc4969"], + ["hangMs", "f6e71aca47bcb3e2"], + ["tests.slowMs", "33494e17be1a5ba5"], + ["tests.loops", "4bc867dd43aad13c"], + ["tests.timerWait", "02f66613e9490d15"], + ["tests.pulse", "04378653193a64e7"] ] } diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index e7d22ad1..2191765a 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -21,6 +21,21 @@ import { drawn, histories } from '../oracle-gaps/gap-report-model-oracle.mjs' // campaign draws 2 histories, and those resolve as on main. const FIXED_SEED = 20261105 const DRAWS = 500 +// The constructed-only test fields (deterministic-cuts change, 2026-11-14): +// constants in every drawn test record, which no generated history sets. A +// test's record hashes without them, so its line is main's, and each one is +// recorded as its own line after the others, with its neutral value checked. +const CONSTRUCTED_TEST_FIELDS = [`slowMs`, `loops`, `timerWait`, `pulse`] +const NEUTRAL_TEST_VALUES: Record = { + slowMs: 3000, + loops: 0, + timerWait: null, + pulse: false, +} +type Sample = Record + +const hashOf = (value: unknown): string => + createHash(`sha1`).update(JSON.stringify(value)).digest(`hex`).slice(0, 16) const fingerprints = ( arbitrary: fc.Arbitrary, @@ -29,14 +44,37 @@ const fingerprints = ( const samples = fc.sample(arbitrary, { seed: FIXED_SEED, numRuns: DRAWS, - }) as Array> - return Object.keys(samples[0] ?? {}).map((key) => [ - key, - createHash(`sha1`) - .update(JSON.stringify(samples.map((history) => history[key]))) - .digest(`hex`) - .slice(0, 16), - ]) + }) as Array + const testsOf = (history: Sample) => history.tests as Array + const withoutConstructed = (test: Sample) => + Object.fromEntries( + Object.entries(test).filter( + ([key]) => !CONSTRUCTED_TEST_FIELDS.includes(key), + ), + ) + const lines: Array<[string, string]> = Object.keys(samples[0] ?? {}).map( + (key) => [ + key, + hashOf( + samples.map((history) => + key === `tests` + ? testsOf(history).map(withoutConstructed) + : history[key], + ), + ), + ], + ) + for (const field of CONSTRUCTED_TEST_FIELDS) { + const values = samples.flatMap((history) => + testsOf(history).map((test) => test[field]), + ) + // Generated tests hold the neutral value: only a constructed history sets it. + expect(values.every((value) => value === NEUTRAL_TEST_VALUES[field])).toBe( + true, + ) + lines.push([`tests.${field}`, hashOf(values)]) + } + return lines } describe(`the gap oracle's history grammar`, () => { From bfca29affa9b2c1e00309fd350f68d729714bbd6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 01:12:51 -0700 Subject: [PATCH 16/32] docs(oracle-gaps): the usage, README, index header and states name the current cuts --help and the index header give the defaults (--test-timeout 65000, --hang-limit 60000) and mark --first-limit and --mutant-limit retired. The README, the changeset and the e2e law 4 text describe the backstop, the call and work counts, and the cause each timeout names. Stale states and names are renamed (confirmations, the first budget, the 15 s limit): a slow kill after a 16 s wait, and a mutant's copies past its work budget. MUTANT_LIMIT_MS, unused, is removed. The determinism twin is described as a determinism check: it passes on its parent, so the outcome rule's RED is the classify oracle's. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/args.mjs | 10 +- packages/oracles/oracle-gaps/check.mjs | 4 +- ...p-report-classify-oracle.property.test.mjs | 13 ++- .../gap-report-oracle.property.test.mjs | 110 +++++++++--------- packages/oracles/oracle-gaps/index.mjs | 16 ++- 5 files changed, 81 insertions(+), 72 deletions(-) diff --git a/packages/oracles/oracle-gaps/args.mjs b/packages/oracles/oracle-gaps/args.mjs index e5b10015..cd3c8b3a 100644 --- a/packages/oracles/oracle-gaps/args.mjs +++ b/packages/oracles/oracle-gaps/args.mjs @@ -39,10 +39,18 @@ export const usage = `Usage: oracle-gaps --package --source [--env-prefix ] [--daemon] [--ordinary] [--seed ] [--mutants ] [--out ] [--only ] [--diff-base ] [--test-timeout ] [--sample ] [--contexts ] [--pool threads|forks] [--config ] - [--first-limit ] [--hang-limit ] [--mutant-limit ] + [--hang-limit ] [--first-limit ] [--mutant-limit ] oracle-gaps --stop-daemon --package oracle-gaps --help +Defaults: --env-prefix ORACLE; --test-timeout 65000 (Vitest's per-test limit, +in ms); --hang-limit 60000 (the base of each copy's backstop, in ms, plus 3 s +per second of its file's unmutated time); --contexts the machine's parallelism +minus 2, at most 16; --pool threads (forks when a package has a native +addon); --sample and --only off; --out a directory under the package's state. +Retired: --first-limit and --mutant-limit are checked and print a note, and +change nothing. A copy has no first budget, and a mutant's work budget bounds it. + The paths in --source and --config are relative to --package. Without --config, the run loads the Vitest config that Vitest discovers for the package: in its directory, or else in the nearest parent directory that diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index eba6f6ed..fdf6649e 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -1274,7 +1274,7 @@ export default async (env) => { const skippedUnit = (item) => ({ file: item.file, skipped: true }) // A copy that ran past its call budget, even one whose throw ended the // loop and let it finish, is a timeout as it stands (a count, not a - // clock: no confirming run). + // clock). const spunUnit = (item) => ({ file: item.file, timedOut: true, @@ -2135,7 +2135,7 @@ export default async (env) => { // hang), so a result never depends on how its evidence was produced. The // ordinary tier, when asked for, runs the same way afterwards. const GROUP = 4 - // A copy that a batch skipped, or cut off for its confirmation, has no + // A copy that a batch skipped, or cut off by a count or a state, has no // verdicts of its own yet. const kills = (u) => !u.skipped && diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index bbae942c..43f97f37 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -2132,6 +2132,8 @@ async function driver(h, observe) { class: r.class, laws: r.laws && [...r.laws].sort(), testOutcome: r.testOutcome, + // A timeout's detail is its cut (`outcomeOf`). + detail: r.outcome === `timeout` ? r.detail : undefined, })), summary: readSummary(summary), messages: messagesOf(summary), @@ -2617,11 +2619,18 @@ function check(h, observations, expected) { ) // Law 8: each gap line is one line, and shows its change shortened: an // operator mutant's original and replacement as two code spans, an - // authored mutant's intent as a code span and its author. The results keep the full text. + // authored mutant's intent as a code span and its author. A timeout's line + // ends with its cause, `(cause: )`, after the change. The results + // keep the full text. for (const owed of gapsOfModel) { const head = `oracle-gaps: gap ${owed.class ?? owed.outcome} L${owed.line} ${owed.id} ` const line = streamed.find((l) => l.startsWith(head)) - const rest = line?.slice(head.length) + const cause = owed.outcome === `timeout` ? ` (cause: ${owed.detail})` : `` + const tail = line?.slice(head.length) + const rest = + tail !== undefined && tail.endsWith(cause) + ? tail.slice(0, tail.length - cause.length) + : undefined let shown if (owed.authored) { // The intent is the author's text, so it sits in a code span. diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index b7a92053..01ebb0e1 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -140,28 +140,24 @@ * decision of 2026-11-06); a mutant that only nondeterministic owner tests * could judge is `indeterminate`. A mutant under which the module throws * while it loads fails every file that imports it before any test runs: - * an error outside any test, a `setup-error`. A run is cut off when it - * passes its first, short budget, and only the copy that ran past it is - * charged with that time (a copy whose file fails to load ends its turn - * there); it then runs again alone with the full limit - * (the short budget with confirmation, maintainer decision of - * 2026-11-06): a run that ends within the full limit is judged as if it - * had never been cut off, so a slow kill is a kill. An owner test that - * does not end within the full limit cuts its file's run off, and the - * clock-cut run records nothing, not even its earlier verdicts; a kill in - * another owner still decides (a kill outranks a timeout, v19), and failing - * one, the mutant is a `timeout`, whatever any other owner reported. The - * confirming runs of one mutant share its budget: the larger of - * `--mutant-limit` (30 s by default) and ten times its owner files' - * unmutated time (maintainer disposition G1, 2026-11-11). Its - * confirmations run in its owner files' order, and each gets at most what - * the budget has left after the seconds of its earlier confirmed runs, - * reused ones included, so the outcome is a function of the runs, not of - * their order or of reuse (law 9 checks a second daemon check of the same - * state); one that the budget cuts, or that gets nothing, is a timeout; so - * a mutant that is slow in two owners, whose confirmations together pass - * the budget, is a `timeout`, and a kill in an earlier owner still - * decides. A copy + * an error outside any test, a `setup-error`. No wall-clock limit decides + * an outcome (deterministic cuts, option A, 2026-11-14): a copy runs to its + * end, or is cut by its call count (a spin past its call budget), by its + * state (no progress: idle, with no timer due in its horizon), or by the + * backstop (`--hang-limit`, 60 s by default, plus 3 s per second of its + * file's unmutated time), which reports a gap. A spin or no-progress cut + * ends only its own copy, charged nothing, and the next owner's copy still + * runs; a backstop cut ends the mutant's later copies. A kill in another + * owner still decides (a kill outranks a timeout), and failing that the + * mutant is a `timeout`, whatever any other owner reported. The copies of + * one mutant share a work budget, counted in site evaluations and not in + * time: the larger of 100,000 and ten times their owner files' unmutated + * work (maintainer disposition G1, 2026-11-11, as a work count). Those + * copies are summed in owner order, and once the sum passes the budget, + * that copy and every later one is a timeout, so a kill among them does not + * count. Every timeout names its cause: `backstop`, `spin`, `no progress`, + * or `work budget`, the first cut in owner order (the CLI's `detail`). A + * copy * cut off for spinning (its call budget ran out, not its clock) is a * timeout as it stands: its rows are not judged, so a verdict it recorded * before the cut does not kill. (The framework's `bounded work` law, which @@ -392,7 +388,7 @@ * Limits. Sources are one file of exported functions and classes over one * number. A hang is generated only as an owner test that waits on a promise * that never settles, under a 20 s full limit - * (`--first-limit`, `--hang-limit`, and a 15 s `--mutant-limit`); a slow run + * (`--hang-limit`); a slow run * that ends is only a witness, a test that waits 3 s where its output * changed and then judges, or 10 s in two one-test owners without judging. Not generated: a busy * loop, a slow run past the full limit that would still end, a hang in an @@ -454,9 +450,8 @@ * after its wait) and a stop that waits for the close are not generated. * Cut-off * charging is judged only where it changes an outcome: under one budget - * (`--first-limit` equal to `--hang-limit`), where a misplaced cut is a - * timeout at once; under two budgets a misplaced cut only costs a - * confirming run, which no law here measures. A copy that failed to load in + * (retired), where a misplaced cut is a timeout at once. A copy that failed + * to load in * a batch cut off later is run again rather than judged from its rows, so * its setup error keeps Vitest's message. Native addons only as a * `binding.gyp` beside the workspace dependency or a prebuilt `.node` file @@ -535,7 +530,6 @@ import { sourceArgument, outsideRepository, ENV_PREFIX, - MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, NO_LATER_STEPS, @@ -1779,7 +1773,7 @@ async function driver(plan, observe) { // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) // The slow twin (`slowTwin`): the same package, where file a's first - // test waits 3 s past its first budget wherever the output differs, and + // test waits 3 s wherever the output differs, and // then judges as before. Its verdicts are the same, so its report must // be too ("Determinism under load", 2026-11-13: an outcome does not // depend on how long a run took). @@ -1810,11 +1804,11 @@ async function driver(plan, observe) { const states = new Set() if (twinSame) states.add(`a slow owner's kill reported as when it did not wait`) - // A kill that waits past Vitest's 15 s test limit, and one whose copy - // waits on a timer, each killed as the report says. + // A kill after a 16 s wait, and one whose copy waits on a timer, each + // killed as the report says. const killed = cli.report?.results?.some((r) => r.outcome === `killed`) - if (killed && plan.history.tests.some((t) => t.slowMs > 15000)) - states.add(`a slow kill past fifteen seconds killed`) + if (killed && plan.history.tests.some((t) => t.slowMs === 16_000)) + states.add(`a slow kill after a 16 s wait killed`) if (killed && plan.history.tests.some((t) => t.timerWait)) states.add(`a copy waiting on a timer killed`) // A copy that waits past the history's backstop is cut off, and no later @@ -2129,7 +2123,7 @@ async function driver(plan, observe) { if (m.function === `w_iowait` && m.outcome === `killed`) states.add(`a copy waiting on I/O killed, not cut for idleness`) if (m.function === `w_slowkill` && m.outcome === `killed`) - states.add(`a slow kill past the first time limit killed`) + states.add(`a slow kill killed`) if (m.function === `w_hang` && m.outcome === `timeout`) states.add(`a hang alone timed out`) if (m.function === `w_hangkill` && m.outcome === `killed`) @@ -2137,7 +2131,7 @@ async function driver(plan, observe) { if (m.function === `w_hangsurvive` && m.outcome === `timeout`) states.add(`a hang in one owner and a survivor in another timed out`) if (m.function === `w_slowpair` && m.outcome === `timeout`) - states.add(`confirmations past a mutant's budget timed out`) + states.add(`a mutant's copies past its work budget timed out`) if (cli.heartbeats.some((beat) => beat.label === m.id)) states.add(`a heartbeat named a busy mutant`) const named = { @@ -3571,7 +3565,7 @@ const required = [ `timeout`, `a kill after a nondeterministic verdict in its file`, `a kill in one owner after a gap in another`, - `a slow kill past the first time limit killed`, + `a slow kill killed`, `a mutant that changes a value only at load killed`, `a mutant that changes no value judged without a run`, `a boolean flip killed`, @@ -3590,12 +3584,12 @@ const required = [ `a hang alone timed out`, `a hang in one owner and a kill in another killed`, `a slow owner's kill reported as when it did not wait`, - `a slow kill past fifteen seconds killed`, + `a slow kill after a 16 s wait killed`, `a copy waiting on a timer killed`, `a kill past the work budget in a later owner does not count`, `a backstop cut ends the copies ahead of a later kill`, `a hang in one owner and a survivor in another timed out`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget timed out`, `a heartbeat named a busy mutant`, `a run stopped by SIGTERM while a worker spun`, `a run stopped by SIGINT while a worker spun`, @@ -3724,7 +3718,7 @@ const separate = [ // The deterministic-cuts states have constructed histories of their own, so // no history that names every state (`every outcome class`) must enter them. `a slow owner's kill reported as when it did not wait`, - `a slow kill past fifteen seconds killed`, + `a slow kill after a 16 s wait killed`, `a copy waiting on a timer killed`, `a kill past the work budget in a later owner does not count`, `a backstop cut ends the copies ahead of a later kill`, @@ -3737,7 +3731,7 @@ const separate = [ `an awaiting config in a package with no module type ran`, `a .cjs or .cts config that calls require ran`, `a .js or .ts config in a package with no module type that calls require ran`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget timed out`, `a second daemon check of the budget witness's state`, `a run with eleven contexts`, `a failing owner replayed under a config that --config names`, @@ -3807,7 +3801,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds a mutant's copies by one work budget per mutant (site evaluations, never time) and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; the retired --first-limit judged only as a note; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with the full limit, since no copy has a first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; the retired --first-limit and --mutant-limit judged only as notes; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its backstop, since a copy has no first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant's work budget is witnessed by w_slowpair and a kill past it, in owner order, each owner's copy counted in site evaluations (not in time), and authored mutants have no copies to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -3873,13 +3867,13 @@ oracleProperty({ }, { // The budget witness checked through the daemon twice with nothing - // changed between: the second check reuses the confirmations the - // first ran, and must report what a cold run reports (law 9, with - // the budget of law 4). + // changed between: the second check reuses the work-budget results + // the first ran, and must report what a cold run reports (law 9, with + // the work budget of law 4). name: `a second daemon check of the budget witness`, enters: [ `a second daemon check of the budget witness's state`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget timed out`, ], build: (generated) => ({ ...generated, @@ -5078,12 +5072,14 @@ oracleProperty({ }), }, { - // File a's random-route test and file b's fixed-seed test both kill - // one mutant (as w_crosskill does). In the slow twin, a's test - // waits 3 s past its first budget before it judges, so its copy is - // cut and queued for a confirmation; b's kill decides the mutant - // first. The report must still be the one the package without the - // wait gives, its `randomAlso` included. + // A determinism check (the determinism under load law, 2026-11-13): + // file a's random-route test and file b's fixed-seed test both kill + // one mutant (as w_crosskill does). In the slow twin, a's test waits + // 3 s before it judges, so its copy runs to its end, and b's kill + // decides the mutant first. The report must still be the one the + // package without the wait gives, its `randomAlso` included. It + // passes on its parent too, so it guards the law and has no RED of + // its own; the outcome rule's RED is the classify oracle's. name: `a slow owner's kill and another owner's kill`, enters: [`a slow owner's kill reported as when it did not wait`], build: (generated) => ({ @@ -5218,12 +5214,11 @@ oracleProperty({ steps: null, }), }, - // A slow kill whose test waits 16 s where its output differs, past - // Vitest's 15 s per-test limit: no wall-clock limit decides an outcome, - // so the mutant is still killed. + // A slow kill whose test waits 16 s where its output differs: no + // wall-clock limit decides an outcome, so the mutant is still killed. { - name: `a slow kill past Vitest's 15 s limit`, - enters: [`a slow kill past fifteen seconds killed`], + name: `a slow kill after a 16 s wait`, + enters: [`a slow kill after a 16 s wait killed`], build: (generated) => soloKill(generated, [ { @@ -5234,9 +5229,8 @@ oracleProperty({ }, ]), }, - // A copy whose test starts a timer due inside its horizon, then spins - // under load, then waits for it: the copy is busy, never idle, and the - // mutant is killed. + // A copy whose test starts a timer due inside its horizon, then waits + // for it: the copy is never idle, and the mutant is killed. { name: `a copy waiting on a timer`, enters: [`a copy waiting on a timer killed`], diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 4321082e..c732e132 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -84,15 +84,13 @@ // [--ordinary] (also run ordinary test files, for information) // [--seed ] [--mutants ] [--out ] [--only ] // [--diff-base ] -// [--first-limit ] (a copy's first budget; default 2000) -// [--hang-limit ] (the full limit a cut copy gets again; default 10000) -// [--mutant-limit ] (the floor of each mutant's budget, which the -// runs that confirm its cut copies share; the -// budget is the larger of this and ten times -// its owner files' unmutated time; default 30000) -// [--test-timeout ] (default 15000; also bounds hooks and teardown. -// Each mutant run also has a wall-clock limit of -// 10 s plus three times its file's unmutated time.) +// [--hang-limit ] (the base of each copy's backstop, which adds +// 3 s per second of its file's unmutated time; +// default 60000) +// [--first-limit ] (retired: checked and noted, no effect) +// [--mutant-limit ] (retired: checked and noted, no effect) +// [--test-timeout ] (Vitest's per-test limit, also bounding hooks +// and teardown; default 65000) // [--sample ] (a fixed-seed sample of n mutants per source; // the report marks the run as partial) // [--contexts ] (warm contexts, one worker each; default From da1bf127ef82a9c7aac0b0f07a5b37e18b989474 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 01:13:07 -0700 Subject: [PATCH 17/32] docs(oracle-gaps): the README and the changeset describe the backstop and counts The README's gap section and the changeset's bullet describe the counts and the backstop, the cause each timeout names (on the gap line too), and the retired options, in place of the confirmation budget. The classify oracle's gap-line law (in bfca29a) accepts a timeout's cause suffix. Co-authored-by: Isaac --- .changeset/gap-cli-runtime.md | 4 ++-- README.md | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.changeset/gap-cli-runtime.md b/.changeset/gap-cli-runtime.md index 9807d437..88e2691c 100644 --- a/.changeset/gap-cli-runtime.md +++ b/.changeset/gap-cli-runtime.md @@ -2,9 +2,9 @@ '@tanstack/oracles': minor --- -The `oracle-gaps` CLI bounds a mutant's confirming runs, reports long runs, stops the processes it started, and writes nothing beside the tests. +The `oracle-gaps` CLI bounds a mutant's copies by counts, not by wall-clock limits, reports long runs, stops the processes it started, and writes nothing beside the tests. -- Each mutant has a budget for the runs that confirm its cut-off copies: the larger of the new `--mutant-limit` (30 s by default) and ten times its owner files' unmutated time. The confirmations run in the owner files' order, each with at most what the mutant's earlier confirmations left, so the outcome depends only on the recorded runs, not on their order or on reuse through the daemon. A mutant whose confirmations pass the budget is a `timeout` gap. With the default limits a single owner's confirmation always fits; the budget binds when several owners need long confirmations, or when `--hang-limit` is above it, which the run notes. To give one copy more time with `--hang-limit`, raise `--mutant-limit` too. +- A mutant's copies share a work budget, counted in site evaluations and not in time: the larger of 100,000 and ten times their owner files' unmutated work. A mutant past it is a `timeout` gap. No wall-clock limit decides an outcome. A copy cut by its call count (`spin`) or by no progress ends only itself, a copy cut by the backstop (`--hang-limit`, 60 s by default, plus 3 s per second of its file's time) ends the mutant's later copies, and every timeout names its cause in `results.json` and on its gap line. `--mutant-limit` and `--first-limit` are retired: they are checked and noted, and change nothing. - While one copy of a test file runs for 30 s or more, the CLI prints `busy: in for s` every 30 s (`baseline` for a run with no mutant). Through `--daemon` the heartbeats arrive when the check ends. - The check runs in a process group of its own, with every Vitest worker and every process a test starts, except one started detached in a session of its own. On SIGINT or SIGTERM the CLI signals the group, kills what is left 2 s later, and exits 130 or 143. Before, a SIGTERM left `forks` workers running. On Windows the run is not supervised. - The runner's copies of the test files exist only in memory. Vitest's read of a copy for its per-file pragmas (`@vitest-environment`, `@module-tag`) is served its text, so a copy runs in its file's environment. The capture setup and Vitest configs are in a session directory of its own while it runs, and one that a stopped run left is removed by the next run. Before, the copies and configs sat beside the tests and the package's config, where a Vitest run started during a check found them and then failed when they were removed. diff --git a/README.md b/README.md index d87f2bb7..8b8f8898 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,20 @@ agent. While one copy of a test file runs for 30 s or more, the CLI prints `oracle-gaps: busy in for s` every 30 s. A run of many fast copies prints none, and a check through `--daemon` prints its heartbeats - only when the check ends. The runs that confirm a mutant's cut-off copies - share one budget: the larger of `--mutant-limit` (30 s by default) and ten - times the owner files' unmutated time. They run in the owner files' - order, each with at most what its mutant's earlier confirmations left. A - mutant whose confirmations pass the budget is a `timeout` gap. With the - default limits one owner's confirmation always fits; the budget binds - when several owners need long confirmations, or when `--hang-limit` is - above it, which the run notes. To give one copy more time with - `--hang-limit`, raise `--mutant-limit` too. + only when the check ends. No wall-clock limit decides an outcome. A copy + is cut in one of three ways: by its call count, when it runs past its + call budget (`spin`); by its state, when it makes no progress (`no progress`); + or by the backstop, when it runs past `--hang-limit` (60 s by default) plus + 3 s per second of its file's unmutated time (`backstop`). A spin or no + progress ends only its own copy, and a backstop ends the mutant's later + copies. The copies of one mutant share a work budget, counted in site + evaluations and not in time: the larger of 100,000 and ten times their owner + files' unmutated work. Past it, the mutant is a `timeout`. A timeout names + its cause in `results.json` (`detail`) and on its gap line, as + `(cause: )`, where the cause is the first cut in owner order, or + `work budget`. A kill in another owner decides a mutant unless a backstop + ended its copies first. `--first-limit` and `--mutant-limit` are retired: + they are checked and noted, and change nothing. The report, on stdout and in `summary.md`, leads with the baseline: each test file's result with its test counts (ordinary files too), and for a @@ -106,7 +111,7 @@ agent. | `baseline` | `oracle-gaps: baseline pass tests/a.test.ts: 3 tests`, `oracle-gaps: baseline fail tests/b.test.ts: 1 of 2 tests failed` | | `baseline` (decisive, before any mutant) | `oracle-gaps: baseline all 4 oracle files pass`, or `oracle-gaps: baseline 1 of 4 oracle files failed (provisional)` | | `blocked` | `oracle-gaps: blocked fails with no mutant active at , so the mutants it owns wait for it: …` | - | `gap` | `oracle-gaps: gap L ` | + | `gap` | `oracle-gaps: gap L `, and for a `timeout` ` (cause: )` | | `progress` | `oracle-gaps: progress judged 25 of 120 mutants: 7 gaps so far, 18 not gaps` | | `busy` | `oracle-gaps: busy in for s` | | `note` | `oracle-gaps: note …`: the `--hang-limit` note, Vite's own messages (`note vite: …`), ordinary files left out, the provisional count, unowned counts, the results' path, and where the tests' console output went (`note test output in `) | From d0df1913c084706f7789f89be68943d8bd2c4b2f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 01:24:30 -0700 Subject: [PATCH 18/32] fix(oracle-gaps): a constructed backstop takes back as the drawn null; the Windows shards list the new histories The constructed backstop (hangMs) is a drawn constant now, so its take-back returns the drawn value (null), which the arbitrary can take back. The fixed-seed shards of main are kept as they are: the new histories fall into the Linux rest shard, and the Windows partition, which is exact, lists them in shard 10. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 7 ++-- packages/oracles/scripts/pr-shards.mjs | 39 ++++--------------- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index c35e81b7..657929e8 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -927,13 +927,14 @@ const histories = drawn.map( ) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) - const { slowTwin, ...drawnValue } = /** @type {Record} */ ( - value - ) + // A constructed backstop (`hangMs`) comes back as the drawn constant (null). + const { slowTwin, hangMs, ...drawnValue } = + /** @type {Record} */ (value) const tests = /** @type {Record[]} */ (drawnValue.tests) const plain = /** @type {{ witness: unknown, steps: unknown[] | null, tornRows: unknown }} */ ({ ...drawnValue, + hangMs: null, tests: tests.map(takenBack), }) // Take-back into the drawn language: a disarmed pause (pauseEngages false, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index bb691d60..efbfb1ce 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -306,16 +306,13 @@ export const partitions = { // Both halves of `every outcome class` (134 s and 155 s on the CI runner), // a second daemon check (101 s), and two small units. The second half // is named here: `every outcome class` alone matches the first half only, - // and the second would fall into the rest (shard 3). The two - // deterministic-cuts histories (a 16 s wait, and a timer) are about 45 s. + // and the second would fall into the rest (shard 3). 2: anyOf([ 'every outcome class', 'every outcome class, the warm and abort stops', 'a second daemon check of the budget witness', 'a declaration fault: brokenVitest', 'an edit that changed the cold result after a hang replaced the only context', - 'a slow kill after a 16 s wait', - 'a copy waiting on a timer', ]), // The rest: noTypescript (174 s on the CI runner), the edit sequence // through the daemon (94 s), brokenOwner (83 s), generated (23 s), the @@ -537,33 +534,6 @@ export const partitions = { 'a second daemon check of the budget witness', 'a declaration fault: missingSource', 'a config fault in a run that owns nothing', - 'a declaration fault: configNoDefault', - 'a declaration fault: nonLiteral', - 'a lazy module loaded before a hang in the same batch', - 'a stale authored mutant and a reformat', - 'an edit to a module that only mutated runs load', - 'an edit during a check through the daemon', - 'a test edits the helper it read, during a check through the daemon', - 'a test edits the data it read, during a check through the daemon', - ]), - 4: anyOf([ - 'a config that --config names', - 'a declaration fault: configNull', - 'a declaration fault: outsideSource', - 'a load failure beside a hang', - 'an ambiguous authored mutant', - "a slow owner's kill and another owner's kill", - 'a slow kill after a 16 s wait', - 'a copy waiting on a timer', - 'a kill past the work budget in a later owner', - 'a backstop cut ends the copies ahead of a later kill', - 'a capture row read while the worker wrote it', - 'two batches each read a capture row while its worker wrote it', - 'every outcome class', - ]), - 5: anyOf([ - 'a declaration fault: apiLessTypescript', - 'a declaration fault: configSyntax', "test.projects in a parent's config", 'a declaration fault: projects', 'a declaration fault: outsideSource', @@ -604,7 +574,14 @@ export const partitions = { 'ordinary tests left out by default', 'a declaration fault: configNull', ]), + // The deterministic-cuts histories (the 16 s wait, the backstop, the work + // budget, the twin and the timer) are short, except the 16 s wait. 10: anyOf([ + "a slow owner's kill and another owner's kill", + 'a backstop cut ends the copies ahead of a later kill', + 'a kill past the work budget in a later owner', + 'a slow kill after a 16 s wait', + 'a copy waiting on a timer', 'an edit that changed the cold result after a hang replaced the only context', 'a test edits the helper it read, during a check through the daemon', 'an edit to data a test reads through fs (callback)', From efc169c9afce681b4ec47eda3d29af3e5f07cd35 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 01:27:28 -0700 Subject: [PATCH 19/32] test(oracle-gaps): the fingerprint helpers do not shadow Vitest's test Co-authored-by: Isaac --- .../oracles/tests/gap-report-history-fingerprint.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index 2191765a..bd8e8adf 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -46,9 +46,9 @@ const fingerprints = ( numRuns: DRAWS, }) as Array const testsOf = (history: Sample) => history.tests as Array - const withoutConstructed = (test: Sample) => + const withoutConstructed = (record: Sample) => Object.fromEntries( - Object.entries(test).filter( + Object.entries(record).filter( ([key]) => !CONSTRUCTED_TEST_FIELDS.includes(key), ), ) @@ -66,7 +66,7 @@ const fingerprints = ( ) for (const field of CONSTRUCTED_TEST_FIELDS) { const values = samples.flatMap((history) => - testsOf(history).map((test) => test[field]), + testsOf(history).map((record) => record[field]), ) // Generated tests hold the neutral value: only a constructed history sets it. expect(values.every((value) => value === NEUTRAL_TEST_VALUES[field])).toBe( From 78347cb9a0aada30add01a1b1486b1ab46ddf8e8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 01:32:44 -0700 Subject: [PATCH 20/32] test(oracle-gaps): a constructed backstop of null is the full limit in the CLI arguments Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index bca05d01..bde5a08f 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -581,7 +581,7 @@ function cliArgs( ordinary = false, sources = `${SOURCE},${LIB}`, budget = `two`, - hangMs = FULL_LIMIT_MS, + hangMs: hangMsOrNull = null, outName = daemon ? `out-daemon` : `out`, config, contexts = 4, @@ -595,6 +595,8 @@ function cliArgs( // relative to there, as an author types them, and the daemon must read // them as the CLI's caller meant them. const path = (absolute) => (daemon ? relative(repo, absolute) : absolute) + // A generated history's backstop is the drawn null: the full limit. + const hangMs = hangMsOrNull ?? FULL_LIMIT_MS const args = [ join(cli, `index.mjs`), ...(daemon ? [`--daemon`] : []), From 784e0563e5f9ac87a540801a56cb1268780e3c9f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 12:55:55 -0700 Subject: [PATCH 21/32] test(oracle-gaps): the held-state witness: a copy waiting on an unref'd timer An unref'd timer is not among the process's active resources, so only the copy's own pending-timer record keeps it busy (the idle rule, held state, M 2026-11-15). The witness is a constructed history of its own; the reach state is recorded. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 2 +- .../gap-report-oracle.property.test.mjs | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 657929e8..7061f633 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -5019,7 +5019,7 @@ function oracleTestFile(plan, file) { // check alone stands between this copy and an idle cut. ...(t.timerWait ? [ - ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.timerWait.waitMs}))`, + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.timerWait.waitMs})${t.timerWait.unref ? `.unref()` : ``})`, ] : []), // A test that waits on I/O (a child process, about 3 s) where the diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index bde5a08f..00b2201a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1950,6 +1950,8 @@ async function driver(plan, observe) { states.add(`a slow kill after a 16 s wait killed`) if (killed && plan.history.tests.some((t) => t.timerWait)) states.add(`a copy waiting on a timer killed`) + if (killed && plan.history.tests.some((t) => t.timerWait?.unref)) + states.add(`a copy waiting on an unref'd timer killed`) // A copy that waits past the history's backstop is cut off, and no later // owner runs, so the mutant is a timeout, not the later owner's kill. if ( @@ -3730,6 +3732,7 @@ const required = [ `a slow owner's kill reported as when it did not wait`, `a slow kill after a 16 s wait killed`, `a copy waiting on a timer killed`, + `a copy waiting on an unref'd timer killed`, `a kill past the work budget in a later owner does not count`, `a backstop cut ends the copies ahead of a later kill`, `a hang in one owner and a survivor in another timed out`, @@ -3862,6 +3865,7 @@ const separate = [ `a slow owner's kill reported as when it did not wait`, `a slow kill after a 16 s wait killed`, `a copy waiting on a timer killed`, + `a copy waiting on an unref'd timer killed`, `a kill past the work budget in a later owner does not count`, `a backstop cut ends the copies ahead of a later kill`, // The config and contexts states have constructed histories of their own. @@ -5509,6 +5513,21 @@ oracleProperty({ }, ]), }, + // The same wait on an unref'd timer: the timer is not listed among the + // process's active resources, so only the copy's own pending timers + // keep it busy (the held state, deterministic cuts, 2026-11-15). + { + name: `a copy waiting on an unref'd timer`, + enters: [`a copy waiting on an unref'd timer killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + timerWait: { waitMs: 3000, unref: true }, + }, + ]), + }, ]), }, resolve: resolveHistory, From 2c83b2f381b96fefc03516a2e72769226c0cd389 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 12:57:28 -0700 Subject: [PATCH 22/32] fix(oracle-gaps): the idle rule is state only: no CPU or lateness condition A copy is idle when no counted timer is due within its horizon, no timer is listed beyond its test's base, and no I/O handle is open (M entry 2026-11-15). Sample lateness and the event loop's utilization no longer reset the window: a busy synchronous or microtask loop blocks the sampler, so it cannot look idle. The held-state witnesses (a copy waiting on a timer, ref'd or unref'd) keep the copy busy, and the mutants that ignore a due timer are killed. Co-authored-by: Isaac --- .../oracles/oracle-gaps/capture-setup.mjs | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index cf56db29..c5f892a6 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -50,7 +50,6 @@ const run = const real = (globalThis.__oracleGapsReal ??= { setInterval: globalThis.setInterval, now: Date.now.bind(Date), - use: performance.eventLoopUtilization.bind(performance), keys: Object.keys, getPrototypeOf: Object.getPrototypeOf, getOwnPropertyDescriptor: Object.getOwnPropertyDescriptor, @@ -64,7 +63,6 @@ const real = (globalThis.__oracleGapsReal ??= { }) const realInterval = real.setInterval const realNow = real.now -const realUse = real.use // A wrapper keeps every own property of the function it wraps, such as // `util.promisify.custom`, so `util.promisify(setTimeout)` still resolves @@ -1027,15 +1025,17 @@ function finishTest(context) { const thisFile = globalThis.__vitest_worker__?.filepath -// I/O and idleness (maintainer decision, 2026-11-07). Each test's row says -// whether the test ever had an I/O handle open: anything but the runner's -// own channel, stdio, and timers. A copy given `idleMs` (a file whose -// unmutated tests did no I/O) is idle when no counted timer is pending, no -// more timers are listed than when its test began (which covers timers made -// outside the wrappers, such as `node:timers/promises`), nothing but the -// channel and stdio is open, and its event loop is under 5% busy. After -// `idleMs` of that it writes an idle row, and the batch's watchdog cuts it -// off as a hang. A late sample means the thread was busy. +// I/O and idleness (maintainer decision, 2026-11-07; state only, 2026-11-15). +// Each test's row says whether the test ever had an I/O handle open: anything +// but the runner's own channel, stdio, and timers. A copy given `idleMs` (a +// file whose unmutated tests did no I/O) is idle, by state alone, when no +// counted timer is due within its horizon, no more timers are listed than +// when its test began (which covers timers made outside the wrappers, such as +// `node:timers/promises`), and nothing but the channel and stdio is open. A +// busy synchronous or microtask loop blocks this sampler, so it cannot look +// idle; it ends by its call budget or the backstop. After `idleMs` of that +// state it writes an idle row, and the batch's watchdog cuts it off as no +// progress. const quiet = new Set([ `MessagePort`, `PipeWrap`, @@ -1049,8 +1049,6 @@ let sawIo = false if (state.capture) { let baseTimers let since - let last - let use = realUse() let reported = false beforeEach(() => { if (continuing) return @@ -1066,10 +1064,6 @@ if (state.capture) { }) realInterval(() => { const now = realNow() - const late = last !== undefined && now - last > 150 - last = now - const used = realUse(use) - use = realUse() if (baseTimers === undefined || reported) return const kinds = process.getActiveResourcesInfo() const io = kinds.some((kind) => !quiet.has(kind)) @@ -1079,13 +1073,7 @@ if (state.capture) { // a timeout at the limit anyway (a collection's five-minute GC timer). const horizon = now + (state.idleHorizonMs ?? Infinity) const timerDue = [...pending.values()].some((due) => due <= horizon) - if ( - late || - io || - used.utilization > 0.05 || - timerDue || - timersListed(kinds) > baseTimers - ) { + if (io || timerDue || timersListed(kinds) > baseTimers) { since = undefined return } From 8ec9d45dffb0354b108573c89298cc3facbbe97e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 14:24:20 -0700 Subject: [PATCH 23/32] test(oracle-gaps): the Windows shards list the unref'd timer history Co-authored-by: Isaac --- packages/oracles/scripts/pr-shards.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index efbfb1ce..8b93aa95 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -577,6 +577,7 @@ export const partitions = { // The deterministic-cuts histories (the 16 s wait, the backstop, the work // budget, the twin and the timer) are short, except the 16 s wait. 10: anyOf([ + "a copy waiting on an unref'd timer", "a slow owner's kill and another owner's kill", 'a backstop cut ends the copies ahead of a later kill', 'a kill past the work budget in a later owner', From 24daca2a0b327f43fceeaad13e1714f3671d08f0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 14:38:20 -0700 Subject: [PATCH 24/32] test(oracle-gaps): the e2e shard list names the 16 s wait and the unref'd timer history The e2e shard check named the removed history 'a slow kill past Vitest's 15 s limit' and had no shard for the unref'd timer history, so CI's test:oracle-gaps:shards failed. The check now partitions the e2e tests exactly. Co-authored-by: Isaac --- packages/oracles/scripts/e2e-shards.mjs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 2e4e36f3..3fd54749 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -63,12 +63,15 @@ export const shards = { `fixed seed 20261105: a slow owner's kill and another owner's kill`, `random seed: a slow owner's kill and another owner's kill`, `determinism: constructed history: a slow owner's kill and another owner's kill`, - `fixed seed 20261105: a slow kill past Vitest's 15 s limit`, - `random seed: a slow kill past Vitest's 15 s limit`, - `determinism: constructed history: a slow kill past Vitest's 15 s limit`, + `fixed seed 20261105: a slow kill after a 16 s wait`, + `random seed: a slow kill after a 16 s wait`, + `determinism: constructed history: a slow kill after a 16 s wait`, `fixed seed 20261105: a copy waiting on a timer`, `random seed: a copy waiting on a timer`, `determinism: constructed history: a copy waiting on a timer`, + `fixed seed 20261105: a copy waiting on an unref'd timer`, + `random seed: a copy waiting on an unref'd timer`, + `determinism: constructed history: a copy waiting on an unref'd timer`, `fixed seed 20261105: a kill past the work budget in a later owner`, `random seed: a kill past the work budget in a later owner`, `determinism: constructed history: a kill past the work budget in a later owner`, From 4768203efe930538f1d3d87e4aaf80d7ab168f39 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 18:04:04 -0600 Subject: [PATCH 25/32] test(oracle-gaps): a slow wait past the horizon is no progress, not a wall-clock cut With the idle rule decided by state (M entry, 2026-11-15), a timer due beyond the copy's horizon does not keep it busy, so it is idle and cut as no progress. The model's wall-clock cut for such a wait was stale; the load experiment's 12 s copies gave the same outcome on three repeats under a concurrent run (the model owed a backstop cut). Co-authored-by: Isaac --- packages/oracles/oracle-gaps/gap-report-model-oracle.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 7061f633..6fcf2cdb 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4064,15 +4064,16 @@ function runsOf(plan, tests, sites, active) { } // A wait past the history's backstop (its `hangMs`, else the full limit) // and a margin of 6 s for three seconds per second of a one-test file - // (the file's time is under 2 s) is a backstop cut: the file's run is cut - // off, as a hang is, with no clock decision in the verdicts. + // (the file's time is under 2 s) is past the copy's horizon: its timer is + // not due inside it, so the copy is idle and cut as no progress, by state + // (M entry, 2026-11-15). Its outcome does not depend on load. const backstopMs = plan.history.hangMs ?? FULL_LIMIT_MS if (changed && t.slow && (t.slowMs ?? SLOW_MS) > backstopMs + 6000) return { t, route, hang: true, - cut: 'wall', + cut: 'state', verdicts: found, changed: true, work: after, From d1ff18b8ffa9d68a41bef375c0a50057f8c3ccaa Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 18:57:21 -0600 Subject: [PATCH 26/32] test(oracle-gaps): the model's slow-wait comment cites the M entry by name Co-authored-by: Isaac --- packages/oracles/oracle-gaps/gap-report-model-oracle.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6fcf2cdb..fd5cd88e 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4066,7 +4066,8 @@ function runsOf(plan, tests, sites, active) { // and a margin of 6 s for three seconds per second of a one-test file // (the file's time is under 2 s) is past the copy's horizon: its timer is // not due inside it, so the copy is idle and cut as no progress, by state - // (M entry, 2026-11-15). Its outcome does not depend on load. + // (M entry "idle is decided by state only", 2026-11-15). Its outcome does + // not depend on load. const backstopMs = plan.history.hangMs ?? FULL_LIMIT_MS if (changed && t.slow && (t.slowMs ?? SLOW_MS) > backstopMs + 6000) return { From 91c15ca876b6637e63d261f9091a93293d2b0e51 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 21:12:03 -0600 Subject: [PATCH 27/32] test(oracle-gaps): RED for idle counting only the test's I/O Two constructed histories, appended after the others (M "Idle counts only the test's I/O", 2026-11-15): - "a copy waiting on a pipe it opened": the test reads the stdout of a child it unrefs, so only its own pipe read is active. The child writes once the copy's sampler has judged the test 20 more times (a count, not a time). The mutant is owed killed. - "a copy beside the runner's pending write": the test writes to the console and waits on an unref'd timer past its horizon, while the runner holds its own pipe write pending through the test. The mutant is owed a timeout by no progress. Two test-only hooks hold those states: ORACLE_GAPS_TEST_SAMPLES (the sampler's count) and ORACLE_GAPS_TEST_RUNNER_WRITE (the runner's pending write, with an idle marker for reach). The idle rule itself is unchanged here, so both histories fail: the test's pipe read is cut as no progress, and the runner's write keeps its copy busy until the backstop. Co-authored-by: Isaac --- README.md | 9 ++ .../oracles/oracle-gaps/capture-setup.mjs | 44 +++++++++ .../oracle-gaps/gap-report-model-oracle.mjs | 72 +++++++++++++-- .../gap-report-oracle.property.test.mjs | 92 +++++++++++++++++-- .../oracle-gaps/history-fingerprint.json | 6 +- packages/oracles/scripts/e2e-shards.mjs | 6 ++ packages/oracles/scripts/pr-shards.mjs | 2 + .../gap-report-history-fingerprint.test.ts | 9 +- 8 files changed, 219 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6368ca79..c146c96d 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,15 @@ agent. watchdog has polled twice. The directory holds a `.begun` marker per batch and a `.reached` or `.unreached` marker per hold (`.unreached` after 60 s with no two polls). + - `ORACLE_GAPS_TEST_SAMPLES` names a directory where each worker's idle + sampler writes, to `-`, how many times it has judged the + running test. A witness holds its own I/O open until that count rises, so + the hold is a count and not a time. + - `ORACLE_GAPS_TEST_RUNNER_WRITE` names a directory. Through each test, the + runner itself holds one write pending on a local pipe whose reader never + reads, as a write that forwards a test's output can stay pending on + win32. A copy that goes idle while that write is pending writes an + `idle--` marker there. A few endings print no done line, or one more line: a SIGKILL of the CLI, or a signal that comes while Node is still starting the CLI, before its diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index c5f892a6..86ed9a15 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -24,10 +24,14 @@ // In a mutant run, fast-check stops a property at its first failure instead // of shrinking it: a kill needs one failing history, not the smallest one. import { createHash } from 'node:crypto' +import { once } from 'node:events' import fs, { appendFileSync, readFileSync } from 'node:fs' import { createRequire, syncBuiltinESMExports } from 'node:module' +import { connect, createServer } from 'node:net' +import { tmpdir } from 'node:os' import { dirname, isAbsolute, join, resolve as resolvePath } from 'node:path' import { fileURLToPath } from 'node:url' +import { threadId } from 'node:worker_threads' import { afterAll, beforeEach, chai, expect, inject, vi } from 'vitest' import { posix, writeWhole } from './paths.mjs' @@ -1045,16 +1049,29 @@ const quiet = new Set([ ]) const timersListed = (kinds) => kinds.filter((kind) => kind === `Timeout` || kind === `Immediate`).length +// Test-only hooks for the CLI's own oracle, not for users. With +// ORACLE_GAPS_TEST_SAMPLES (a directory), the sampler writes how many times +// it has judged the running test to `-` there, so a witness +// can hold its own I/O open for a count of samples rather than for a time. +// With ORACLE_GAPS_TEST_RUNNER_WRITE (a directory), the runner itself holds +// one write pending through each test, on a local pipe whose reader never +// reads, as a write that forwards output can stay pending on win32; a copy +// that goes idle while it is pending marks the directory. +const samplesAt = process.env.ORACLE_GAPS_TEST_SAMPLES +const runnerWrites = process.env.ORACLE_GAPS_TEST_RUNNER_WRITE +let held let sawIo = false if (state.capture) { let baseTimers let since let reported = false + let samples = 0 beforeEach(() => { if (continuing) return baseTimers = timersListed(process.getActiveResourcesInfo()) since = undefined sawIo = false + samples = 0 }) beforeEach((context) => { if (continuing) return @@ -1062,9 +1079,34 @@ if (state.capture) { baseTimers = undefined }) }) + if (runnerWrites) + beforeEach(async (context) => { + if (continuing) return + const name = `oracle-gaps-${process.pid}-${threadId}-${(globalThis.__oracleGapsHolds = (globalThis.__oracleGapsHolds ?? 0) + 1)}` + const server = createServer({ pauseOnConnect: true }).listen( + process.platform === `win32` + ? `\\\\.\\pipe\\${name}` + : join(tmpdir(), `${name}.sock`), + ) + await once(server, `listening`) + const accepted = once(server, `connection`) + const client = connect(server.address()) + await once(client, `connect`) + const [peer] = await accepted + client.write(Buffer.alloc(1 << 22)) + held = client + context.onTestFinished(() => { + held = undefined + client.destroy() + peer.destroy() + server.close() + }) + }) realInterval(() => { const now = realNow() if (baseTimers === undefined || reported) return + if (samplesAt) + writeWhole(join(samplesAt, `${process.pid}-${threadId}`), `${++samples}`) const kinds = process.getActiveResourcesInfo() const io = kinds.some((kind) => !quiet.has(kind)) sawIo ||= io @@ -1080,6 +1122,8 @@ if (state.capture) { since ??= now if (!state.idleMs || now - since < state.idleMs) return reported = true + if (held?.writableLength) + writeWhole(join(runnerWrites, `idle-${process.pid}-${threadId}`), ``) appendFileSync( state.capture, `${JSON.stringify({ file: thisFile, idle: true })}\n`, diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index fd5cd88e..c194c461 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -134,6 +134,14 @@ const caseless = existsSync(fileURLToPath(import.meta.url).toUpperCase()) const FULL_LIMIT_MS = 20000 // How long a slow test waits, in real milliseconds, where its output differs. const SLOW_MS = 3000 +// The `pendingIo: 'test'` witness's child (run with `node -e`, given the +// sampler's count file): it writes to its stdout once the count has risen by +// 20, and gives up after 60 s, a backstop that leaves the read pending, so the +// witness then fails loudly. +const SAMPLE_HOLD = `const read = () => { try { return Number(require('fs').readFileSync(process.argv[1], 'utf8')) } catch { return 0 } }; const from = read(); setInterval(() => { if (read() >= from + 20) process.stdout.write('go', () => process.exit(0)) }, 10); setTimeout(() => process.exit(1), 60000)` +// A wait long past any copy's horizon (the full limit and 3 s per second of +// the file's time). +const PAST_HORIZON_MS = 600_000 // The calls that each copy of `w_slowpair` makes beyond its first: its test // calls the function this many more times where its output differs. A call // evaluates two or three instrumented sites, so a copy makes 60,000 to 90,000 @@ -418,13 +426,15 @@ const testCase = fc.record({ // turn: its work could follow the clock. timed: fc.constant(false), // The constructed-only fields (see `histories`): a witness's wait where its - // output differs, its repeated calls, its load, and a repeating timer kept - // due while a hang waits. A generated test has the neutral values, which - // change nothing. + // output differs, its repeated calls, its load, a repeating timer kept due + // while a hang waits, and whose I/O is pending while it waits (`pendingIo`, + // the test's own or the runner's). A generated test has the neutral values, + // which change nothing. slowMs: fc.constant(SLOW_MS), loops: fc.constant(0), timerWait: fc.constant(null), pulse: fc.constant(false), + pendingIo: fc.constant(null), }) // Law 9's edits. Each content edit changes what some evidence depends on: // a test file, a source statement, the source's line numbers, an owns list, @@ -907,9 +917,9 @@ const keepsSteps = (history) => { // take-back reads it as the drawn constant `false` (as `claimScenario` does): a // shrink is torn-row-free. A constructed history may set `slowTwin`, which no // generated history draws, and which is taken back without it. Its tests may -// set the constructed-only fields `slowMs`, `loops`, `timerWait` and `pulse`, -// which the drawn test record holds at neutral values (the generator rule, -// 2026-11-14). +// set the constructed-only fields `slowMs`, `loops`, `timerWait`, `pulse` and +// `pendingIo`, which the drawn test record holds at neutral values (the +// generator rule, 2026-11-14). const takenBack = (t) => ({ ...t, slow: false, @@ -917,6 +927,7 @@ const takenBack = (t) => ({ loops: 0, timerWait: null, pulse: false, + pendingIo: null, }) const histories = drawn.map( (history) => (keepsSteps(history) ? history : { ...history, steps: null }), @@ -4080,6 +4091,22 @@ function runsOf(plan, tests, sites, active) { work: after, before, } + // A test that writes to the console and then waits on a timer due past + // its horizon, while the runner's own write is pending (`pendingIo: + // 'runner'`), cannot make progress of its own: the runner's I/O is not + // the test's, so the copy is idle and cut as no progress, by state (M + // entry "Idle counts only the test's I/O", 2026-11-15). + if (changed && t.pendingIo === `runner`) + return { + t, + route, + hang: true, + cut: 'state', + verdicts: found, + changed: true, + work: after, + before, + } // A loop that never ends cuts the file's run off, as a hang does. if (mutated.spun) return { @@ -4850,9 +4877,17 @@ function oracleTestFile(plan, file) { ...(plan.tests.some((t) => t.file === file && t.shrinks) ? [`import fc from 'fast-check'`] : []), - ...(plan.tests.some((t) => t.file === file && (t.ioWait || t.stops)) + ...(plan.tests.some( + (t) => t.file === file && (t.ioWait || t.stops || t.pendingIo === `test`), + ) ? [`import { spawn } from 'node:child_process'`] : []), + ...(plan.tests.some((t) => t.file === file && t.pendingIo === `test`) + ? [ + `import { join as samplesPath } from 'node:path'`, + `import { threadId } from 'node:worker_threads'`, + ] + : []), ...(plan.tests.some((t) => t.file === file && t.timed === `sleep`) ? [`import { setTimeout as sleep } from 'node:timers/promises'`] : []), @@ -5024,6 +5059,27 @@ function oracleTestFile(plan, file) { ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.timerWait.waitMs})${t.timerWait.unref ? `.unref()` : ``})`, ] : []), + // Where its output differs, a test that reads a pipe it opened: the + // stdout of a child it unrefs, so that only the read is active. The + // child writes once the copy's sampler has judged the test 20 more + // times (ORACLE_GAPS_TEST_SAMPLES), a count of samples at least 50 ms + // apart, past the 500 ms idle window, so the state is held by a count + // and not by a time. A copy that wrongly goes idle stops its sampler's + // count, so the read never ends and the copy is cut. The test then + // judges as usual. + ...(t.pendingIo === `test` + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => { const child = spawn(process.execPath, ['-e', ${JSON.stringify(SAMPLE_HOLD)}, samplesPath(process.env.ORACLE_GAPS_TEST_SAMPLES ?? '', \`\${process.pid}-\${threadId}\`)], { stdio: ['ignore', 'pipe', 'ignore'] }); child.unref(); child.stdout.once('data', r) })`, + ] + : []), + // Where its output differs, a test that writes to the console and then + // waits on an unref'd timer due long past its horizon, while the + // runner holds its own write pending (ORACLE_GAPS_TEST_RUNNER_WRITE). + ...(t.pendingIo === `runner` + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { console.log('waiting past the horizon'); await new Promise((r) => setTimeout(r, ${PAST_HORIZON_MS}).unref()) }`, + ] + : []), // A test that waits on I/O (a child process, about 3 s) where the // output differs, then judges as usual. ...(t.ioWait @@ -5090,7 +5146,7 @@ function oracleTestFile(plan, file) { : []), t.asProp ? ` fcTest.prop([fcv.integer({ min: 0, max: 10 })], { numRuns: 20, seed: 1 })('${testName(t, index)}', async (n) => {` - : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timerWait || t.timed || lazyGated(plan, file, t) || fsGated(plan, file, t) ? `async ` : ``}() => {`, + : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timerWait || t.pendingIo || t.timed || lazyGated(plan, file, t) || fsGated(plan, file, t) ? `async ` : ``}() => {`, ...partsOf(t).flatMap(campaign), ` })`, `})`, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 00b2201a..2042026f 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1866,6 +1866,17 @@ async function driver(plan, observe) { // and holds each batch's count and its witness markers. const tornPolls = `${dir}.torn-polls` if (plan.tornRows) mkdirSync(tornPolls, { recursive: true }) + // A test that holds its own I/O open for a count of the sampler's samples + // (`pendingIo: 'test'`), and the runner's own pending write beside a test + // that can make no progress (`pendingIo: 'runner'`): each hook's directory + // sits beside the package, and the runner's marks each copy that went idle + // while its write was pending. + const pendingIo = (owner) => + plan.history.tests.some((t) => t.pendingIo === owner) + const samples = `${dir}.samples` + const runnerWrites = `${dir}.runner-writes` + if (pendingIo(`test`)) mkdirSync(samples, { recursive: true }) + if (pendingIo(`runner`)) mkdirSync(runnerWrites, { recursive: true }) const how = { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), @@ -1874,16 +1885,21 @@ async function driver(plan, observe) { cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), } - const { replay, ...cli } = runCli( - dir, - files, - plan.tornRows - ? { - ...how, - env: { ORACLE_GAPS_TEST_TORN_ROWS: tornPolls }, - } - : how, - ) + const { replay, ...cli } = runCli(dir, files, { + ...how, + env: { + ...(plan.tornRows ? { ORACLE_GAPS_TEST_TORN_ROWS: tornPolls } : {}), + ...(pendingIo(`test`) ? { ORACLE_GAPS_TEST_SAMPLES: samples } : {}), + ...(pendingIo(`runner`) + ? { ORACLE_GAPS_TEST_RUNNER_WRITE: runnerWrites } + : {}), + }, + }) + const idleBesideRunnerWrite = pendingIo(`runner`) + ? readdirSync(runnerWrites).length + : 0 + rmSync(samples, { recursive: true, force: true }) + rmSync(runnerWrites, { recursive: true, force: true }) // The watchdog read a closing row while the worker wrote it: a hold // marks `reached` once two polls came after its first half. A hold that // waited 60 s with no poll marks `unreached`, and the witness fails @@ -1952,6 +1968,19 @@ async function driver(plan, observe) { states.add(`a copy waiting on a timer killed`) if (killed && plan.history.tests.some((t) => t.timerWait?.unref)) states.add(`a copy waiting on an unref'd timer killed`) + // A copy that read a pipe it opened, past the idle window by count, was + // kept busy by that read alone, and its kill decided. + if (killed && pendingIo(`test`)) + states.add(`a copy waiting on I/O it opened killed`) + // A copy went idle while the runner's own write was pending (its mark), + // and the mutant is a timeout by no progress, not a backstop gap. + if ( + idleBesideRunnerWrite > 0 && + cli.report?.results?.some( + (r) => r.outcome === `timeout` && r.cause === `no progress`, + ) + ) + states.add(`a copy beside the runner's pending write cut as no progress`) // A copy that waits past the history's backstop is cut off, and no later // owner runs, so the mutant is a timeout, not the later owner's kill. if ( @@ -3735,6 +3764,8 @@ const required = [ `a copy waiting on an unref'd timer killed`, `a kill past the work budget in a later owner does not count`, `a backstop cut ends the copies ahead of a later kill`, + `a copy waiting on I/O it opened killed`, + `a copy beside the runner's pending write cut as no progress`, `a hang in one owner and a survivor in another timed out`, `a mutant's copies past its work budget timed out`, `a heartbeat named a busy mutant`, @@ -3868,6 +3899,8 @@ const separate = [ `a copy waiting on an unref'd timer killed`, `a kill past the work budget in a later owner does not count`, `a backstop cut ends the copies ahead of a later kill`, + `a copy waiting on I/O it opened killed`, + `a copy beside the runner's pending write cut as no progress`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, `a package with no config ran`, @@ -5528,6 +5561,45 @@ oracleProperty({ }, ]), }, + // Idle counts only the test's I/O (M entry, 2026-11-15). A copy whose + // test reads a pipe it opened, the stdout of a child it unrefs, is busy + // while the read is pending, and nothing else keeps it busy: the read is + // released by a count of the sampler's samples past the idle window, not + // by a time. The mutant is killed. + { + name: `a copy waiting on a pipe it opened`, + enters: [`a copy waiting on I/O it opened killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + pendingIo: `test`, + }, + ]), + }, + // A copy whose test writes to the console and then waits on an unref'd + // timer due long past its horizon, while the runner holds its own write + // pending through the test, as Vitest's forwarded output can be on + // win32: the runner's write is not the test's I/O, so the copy is idle + // and the mutant is a timeout by no progress, not a backstop gap. The + // package has a native addon, so its tests run in child processes, whose + // output Vitest forwards over their IPC channel, as on Windows shard 7. + { + name: `a copy beside the runner's pending write`, + enters: [`a copy beside the runner's pending write cut as no progress`], + build: (generated) => ({ + ...soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + pendingIo: `runner`, + }, + ]), + native: `gyp`, + }), + }, ]), }, resolve: resolveHistory, diff --git a/packages/oracles/oracle-gaps/history-fingerprint.json b/packages/oracles/oracle-gaps/history-fingerprint.json index 1007b123..fe9b18bb 100644 --- a/packages/oracles/oracle-gaps/history-fingerprint.json +++ b/packages/oracles/oracle-gaps/history-fingerprint.json @@ -59,7 +59,8 @@ ["tests.slowMs", "33494e17be1a5ba5"], ["tests.loops", "4bc867dd43aad13c"], ["tests.timerWait", "02f66613e9490d15"], - ["tests.pulse", "04378653193a64e7"] + ["tests.pulse", "04378653193a64e7"], + ["tests.pendingIo", "02f66613e9490d15"] ], "histories": [ ["functions", "66c0e9c56951c377"], @@ -121,6 +122,7 @@ ["tests.slowMs", "33494e17be1a5ba5"], ["tests.loops", "4bc867dd43aad13c"], ["tests.timerWait", "02f66613e9490d15"], - ["tests.pulse", "04378653193a64e7"] + ["tests.pulse", "04378653193a64e7"], + ["tests.pendingIo", "02f66613e9490d15"] ] } diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 3fd54749..2aca8726 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -72,6 +72,12 @@ export const shards = { `fixed seed 20261105: a copy waiting on an unref'd timer`, `random seed: a copy waiting on an unref'd timer`, `determinism: constructed history: a copy waiting on an unref'd timer`, + `fixed seed 20261105: a copy waiting on a pipe it opened`, + `random seed: a copy waiting on a pipe it opened`, + `determinism: constructed history: a copy waiting on a pipe it opened`, + `fixed seed 20261105: a copy beside the runner's pending write`, + `random seed: a copy beside the runner's pending write`, + `determinism: constructed history: a copy beside the runner's pending write`, `fixed seed 20261105: a kill past the work budget in a later owner`, `random seed: a kill past the work budget in a later owner`, `determinism: constructed history: a kill past the work budget in a later owner`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 8b93aa95..bf7db89f 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -578,6 +578,8 @@ export const partitions = { // budget, the twin and the timer) are short, except the 16 s wait. 10: anyOf([ "a copy waiting on an unref'd timer", + 'a copy waiting on a pipe it opened', + "a copy beside the runner's pending write", "a slow owner's kill and another owner's kill", 'a backstop cut ends the copies ahead of a later kill', 'a kill past the work budget in a later owner', diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index bd8e8adf..c0dff580 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -25,12 +25,19 @@ const DRAWS = 500 // constants in every drawn test record, which no generated history sets. A // test's record hashes without them, so its line is main's, and each one is // recorded as its own line after the others, with its neutral value checked. -const CONSTRUCTED_TEST_FIELDS = [`slowMs`, `loops`, `timerWait`, `pulse`] +const CONSTRUCTED_TEST_FIELDS = [ + `slowMs`, + `loops`, + `timerWait`, + `pulse`, + `pendingIo`, +] const NEUTRAL_TEST_VALUES: Record = { slowMs: 3000, loops: 0, timerWait: null, pulse: false, + pendingIo: null, } type Sample = Record From a9df3f4b95eda7350883b32151a0d46991594f96 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 21:16:53 -0600 Subject: [PATCH 28/32] test(oracle-gaps): the runner-write witness reads its cut from the result's detail Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 2042026f..fbbd0be4 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1977,7 +1977,7 @@ async function driver(plan, observe) { if ( idleBesideRunnerWrite > 0 && cli.report?.results?.some( - (r) => r.outcome === `timeout` && r.cause === `no progress`, + (r) => r.outcome === `timeout` && r.detail === `no progress`, ) ) states.add(`a copy beside the runner's pending write cut as no progress`) From 1bb0b982020f70808d3c00d897e717e3f8420e20 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 21:32:17 -0600 Subject: [PATCH 29/32] fix(oracle-gaps): idle counts only the test's own I/O M "Idle counts only the test's I/O" (2026-11-15). The idle sampler no longer reads I/O from resource names (the `quiet` list). It records, once per worker, which code started each handle and request (async_hooks init): the test's when the test's or production's code is on the stack then, or when the resource that triggered it is the test's. Node triggers a write by the stream it writes to, so the runner's own pending write (Vitest forwarding output over its IPC pipe on Windows, the SimpleWriteWrap of Windows shard 7) is the runner's, and keeps neither a copy busy nor a file in the I/O set. The same rule holds on every OS. The test's own pipe read keeps its copy busy, where the name list took a PipeWrap as the runner's. Both witnesses of the previous commit pass. Co-authored-by: Isaac --- .changeset/gap-cli-runtime.md | 2 +- README.md | 6 +- .../oracles/oracle-gaps/capture-setup.mjs | 87 ++++++++++++++----- packages/oracles/oracle-gaps/check.mjs | 7 +- .../gap-report-oracle.property.test.mjs | 2 +- 5 files changed, 76 insertions(+), 28 deletions(-) diff --git a/.changeset/gap-cli-runtime.md b/.changeset/gap-cli-runtime.md index 88e2691c..a32a19cd 100644 --- a/.changeset/gap-cli-runtime.md +++ b/.changeset/gap-cli-runtime.md @@ -4,7 +4,7 @@ The `oracle-gaps` CLI bounds a mutant's copies by counts, not by wall-clock limits, reports long runs, stops the processes it started, and writes nothing beside the tests. -- A mutant's copies share a work budget, counted in site evaluations and not in time: the larger of 100,000 and ten times their owner files' unmutated work. A mutant past it is a `timeout` gap. No wall-clock limit decides an outcome. A copy cut by its call count (`spin`) or by no progress ends only itself, a copy cut by the backstop (`--hang-limit`, 60 s by default, plus 3 s per second of its file's time) ends the mutant's later copies, and every timeout names its cause in `results.json` and on its gap line. `--mutant-limit` and `--first-limit` are retired: they are checked and noted, and change nothing. +- A mutant's copies share a work budget, counted in site evaluations and not in time: the larger of 100,000 and ten times their owner files' unmutated work. A mutant past it is a `timeout` gap. No wall-clock limit decides an outcome. A copy cut by its call count (`spin`) or by no progress ends only itself, a copy cut by the backstop (`--hang-limit`, 60 s by default, plus 3 s per second of its file's time) ends the mutant's later copies, and every timeout names its cause in `results.json` and on its gap line. A copy makes no progress when no timer of it is due before its backstop and none of its test's own I/O is pending; I/O the runner makes for itself, such as a pending write that forwards the test's console output on Windows, does not keep it busy. `--mutant-limit` and `--first-limit` are retired: they are checked and noted, and change nothing. - While one copy of a test file runs for 30 s or more, the CLI prints `busy: in for s` every 30 s (`baseline` for a run with no mutant). Through `--daemon` the heartbeats arrive when the check ends. - The check runs in a process group of its own, with every Vitest worker and every process a test starts, except one started detached in a session of its own. On SIGINT or SIGTERM the CLI signals the group, kills what is left 2 s later, and exits 130 or 143. Before, a SIGTERM left `forks` workers running. On Windows the run is not supervised. - The runner's copies of the test files exist only in memory. Vitest's read of a copy for its per-file pragmas (`@vitest-environment`, `@module-tag`) is served its text, so a copy runs in its file's environment. The capture setup and Vitest configs are in a session directory of its own while it runs, and one that a stopped run left is removed by the next run. Before, the copies and configs sat beside the tests and the package's config, where a Vitest run started during a check found them and then failed when they were removed. diff --git a/README.md b/README.md index c146c96d..e383b989 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,10 @@ agent. copies prints none, and a check through `--daemon` prints its heartbeats only when the check ends. No wall-clock limit decides an outcome. A copy is cut in one of three ways: by its call count, when it runs past its - call budget (`spin`); by its state, when it makes no progress (`no progress`); - or by the backstop, when it runs past `--hang-limit` (60 s by default) plus + call budget (`spin`); by its state, when it makes no progress (`no progress`: + no timer is due before its backstop, and none of its test's own I/O is + pending, while I/O the runner makes for itself, such as forwarding the + test's console output, does not count); or by the backstop, when it runs past `--hang-limit` (60 s by default) plus 3 s per second of its file's unmutated time (`backstop`). A spin or no progress ends only its own copy, and a backstop ends the mutant's later copies. The copies of one mutant share a work budget, counted in site diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 86ed9a15..674ee56b 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -23,6 +23,7 @@ // // In a mutant run, fast-check stops a property at its first failure instead // of shrinking it: a kill needs one failing history, not the smallest one. +import { createHook } from 'node:async_hooks' import { createHash } from 'node:crypto' import { once } from 'node:events' import fs, { appendFileSync, readFileSync } from 'node:fs' @@ -891,12 +892,10 @@ const SEP = process.platform === `win32` ? `[\\\\/]` : `/` const machinery = new RegExp( `${SEP}node_modules${SEP}(?:vitest|@vitest|fast-check|@fast-check|tinypool|tinyspy|chai|@tanstack${SEP}oracles)${SEP}|${SEP}packages${SEP}oracles${SEP}src${SEP}`, ) -function fromTest() { - const limit = Error.stackTraceLimit - Error.stackTraceLimit = 50 - const stack = String(new Error().stack) - Error.stackTraceLimit = limit - return stack +// Whether a stack names the test's or production's code: a frame outside +// Node, this file, and the machinery. +const namesTest = (stack) => + String(stack) .split(`\n`) .slice(1) .some((frame) => { @@ -910,6 +909,12 @@ function fromTest() { !machinery.test(file) ) }) +function fromTest() { + const limit = Error.stackTraceLimit + Error.stackTraceLimit = 50 + const stack = new Error().stack + Error.stackTraceLimit = limit + return namesTest(stack) } globalThis.__oracleGapsClock = () => { if (inTest && !clocked && fromTest()) clocked = true @@ -1029,24 +1034,64 @@ function finishTest(context) { const thisFile = globalThis.__vitest_worker__?.filepath -// I/O and idleness (maintainer decision, 2026-11-07; state only, 2026-11-15). -// Each test's row says whether the test ever had an I/O handle open: anything -// but the runner's own channel, stdio, and timers. A copy given `idleMs` (a -// file whose unmutated tests did no I/O) is idle, by state alone, when no -// counted timer is due within its horizon, no more timers are listed than -// when its test began (which covers timers made outside the wrappers, such as -// `node:timers/promises`), and nothing but the channel and stdio is open. A +// I/O and idleness (maintainer decision, 2026-11-07; state only, 2026-11-15; +// only the test's I/O, 2026-11-15). Each test's row says whether the test had +// I/O of its own active: a handle or request that the test's or production's +// code started, never one the runner made for itself (its channel, stdio, a +// write that forwards the test's console output). A copy given `idleMs` (a +// file whose unmutated tests had no I/O of their own) is idle, by state alone, +// when no counted timer is due within its horizon, no more timers are listed +// than when its test began (which covers timers made outside the wrappers, +// such as `node:timers/promises`), and none of its test's I/O is active. A // busy synchronous or microtask loop blocks this sampler, so it cannot look // idle; it ends by its call budget or the backstop. After `idleMs` of that // state it writes an idle row, and the batch's watchdog cuts it off as no // progress. -const quiet = new Set([ - `MessagePort`, - `PipeWrap`, - `TTYWrap`, - `Timeout`, - `Immediate`, -]) +// +// Who started each handle and request is recorded once per worker, when Node +// makes it: the test, when the test's or production's code is on the stack +// then (`namesTest`), or when the resource that triggered it is the test's. +// Node triggers a write by the stream it writes to, so the runner's write on +// its own channel is the runner's even when a test's console.log led to it. +// The stack is read only when the sampler meets the resource, or when the +// resource triggers another. Promises are not recorded. +const startedBy = (globalThis.__oracleGapsStartedBy ??= (() => { + const records = new WeakMap() + const live = new Map() + createHook({ + init(id, type, trigger, resource) { + if (type === `PROMISE`) return + const parent = live.get(trigger) + const record = {} + if (parent !== undefined && owned(parent)) record.test = true + else { + const limit = Error.stackTraceLimit + Error.stackTraceLimit = 50 + Error.captureStackTrace(record) + Error.stackTraceLimit = limit + } + live.set(id, record) + records.set(resource, record) + }, + destroy(id) { + live.delete(id) + }, + trackPromises: false, + }).enable() + return records +})()) +function owned(record) { + return (record.test ??= namesTest(record.stack)) +} +// Node lists a handle by its owner (a socket, a child process), and records +// the handle itself. +const testsIo = () => + [...process._getActiveRequests(), ...process._getActiveHandles()].some( + (resource) => { + const record = startedBy.get(resource) ?? startedBy.get(resource?._handle) + return record !== undefined && owned(record) + }, + ) const timersListed = (kinds) => kinds.filter((kind) => kind === `Timeout` || kind === `Immediate`).length // Test-only hooks for the CLI's own oracle, not for users. With @@ -1108,7 +1153,7 @@ if (state.capture) { if (samplesAt) writeWhole(join(samplesAt, `${process.pid}-${threadId}`), `${++samples}`) const kinds = process.getActiveResourcesInfo() - const io = kinds.some((kind) => !quiet.has(kind)) + const io = testsIo() sawIo ||= io // A timer due after the copy's full time limit cannot end its wait in // time, so it does not keep the copy busy: a test waiting on it would be diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 50e034b0..4137c9b9 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -2067,9 +2067,10 @@ export default async (env) => { valueChanges.get(file).add(number) } const unmutatedUnits = new Map(first.units.map((u) => [u.file, u])) - // Files whose unmutated tests had an I/O handle open (a socket, a child - // process, file I/O): a test there may wait on something outside the - // worker, so no copy of it is cut for idleness (capture-setup.mjs). + // Files whose unmutated tests had I/O of their own open (a socket, a child + // process, file I/O, never the runner's): a test there may wait on + // something outside the worker, so no copy of it is cut for idleness + // (capture-setup.mjs). const ioFiles = new Set() for (const rows of [first.rows, second.rows]) for (const [key, row] of rows) if (row.io) ioFiles.add(fileOf(key)) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index fbbd0be4..0b46c7c6 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -4020,7 +4020,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds a mutant's copies by one work budget per mutant (site evaluations, never time) and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its backstop, since a copy has no first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant's work budget is witnessed by w_slowpair and a kill past it, in owner order, each owner's copy counted in site evaluations (not in time), and authored mutants have no copies to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Windows idle gap (recorded, not witnessed): the state \`a hang cut off for idleness\` is required on every platform, and win32 reaches it (run 43683794816, job 132719872988), so it is not platform-scoped. The sampler (capture-setup.mjs) counts a copy idle only after 500 ms with no late tick, no non-quiet handle, utilization at most 5%, and no timer due, so \`every outcome class\` reaches the state only when the sampler sees that window. On win32 the check missed it once (run 43683794816, job 132719872940); the cause is not confirmed (a late tick on a shared runner, or a stdio handle outside \`quiet\` for this file's console output, are the candidates). Its witness, a test-only hold at the hang until the window is seen, is owed, and that is a timing dependency the determinism-under-load rule forbids; the maintainer decides whether it lands before the Windows job becomes blocking.`, + limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its backstop, since a copy has no first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant's work budget is witnessed by w_slowpair and a kill past it, in owner order, each owner's copy counted in site evaluations (not in time), and authored mutants have no copies to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Idle and the runner's I/O (M entries "idle is decided by state only" and "Idle counts only the test's I/O", 2026-11-15): the sampler (capture-setup.mjs) counts a copy idle after 500 ms with no timer due inside its horizon, no more timers listed than at its test's start, and none of its test's own I/O active, where a handle or request is the test's when the test's or production's code was on the stack when Node made it, or when the resource that triggered it is the test's. Two constructed histories hold the states by count and by state: \`a copy waiting on a pipe it opened\` (the test's own pipe read, released after 20 samples) and \`a copy beside the runner's pending write\` (a write the runner holds pending through the test, as Vitest's forwarded output can stay pending on win32; Windows shard 7, run 43778323034). Not witnessed: a handle whose owner keeps its wrap elsewhere than \`_handle\` (a dgram socket) is not found as the test's, so a test that waits only on it looks idle; a test that awaits only a reply over the runner's own channel (a file snapshot) looks idle on every platform, as it did on POSIX before; a request whose stack frames all lie in node_modules machinery (vitest, fast-check, chai, this package) is the runner's even when a test's call led to it; and a timer made through the wrappers counts in the listed-timer check whatever its due time, so a ref'd timer past the horizon keeps a copy busy unless the listed count at the test's start held a timer of the runner's, the reason the runner-write witness waits on an unref'd timer.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs From e4f5c9fec72b84d5a8d92d4ce52847439c412355 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 21:59:36 -0600 Subject: [PATCH 30/32] scratch: run only the Windows jobs, for #99's idle-ownership check (not for the PR) Co-authored-by: Isaac --- .github/workflows/pr.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index edf59139..0041e002 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,6 +20,7 @@ jobs: # 21 minutes when they ran as one job. Each job should stay well under the # 10-minute limit; pr-shards.mjs records how each suite is split. test: + if: false name: Test runs-on: ubuntu-latest steps: @@ -46,6 +47,7 @@ jobs: - run: pnpm run test:publint - run: pnpm run test:sherif test-eslint-plugin: + if: false name: Test eslint-plugin-oracles runs-on: ubuntu-latest steps: @@ -60,6 +62,7 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: + if: false name: Framework self-oracle (${{ matrix.shard }}/8) runs-on: ubuntu-latest strategy: @@ -81,6 +84,7 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-fast: + if: false name: Gap report oracles (${{ matrix.shard }}/3) runs-on: ubuntu-latest strategy: @@ -106,6 +110,7 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-end-to-end: + if: false name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) runs-on: ubuntu-latest strategy: @@ -126,6 +131,7 @@ jobs: env: SHARD: ${{ matrix.shard }} test-vitest-5: + if: false name: Test on Vitest 5 (${{ matrix.suite }}) # The package's vitest peer range includes 5; this job is the evidence. # The unit tests and each self-oracle shard are their own jobs. @@ -159,6 +165,7 @@ jobs: env: SUITE: ${{ matrix.suite }} preview: + if: false name: Publish preview # Only a commit that passes every check gets an installable preview. needs: @@ -190,6 +197,7 @@ jobs: # npm token, so npm trusted publishing does not apply. - run: pnpm dlx pkg-pr-new@0.0.92 publish --pnpm './packages/*' # zizmor: ignore[use-trusted-publishing] zizmor: + if: false name: zizmor runs-on: ubuntu-latest steps: From 8183c12e62b0719ab5e0274420f32ad0cc11deb6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 22:34:27 -0600 Subject: [PATCH 31/32] test(oracle-gaps): readGapLine reads a timeout's cause, judged against results.json #115 (merged as eec7f77) reads each stdout gap line against results.json and fails loudly on any text after the change, so #99's ` (cause: )` suffix on timeout lines did not read: on the merge, "a copy beside the runner's pending write" fails with `does not read as a kind, a line, an id, and a change`. #99 lands second, so it extends the reader: a timeout's line names its cause after its change, equal to results.json's `detail`, and no other line names one. The calibration "a stdout gap line names another cause" swaps one timeout line's cause (backstop and no progress); the stream law rejects it. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 61 ++++++++++++++++--- packages/oracles/scripts/e2e-shards.mjs | 1 + 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index d14deee8..898a1c1a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -3114,7 +3114,9 @@ function streamBreach(checkpoint, run, expected, plan) { * `, and it shows the gap's kind (its class, or its outcome), its * line (`?` when the mutant never applied), and its change: two code spans * around ` → `, the code and its replacement, or the intent's span and - * ` (authored by )`. A span shows its code on one line, with each + * ` (authored by )`. A timeout's line then names its cause, the cut + * that ended it, as ` (cause: )`, which is results.json's `detail`, + * and no other line names one. A span shows its code on one line, with each * run of whitespace as one space, whole or cut and ended with `...`; the * cut's length is the printer's choice. The comparison is by id, so a run * that prints one gap twice and drops another breaks it, though its count @@ -3149,6 +3151,7 @@ function stdoutGapBreach(stdoutGaps, results) { const owed = { kind: gap.class ?? gap.outcome, line: gap.line === undefined ? `?` : String(gap.line), + cause: gap.outcome === `timeout` ? gap.detail : undefined, } const shows = gap.authored ? read.intent !== undefined && @@ -3157,9 +3160,14 @@ function stdoutGapBreach(stdoutGaps, results) { : read.original !== undefined && showsCode(read.original, gap.original) && showsCode(read.replacement, gap.replacement) - if (read.kind !== owed.kind || read.line !== owed.line || !shows) + if ( + read.kind !== owed.kind || + read.line !== owed.line || + read.cause !== owed.cause || + !shows + ) return { - difference: `the stdout's gap line for ${read.id} is ${JSON.stringify(line)}, and results.json reports it as ${owed.kind} at line ${owed.line}, changing ${JSON.stringify(gap.authored ? gap.intent : `${gap.original} → ${gap.replacement}`)}`, + difference: `the stdout's gap line for ${read.id} is ${JSON.stringify(line)}, and results.json reports it as ${owed.kind}${owed.cause === undefined ? `` : ` (cause: ${owed.cause})`} at line ${owed.line}, changing ${JSON.stringify(gap.authored ? gap.intent : `${gap.original} → ${gap.replacement}`)}`, extra: { actual: line, expected: { ...owed, id: read.id } }, } } @@ -3174,7 +3182,8 @@ function stdoutGapBreach(stdoutGaps, results) { /** * A gap line's fields, read from its text as an agent reads it: its kind, - * line, id, and change. Undefined when the line does not read so. + * line, id, change, and a timeout's cause (` (cause: )` after the + * change). Undefined when the line does not read so. */ function readGapLine(line) { const head = line.match(/^oracle-gaps: gap ([\w-]+) L(\d+|\?) (\S+) (.*)$/) @@ -3182,18 +3191,31 @@ function readGapLine(line) { const [, kind, at, id, change] = head const first = readCodeSpan(change) if (!first) return undefined - const authored = first.after.match(/^ \(authored by (.*)\)$/) + // What follows the change's last part: nothing, or the cause of a timeout. + const ending = (after) => after.match(/^(.*?)(?: \(cause: ([^()]+)\))?$/) + const [, afterIntent, intentCause] = ending(first.after) + const authored = afterIntent.match(/^ \(authored by (.*)\)$/) if (authored) - return { kind, line: at, id, intent: first.code, author: authored[1] } + return { + kind, + line: at, + id, + intent: first.code, + author: authored[1], + cause: intentCause, + } if (!first.after.startsWith(` → `)) return undefined const second = readCodeSpan(first.after.slice(` → `.length)) - if (!second || second.after !== ``) return undefined + if (!second) return undefined + const [, rest, cause] = ending(second.after) + if (rest !== ``) return undefined return { kind, line: at, id, original: first.code, replacement: second.code, + cause, } } @@ -3778,6 +3800,29 @@ calibrations.push( return { ...cli, stdoutGaps: [renamed, ...rest] } }), }, + { + name: `a stdout gap line names another cause`, + law: `greppable stream`, + // A timeout's line whose cause is another cut than results.json's + // `detail`: the line still reads, so only the cause comparison rejects it. + corrupt: corruptCli((cli) => { + const at = cli.stdoutGaps.findIndex((line) => + / \(cause: [^()]+\)$/.test(line), + ) + if (cli.exit !== 0 || at < 0) return undefined + const recaused = cli.stdoutGaps[at].replace( + / \(cause: ([^()]+)\)$/, + (_, cause) => + cause === `backstop` ? ` (cause: no progress)` : ` (cause: backstop)`, + ) + return { + ...cli, + stdoutGaps: cli.stdoutGaps.map((line, i) => + i === at ? recaused : line, + ), + } + }), + }, { name: `a stdout gap line does not read as a gap`, law: `greppable stream`, @@ -4277,7 +4322,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds a mutant's copies by one work budget per mutant (site evaluations, never time) and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A line with any text after its change does not read and fails loudly: the \` (cause: )\` suffix that #99 adds to timeout lines is not read yet, so whichever of #99 and #115 lands second extends readGapLine. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its backstop, since a copy has no first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant's work budget is witnessed by w_slowpair and a kill past it, in owner order, each owner's copy counted in site evaluations (not in time), and authored mutants have no copies to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Idle and the runner's I/O (M entries "idle is decided by state only" and "Idle counts only the test's I/O", 2026-11-15): the sampler (capture-setup.mjs) counts a copy idle after 500 ms with no timer due inside its horizon, no more timers listed than at its test's start, and none of its test's own I/O active, where a handle or request is the test's when the test's or production's code was on the stack when Node made it, or when the resource that triggered it is the test's. Two constructed histories hold the states by count and by state: \`a copy waiting on a pipe it opened\` (the test's own pipe read, released after 20 samples) and \`a copy beside the runner's pending write\` (a write the runner holds pending through the test, as Vitest's forwarded output can stay pending on win32; Windows shard 7, run 43778323034). Not witnessed: a handle whose owner keeps its wrap elsewhere than \`_handle\` (a dgram socket) is not found as the test's, so a test that waits only on it looks idle; a test that awaits only a reply over the runner's own channel (a file snapshot) looks idle on every platform, as it did on POSIX before; a request whose stack frames all lie in node_modules machinery (vitest, fast-check, chai, this package) is the runner's even when a test's call led to it; and a timer made through the wrappers counts in the listed-timer check whatever its due time, so a ref'd timer past the horizon keeps a copy busy unless the listed count at the test's start held a timer of the runner's, the reason the runner-write witness waits on an unref'd timer.`, + limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A timeout's line names its cause as \` (cause: )\` after its change, judged equal to results.json's \`detail\`; a line with any other text after its change does not read and fails loudly. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its backstop, since a copy has no first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant's work budget is witnessed by w_slowpair and a kill past it, in owner order, each owner's copy counted in site evaluations (not in time), and authored mutants have no copies to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Idle and the runner's I/O (M entries "idle is decided by state only" and "Idle counts only the test's I/O", 2026-11-15): the sampler (capture-setup.mjs) counts a copy idle after 500 ms with no timer due inside its horizon, no more timers listed than at its test's start, and none of its test's own I/O active, where a handle or request is the test's when the test's or production's code was on the stack when Node made it, or when the resource that triggered it is the test's. Two constructed histories hold the states by count and by state: \`a copy waiting on a pipe it opened\` (the test's own pipe read, released after 20 samples) and \`a copy beside the runner's pending write\` (a write the runner holds pending through the test, as Vitest's forwarded output can stay pending on win32; Windows shard 7, run 43778323034). Not witnessed: a handle whose owner keeps its wrap elsewhere than \`_handle\` (a dgram socket) is not found as the test's, so a test that waits only on it looks idle; a test that awaits only a reply over the runner's own channel (a file snapshot) looks idle on every platform, as it did on POSIX before; a request whose stack frames all lie in node_modules machinery (vitest, fast-check, chai, this package) is the runner's even when a test's call led to it; and a timer made through the wrappers counts in the listed-timer check whatever its due time, so a ref'd timer past the horizon keeps a copy busy unless the listed count at the test's start held a timer of the runner's, the reason the runner-write witness waits on an unref'd timer.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 5f92e875..e006d8fc 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -285,6 +285,7 @@ export const shards = { `calibration: the stdout drops a gap line`, `calibration: a stdout gap line shows another change`, `calibration: a stdout gap line names another kind`, + `calibration: a stdout gap line names another cause`, `calibration: a stdout gap line does not read as a gap`, `calibration: the stdout has a gap line for a mutant that is not a gap`, `calibration: a check through the daemon drops a gap line`, From 325aeb1077cb771da9d8832a0cdb602c329f7123 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 15 Nov 2026 22:43:59 -0600 Subject: [PATCH 32/32] test(oracle-gaps): every run of an I/O-owner history sets its hooks The random-seed history of "a copy waiting on a pipe it opened" also runs a partial check, which ran without ORACLE_GAPS_TEST_SAMPLES: its child never saw the count rise, so the pipe read held the copy busy until the backstop and the partial run judged a timeout where the full run killed (law: baseline lead). The hooks now ride on the history's shared run options, so the partial run, the slow twin and the stability rerun set them too; their directories are removed with the scratch package. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 898a1c1a..9e8c82de 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1885,7 +1885,8 @@ async function driver(plan, observe) { // (`pendingIo: 'test'`), and the runner's own pending write beside a test // that can make no progress (`pendingIo: 'runner'`): each hook's directory // sits beside the package, and the runner's marks each copy that went idle - // while its write was pending. + // while its write was pending. Every run of the history sets the hooks (its + // partial run, twin and stability rerun too), since the test needs them. const pendingIo = (owner) => plan.history.tests.some((t) => t.pendingIo === owner) const samples = `${dir}.samples` @@ -1899,22 +1900,26 @@ async function driver(plan, observe) { hangMs: plan.history.hangMs, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), - } - const { replay, ...cli } = runCli(dir, files, { - ...how, env: { - ...(plan.tornRows ? { ORACLE_GAPS_TEST_TORN_ROWS: tornPolls } : {}), ...(pendingIo(`test`) ? { ORACLE_GAPS_TEST_SAMPLES: samples } : {}), ...(pendingIo(`runner`) ? { ORACLE_GAPS_TEST_RUNNER_WRITE: runnerWrites } : {}), }, - }) + } + const { replay, ...cli } = runCli( + dir, + files, + plan.tornRows + ? { + ...how, + env: { ...how.env, ORACLE_GAPS_TEST_TORN_ROWS: tornPolls }, + } + : how, + ) const idleBesideRunnerWrite = pendingIo(`runner`) ? readdirSync(runnerWrites).length : 0 - rmSync(samples, { recursive: true, force: true }) - rmSync(runnerWrites, { recursive: true, force: true }) // The watchdog read a closing row while the worker wrote it: a hold // marks `reached` once two polls came after its first half. A hold that // waited 60 s with no poll marks `unreached`, and the witness fails @@ -2461,6 +2466,8 @@ async function driver(plan, observe) { for (const state of states) observe(`reach`, state) } finally { if (copy) rmSync(copy, { recursive: true, force: true }) + rmSync(`${dir}.samples`, { recursive: true, force: true }) + rmSync(`${dir}.runner-writes`, { recursive: true, force: true }) removePath(`${dir}.link`) written.delete(dir) try {