diff --git a/.changeset/gap-cli-runtime.md b/.changeset/gap-cli-runtime.md index 9807d437..a32a19cd 100644 --- a/.changeset/gap-cli-runtime.md +++ b/.changeset/gap-cli-runtime.md @@ -2,9 +2,9 @@ '@tanstack/oracles': minor --- -The `oracle-gaps` CLI bounds a mutant's confirming runs, reports long runs, stops the processes it started, and writes nothing beside the tests. +The `oracle-gaps` CLI bounds a mutant's copies by counts, not by wall-clock limits, reports long runs, stops the processes it started, and writes nothing beside the tests. -- Each mutant has a budget for the runs that confirm its cut-off copies: the larger of the new `--mutant-limit` (30 s by default) and ten times its owner files' unmutated time. The confirmations run in the owner files' order, each with at most what the mutant's earlier confirmations left, so the outcome depends only on the recorded runs, not on their order or on reuse through the daemon. A mutant whose confirmations pass the budget is a `timeout` gap. With the default limits a single owner's confirmation always fits; the budget binds when several owners need long confirmations, or when `--hang-limit` is above it, which the run notes. To give one copy more time with `--hang-limit`, raise `--mutant-limit` too. +- A mutant's copies share a work budget, counted in site evaluations and not in time: the larger of 100,000 and ten times their owner files' unmutated work. A mutant past it is a `timeout` gap. No wall-clock limit decides an outcome. A copy cut by its call count (`spin`) or by no progress ends only itself, a copy cut by the backstop (`--hang-limit`, 60 s by default, plus 3 s per second of its file's time) ends the mutant's later copies, and every timeout names its cause in `results.json` and on its gap line. A copy makes no progress when no timer of it is due before its backstop and none of its test's own I/O is pending; I/O the runner makes for itself, such as a pending write that forwards the test's console output on Windows, does not keep it busy. `--mutant-limit` and `--first-limit` are retired: they are checked and noted, and change nothing. - While one copy of a test file runs for 30 s or more, the CLI prints `busy: in for s` every 30 s (`baseline` for a run with no mutant). Through `--daemon` the heartbeats arrive when the check ends. - The check runs in a process group of its own, with every Vitest worker and every process a test starts, except one started detached in a session of its own. On SIGINT or SIGTERM the CLI signals the group, kills what is left 2 s later, and exits 130 or 143. Before, a SIGTERM left `forks` workers running. On Windows the run is not supervised. - The runner's copies of the test files exist only in memory. Vitest's read of a copy for its per-file pragmas (`@vitest-environment`, `@module-tag`) is served its text, so a copy runs in its file's environment. The capture setup and Vitest configs are in a session directory of its own while it runs, and one that a stopped run left is removed by the next run. Before, the copies and configs sat beside the tests and the package's config, where a Vitest run started during a check found them and then failed when they were removed. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index edf59139..0041e002 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,6 +20,7 @@ jobs: # 21 minutes when they ran as one job. Each job should stay well under the # 10-minute limit; pr-shards.mjs records how each suite is split. test: + if: false name: Test runs-on: ubuntu-latest steps: @@ -46,6 +47,7 @@ jobs: - run: pnpm run test:publint - run: pnpm run test:sherif test-eslint-plugin: + if: false name: Test eslint-plugin-oracles runs-on: ubuntu-latest steps: @@ -60,6 +62,7 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: + if: false name: Framework self-oracle (${{ matrix.shard }}/8) runs-on: ubuntu-latest strategy: @@ -81,6 +84,7 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-fast: + if: false name: Gap report oracles (${{ matrix.shard }}/3) runs-on: ubuntu-latest strategy: @@ -106,6 +110,7 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-end-to-end: + if: false name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) runs-on: ubuntu-latest strategy: @@ -126,6 +131,7 @@ jobs: env: SHARD: ${{ matrix.shard }} test-vitest-5: + if: false name: Test on Vitest 5 (${{ matrix.suite }}) # The package's vitest peer range includes 5; this job is the evidence. # The unit tests and each self-oracle shard are their own jobs. @@ -159,6 +165,7 @@ jobs: env: SUITE: ${{ matrix.suite }} preview: + if: false name: Publish preview # Only a commit that passes every check gets an installable preview. needs: @@ -190,6 +197,7 @@ jobs: # npm token, so npm trusted publishing does not apply. - run: pnpm dlx pkg-pr-new@0.0.92 publish --pnpm './packages/*' # zizmor: ignore[use-trusted-publishing] zizmor: + if: false name: zizmor runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 94e550b6..e383b989 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,22 @@ agent. While one copy of a test file runs for 30 s or more, the CLI prints `oracle-gaps: busy in for s` every 30 s. A run of many fast copies prints none, and a check through `--daemon` prints its heartbeats - only when the check ends. The runs that confirm a mutant's cut-off copies - share one budget: the larger of `--mutant-limit` (30 s by default) and ten - times the owner files' unmutated time. They run in the owner files' - order, each with at most what its mutant's earlier confirmations left. A - mutant whose confirmations pass the budget is a `timeout` gap. With the - default limits one owner's confirmation always fits; the budget binds - when several owners need long confirmations, or when `--hang-limit` is - above it, which the run notes. To give one copy more time with - `--hang-limit`, raise `--mutant-limit` too. + only when the check ends. No wall-clock limit decides an outcome. A copy + is cut in one of three ways: by its call count, when it runs past its + call budget (`spin`); by its state, when it makes no progress (`no progress`: + no timer is due before its backstop, and none of its test's own I/O is + pending, while I/O the runner makes for itself, such as forwarding the + test's console output, does not count); or by the backstop, when it runs past `--hang-limit` (60 s by default) plus + 3 s per second of its file's unmutated time (`backstop`). A spin or no + progress ends only its own copy, and a backstop ends the mutant's later + copies. The copies of one mutant share a work budget, counted in site + evaluations and not in time: the larger of 100,000 and ten times their owner + files' unmutated work. Past it, the mutant is a `timeout`. A timeout names + its cause in `results.json` (`detail`) and on its gap line, as + `(cause: )`, where the cause is the first cut in owner order, or + `work budget`. A kill in another owner decides a mutant unless a backstop + ended its copies first. `--first-limit` and `--mutant-limit` are retired: + they are checked and noted, and change nothing. The report, on stdout and in `summary.md`, leads with the baseline: each test file's result with its test counts (ordinary files too), and for a @@ -106,7 +113,7 @@ agent. | `baseline` | `oracle-gaps: baseline pass tests/a.test.ts: 3 tests`, `oracle-gaps: baseline fail tests/b.test.ts: 1 of 2 tests failed` | | `baseline` (decisive, before any mutant) | `oracle-gaps: baseline all 4 oracle files pass`, or `oracle-gaps: baseline 1 of 4 oracle files failed (provisional)` | | `blocked` | `oracle-gaps: blocked fails with no mutant active at , so the mutants it owns wait for it: …` | - | `gap` | `oracle-gaps: gap L ` | + | `gap` | `oracle-gaps: gap L `, and for a `timeout` ` (cause: )` | | `progress` | `oracle-gaps: progress judged 25 of 120 mutants: 7 gaps so far, 18 not gaps` | | `busy` | `oracle-gaps: busy in for s` | | `note` | `oracle-gaps: note …`: the `--hang-limit` note, Vite's own messages (`note vite: …`), ordinary files left out, the provisional count, unowned counts, the results' path, and where the tests' console output went (`note test output in `) | @@ -157,6 +164,15 @@ agent. watchdog has polled twice. The directory holds a `.begun` marker per batch and a `.reached` or `.unreached` marker per hold (`.unreached` after 60 s with no two polls). + - `ORACLE_GAPS_TEST_SAMPLES` names a directory where each worker's idle + sampler writes, to `-`, how many times it has judged the + running test. A witness holds its own I/O open until that count rises, so + the hold is a count and not a time. + - `ORACLE_GAPS_TEST_RUNNER_WRITE` names a directory. Through each test, the + runner itself holds one write pending on a local pipe whose reader never + reads, as a write that forwards a test's output can stay pending on + win32. A copy that goes idle while that write is pending writes an + `idle--` marker there. A few endings print no done line, or one more line: a SIGKILL of the CLI, or a signal that comes while Node is still starting the CLI, before its diff --git a/packages/oracles/oracle-gaps/args.mjs b/packages/oracles/oracle-gaps/args.mjs index d8f7d02f..cd3c8b3a 100644 --- a/packages/oracles/oracle-gaps/args.mjs +++ b/packages/oracles/oracle-gaps/args.mjs @@ -39,10 +39,18 @@ export const usage = `Usage: oracle-gaps --package --source [--env-prefix ] [--daemon] [--ordinary] [--seed ] [--mutants ] [--out ] [--only ] [--diff-base ] [--test-timeout ] [--sample ] [--contexts ] [--pool threads|forks] [--config ] - [--first-limit ] [--hang-limit ] [--mutant-limit ] + [--hang-limit ] [--first-limit ] [--mutant-limit ] oracle-gaps --stop-daemon --package oracle-gaps --help +Defaults: --env-prefix ORACLE; --test-timeout 65000 (Vitest's per-test limit, +in ms); --hang-limit 60000 (the base of each copy's backstop, in ms, plus 3 s +per second of its file's unmutated time); --contexts the machine's parallelism +minus 2, at most 16; --pool threads (forks when a package has a native +addon); --sample and --only off; --out a directory under the package's state. +Retired: --first-limit and --mutant-limit are checked and print a note, and +change nothing. A copy has no first budget, and a mutant's work budget bounds it. + The paths in --source and --config are relative to --package. Without --config, the run loads the Vitest config that Vitest discovers for the package: in its directory, or else in the nearest parent directory that @@ -210,17 +218,20 @@ function optionsOf(args, cwd) { sample: args.sample ? Number(args.sample) : undefined, // A mutant can make a test hang. Oracles that run in milliseconds do not // need Vitest's default limits, which would make each hang cost seconds. - testTimeout: Number(args[`test-timeout`] ?? 15000), - // The base of each mutant run's time limit, in milliseconds; the limit - // adds three times the file's unmutated duration (see limitFor). - hangLimit: Number(args[`hang-limit`] ?? 10000), - // The base of a mutant run's first, short budget; a copy past it runs - // again alone with the full limit before it can be a timeout. - firstLimit: Number(args[`first-limit`] ?? 2000), - // The floor of a mutant's budget, which the runs that confirm its - // cut-off copies share (check.mjs, `budgetOf`): 30 s by default, room - // for three confirmations at the default hang limit. - mutantLimit: Number(args[`mutant-limit`] ?? 30000), + // Vitest's per-test limit, above the copies' backstop (`hangLimit` plus + // five seconds, so a copy that waits for its backstop is cut by it first). + testTimeout: Number(args[`test-timeout`] ?? 65000), + // The base of each mutant run's backstop time limit, in milliseconds; the + // limit adds three times the file's unmutated duration (see limitFor). A + // backstop reports a gap and never decides a kill (deterministic cuts, + // option A, 2026-11-14). + hangLimit: Number(args[`hang-limit`] ?? 60000), + // The `--first-limit` option is retired (option A): it is validated and + // noted, and changes nothing. + retiredFirstLimit: args[`first-limit`] !== undefined, + // `--mutant-limit` is retired too (deterministic cuts, option A): the work + // budget bounds a mutant, so it is validated and noted, and changes nothing. + retiredMutantLimit: args[`mutant-limit`] !== undefined, // Vitest's worker pool. Threads start much faster than child processes; // a package with a native addon in its tree runs with `forks`. pool: args.pool ?? defaultPool(pkg), diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index cf56db29..674ee56b 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -23,11 +23,16 @@ // // In a mutant run, fast-check stops a property at its first failure instead // of shrinking it: a kill needs one failing history, not the smallest one. +import { createHook } from 'node:async_hooks' import { createHash } from 'node:crypto' +import { once } from 'node:events' import fs, { appendFileSync, readFileSync } from 'node:fs' import { createRequire, syncBuiltinESMExports } from 'node:module' +import { connect, createServer } from 'node:net' +import { tmpdir } from 'node:os' import { dirname, isAbsolute, join, resolve as resolvePath } from 'node:path' import { fileURLToPath } from 'node:url' +import { threadId } from 'node:worker_threads' import { afterAll, beforeEach, chai, expect, inject, vi } from 'vitest' import { posix, writeWhole } from './paths.mjs' @@ -50,7 +55,6 @@ const run = const real = (globalThis.__oracleGapsReal ??= { setInterval: globalThis.setInterval, now: Date.now.bind(Date), - use: performance.eventLoopUtilization.bind(performance), keys: Object.keys, getPrototypeOf: Object.getPrototypeOf, getOwnPropertyDescriptor: Object.getOwnPropertyDescriptor, @@ -64,7 +68,6 @@ const real = (globalThis.__oracleGapsReal ??= { }) const realInterval = real.setInterval const realNow = real.now -const realUse = real.use // A wrapper keeps every own property of the function it wraps, such as // `util.promisify.custom`, so `util.promisify(setTimeout)` still resolves @@ -889,12 +892,10 @@ const SEP = process.platform === `win32` ? `[\\\\/]` : `/` const machinery = new RegExp( `${SEP}node_modules${SEP}(?:vitest|@vitest|fast-check|@fast-check|tinypool|tinyspy|chai|@tanstack${SEP}oracles)${SEP}|${SEP}packages${SEP}oracles${SEP}src${SEP}`, ) -function fromTest() { - const limit = Error.stackTraceLimit - Error.stackTraceLimit = 50 - const stack = String(new Error().stack) - Error.stackTraceLimit = limit - return stack +// Whether a stack names the test's or production's code: a frame outside +// Node, this file, and the machinery. +const namesTest = (stack) => + String(stack) .split(`\n`) .slice(1) .some((frame) => { @@ -908,6 +909,12 @@ function fromTest() { !machinery.test(file) ) }) +function fromTest() { + const limit = Error.stackTraceLimit + Error.stackTraceLimit = 50 + const stack = new Error().stack + Error.stackTraceLimit = limit + return namesTest(stack) } globalThis.__oracleGapsClock = () => { if (inTest && !clocked && fromTest()) clocked = true @@ -1027,36 +1034,89 @@ function finishTest(context) { const thisFile = globalThis.__vitest_worker__?.filepath -// I/O and idleness (maintainer decision, 2026-11-07). Each test's row says -// whether the test ever had an I/O handle open: anything but the runner's -// own channel, stdio, and timers. A copy given `idleMs` (a file whose -// unmutated tests did no I/O) is idle when no counted timer is pending, no -// more timers are listed than when its test began (which covers timers made -// outside the wrappers, such as `node:timers/promises`), nothing but the -// channel and stdio is open, and its event loop is under 5% busy. After -// `idleMs` of that it writes an idle row, and the batch's watchdog cuts it -// off as a hang. A late sample means the thread was busy. -const quiet = new Set([ - `MessagePort`, - `PipeWrap`, - `TTYWrap`, - `Timeout`, - `Immediate`, -]) +// I/O and idleness (maintainer decision, 2026-11-07; state only, 2026-11-15; +// only the test's I/O, 2026-11-15). Each test's row says whether the test had +// I/O of its own active: a handle or request that the test's or production's +// code started, never one the runner made for itself (its channel, stdio, a +// write that forwards the test's console output). A copy given `idleMs` (a +// file whose unmutated tests had no I/O of their own) is idle, by state alone, +// when no counted timer is due within its horizon, no more timers are listed +// than when its test began (which covers timers made outside the wrappers, +// such as `node:timers/promises`), and none of its test's I/O is active. A +// busy synchronous or microtask loop blocks this sampler, so it cannot look +// idle; it ends by its call budget or the backstop. After `idleMs` of that +// state it writes an idle row, and the batch's watchdog cuts it off as no +// progress. +// +// Who started each handle and request is recorded once per worker, when Node +// makes it: the test, when the test's or production's code is on the stack +// then (`namesTest`), or when the resource that triggered it is the test's. +// Node triggers a write by the stream it writes to, so the runner's write on +// its own channel is the runner's even when a test's console.log led to it. +// The stack is read only when the sampler meets the resource, or when the +// resource triggers another. Promises are not recorded. +const startedBy = (globalThis.__oracleGapsStartedBy ??= (() => { + const records = new WeakMap() + const live = new Map() + createHook({ + init(id, type, trigger, resource) { + if (type === `PROMISE`) return + const parent = live.get(trigger) + const record = {} + if (parent !== undefined && owned(parent)) record.test = true + else { + const limit = Error.stackTraceLimit + Error.stackTraceLimit = 50 + Error.captureStackTrace(record) + Error.stackTraceLimit = limit + } + live.set(id, record) + records.set(resource, record) + }, + destroy(id) { + live.delete(id) + }, + trackPromises: false, + }).enable() + return records +})()) +function owned(record) { + return (record.test ??= namesTest(record.stack)) +} +// Node lists a handle by its owner (a socket, a child process), and records +// the handle itself. +const testsIo = () => + [...process._getActiveRequests(), ...process._getActiveHandles()].some( + (resource) => { + const record = startedBy.get(resource) ?? startedBy.get(resource?._handle) + return record !== undefined && owned(record) + }, + ) const timersListed = (kinds) => kinds.filter((kind) => kind === `Timeout` || kind === `Immediate`).length +// Test-only hooks for the CLI's own oracle, not for users. With +// ORACLE_GAPS_TEST_SAMPLES (a directory), the sampler writes how many times +// it has judged the running test to `-` there, so a witness +// can hold its own I/O open for a count of samples rather than for a time. +// With ORACLE_GAPS_TEST_RUNNER_WRITE (a directory), the runner itself holds +// one write pending through each test, on a local pipe whose reader never +// reads, as a write that forwards output can stay pending on win32; a copy +// that goes idle while it is pending marks the directory. +const samplesAt = process.env.ORACLE_GAPS_TEST_SAMPLES +const runnerWrites = process.env.ORACLE_GAPS_TEST_RUNNER_WRITE +let held let sawIo = false if (state.capture) { let baseTimers let since - let last - let use = realUse() let reported = false + let samples = 0 beforeEach(() => { if (continuing) return baseTimers = timersListed(process.getActiveResourcesInfo()) since = undefined sawIo = false + samples = 0 }) beforeEach((context) => { if (continuing) return @@ -1064,34 +1124,51 @@ if (state.capture) { baseTimers = undefined }) }) + if (runnerWrites) + beforeEach(async (context) => { + if (continuing) return + const name = `oracle-gaps-${process.pid}-${threadId}-${(globalThis.__oracleGapsHolds = (globalThis.__oracleGapsHolds ?? 0) + 1)}` + const server = createServer({ pauseOnConnect: true }).listen( + process.platform === `win32` + ? `\\\\.\\pipe\\${name}` + : join(tmpdir(), `${name}.sock`), + ) + await once(server, `listening`) + const accepted = once(server, `connection`) + const client = connect(server.address()) + await once(client, `connect`) + const [peer] = await accepted + client.write(Buffer.alloc(1 << 22)) + held = client + context.onTestFinished(() => { + held = undefined + client.destroy() + peer.destroy() + server.close() + }) + }) realInterval(() => { const now = realNow() - const late = last !== undefined && now - last > 150 - last = now - const used = realUse(use) - use = realUse() if (baseTimers === undefined || reported) return + if (samplesAt) + writeWhole(join(samplesAt, `${process.pid}-${threadId}`), `${++samples}`) const kinds = process.getActiveResourcesInfo() - const io = kinds.some((kind) => !quiet.has(kind)) + const io = testsIo() sawIo ||= io // A timer due after the copy's full time limit cannot end its wait in // time, so it does not keep the copy busy: a test waiting on it would be // a timeout at the limit anyway (a collection's five-minute GC timer). const horizon = now + (state.idleHorizonMs ?? Infinity) const timerDue = [...pending.values()].some((due) => due <= horizon) - if ( - late || - io || - used.utilization > 0.05 || - timerDue || - timersListed(kinds) > baseTimers - ) { + if (io || timerDue || timersListed(kinds) > baseTimers) { since = undefined return } since ??= now if (!state.idleMs || now - since < state.idleMs) return reported = true + if (held?.writableLength) + writeWhole(join(runnerWrites, `idle-${process.pid}-${threadId}`), ``) appendFileSync( state.capture, `${JSON.stringify({ file: thisFile, idle: true })}\n`, diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 8b0eac0b..4137c9b9 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -96,6 +96,11 @@ const IDLE_MS = 500 // in one test before it counts as a loop that never ends. A file's budget is // a hundred times the most calls any of its unmutated tests made, if more. const CALL_FLOOR = 100_000 +// The least work (site evaluations) a mutant's copies may do together: the +// larger of this and ten times their owner files' unmutated work (`workBudgetOf`). +// It is the call floor: two copies that each stay inside their own call budget +// can pass it together. +const WORK_FLOOR = CALL_FLOOR const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) /** The host that a session directory names, if it names one. */ function sessionHostOf(dir) { @@ -797,14 +802,25 @@ export default async (env) => { } contextSpecs.set(context, map) } + // A context's close, up to the backstop (60 s). A close still pending past it + // is a loud backstop gap (register L8), never a silent one: the caller never + // warms a replacement while the old context is still closing. + async function closeWithin(context, label) { + const closed = await Promise.race([ + closeContext(context).then( + () => true, + () => true, + ), + sleep(60_000).then(() => false), + ]) + if (!closed) + throw new Error( + `oracle-gaps: gap backstop: a context close (${label}) is still pending after 60 s; no replacement is warmed while it closes`, + ) + } async function closeContexts() { await Promise.all( - contexts.map((context) => - Promise.race([ - closeContext(context).catch(() => undefined), - sleep(10_000), - ]), - ), + contexts.map((context) => closeWithin(context, `end of run`)), ) contexts = [] lastServed.clear() @@ -956,7 +972,12 @@ export default async (env) => { forceCancel(context) span(`run`, started, { ...profiled, timedOut: true }) rmSync(capture, { force: true }) - return { file, timedOut: true, seconds, ...(spun ? { spin: true } : {}) } + return { + file, + timedOut: true, + seconds, + cause: spun ? `spin` : `backstop`, + } } // The result also lists test modules from earlier runs, with their old // results. Only this run's file counts. @@ -995,7 +1016,7 @@ export default async (env) => { return { file, timedOut: true, - spin: true, + cause: `spin`, seconds, inputs: importsOf(file, modules, closing), readAt: closing.readAt, @@ -1184,6 +1205,10 @@ export default async (env) => { let current = 0 let copyStarted = Date.now() let timer + // The cut that ended the running copy, if the watchdog cut it off, and how + // long the copy had run then. + let cutCause + let cutMs const beat = heartbeat() // Each poll counts itself, after it has read the capture (the hold in // capture-setup.mjs waits for two of these). @@ -1202,12 +1227,18 @@ export default async (env) => { items[current].file, copyStarted, ) - if ( - idle.has(copies[current]) || - Date.now() - (spunAt.get(copies[current]) ?? Infinity) > IDLE_MS || - Date.now() - copyStarted > items[current].limitMs - ) + // The cut that ends the copy, in this order: a spin past the call + // budget (a count), no progress (a state), then the backstop (a clock). + const copy = copies[current] + if (Date.now() - (spunAt.get(copy) ?? Infinity) > IDLE_MS) + cutCause = `spin` + else if (idle.has(copy)) cutCause = `no progress` + else if (Date.now() - copyStarted > items[current].limitMs) + cutCause = `backstop` + if (cutCause !== undefined) { + cutMs = Date.now() - copyStarted return resolve(undefined) + } timer = setTimeout(poll, 50) } timer = setTimeout(poll, 50) @@ -1243,11 +1274,11 @@ export default async (env) => { const skippedUnit = (item) => ({ file: item.file, skipped: true }) // A copy that ran past its call budget, even one whose throw ended the // loop and let it finish, is a timeout as it stands (a count, not a - // clock: no confirming run). + // clock). const spunUnit = (item) => ({ file: item.file, timedOut: true, - spin: true, + cause: `spin`, seconds, }) try { @@ -1301,8 +1332,8 @@ export default async (env) => { } }), timedOutAt: current, - idle: idle.has(copies[current]), - spin: spun.has(copies[current]), + cause: cutCause, + cutMs, } } // Errors outside any test cannot be attributed to one copy, so the @@ -1364,9 +1395,13 @@ export default async (env) => { async function check(options, store, log = console.error) { const authored = options.mutants ? loadAuthored(options.mutants) : [] seedVariable = `${options.envPrefix}_SEED` - if (options.hangLimit > options.mutantLimit) + if (options.retiredFirstLimit) log( - `oracle-gaps: note --hang-limit ${options.hangLimit} ms is above --mutant-limit ${options.mutantLimit} ms; a mutant's confirmations together get at most the larger of --mutant-limit and ten times its owner files' unmutated time`, + `oracle-gaps: note --first-limit is retired: a copy has no first budget, so it changes nothing (deterministic cuts, option A)`, + ) + if (options.retiredMutantLimit) + log( + `oracle-gaps: note --mutant-limit is retired: a mutant's work budget bounds it, so it changes nothing (deterministic cuts, option A)`, ) pulse = { log, @@ -1477,7 +1512,6 @@ export default async (env) => { settings, // A run cut off under one limit may end under a longer one. options.hangLimit, - options.mutantLimit, ]), ) // How the evidence was produced: units reused and computed, runs cut off @@ -1492,7 +1526,7 @@ export default async (env) => { spin: 0, rerun: 0, unchanged: 0, - // Confirmations that a mutant's budget cut short or left no time. + // Copies past their mutant's work budget (each a timeout). budget: 0, } // Each file's imports in its unmutated run. A run that timed out reports @@ -1585,8 +1619,7 @@ export default async (env) => { ) return { ...batch, inputs } } - // A unit that depends on more than its inputs (a confirmation that its - // mutant's budget cut short) is not kept (`keep`). + // A unit that depends on more than its inputs is not kept (`keep`). async function unit(context, file, spec, keep, runOptions = {}) { const index = indexOf.get(context) const id = idOf(file, spec, runOptions.select, runOptions.ignore) @@ -1629,13 +1662,29 @@ export default async (env) => { return result } // Units for many requests at once. Each is looked up first; the rest run - // as one batch (runBatch). A batch cut off by a hang queues that item's - // confirming run, which any free slot takes, and reruns the rest in a - // fresh context; a batch with + // as one batch (runBatch). A batch cut off (a copy's backstop, idleness, + // or a spin) settles that copy as a timeout and reruns the rest in a fresh + // context; a batch with // errors outside any test runs its items alone. `needed` says, from the // units found so far, whether a request must still run; a request that // need not run, gets no unit; a copy that skipped its tests gets a // skipped marker, which its caller runs again (runOwners). + // A profile row for one copy (ORACLE_GAPS_PROFILE): its mutant, its file's + // measured time, its limit, the cause that ended it or `completed`, and its + // duration. + const copyRow = (request, { cause, durationMs }) => + appendFileSync( + profileFile, + `${JSON.stringify({ + kind: `copy`, + mutant: request.label, + file: baseOf(request.file), + cause, + fileMs: Math.round((fileSeconds.get(request.file) ?? 0) * 1000), + limitMs: request.limitMs, + durationMs, + })}\n`, + ) async function unitsBatch(slot, requests, needed = () => true) { const index = indexOf.get(slot.context) const got = new Array(requests.length) @@ -1673,30 +1722,42 @@ export default async (env) => { // A skipped copy keeps its marker, so the caller can run it again. got[rest[k].i] = u if (u.skipped) return - if (u.spin) evidence.spin += 1 + if (u.cause === `spin`) evidence.spin += 1 remember(rest[k].id, u, batch.inputs[k]) evidence.computed += 1 }) + // Per-copy rows for ORACLE_GAPS_PROFILE: each copy's cut or completion, + // with its file's measured time and its limit, to see what a cut was. + if (profileFile) + batch.units.forEach((u, k) => { + if (u === undefined || u.skipped) return + copyRow(rest[k], { + cause: u.cause ?? `completed`, + durationMs: Math.round(u.seconds * 1000), + }) + }) if (batch.timedOutAt === undefined) break - // A copy past its first budget may only be slow: a loaded machine - // slows a run that ends. It runs again alone, in a fresh context, - // with the full limit, and only a run cut off there too is a timeout. + // A copy cut off by its backstop, for idleness, or for spinning is a + // timeout as it stands: no clock decides a kill (deterministic cuts). const hung = rest[batch.timedOutAt] - const full = limitFor(hung.file) + if (profileFile) + copyRow(hung, { cause: batch.cause, durationMs: batch.cutMs }) evidence.cut += 1 - if (batch.idle) evidence.idle += 1 - if (batch.spin) evidence.spin += 1 + if (batch.cause === `no progress`) evidence.idle += 1 + if (batch.cause === `spin`) evidence.spin += 1 await replace(slot) - // A copy that already had the full limit, or that sat idle, is - // final; any other comes back marked `cut`, for its mutant's own - // confirmation (`runOwners`). - if (hung.limitMs >= full || batch.idle || batch.spin) - settle( - hung, - { file: hung.file, timedOut: true, seconds: full / 1000 }, - batch.inputs[batch.timedOutAt], - ) - else got[hung.i] = { file: hung.file, cut: true } + // A count or state cut (a spin, or no progress) is a timeout for its + // copy alone; only a wall-clock cut ends the mutant's copies (cutByTime). + settle( + hung, + { + file: hung.file, + timedOut: true, + seconds: hung.limitMs / 1000, + cause: batch.cause, + }, + batch.inputs[batch.timedOutAt], + ) rest = [ ...rest.slice(0, batch.timedOutAt).filter((_, k) => !batch.units[k]), ...rest.slice(batch.timedOutAt + 1), @@ -1717,57 +1778,10 @@ export default async (env) => { limitMs: r.limitMs, callBudget: r.callBudget, }) - if (u.spin) evidence.spin += 1 + if (u.cause === `spin`) evidence.spin += 1 if (u.timedOut) await replace(slot) return u } - // A hung copy's confirming run: alone, in a clean context, with the full - // limit, or with what is left of its mutant's budget when that is less - // (`left`, in milliseconds). Only a run cut off there too is a timeout. - // An idle copy in a file with no I/O is final at once (see unitsBatch): - // it waits on nothing that can come. The unit is marked `confirmed`, so - // its seconds count against the budget whenever it is used, reused ones - // too. One that the budget cut short, or left no time, depends on the - // mutant's other units, so it is not kept. - async function confirmAlone(slot, request, left) { - // The confirmation is kept under the request's own identity, so a - // later check reuses it in place of the request's first run. - const hung = { - ...request, - id: idOf(request.file, request.spec, request.select, request.ignore), - } - const full = limitFor(hung.file) - const limit = Math.min(full, left) - const confirmed = (u) => ({ ...u, confirmed: true }) - if (limit <= 0) { - evidence.budget += 1 - return confirmed({ file: hung.file, timedOut: true, seconds: 0 }) - } - const alone = await batchIn(slot, [{ ...hung, limitMs: limit }]) - if (alone.unattributed) - return confirmed( - await runAlone(slot, { ...hung, limitMs: limit }, limit >= full), - ) - const [inputs] = alone.inputs - if (alone.timedOutAt === 0) { - evidence.cut += 1 - if (alone.idle) evidence.idle += 1 - if (limit < full) evidence.budget += 1 - const u = confirmed({ - file: hung.file, - timedOut: true, - seconds: limit / 1000, - }) - if (limit >= full) remember(hung.id, u, inputs) - await replace(slot) - return u - } - const u = confirmed(alone.units[0]) - remember(hung.id, u, inputs) - evidence.computed += 1 - return u - } - const vitest = contexts[0] const fileSpecs = new Map(contextSpecs.get(vitest)) const relative = (file) => relativeWithin(pkg, file) ?? posix(file) @@ -1785,49 +1799,45 @@ export default async (env) => { // detect anything, and it may read source text that instrumentation // changes. Each file's unmutated duration sets a mutant run's time limit. const fileSeconds = new Map() - // A mutant can slow a run that still ends, for example by lengthening a - // retry backoff, and a loaded machine slows every run. A mutant run - // first gets a short budget (`--first-limit`); a copy cut off there runs - // again alone, with the full limit (`--hang-limit`, 10 s by default: - // a 5 s base once turned a slow kill into a timeout), and only a run - // cut off there too is a timeout (see unitsBatch). + // A copy's time limit is a backstop: the full limit, which reports a gap and + // never decides a kill (deterministic cuts, option A, 2026-11-14). There is + // no first budget, so a copy runs once, to its end. const limitFor = (file) => options.hangLimit + 3000 * (fileSeconds.get(file) ?? 10) - const firstLimitFor = (file) => - Math.min( - limitFor(file), - options.firstLimit + 3000 * (fileSeconds.get(file) ?? 10), - ) - // Each mutant's budget (maintainer disposition G1, 2026-11-11), for the - // runs that confirm its cut-off copies: the larger of `--mutant-limit` - // (30 s by default) and ten times its owner files' unmutated time. Its - // confirmations run in its owner files' order, and each gets at most the - // smaller of its full limit and what the budget has left after the - // seconds of the mutant's earlier confirmed units, reused ones included - // (`runOwners`), so the outcome is a function of the units, whatever ran - // first or was reused. With the default limits the budget cuts a - // confirmation below its full limit (10 s plus three times its file's - // time t) only when the mutant's confirmations together pass it: for - // one owner file that is never, since 10t >= 10 + 3t once t >= 1.43 s - // and the 30 s floor covers a smaller t; it binds when `--hang-limit` is - // raised past the budget, or when several owner files need long - // confirmations. It does not bound a mutant's first runs (each has its - // first budget) or the context replacements after cuts (each up to a - // 10 s close and a warm start). - const budgetOf = (files) => + // A mutant's work budget (maintainer disposition G1, option A): the larger of + // WORK_FLOOR and ten times its owner files' unmutated work, in site + // evaluations, each file counted by its most-working unmutated test. + const workBudgetOf = (files) => Math.max( - options.mutantLimit, - 10_000 * - files.reduce((sum, file) => sum + (fileSeconds.get(file) ?? 0), 0), + WORK_FLOOR, + 10 * files.reduce((sum, file) => sum + (mostCalls.get(file) ?? 0), 0), ) + // A copy cut by a clock or by its call count: its own run's timeout, or + // Vitest's own limit ("Test timed out"), which the copy outlived. Such a + // copy is never charged, and it ends the mutant's copies (the outcome rule, + // register 2026-11-14, `Deterministic cuts, outcome rule`). + const cutByTime = (unit) => + unit.cause === `backstop` || + (unit.tests ?? []).some( + (test) => + test.state === `failed` && /Test timed out/.test(test.message), + ) + // A copy's work: the calls of its deterministic, unclocked tests + // (`baseline.work`, classify.mjs), so a row that a clock could move counts + // nothing. + const workOf = (unit) => + cutByTime(unit) + ? 0 + : (unit.rows ?? []).reduce( + (sum, [key, row]) => + sum + (baseline.work.has(key) ? (row.calls ?? 0) : 0), + 0, + ) // A run cut off is still running, so its context is replaced; closing // it stops the worker. const replace = async (slot) => { const started = Date.now() - await Promise.race([ - closeContext(slot.context).catch(() => undefined), - sleep(10_000), - ]) + await closeWithin(slot.context, `slot ${slot.index}`) slot.context = await warmContext(settings, slot.index) await globSpecs(slot.context) contexts[slot.index] = slot.context @@ -2057,9 +2067,10 @@ export default async (env) => { valueChanges.get(file).add(number) } const unmutatedUnits = new Map(first.units.map((u) => [u.file, u])) - // Files whose unmutated tests had an I/O handle open (a socket, a child - // process, file I/O): a test there may wait on something outside the - // worker, so no copy of it is cut for idleness (capture-setup.mjs). + // Files whose unmutated tests had I/O of their own open (a socket, a child + // process, file I/O, never the runner's): a test there may wait on + // something outside the worker, so no copy of it is cut for idleness + // (capture-setup.mjs). const ioFiles = new Set() for (const rows of [first.rows, second.rows]) for (const [key, row] of rows) if (row.io) ioFiles.add(fileOf(key)) @@ -2067,13 +2078,11 @@ export default async (env) => { // any of its unmutated tests made, and at least CALL_FLOOR. A copy past it // is spinning (capture-setup.mjs) and is a timeout as it stands. const mostCalls = new Map() - for (const rows of [first.rows, second.rows]) - for (const [key, row] of rows) - if (!row.background) - mostCalls.set( - fileOf(key), - Math.max(mostCalls.get(fileOf(key)) ?? 0, row.calls ?? 0), - ) + for (const [key, calls] of baseline.work) + mostCalls.set( + fileOf(key), + Math.max(mostCalls.get(fileOf(key)) ?? 0, calls), + ) const callBudgetFor = (file) => Math.max(CALL_FLOOR, 100 * (mostCalls.get(file) ?? 0)) const changesNothing = (mutant, targets) => @@ -2127,12 +2136,16 @@ export default async (env) => { // hang), so a result never depends on how its evidence was produced. The // ordinary tier, when asked for, runs the same way afterwards. const GROUP = 4 - // A copy that a batch skipped, or cut off for its confirmation, has no + // A copy that a batch skipped, or cut off by a count or a state, has no // verdicts of its own yet. const kills = (u) => !u.skipped && - !u.cut && outcomeOf([u], baseline.nondeterministic).outcome === `killed` + // A mutant's copies run in their owner files' order and are summed in that + // order (the work budget, below). The order is the report's, so it does not + // depend on the timing of the copies. A later owner's kill does not count + // once the budget is passed; that is deterministic because the work is a + // count, and it is the rule the design note (option A) states. const runOwners = async (slot, states, filesOf, keysOf) => { const active = states.filter((st) => !st.done) const requests = [] @@ -2149,7 +2162,7 @@ export default async (env) => { spec: `mutant:${st.mutant.number}`, mutant: st.mutant.number, select: await selectionFor(slot, file, keysOf(st)), - limitMs: firstLimitFor(file), + limitMs: limitFor(file), // Idle detection only where no unmutated test of the file did I/O. idleMs: ioFiles.has(file) ? null : IDLE_MS, // A loop that spins on instrumented code is cut by count. @@ -2175,43 +2188,45 @@ export default async (env) => { (request, found) => !killedBefore(request, found), ) for (const st of active) { - // The mutant's budget, spent in its files' order by the seconds of - // each confirmed unit, reused or run now, so its outcome is a - // function of its units, whatever ran first or was reused. - let left = budgetOf(filesOf(st)) + const files = filesOf(st) + // The mutant's copies in their files' order, judged by count: each copy's + // work is added, and once the sum passes the work budget, that copy and + // every later one is a timeout, so a kill among them does not count. + const budget = workBudgetOf(files) + let work = 0 for (const [i, request] of requests.entries()) { if (request.st !== st) continue - // A copy its batch skipped (an earlier copy killed the mutant) - // runs again through unitsBatch, so it is confirmed like any copy - // while no kill is in hand. + // A copy its batch skipped (an earlier copy killed the mutant) runs + // again through unitsBatch, while no kill is in hand. if (got[i]?.skipped) { if (st.units.some(kills)) break got[i] = (await unitsBatch(slot, [request], () => true))[0] } - // A kill earlier in the mutant's files leaves a file without a - // unit. Any other missing unit stops the run. + // A kill earlier in the mutant's files leaves a file without a unit. + // Any other missing unit stops the run. if (got[i] === undefined || got[i].skipped) { if (st.units.some(kills)) break throw new Error( `${st.mutant.id}: ${relative(request.file)} has no unit and no earlier kill`, ) } - // A cut copy is confirmed only while no later owner's unit already - // killed the mutant: a kill decides, so its confirmation would - // spend time for nothing. - if ( - got[i].cut && - requests.some( - (other, j) => - other.st === st && - j > i && - got[j] !== undefined && - kills(got[j]), - ) - ) - continue - if (got[i].cut) got[i] = await confirmAlone(slot, request, left) - if (got[i].confirmed) left -= got[i].seconds * 1000 + // A copy cut by a clock or its count is a timeout, uncharged, and no + // later copy runs (the outcome rule). + if (cutByTime(got[i])) { + st.units.push(got[i]) + break + } + work += workOf(got[i]) + if (work > budget) { + evidence.budget += 1 + st.units.push({ + file: request.file, + timedOut: true, + seconds: 0, + cause: `work budget`, + }) + break + } st.units.push(got[i]) if (kills(got[i])) break } diff --git a/packages/oracles/oracle-gaps/classify.mjs b/packages/oracles/oracle-gaps/classify.mjs index 72da6178..ea4e84dc 100644 --- a/packages/oracles/oracle-gaps/classify.mjs +++ b/packages/oracles/oracle-gaps/classify.mjs @@ -47,10 +47,19 @@ export function outcomeOf( assertions = false, pastBound = () => false, ) { - // A run that timed out recorded nothing. A kill in another run still - // decides; failing that, the mutant is `timeout`. - const timeout = units.some((unit) => unit.timedOut) - ? { outcome: `timeout`, detail: `run time limit` } + // A run that timed out recorded nothing, and a test that Vitest's own limit + // ended (`Test timed out`) is the same kind of backstop (deterministic cuts, + // 2026-11-14): a gap that never keeps a kill from deciding. A kill in another + // run still decides; failing that, the mutant is `timeout`. + const timedOutTest = (test) => + test.state === `failed` && /Test timed out/.test(test.message) + // The timeout's detail is the cause of the first cut in report order: the + // copy's `cause` (`backstop`, `spin`, `no progress`, or `work budget`). + const cut = [...units] + .sort((a, b) => byPath(a.file, b.file)) + .find((unit) => unit.timedOut || (unit.tests ?? []).some(timedOutTest)) + const timeout = cut + ? { outcome: `timeout`, detail: cut.timedOut ? cut.cause : `backstop` } : undefined const failures = [] // By file path (report order, `inReportOrder`), in whatever order the @@ -87,7 +96,6 @@ export function outcomeOf( // fast-check's wrapper text ("Property failed …") is not an // assertion; the wrapped error, which its message quotes, decides. /AssertionError|expected .+ to /.test(test.message), - timeout: /Test timed out/.test(test.message), // fast-check quotes the error its predicate threw after its own // header; that error is the cause the report shows. message: ( @@ -153,9 +161,7 @@ export function outcomeOf( laws: lawsOf(unregistered), } const tests = failures.filter((f) => f.test) - if (tests.length > 0 && tests.every((f) => f.timeout)) - return { outcome: `timeout`, by: tests.map((f) => f.test) } - const crashed = tests.filter((f) => f.route !== `framework` && !f.timeout) + const crashed = tests.filter((f) => f.route !== `framework`) if (crashed.length > 0) return { outcome: `crashed`, @@ -278,7 +284,9 @@ export function targetsOf(mutant, baseline, owners, oracleFiles) { return { oracleKeys, testKeys, - oracleTargets: [...new Set(oracleKeys.map(fileOf))], + // Report order (byPath), so a mutant's owners are summed in one order + // however its copies ran. + oracleTargets: [...new Set(oracleKeys.map(fileOf))].sort(byPath), testTargets: [...new Set(testKeys.map(fileOf))], } } diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 5765025d..43f97f37 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -36,8 +36,8 @@ * so a kill names a non-empty part of them. An operator mutant's owner * files run one after another in report order (law 7), and the runner * stops at the first that kills, so its kill names laws of that file - * alone. Failing that, a run that - * timed out makes it `timeout`. Failing that, reproducible verdicts under + * alone. Failing that, a run that timed out, or a test that Vitest's limit + * ended, makes it `timeout`. Failing that, reproducible verdicts under * reach agreement alone make it an `agreement-only-kill`, which names * them all. Failing that, a verdict on the random campaign * makes it a `random-only-kill`, and one on an unknown or unregistered @@ -62,7 +62,10 @@ * and the design decided with it that day: a speed shortcut is owed a * cost law, not an exit, so a regression that deletes it must be * catchable, by a deterministic count and never by wall-clock time. - * 4. Failing that, if every failing test timed out, it is `timeout`; a + * 4. Failing that, a test that Vitest's own limit ended (its message is + * `Test timed out`) is a backstop, with a run that timed out: the mutant + * is `timeout`, a gap that never keeps a kill from deciding (deterministic + * cuts, option A, 2026-11-14). Otherwise a * failing test outside the framework's own checks makes it `crashed`; a * failing framework check alone (the `calibration` route, which covers * calibrations, probe calibrations, and the determinism test) makes it a @@ -720,7 +723,14 @@ function boundBelow(before, after) { // at a kill reads no work. function failureOf(h, slot, tests, work = false) { const under = ({ t }) => t.under[slot].kind - const timedOut = tests.some((x) => under(x) === `runTimeout`) + // A test that Vitest's own limit ended is a backstop, as a cut run is, + // in a file that loaded (a file that failed to load has no tests to end). + const timedOut = tests.some( + (x) => + under(x) === `runTimeout` || + (under(x) === `timeoutMessage` && + !h.files[x.fi].tests.some((t) => t.under[slot].kind === `setupError`)), + ) const setup = tests.some((x) => under(x) === `setupError`) const ran = tests.filter( (x) => @@ -762,7 +772,8 @@ function failureOf(h, slot, tests, work = false) { laws: [...new Set(behavioral)].sort(), lawsMayBePart: true, }) - if (timedOut) return decided({ outcome: `timeout` }) + // A Vitest limit is a backstop cut: the timeout names it (classify.mjs). + if (timedOut) return decided({ outcome: `timeout`, detail: `backstop` }) if (kills.length > 0) return naming(`agreement-only-kill`, verdictsIn(`reproducible`), { laws: [`reach agreement`], @@ -771,11 +782,7 @@ function failureOf(h, slot, tests, work = false) { return naming(`random-only-kill`, verdictsIn(`random`)) if (verdictsIn(`unknown`).length > 0) return naming(`unregistered-kill`, verdictsIn(`unknown`)) - if (failing.length > 0 && failing.every((x) => under(x) === `timeoutMessage`)) - return decided({ outcome: `timeout` }) - const crashed = failing.filter( - (x) => under(x) !== `timeoutMessage` && kindOf(x.t.route) !== `framework`, - ) + const crashed = failing.filter((x) => kindOf(x.t.route) !== `framework`) if (crashed.length > 0) return naming(`crashed`, crashed) if (failing.length > 0) return naming(`framework-error`, failing) const unhandled = ran.some((x) => under(x) === `unhandled`) @@ -1102,7 +1109,14 @@ function operandsSeen(h, mi, owners) { function ordinaryOutcome(h, mi, files) { const tests = files.flatMap((fi) => h.files[fi].tests.map((t) => ({ t, fi }))) const under = ({ t }) => t.under[mi].kind - const timedOut = tests.some((x) => under(x) === `runTimeout`) + // A test that Vitest's own limit ended is a backstop, as a cut run is, + // in a file that loaded (a file that failed to load has no tests to end). + const timedOut = tests.some( + (x) => + under(x) === `runTimeout` || + (under(x) === `timeoutMessage` && + !h.files[x.fi].tests.some((t) => t.under[mi].kind === `setupError`)), + ) const ran = tests.filter( (x) => !h.files[x.fi].tests.some((t) => @@ -1130,14 +1144,7 @@ function ordinaryOutcome(h, mi, files) { if (kills.length > 0) return `agreement-only-kill` if (judgedOn(`random`).length > 0) return `random-only-kill` if (judgedOn(`unknown`).length > 0) return `unregistered-kill` - if (failing.length > 0 && failing.every((x) => under(x) === `timeoutMessage`)) - return `timeout` - if ( - failing.some( - (x) => under(x) !== `timeoutMessage` && kindOf(x.t.route) !== `framework`, - ) - ) - return `crashed` + if (failing.some((x) => kindOf(x.t.route) !== `framework`)) return `crashed` if (failing.length > 0) return `framework-error` if (tests.some((x) => under(x) === `setupError` || under(x) === `unhandled`)) return `setup-error` @@ -1487,7 +1494,7 @@ const messages = { verdict: `Error: violation`, assertion: `AssertionError: expected [ 'a' ] to deeply equal [ 'b' ]`, crash: `TypeError: x.y is not a function`, - timeoutMessage: `Error: Test timed out in 15000ms.`, + timeoutMessage: `Error: Test timed out in 65000ms.`, } // The capture's comparison and the swap a comparison mutant states, as the // instrumented code evaluates them (capture-setup.mjs `cmp`). @@ -1633,7 +1640,12 @@ function unitOf(h, fi, slot) { const f = h.files[fi] const under = (t) => t.under[slot] if (f.tests.some((t) => under(t).kind === `runTimeout`)) - return { file: fileName(h, fi), timedOut: true, seconds: 1 } + return { + file: fileName(h, fi), + timedOut: true, + cause: `backstop`, + seconds: 1, + } const loadFailed = f.tests.some((t) => under(t).kind === `setupError`) const tests = loadFailed ? [] : f.tests return { @@ -2120,6 +2132,8 @@ async function driver(h, observe) { class: r.class, laws: r.laws && [...r.laws].sort(), testOutcome: r.testOutcome, + // A timeout's detail is its cut (`outcomeOf`). + detail: r.outcome === `timeout` ? r.detail : undefined, })), summary: readSummary(summary), messages: messagesOf(summary), @@ -2259,8 +2273,14 @@ function check(h, observations, expected) { return violation(`classified`, `nothing was classified`, `completion`) // A kill names a non-empty part of its behavioral laws (the runner may // stop at the first); an agreement-only kill names reach agreement. + // A timeout names the cut that ended it (its `detail`, the cause). const view = (o) => - JSON.stringify([o.outcome, o.class, o.testOutcome ?? null]) + JSON.stringify([ + o.outcome, + o.class, + o.testOutcome ?? null, + o.outcome === `timeout` ? o.detail : null, + ]) for (const [mi, owed] of expected.outcomes.entries()) { const actual = got.outcomes[mi] const lawsFit = @@ -2599,11 +2619,18 @@ function check(h, observations, expected) { ) // Law 8: each gap line is one line, and shows its change shortened: an // operator mutant's original and replacement as two code spans, an - // authored mutant's intent as a code span and its author. The results keep the full text. + // authored mutant's intent as a code span and its author. A timeout's line + // ends with its cause, `(cause: )`, after the change. The results + // keep the full text. for (const owed of gapsOfModel) { const head = `oracle-gaps: gap ${owed.class ?? owed.outcome} L${owed.line} ${owed.id} ` const line = streamed.find((l) => l.startsWith(head)) - const rest = line?.slice(head.length) + const cause = owed.outcome === `timeout` ? ` (cause: ${owed.detail})` : `` + const tail = line?.slice(head.length) + const rest = + tail !== undefined && tail.endsWith(cause) + ? tail.slice(0, tail.length - cause.length) + : undefined let shown if (owed.authored) { // The intent is the author's text, so it sits in a code span. diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6c886733..0f734390 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -129,30 +129,41 @@ const TOP_LOOP = `@..gen` // Whether this filesystem ignores letter case, so that `src/Sut.ts` names // `src/sut.ts`; the `cased` spelling applies only where it does. const caseless = existsSync(fileURLToPath(import.meta.url).toUpperCase()) -// The full limit of a cut-off copy's confirmation that the driver passes -// (`--hang-limit`), and, under one budget, the first budget too. +// The backstop base that the driver passes (`--hang-limit`): a copy that runs +// past it, plus three seconds per second of its file's time, is cut. const FULL_LIMIT_MS = 20000 -// How long a slow test waits, in real milliseconds: past the 1 s first -// budget that the driver passes (`--first-limit`), within the full limit. +// How long a slow test waits, in real milliseconds, where its output differs. const SLOW_MS = 3000 -// How long each slow test of `w_slowpair` waits: past its file's first -// budget (the 1 s `--first-limit` plus three times its one-test file's -// unmutated time t), so its copy is always cut and confirmed; past half the -// mutant's budget, so two such confirmations never fit it; and under the -// full limit, so each would fit alone. The window holds while t < 3 s and -// ten times the two files' time stays under 2 × 10 s (their sum under 2 s). -const PAIR_SLOW_MS = 10000 -// A mutant's budget for the runs that confirm its cut-off copies, which the -// driver passes (`--mutant-limit`): one slow owner's confirmation fits it -// (SLOW_MS), and two of `w_slowpair`'s do not. Ten times the fixture's -// one-test files' time stays under it, so the floor is the budget. -const MUTANT_LIMIT_MS = 15000 +// The `pendingIo: 'test'` witness's child (run with `node -e`, given the +// sampler's count file): it writes to its stdout once the count has risen by +// 20, and gives up after 60 s, a backstop that leaves the read pending, so the +// witness then fails loudly. +const SAMPLE_HOLD = `const read = () => { try { return Number(require('fs').readFileSync(process.argv[1], 'utf8')) } catch { return 0 } }; const from = read(); setInterval(() => { if (read() >= from + 20) process.stdout.write('go', () => process.exit(0)) }, 10); setTimeout(() => process.exit(1), 60000)` +// A wait long past any copy's horizon (the full limit and 3 s per second of +// the file's time). +const PAST_HORIZON_MS = 600_000 +// The calls that each copy of `w_slowpair` makes beyond its first: its test +// calls the function this many more times where its output differs. A call +// evaluates two or three instrumented sites, so a copy makes 60,000 to 90,000 +// site evaluations: inside its call budget (CALL_FLOOR in check.mjs, 100,000), +// and the two copies together past the mutant's work budget (WORK_FLOOR). +// Work is counted, not timed, so the outcome is the same on any machine. +const PAIR_LOOPS = 30_000 +// A mutant's work budget (maintainer disposition G1, under the deterministic +// cuts decision of 2026-11-14): the larger of this floor and ten times its +// owner files' unmutated work, in site evaluations. The floor is the copies' +// call budget, so two copies that each stay inside theirs can pass it. +const WORK_FLOOR = 100_000 +// The least call budget a copy gets (check.mjs CALL_FLOOR): site evaluations in +// one copy's runs before it counts as a loop that never ends. A file's budget +// is a hundred times the most its unmutated tests did, if more. Past it, the +// copy is cut by count (a spin) and charged nothing (the outcome rule, item 4). +const CALL_FLOOR = 100_000 // How often the CLI says which mutant a busy copy runs (its heartbeat): the // driver shortens it from 30 s (`ORACLE_GAPS_HEARTBEAT_MS`), so that a slow // test's wait of SLOW_MS passes it. A copy beats once its own run has lasted -// an interval, so the interval is twice the 1 s first budget: a copy cut at -// its first budget never lasts it, and whether a run beats is a function of -// the history, not of the machine's load, as the determinism tests require. +// an interval, and whether a run beats is a function of the history, not of +// the machine's load, as the determinism tests require. const HEARTBEAT_MS = 2000 const ORACLE_FILES = [`a`, `b`] // The owners of the budget witness (`w_slowpair`): two oracle files of one @@ -414,6 +425,16 @@ const testCase = fc.record({ // Only a witness waits on a timer, sleeps, reads the clock, or waits a // turn: its work could follow the clock. timed: fc.constant(false), + // The constructed-only fields (see `histories`): a witness's wait where its + // output differs, its repeated calls, its load, a repeating timer kept due + // while a hang waits, and whose I/O is pending while it waits (`pendingIo`, + // the test's own or the runner's). A generated test has the neutral values, + // which change nothing. + slowMs: fc.constant(SLOW_MS), + loops: fc.constant(0), + timerWait: fc.constant(null), + pulse: fc.constant(false), + pendingIo: fc.constant(null), }) // Law 9's edits. Each content edit changes what some evidence depends on: // a test file, a source statement, the source's line numbers, an owns list, @@ -559,9 +580,10 @@ const drawn = fc.record({ tests: fc.array(testCase, { minLength: 1, maxLength: 5 }), // The witness module: all of it, or only the functions whose outcomes // need real test runs (see `captureWitness`). - // The budget witness alone (`budget`) costs two 10 s confirmations per - // mutant in every CLI run of a history, so generated histories rarely - // draw it; its constructed history (the daemon recheck) always does. + // The budget witness alone (`budget`) makes two copies of about 90,000 + // site evaluations each in every CLI run of a history, so generated + // histories rarely draw it; its constructed history (the daemon recheck) + // always does. witness: fc.oneof( { weight: 30, arbitrary: fc.constantFrom(false, true, `capture`) }, { weight: 1, arbitrary: fc.constant(`budget`) }, @@ -681,9 +703,9 @@ const drawn = fc.record({ // module loads, with 0. A mutant under which it throws then fails every // file that imports the module, before any test runs. boot: fc.constantFrom(false, false, false, true), - // The run's time budgets: a short first budget confirmed by a run alone - // with the full limit, or one budget (`--first-limit` equal to - // `--hang-limit`), where a run past it is a timeout at once. + // The run's budget mode: `one` passes the retired `--first-limit`, which + // the CLI checks and notes and which changes nothing; `two` passes no + // retired option. budget: fc.constantFrom(`two`, `two`, `two`, `one`), // Runs that the driver stops by a signal while the stop witness's test // spins (`w_stop`), each in a pool: Vitest's workers as child processes or @@ -848,6 +870,9 @@ const drawn = fc.record({ // Whether the warm daemon is replaced after the warm check (review of #73, B2). // Only the constructed history of a replaced warm daemon sets it. warmReplaced: fc.constant(false), + // A constructed history's backstop (`--hang-limit`), which no generated + // history sets: null is the full limit. Recorded by the canary as well. + hangMs: fc.constant(null), }) // The edit sequence over the witness module, the one list of steps that a @@ -856,7 +881,7 @@ const drawn = fc.record({ // only file b, so the daemon reuses file a's timed-out runs; the revert // returns to a state that the daemon has seen. The budget witness alone // (`budget`) takes its own one step instead: a second daemon check with -// nothing changed, which reuses the confirmations that the first check ran. +// nothing changed, which reuses the results that the first check ran. const witnessSteps = [`test`, `helper`, `revert`] const budgetSteps = [`recheck`] const keepsSteps = (history) => { @@ -877,20 +902,33 @@ const keepsSteps = (history) => { // witness module" takes `witnessSteps`, so every campaign still judges law // 9 over the witness module's outcome classes, along one short sequence // (maintainer decision, 2026-11-11, option M of the CI plan; a stated -// limit of the property). The budget witness's confirmations cost about -// 15 s a mutant, so its histories follow the same rule, and its constructed +// limit of the property). The budget witness's two copies make about 90,000 +// site evaluations each, so its histories follow the same rule, and its constructed // history "a second daemon check of the budget witness" takes // `budgetSteps` (the review of the gap CLI's runtime fixes, 2026-11-12, // P1-a: the budget witness only in its own histories). The map is its own // inverse on the histories it keeps, so fast-check takes every one of them // back and shrinks it. // A history is drawn as on main; the plan takes its later edits after its -// steps (`laterStepsOf`). Taking a history back returns it unchanged, -// except a constructed one whose `steps` hold a later edit, which the -// steps grammar cannot draw: its later edits move to `laterSteps`. A -// constructed history may also set `tornRows` (issue #83), which `drawn` -// cannot produce, so its take-back reads it as the drawn constant `false` -// (as `claimScenario` does): a shrink is torn-row-free. +// steps (`laterStepsOf`). Taking a history back returns it unchanged, except +// a constructed one whose `steps` hold a later edit, which the steps grammar +// cannot draw: its later edits move to `laterSteps`. A constructed history may +// also set `tornRows` (issue #83), which `drawn` cannot produce, so its +// take-back reads it as the drawn constant `false` (as `claimScenario` does): a +// shrink is torn-row-free. A constructed history may set `slowTwin`, which no +// generated history draws, and which is taken back without it. Its tests may +// set the constructed-only fields `slowMs`, `loops`, `timerWait`, `pulse` and +// `pendingIo`, which the drawn test record holds at neutral values (the +// generator rule, 2026-11-14). +const takenBack = (t) => ({ + ...t, + slow: false, + slowMs: SLOW_MS, + loops: 0, + timerWait: null, + pulse: false, + pendingIo: null, +}) const histories = drawn.map( (history) => (keepsSteps(history) ? history : { ...history, steps: null }), (value) => { @@ -900,17 +938,27 @@ const histories = drawn.map( ) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) + // A constructed backstop (`hangMs`) comes back as the drawn constant (null). + const { slowTwin, hangMs, ...drawnValue } = + /** @type {Record} */ (value) + const tests = /** @type {Record[]} */ (drawnValue.tests) + const plain = + /** @type {{ witness: unknown, steps: unknown[] | null, tornRows: unknown }} */ ({ + ...drawnValue, + hangMs: null, + tests: tests.map(takenBack), + }) // Take-back into the drawn language: a disarmed pause (pauseEngages false, // only the constructed history sets it) comes back as engaged, so the // arbitrary can shrink it. The disarm is therefore not kept by a shrink. const taken = - history.tornRows === false ? value : { ...history, tornRows: false } - const later = (history.steps ?? []).filter((step) => + plain.tornRows === false ? plain : { ...plain, tornRows: false } + const later = (plain.steps ?? []).filter((step) => laterEdits.includes(step), ) if (later.length === 0) return { ...taken, pauseEngages: true, warmReplaced: false } - const earlier = history.steps.filter((step) => !laterEdits.includes(step)) + const earlier = plain.steps.filter((step) => !laterEdits.includes(step)) return { ...taken, pauseEngages: true, @@ -1700,33 +1748,24 @@ const witness = [ body: [kill], tests: [judging(`a`, `random seed`, 7), judging(`b`, `fixed seed`, 7)], }, - // A slow kill: the run ends after the first budget, within the full - // limit, so the confirming rerun kills it (a run is a timeout only when - // it passes the full limit alone). + // A slow kill: its owner test waits 3 s where its output differs, then + // judges, so its copy runs to its end and the kill decides. { id: `w_slowkill`, owners: [`a`], body: [kill], tests: [judging(`a`, `fixed seed`, 7, { slow: true })], }, - // Slow in both owners, and judged by neither: each owner's copy is cut at - // its first budget, and the two confirmations together need more than the - // mutant's budget, so the mutant is a timeout, not a survivor. + // Slow in both owners, and judged by neither: each owner's copy makes + // 30,000 more calls, and the two copies together pass the work budget, so + // the mutant is a timeout by count, not a survivor. { id: `w_slowpair`, owners: PAIR_FILES, body: [kill], tests: [ - judging(`p`, `fixed seed`, 7, { - judges: false, - slow: true, - slowMs: PAIR_SLOW_MS, - }), - judging(`q`, `fixed seed`, 7, { - judges: false, - slow: true, - slowMs: PAIR_SLOW_MS, - }), + judging(`p`, `fixed seed`, 7, { judges: false, loops: PAIR_LOOPS }), + judging(`q`, `fixed seed`, 7, { judges: false, loops: PAIR_LOOPS }), ], }, // A kill as usual. In a run that the driver stops (`GAP_STOP_DIR` set), @@ -1819,8 +1858,8 @@ const witness = [ tests: [judging(`a`, `fixed seed`, 1)], }, // A test that waits on a child process where the output differs: an I/O - // handle is open, so its copy is busy, never cut for idleness, and the - // confirming run kills it. + // handle is open, so its copy is busy, never cut for idleness, and its + // kill decides. { id: `w_iowait`, owners: [`a`], @@ -2578,8 +2617,8 @@ function resolveHistory(history) { ? witness .filter( (w) => - // The budget witness costs two 10 s confirmations per mutant, - // so only histories of the `budget` mode carry it. + // The budget witness's two copies make about 90,000 site + // evaluations each, so only histories of the `budget` mode carry it. (history.witness === true && w.id !== `w_slowpair`) || (history.witness === `capture` && captureWitness.has(w.id)) || (history.witness === `budget` && w.id === `w_slowpair`), @@ -4025,16 +4064,61 @@ function runsOf(plan, tests, sites, active) { for (const part of partsOf(t)) { const base = interpret(plan, t.fn, part.input, sites) const mutated = interpret(plan, t.fn, part.input, sites, active) + const route = routeOf(t, part) + const changed = + !mutated.threw && mutated.trace.join() !== base.trace.join() + // Where the output differs, a `loops` witness calls the function that + // many more times, so its work is the call's work once per call. if (!t.judgesShared) { before += base.work - after += mutated.work + after += mutated.work * (1 + (changed ? (t.loops ?? 0) : 0)) } - const route = routeOf(t, part) + // A wait past the history's backstop (its `hangMs`, else the full limit) + // and a margin of 6 s for three seconds per second of a one-test file + // (the file's time is under 2 s) is past the copy's horizon: its timer is + // not due inside it, so the copy is idle and cut as no progress, by state + // (M entry "idle is decided by state only", 2026-11-15). Its outcome does + // not depend on load. + const backstopMs = plan.history.hangMs ?? FULL_LIMIT_MS + if (changed && t.slow && (t.slowMs ?? SLOW_MS) > backstopMs + 6000) + return { + t, + route, + hang: true, + cut: 'state', + verdicts: found, + changed: true, + work: after, + before, + } + // A test that writes to the console and then waits on a timer due past + // its horizon, while the runner's own write is pending (`pendingIo: + // 'runner'`), cannot make progress of its own: the runner's I/O is not + // the test's, so the copy is idle and cut as no progress, by state (M + // entry "Idle counts only the test's I/O", 2026-11-15). + if (changed && t.pendingIo === `runner`) + return { + t, + route, + hang: true, + cut: 'state', + verdicts: found, + changed: true, + work: after, + before, + } // A loop that never ends cuts the file's run off, as a hang does. if (mutated.spun) - return { t, route, hang: true, verdicts: found, changed: true } - const changed = - !mutated.threw && mutated.trace.join() !== base.trace.join() + return { + t, + route, + hang: true, + cut: 'count', + verdicts: found, + changed: true, + work: after, + before, + } changedAny ||= changed slowed ||= changed && (t.slow || t.ioWait) if (mutated.threw) @@ -4045,17 +4129,38 @@ function runsOf(plan, tests, sites, active) { verdicts: found, changed: changedAny, slowed, + work: after, + before, } - // A test that hangs where another would judge never ends. + // A test that hangs where another would judge never ends. Without a + // `pulse` it goes idle (no progress); with one, its timer keeps it busy, + // so only the backstop cuts it (a wall-clock cut). if (changed && t.hangs) - return { t, route, hang: true, verdicts: found, changed: true } + return { + t, + route, + hang: true, + cut: t.pulse ? 'wall' : 'state', + verdicts: found, + changed: true, + work: after, + before, + } if (changed && t.judges) { found.push({ route, law: t.judgeBy === `verdict` ? t.law : `(unnamed)`, }) if (t.judgeBy !== `soft`) - return { t, route, verdicts: found, changed: true, slowed } + return { + t, + route, + verdicts: found, + changed: true, + slowed, + work: after, + before, + } } } // A test that passed: its route is the last one it registered, and @@ -4084,6 +4189,8 @@ function runsOf(plan, tests, sites, active) { changed: changedAny, slowed, rose: judgesWork && after > before, + work: after, + before, } }) } @@ -4127,12 +4234,16 @@ function slowedFiles(plan, tests, sites, active) { return slowed } +// The timeout cause that each kind of cut names (the CLI's `cause`, `backstop`, +// `spin` and `no progress`; `work budget` is the mutant's own count). +const CUT_CAUSE = { wall: `backstop`, count: `spin`, state: `no progress` } + /** - * A mutant's kind from its owner tests' runs. An operator mutant's - * confirming runs share its budget (`budgeted`); an authored mutant's runs - * are each final at the full limit, so it has no confirmations to budget. + * A mutant's kind from its owner tests' runs. An operator mutant's copies + * share its work budget (`budgeted`); an authored mutant's copies are judged + * alone, with no budget. */ -function judge(plan, tests, sites, active, { budgeted = true } = {}) { +function judge(plan, tests, sites, active, { budgeted = true, detail } = {}) { const runs = runsOf(plan, tests, sites, active) // A file's tests run in order, and a kill (a behavioral verdict on a // reproducible route) stops the file. A hang before that cuts the file's @@ -4140,35 +4251,72 @@ function judge(plan, tests, sites, active, { budgeted = true } = {}) { // earlier verdicts. A kill in another file still decides (a kill outranks // a timeout); failing that, the mutant is a timeout. const hung = new Set() - const slowed = new Map() - for (const file of new Set(runs.map((r) => r.t.file))) + // The cause of a timeout: the first cut in the order the copies run, which + // the CLI names on the mutant's timeout (`detail.cause`). + let cut + // Only a wall-clock cut (the backstop) ends the owner files after it. A count + // or a state cut (a spin, or no progress) ends its own copy only (the outcome + // rule, 2026-11-14, as refined: the next owner's copy still runs). + const wallHung = new Set() + // The mutant's work budget (G1): its owner files' copies are counted in + // their order, each copy's work being the site evaluations of its runs up + // to a hang or a kill. A copy past the budget is a timeout, and so is every + // copy after it, so a kill among them does not count. The budget is the + // larger of WORK_FLOOR and ten times the owner files' unmutated work, where + // a file's unmutated work is its most-working test's. + const files = [...new Set(runs.map((r) => r.t.file))] + // A file's most-working unmutated test, in site evaluations. + const mostBefore = (file) => + Math.max(0, ...runs.filter((r) => r.t.file === file).map((r) => r.before)) + const unmutated = files.reduce((sum, file) => sum + mostBefore(file), 0) + const workBudget = Math.max(WORK_FLOOR, 10 * unmutated) + const counted = new Set() + let spent = 0 + let exhausted = false + for (const file of files) { + let work = 0 + const callBudget = Math.max(CALL_FLOOR, 100 * mostBefore(file)) for (const r of runs.filter((x) => x.t.file === file)) { - if (r.slowed) slowed.set(file, r.t.slowMs ?? SLOW_MS) - if (r.hang) hung.add(file) - if (r.hang || killsIn(r)) break + // A copy whose runs pass its call budget is cut by count, where the run + // goes past it (the CLI's spin row). + const spun = !r.hang && work + r.work > callBudget + if (r.hang || spun) { + hung.add(file) + cut ??= CUT_CAUSE[spun ? `count` : r.cut] + } + if (r.hang && r.cut === `wall`) wallHung.add(file) + // A cut copy records no work (its calls are not rows the CLI keeps). + work += r.hang || spun ? 0 : r.work + if (r.hang || spun || killsIn(r)) break + } + // A copy cut by a clock ends the mutant's copies here: no later copy runs. + if (wallHung.has(file)) { + exhausted = true + break + } + if (budgeted && spent + work > workBudget) { + exhausted = true + cut ??= `work budget` + break } - const live = runs.filter((r) => !hung.has(r.t.file)) + spent += work + counted.add(file) + } + const live = runs.filter((r) => counted.has(r.t.file) && !hung.has(r.t.file)) const verdicts = (route) => live.flatMap((r) => r.verdicts.filter((v) => v.route === route).map((v) => v.law), ) const reproducible = verdicts(`reproducible`) if (reproducible.some((law) => law !== `reach agreement`)) return `killed` - if (hung.size > 0) return `timeout` - // The runs that confirm a mutant's cut-off copies share its budget - // (`--mutant-limit`, MUTANT_LIMIT_MS here; maintainer disposition G1, - // 2026-11-11). Under two budgets, a slow owner file is cut at its first - // budget and confirmed alone, and its confirmation takes longer than the - // time its slow test waits. When the confirmations of every slow owner - // file do not fit the budget together, one is cut, and the mutant is a - // timeout; a kill still decides first. Under one budget no copy is - // confirmed, so the budget never binds. - if ( - budgeted && - plan.budget === `two` && - [...slowed.values()].reduce((sum, ms) => sum + ms, 0) >= MUTANT_LIMIT_MS - ) + if (hung.size > 0 || exhausted) { + if (detail) detail.cause = cut return `timeout` + } + // No wall-clock budget decides here (maintainer decision, deterministic + // cuts, option A, 2026-11-14): a slow copy runs to its end, and the mutant + // budget is a work count (above). The wall-clock backstops report a gap no + // history here reaches, so a slow owner's wait never makes a timeout. if (reproducible.length > 0) return `agreement-only-kill` if (verdicts(`random`).length > 0) return `random-only-kill` if (verdicts(`unknown`).length > 0) return `unregistered-kill` @@ -4506,6 +4654,12 @@ function model(plan) { `indeterminate`, `unreached`, ]) + // A timeout's cause: the cut that the copies ran into first (`judge`). + const causeOf = (tests, active, options) => { + const detail = {} + judge(plan, tests, sites, active, { ...options, detail }) + return detail.cause + } const mutants = sites.map((site) => { const kind = site.probe ? undefined @@ -4522,6 +4676,10 @@ function model(plan) { ordinal: site.ordinal, probe: site.probe, kind, + cause: + kind === `timeout` + ? causeOf(ownerTests(site.fnId), { site: site.id }) + : undefined, // A mutant whose run reaches a slow owner test keeps a copy busy for // SLOW_MS, so the CLI must name it in a heartbeat while it runs. heartbeat: @@ -4557,6 +4715,10 @@ function model(plan) { ), ), line: a.applies ? a.line : undefined, + cause: + a.applies && owners(a.fn).size > 0 + ? causeOf(ownerTests(a.fn), { authored: a.label }, { budgeted: false }) + : undefined, })) const authored = authoredAll[0]?.kind const isGap = (kind) => @@ -4715,9 +4877,17 @@ function oracleTestFile(plan, file) { ...(plan.tests.some((t) => t.file === file && t.shrinks) ? [`import fc from 'fast-check'`] : []), - ...(plan.tests.some((t) => t.file === file && (t.ioWait || t.stops)) + ...(plan.tests.some( + (t) => t.file === file && (t.ioWait || t.stops || t.pendingIo === `test`), + ) ? [`import { spawn } from 'node:child_process'`] : []), + ...(plan.tests.some((t) => t.file === file && t.pendingIo === `test`) + ? [ + `import { join as samplesPath } from 'node:path'`, + `import { threadId } from 'node:worker_threads'`, + ] + : []), ...(plan.tests.some((t) => t.file === file && t.timed === `sleep`) ? [`import { setTimeout as sleep } from 'node:timers/promises'`] : []), @@ -4866,13 +5036,50 @@ function oracleTestFile(plan, file) { ` if (reads${i} !== 0) throw new Error('the payload was read')`, ] : []), - // A slow test waits past the first budget, within the full limit, - // where the output differs; it then judges as usual. + // A slow test waits where the output differs; it then judges as usual. ...(t.slow ? [ ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.slowMs ?? SLOW_MS}))`, ] : []), + // Where its output differs, a test that calls the function `loops` + // more times: each call is work the copy does, and the work budget + // counts it. + ...(t.loops + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) for (let k${i} = 0; k${i} < ${t.loops}; k${i} += 1) ${callFor(t.fn)}(${part.input}, [])`, + ] + : []), + // Where its output differs, a test that waits on one timer due well + // inside its horizon. Nothing else keeps it busy: no load is in the test + // (a loaded variant would be a CPU hog outside the worker), so the timer + // check alone stands between this copy and an idle cut. + ...(t.timerWait + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.timerWait.waitMs})${t.timerWait.unref ? `.unref()` : ``})`, + ] + : []), + // Where its output differs, a test that reads a pipe it opened: the + // stdout of a child it unrefs, so that only the read is active. The + // child writes once the copy's sampler has judged the test 20 more + // times (ORACLE_GAPS_TEST_SAMPLES), a count of samples at least 50 ms + // apart, past the 500 ms idle window, so the state is held by a count + // and not by a time. A copy that wrongly goes idle stops its sampler's + // count, so the read never ends and the copy is cut. The test then + // judges as usual. + ...(t.pendingIo === `test` + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => { const child = spawn(process.execPath, ['-e', ${JSON.stringify(SAMPLE_HOLD)}, samplesPath(process.env.ORACLE_GAPS_TEST_SAMPLES ?? '', \`\${process.pid}-\${threadId}\`)], { stdio: ['ignore', 'pipe', 'ignore'] }); child.unref(); child.stdout.once('data', r) })`, + ] + : []), + // Where its output differs, a test that writes to the console and then + // waits on an unref'd timer due long past its horizon, while the + // runner holds its own write pending (ORACLE_GAPS_TEST_RUNNER_WRITE). + ...(t.pendingIo === `runner` + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { console.log('waiting past the horizon'); await new Promise((r) => setTimeout(r, ${PAST_HORIZON_MS}).unref()) }`, + ] + : []), // A test that waits on I/O (a child process, about 3 s) where the // output differs, then judges as usual. ...(t.ioWait @@ -4901,10 +5108,14 @@ function oracleTestFile(plan, file) { ` }`, ] : []), - // A flaky test never hangs: the model leaves flaky tests out. + // A flaky test never hangs: the model leaves flaky tests out. A + // `pulse` hang keeps a repeating timer due inside its horizon, so the + // copy is never idle, and it never finishes: only the backstop cuts it. ...(t.hangs && !t.flaky ? [ - ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise(() => {})`, + t.pulse + ? ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { setInterval(() => {}, 100); await new Promise(() => {}) }` + : ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise(() => {})`, ] : []), ...(!t.judges @@ -4935,7 +5146,7 @@ function oracleTestFile(plan, file) { : []), t.asProp ? ` fcTest.prop([fcv.integer({ min: 0, max: 10 })], { numRuns: 20, seed: 1 })('${testName(t, index)}', async (n) => {` - : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timed || lazyGated(plan, file, t) || fsGated(plan, file, t) ? `async ` : ``}() => {`, + : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timerWait || t.pendingIo || t.timed || lazyGated(plan, file, t) || fsGated(plan, file, t) ? `async ` : ``}() => {`, ...partsOf(t).flatMap(campaign), ` })`, `})`, @@ -5922,7 +6133,6 @@ export { interpret, judge, slowedFiles, - MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, refusal, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 31b94b29..9e8c82de 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -140,28 +140,24 @@ * decision of 2026-11-06); a mutant that only nondeterministic owner tests * could judge is `indeterminate`. A mutant under which the module throws * while it loads fails every file that imports it before any test runs: - * an error outside any test, a `setup-error`. A run is cut off when it - * passes its first, short budget, and only the copy that ran past it is - * charged with that time (a copy whose file fails to load ends its turn - * there); it then runs again alone with the full limit - * (the short budget with confirmation, maintainer decision of - * 2026-11-06): a run that ends within the full limit is judged as if it - * had never been cut off, so a slow kill is a kill. An owner test that - * does not end within the full limit cuts its file's run off, and the - * clock-cut run records nothing, not even its earlier verdicts; a kill in - * another owner still decides (a kill outranks a timeout, v19), and failing - * one, the mutant is a `timeout`, whatever any other owner reported. The - * confirming runs of one mutant share its budget: the larger of - * `--mutant-limit` (30 s by default) and ten times its owner files' - * unmutated time (maintainer disposition G1, 2026-11-11). Its - * confirmations run in its owner files' order, and each gets at most what - * the budget has left after the seconds of its earlier confirmed runs, - * reused ones included, so the outcome is a function of the runs, not of - * their order or of reuse (law 9 checks a second daemon check of the same - * state); one that the budget cuts, or that gets nothing, is a timeout; so - * a mutant that is slow in two owners, whose confirmations together pass - * the budget, is a `timeout`, and a kill in an earlier owner still - * decides. A copy + * an error outside any test, a `setup-error`. No wall-clock limit decides + * an outcome (deterministic cuts, option A, 2026-11-14): a copy runs to its + * end, or is cut by its call count (a spin past its call budget), by its + * state (no progress: idle, with no timer due in its horizon), or by the + * backstop (`--hang-limit`, 60 s by default, plus 3 s per second of its + * file's unmutated time), which reports a gap. A spin or no-progress cut + * ends only its own copy, charged nothing, and the next owner's copy still + * runs; a backstop cut ends the mutant's later copies. A kill in another + * owner still decides (a kill outranks a timeout), and failing that the + * mutant is a `timeout`, whatever any other owner reported. The copies of + * one mutant share a work budget, counted in site evaluations and not in + * time: the larger of 100,000 and ten times their owner files' unmutated + * work (maintainer disposition G1, 2026-11-11, as a work count). Those + * copies are summed in owner order, and once the sum passes the budget, + * that copy and every later one is a timeout, so a kill among them does not + * count. Every timeout names its cause: `backstop`, `spin`, `no progress`, + * or `work budget`, the first cut in owner order (the CLI's `detail`). A + * copy * cut off for spinning (its call budget ran out, not its clock) is a * timeout as it stands: its rows are not judged, so a verdict it recorded * before the cut does not kill. (The framework's `bounded work` law, which @@ -400,8 +396,8 @@ * * Limits. Sources are one file of exported functions and classes over one * number. A hang is generated only as an owner test that waits on a promise - * that never settles, under a 1 s first budget and a 20 s full limit - * (`--first-limit`, `--hang-limit`, and a 15 s `--mutant-limit`); a slow run + * that never settles, under a 20 s full limit + * (`--hang-limit`); a slow run * that ends is only a witness, a test that waits 3 s where its output * changed and then judges, or 10 s in two one-test owners without judging. Not generated: a busy * loop, a slow run past the full limit that would still end, a hang in an @@ -463,9 +459,8 @@ * after its wait) and a stop that waits for the close are not generated. * Cut-off * charging is judged only where it changes an outcome: under one budget - * (`--first-limit` equal to `--hang-limit`), where a misplaced cut is a - * timeout at once; under two budgets a misplaced cut only costs a - * confirming run, which no law here measures. A copy that failed to load in + * (retired), where a misplaced cut is a timeout at once. A copy that failed + * to load in * a batch cut off later is run again rather than judged from its rows, so * its setup error keeps Vitest's message. Native addons only as a * `binding.gyp` beside the workspace dependency or a prebuilt `.node` file @@ -546,7 +541,6 @@ import { sourceArgument, outsideRepository, ENV_PREFIX, - MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, NO_LATER_STEPS, @@ -597,6 +591,7 @@ function cliArgs( ordinary = false, sources = `${SOURCE},${LIB}`, budget = `two`, + hangMs: hangMsOrNull = null, outName = daemon ? `out-daemon` : `out`, config, contexts = 4, @@ -610,6 +605,8 @@ function cliArgs( // relative to there, as an author types them, and the daemon must read // them as the CLI's caller meant them. const path = (absolute) => (daemon ? relative(repo, absolute) : absolute) + // A generated history's backstop is the drawn null: the full limit. + const hangMs = hangMsOrNull ?? FULL_LIMIT_MS const args = [ join(cli, `index.mjs`), ...(daemon ? [`--daemon`] : []), @@ -626,18 +623,13 @@ function cliArgs( // in a fixed order; results do not depend on the count. `--contexts`, String(contexts), - // A fixture's tests run in milliseconds. A 1 s first budget cuts off - // the slow witnesses, and the full limit lets them end; a hang is cut - // for idleness. With one budget, a run past the full limit is a timeout - // at once. - `--first-limit`, - budget === `one` ? String(FULL_LIMIT_MS) : `1000`, + // A fixture's tests run in milliseconds. A run's copies have no first + // budget (option A): the full limit is the only time limit, and a hang + // is cut for idleness. `--first-limit` is retired; the budget `one` + // history passes it, and the CLI notes that it changes nothing. + ...(budget === `one` ? [`--first-limit`, `1000`] : []), `--hang-limit`, - String(FULL_LIMIT_MS), - // The budget that a mutant's confirming runs share, the full limit of - // one copy: one slow owner's confirmation fits it, two do not (law 4). - `--mutant-limit`, - String(MUTANT_LIMIT_MS), + String(hangMs), `--out`, path(out), ] @@ -1223,6 +1215,8 @@ function readRun(dir, out, status, stderr, cli, stdout) { outcome: r.outcome, class: r.class, laws: r.laws, + // The cut that ended a timeout (classify.mjs `outcomeOf`). + cause: r.outcome === `timeout` ? r.detail : undefined, probe: r.probeGuard === true, ordinaryOutcome: r.testOutcome !== undefined, })), @@ -1400,6 +1394,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + hangMs: plan.history.hangMs, ...configArgs(plan), // After the `config` edit, `--config` names the second config. ...(state.altConfig ? { config: altConfigPath(plan) } : {}), @@ -1886,12 +1881,31 @@ async function driver(plan, observe) { // and holds each batch's count and its witness markers. const tornPolls = `${dir}.torn-polls` if (plan.tornRows) mkdirSync(tornPolls, { recursive: true }) + // A test that holds its own I/O open for a count of the sampler's samples + // (`pendingIo: 'test'`), and the runner's own pending write beside a test + // that can make no progress (`pendingIo: 'runner'`): each hook's directory + // sits beside the package, and the runner's marks each copy that went idle + // while its write was pending. Every run of the history sets the hooks (its + // partial run, twin and stability rerun too), since the test needs them. + const pendingIo = (owner) => + plan.history.tests.some((t) => t.pendingIo === owner) + const samples = `${dir}.samples` + const runnerWrites = `${dir}.runner-writes` + if (pendingIo(`test`)) mkdirSync(samples, { recursive: true }) + if (pendingIo(`runner`)) mkdirSync(runnerWrites, { recursive: true }) const how = { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + hangMs: plan.history.hangMs, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), + env: { + ...(pendingIo(`test`) ? { ORACLE_GAPS_TEST_SAMPLES: samples } : {}), + ...(pendingIo(`runner`) + ? { ORACLE_GAPS_TEST_RUNNER_WRITE: runnerWrites } + : {}), + }, } const { replay, ...cli } = runCli( dir, @@ -1899,10 +1913,13 @@ async function driver(plan, observe) { plan.tornRows ? { ...how, - env: { ORACLE_GAPS_TEST_TORN_ROWS: tornPolls }, + env: { ...how.env, ORACLE_GAPS_TEST_TORN_ROWS: tornPolls }, } : how, ) + const idleBesideRunnerWrite = pendingIo(`runner`) + ? readdirSync(runnerWrites).length + : 0 // The watchdog read a closing row while the worker wrote it: a hold // marks `reached` once two polls came after its first half. A hold that // waited 60 s with no poll marks `unreached`, and the witness fails @@ -1930,11 +1947,75 @@ async function driver(plan, observe) { // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) + // The slow twin (`slowTwin`): the same package, where file a's first + // test waits 3 s wherever the output differs, and + // then judges as before. Its verdicts are the same, so its report must + // be too ("Determinism under load", 2026-11-13: an outcome does not + // depend on how long a run took). + let twinSame = false + if (plan.history.slowTwin) { + const twin = resolveHistory({ + ...plan.history, + slowTwin: undefined, + tests: plan.history.tests.map((t, i) => + i === 0 ? { ...t, slow: true } : t, + ), + }) + const slow = runCli(dir, fixtureFiles(twin), how) + twinSame = + slow.exit === cli.exit && + JSON.stringify(slow.report?.results) === + JSON.stringify(cli.report?.results) + observe(`twin`, { + fast: cli.report?.results, + slow: slow.report?.results, + exit: slow.exit, + }) + } // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if (twinSame) + states.add(`a slow owner's kill reported as when it did not wait`) + // A kill after a 16 s wait, and one whose copy waits on a timer, each + // killed as the report says. + const killed = cli.report?.results?.some((r) => r.outcome === `killed`) + if (killed && plan.history.tests.some((t) => t.slowMs === 16_000)) + states.add(`a slow kill after a 16 s wait killed`) + if (killed && plan.history.tests.some((t) => t.timerWait)) + states.add(`a copy waiting on a timer killed`) + if (killed && plan.history.tests.some((t) => t.timerWait?.unref)) + states.add(`a copy waiting on an unref'd timer killed`) + // A copy that read a pipe it opened, past the idle window by count, was + // kept busy by that read alone, and its kill decided. + if (killed && pendingIo(`test`)) + states.add(`a copy waiting on I/O it opened killed`) + // A copy went idle while the runner's own write was pending (its mark), + // and the mutant is a timeout by no progress, not a backstop gap. + if ( + idleBesideRunnerWrite > 0 && + cli.report?.results?.some( + (r) => r.outcome === `timeout` && r.detail === `no progress`, + ) + ) + states.add(`a copy beside the runner's pending write cut as no progress`) + // A copy that waits past the history's backstop is cut off, and no later + // owner runs, so the mutant is a timeout, not the later owner's kill. + if ( + plan.history.hangMs && + cli.report?.results?.some((r) => r.outcome === `timeout`) + ) + states.add(`a backstop cut ends the copies ahead of a later kill`) + // Two owners that each make 30,000 extra calls: the second owner's copy + // passes the work budget, and its kill does not count (the outcome rule). + if ( + plan.history.tests.length === 2 && + plan.history.tests.every((t) => t.loops === 30_000) && + cli.report?.results?.some((r) => r.outcome === `timeout`) + ) + states.add(`a kill past the work budget in a later owner does not count`) if (tornRead && cli.reported) states.add(`a capture row read while the worker wrote it`) if (cli.exit === 0 && !plan.consoleOutput) @@ -2232,7 +2313,7 @@ async function driver(plan, observe) { if (m.function === `w_iowait` && m.outcome === `killed`) states.add(`a copy waiting on I/O killed, not cut for idleness`) if (m.function === `w_slowkill` && m.outcome === `killed`) - states.add(`a slow kill past the first time limit killed`) + states.add(`a slow kill killed`) if (m.function === `w_hang` && m.outcome === `timeout`) states.add(`a hang alone timed out`) if (m.function === `w_hangkill` && m.outcome === `killed`) @@ -2240,7 +2321,7 @@ async function driver(plan, observe) { if (m.function === `w_hangsurvive` && m.outcome === `timeout`) states.add(`a hang in one owner and a survivor in another timed out`) if (m.function === `w_slowpair` && m.outcome === `timeout`) - states.add(`confirmations past a mutant's budget timed out`) + states.add(`a mutant's copies past its work budget timed out`) if (cli.heartbeats.some((beat) => beat.label === m.id)) states.add(`a heartbeat named a busy mutant`) const named = { @@ -2385,6 +2466,8 @@ async function driver(plan, observe) { for (const state of states) observe(`reach`, state) } finally { if (copy) rmSync(copy, { recursive: true, force: true }) + rmSync(`${dir}.samples`, { recursive: true, force: true }) + rmSync(`${dir}.runner-writes`, { recursive: true, force: true }) removePath(`${dir}.link`) written.delete(dir) try { @@ -2541,6 +2624,23 @@ function daemonCheck(observations) { function check(plan, observations, expected) { const cli = observations.find((o) => o.checkpoint === `cli`)?.value if (!cli) return violation(`cli`, `no CLI run was observed`, `completion`) + // A run's outcome does not depend on how long its runs took: the slow + // twin reports each mutant as the package does (determinism under load). + const twin = observations.find((o) => o.checkpoint === `twin`)?.value + if (twin) { + const at = (twin.fast ?? []).findIndex( + (r, i) => JSON.stringify(r) !== JSON.stringify(twin.slow?.[i]), + ) + if (twin.exit !== cli.exit || at >= 0) + return violation( + `twin`, + `the slow twin reports ${at >= 0 ? twin.fast[at].key : `exit ${twin.exit}`} otherwise than the package whose test does not wait`, + `determinism under load`, + at >= 0 + ? { actual: twin.slow?.[at], expected: twin.fast[at] } + : undefined, + ) + } // An internal error comes first: it also leaves the session directory, so // the namespace law below would name only its symptom (issue #83). The // message carries the run's error.log (readRun). @@ -2726,6 +2826,14 @@ function check(plan, observations, expected) { : `classification`, { actual: got, expected: owed }, ) + // A timeout names the cut that ended it (its `detail`). + if (kind === `timeout` && got.cause !== owed.cause) + return violation( + `cli`, + `${site(got)}: its timeout is cut by ${got.cause}, owed ${owed.cause}`, + `classification`, + { actual: got, expected: owed }, + ) } // Each gap is streamed once as its mutant is judged (stream.mjs), and the // streamed gaps are the report's gaps, each once. @@ -2795,6 +2903,13 @@ function check(plan, observations, expected) { `authored mutants`, { actual: kind, expected: owed.kind }, ) + if (kind === `timeout` && got.cause !== owed.cause) + return violation( + `cli`, + `the authored mutant ${owed.id} is cut by ${got.cause}, owed ${owed.cause}`, + `authored mutants`, + { actual: got.cause, expected: owed.cause }, + ) if (owed.line !== undefined && got.line !== owed.line) return violation( `cli`, @@ -3006,7 +3121,9 @@ function streamBreach(checkpoint, run, expected, plan) { * `, and it shows the gap's kind (its class, or its outcome), its * line (`?` when the mutant never applied), and its change: two code spans * around ` → `, the code and its replacement, or the intent's span and - * ` (authored by )`. A span shows its code on one line, with each + * ` (authored by )`. A timeout's line then names its cause, the cut + * that ended it, as ` (cause: )`, which is results.json's `detail`, + * and no other line names one. A span shows its code on one line, with each * run of whitespace as one space, whole or cut and ended with `...`; the * cut's length is the printer's choice. The comparison is by id, so a run * that prints one gap twice and drops another breaks it, though its count @@ -3041,6 +3158,7 @@ function stdoutGapBreach(stdoutGaps, results) { const owed = { kind: gap.class ?? gap.outcome, line: gap.line === undefined ? `?` : String(gap.line), + cause: gap.outcome === `timeout` ? gap.detail : undefined, } const shows = gap.authored ? read.intent !== undefined && @@ -3049,9 +3167,14 @@ function stdoutGapBreach(stdoutGaps, results) { : read.original !== undefined && showsCode(read.original, gap.original) && showsCode(read.replacement, gap.replacement) - if (read.kind !== owed.kind || read.line !== owed.line || !shows) + if ( + read.kind !== owed.kind || + read.line !== owed.line || + read.cause !== owed.cause || + !shows + ) return { - difference: `the stdout's gap line for ${read.id} is ${JSON.stringify(line)}, and results.json reports it as ${owed.kind} at line ${owed.line}, changing ${JSON.stringify(gap.authored ? gap.intent : `${gap.original} → ${gap.replacement}`)}`, + difference: `the stdout's gap line for ${read.id} is ${JSON.stringify(line)}, and results.json reports it as ${owed.kind}${owed.cause === undefined ? `` : ` (cause: ${owed.cause})`} at line ${owed.line}, changing ${JSON.stringify(gap.authored ? gap.intent : `${gap.original} → ${gap.replacement}`)}`, extra: { actual: line, expected: { ...owed, id: read.id } }, } } @@ -3066,7 +3189,8 @@ function stdoutGapBreach(stdoutGaps, results) { /** * A gap line's fields, read from its text as an agent reads it: its kind, - * line, id, and change. Undefined when the line does not read so. + * line, id, change, and a timeout's cause (` (cause: )` after the + * change). Undefined when the line does not read so. */ function readGapLine(line) { const head = line.match(/^oracle-gaps: gap ([\w-]+) L(\d+|\?) (\S+) (.*)$/) @@ -3074,18 +3198,31 @@ function readGapLine(line) { const [, kind, at, id, change] = head const first = readCodeSpan(change) if (!first) return undefined - const authored = first.after.match(/^ \(authored by (.*)\)$/) + // What follows the change's last part: nothing, or the cause of a timeout. + const ending = (after) => after.match(/^(.*?)(?: \(cause: ([^()]+)\))?$/) + const [, afterIntent, intentCause] = ending(first.after) + const authored = afterIntent.match(/^ \(authored by (.*)\)$/) if (authored) - return { kind, line: at, id, intent: first.code, author: authored[1] } + return { + kind, + line: at, + id, + intent: first.code, + author: authored[1], + cause: intentCause, + } if (!first.after.startsWith(` → `)) return undefined const second = readCodeSpan(first.after.slice(` → `.length)) - if (!second || second.after !== ``) return undefined + if (!second) return undefined + const [, rest, cause] = ending(second.after) + if (rest !== ``) return undefined return { kind, line: at, id, original: first.code, replacement: second.code, + cause, } } @@ -3670,6 +3807,29 @@ calibrations.push( return { ...cli, stdoutGaps: [renamed, ...rest] } }), }, + { + name: `a stdout gap line names another cause`, + law: `greppable stream`, + // A timeout's line whose cause is another cut than results.json's + // `detail`: the line still reads, so only the cause comparison rejects it. + corrupt: corruptCli((cli) => { + const at = cli.stdoutGaps.findIndex((line) => + / \(cause: [^()]+\)$/.test(line), + ) + if (cli.exit !== 0 || at < 0) return undefined + const recaused = cli.stdoutGaps[at].replace( + / \(cause: ([^()]+)\)$/, + (_, cause) => + cause === `backstop` ? ` (cause: no progress)` : ` (cause: backstop)`, + ) + return { + ...cli, + stdoutGaps: cli.stdoutGaps.map((line, i) => + i === at ? recaused : line, + ), + } + }), + }, { name: `a stdout gap line does not read as a gap`, law: `greppable stream`, @@ -3889,7 +4049,7 @@ const required = [ `timeout`, `a kill after a nondeterministic verdict in its file`, `a kill in one owner after a gap in another`, - `a slow kill past the first time limit killed`, + `a slow kill killed`, `a mutant that changes a value only at load killed`, `a mutant that changes no value judged without a run`, `a boolean flip killed`, @@ -3907,8 +4067,16 @@ const required = [ `each owner's value change killed its own mutant`, `a hang alone timed out`, `a hang in one owner and a kill in another killed`, + `a slow owner's kill reported as when it did not wait`, + `a slow kill after a 16 s wait killed`, + `a copy waiting on a timer killed`, + `a copy waiting on an unref'd timer killed`, + `a kill past the work budget in a later owner does not count`, + `a backstop cut ends the copies ahead of a later kill`, + `a copy waiting on I/O it opened killed`, + `a copy beside the runner's pending write cut as no progress`, `a hang in one owner and a survivor in another timed out`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget timed out`, `a heartbeat named a busy mutant`, `a run stopped by SIGINT while a worker spun`, `a stop signal after the completed line`, @@ -4032,6 +4200,16 @@ const separate = [ `a session failure larger than a socket buffer`, `a config fault in a run that owns nothing`, `a refused run whose package lies inside the CLI's directory`, + // The deterministic-cuts states have constructed histories of their own, so + // no history that names every state (`every outcome class`) must enter them. + `a slow owner's kill reported as when it did not wait`, + `a slow kill after a 16 s wait killed`, + `a copy waiting on a timer killed`, + `a copy waiting on an unref'd timer killed`, + `a kill past the work budget in a later owner does not count`, + `a backstop cut ends the copies ahead of a later kill`, + `a copy waiting on I/O it opened killed`, + `a copy beside the runner's pending write cut as no progress`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, `a package with no config ran`, @@ -4041,7 +4219,7 @@ const separate = [ `an awaiting config in a package with no module type ran`, `a .cjs or .cts config that calls require ran`, `a .js or .ts config in a package with no module type that calls require ran`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget timed out`, `a second daemon check of the budget witness's state`, `a run with eleven contexts`, `a failing owner replayed under a config that --config names`, @@ -4112,12 +4290,46 @@ const withDeclaration = (constructed) => }), })) +// A constructed history whose only owned function is a kill, owned by file a +// alone, with the tests given (the rest of the generated base is left as the +// grammar draws it, as the histories above do). +const soloKill = (generated, tests) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests, + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, +}) oracleProperty({ id: `oracle-gaps.report`, law: { - statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, + statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds a mutant's copies by one work budget per mutant (site evaluations, never time) and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A line with any text after its change does not read and fails loudly: the \` (cause: )\` suffix that #99 adds to timeout lines is not read yet, so whichever of #99 and #115 lands second extends readGapLine. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Windows idle gap (recorded, not witnessed): the state \`a hang cut off for idleness\` is required on every platform, and win32 reaches it (run 43683794816, job 132719872988), so it is not platform-scoped. The sampler (capture-setup.mjs) counts a copy idle only after 500 ms with no late tick, no non-quiet handle, utilization at most 5%, and no timer due, so \`every outcome class\` reaches the state only when the sampler sees that window. On win32 the check missed it once (run 43683794816, job 132719872940); the cause is not confirmed (a late tick on a shared runner, or a stdio handle outside \`quiet\` for this file's console output, are the candidates). Its witness, a test-only hold at the hang until the window is seen, is owed, and that is a timing dependency the determinism-under-load rule forbids; the maintainer decides whether it lands before the Windows job becomes blocking.`, + limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A timeout's line names its cause as \` (cause: )\` after its change, judged equal to results.json's \`detail\`; a line with any other text after its change does not read and fails loudly. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its backstop, since a copy has no first budget (option A); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant's work budget is witnessed by w_slowpair and a kill past it, in owner order, each owner's copy counted in site evaluations (not in time), and authored mutants have no copies to budget; the witness w_slowpair is a work-budget witness (no wall-clock window): each of its two copies calls the function 30,000 more times, 60,000 to 90,000 site evaluations, inside its call budget, and the two together pass the work budget, so the mutant is a timeout by count on any machine; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Idle and the runner's I/O (M entries "idle is decided by state only" and "Idle counts only the test's I/O", 2026-11-15): the sampler (capture-setup.mjs) counts a copy idle after 500 ms with no timer due inside its horizon, no more timers listed than at its test's start, and none of its test's own I/O active, where a handle or request is the test's when the test's or production's code was on the stack when Node made it, or when the resource that triggered it is the test's. Two constructed histories hold the states by count and by state: \`a copy waiting on a pipe it opened\` (the test's own pipe read, released after 20 samples) and \`a copy beside the runner's pending write\` (a write the runner holds pending through the test, as Vitest's forwarded output can stay pending on win32; Windows shard 7, run 43778323034). Not witnessed: a handle whose owner keeps its wrap elsewhere than \`_handle\` (a dgram socket) is not found as the test's, so a test that waits only on it looks idle; a test that awaits only a reply over the runner's own channel (a file snapshot) looks idle on every platform, as it did on POSIX before; a request whose stack frames all lie in node_modules machinery (vitest, fast-check, chai, this package) is the runner's even when a test's call led to it; and a timer made through the wrappers counts in the listed-timer check whatever its due time, so a ref'd timer past the horizon keeps a copy busy unless the listed count at the test's start held a timer of the runner's, the reason the runner-write witness waits on an unref'd timer.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4184,13 +4396,13 @@ oracleProperty({ }, { // The budget witness checked through the daemon twice with nothing - // changed between: the second check reuses the confirmations the - // first ran, and must report what a cold run reports (law 9, with - // the budget of law 4). + // changed between: the second check reuses the work-budget results + // the first ran, and must report what a cold run reports (law 9, with + // the work budget of law 4). name: `a second daemon check of the budget witness`, enters: [ `a second daemon check of the budget witness's state`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget timed out`, ], build: (generated) => ({ ...generated, @@ -5471,6 +5683,232 @@ oracleProperty({ warmReplaced: true, }), }, + { + // A determinism check (the determinism under load law, 2026-11-13): + // file a's random-route test and file b's fixed-seed test both kill + // one mutant (as w_crosskill does). In the slow twin, a's test waits + // 3 s before it judges, so its copy runs to its end, and b's kill + // decides the mutant first. The report must still be the one the + // package without the wait gives, its `randomAlso` included. It + // passes on its parent too, so it guards the law and has no RED of + // its own; the outcome rule's RED is the classify oracle's. + name: `a slow owner's kill and another owner's kill`, + enters: [`a slow owner's kill reported as when it did not wait`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `random seed`, 7), fn: 0 }, + { ...judging(`b`, `fixed seed`, 7), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + slowTwin: true, + }), + }, + // Constructed histories added by the deterministic-cuts change follow. + // Owner a's copy never finishes and never goes idle: it awaits a promise + // that never resolves, while a repeating timer stays due inside its + // horizon (`pulse`). So it is cut only by the 2 s backstop, on every + // machine, and owner b would kill. The cut ends the copies, so b does + // not run: the mutant is a backstop gap (a timeout), never a kill. + { + name: `a backstop cut ends the copies ahead of a later kill`, + enters: [`a backstop cut ends the copies ahead of a later kill`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + hangs: true, + pulse: true, + }, + { ...judging(`b`, `fixed seed`, 7), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + hangMs: 2_000, + }), + }, + // Owner a's copy makes 30,000 extra calls (under its budget), and so does + // owner b's, which would kill. Together they pass the work budget, so b's + // kill does not count: the mutant is a timeout by count. + { + name: `a kill past the work budget in a later owner`, + enters: [`a kill past the work budget in a later owner does not count`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + loops: 30_000, + }, + { ...judging(`b`, `fixed seed`, 7), fn: 0, loops: 30_000 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + }), + }, + // A slow kill whose test waits 16 s where its output differs: no + // wall-clock limit decides an outcome, so the mutant is still killed. + { + name: `a slow kill after a 16 s wait`, + enters: [`a slow kill after a 16 s wait killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + slow: true, + slowMs: 16000, + }, + ]), + }, + // A copy whose test starts a timer due inside its horizon, then waits + // for it: the copy is never idle, and the mutant is killed. + { + name: `a copy waiting on a timer`, + enters: [`a copy waiting on a timer killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + timerWait: { waitMs: 3000 }, + }, + ]), + }, + // The same wait on an unref'd timer: the timer is not listed among the + // process's active resources, so only the copy's own pending timers + // keep it busy (the held state, deterministic cuts, 2026-11-15). + { + name: `a copy waiting on an unref'd timer`, + enters: [`a copy waiting on an unref'd timer killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + timerWait: { waitMs: 3000, unref: true }, + }, + ]), + }, + // Idle counts only the test's I/O (M entry, 2026-11-15). A copy whose + // test reads a pipe it opened, the stdout of a child it unrefs, is busy + // while the read is pending, and nothing else keeps it busy: the read is + // released by a count of the sampler's samples past the idle window, not + // by a time. The mutant is killed. + { + name: `a copy waiting on a pipe it opened`, + enters: [`a copy waiting on I/O it opened killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + pendingIo: `test`, + }, + ]), + }, + // A copy whose test writes to the console and then waits on an unref'd + // timer due long past its horizon, while the runner holds its own write + // pending through the test, as Vitest's forwarded output can be on + // win32: the runner's write is not the test's I/O, so the copy is idle + // and the mutant is a timeout by no progress, not a backstop gap. The + // package has a native addon, so its tests run in child processes, whose + // output Vitest forwards over their IPC channel, as on Windows shard 7. + { + name: `a copy beside the runner's pending write`, + enters: [`a copy beside the runner's pending write cut as no progress`], + build: (generated) => ({ + ...soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + pendingIo: `runner`, + }, + ]), + native: `gyp`, + }), + }, ]), }, resolve: resolveHistory, diff --git a/packages/oracles/oracle-gaps/history-fingerprint.json b/packages/oracles/oracle-gaps/history-fingerprint.json index ead6dcce..fe9b18bb 100644 --- a/packages/oracles/oracle-gaps/history-fingerprint.json +++ b/packages/oracles/oracle-gaps/history-fingerprint.json @@ -54,7 +54,13 @@ ["laterSteps", "fb28f2575c00f223"], ["tornRows", "ed6866229ecc4969"], ["pauseEngages", "c93e2e40cd0c4b9f"], - ["warmReplaced", "ed6866229ecc4969"] + ["warmReplaced", "ed6866229ecc4969"], + ["hangMs", "f6e71aca47bcb3e2"], + ["tests.slowMs", "33494e17be1a5ba5"], + ["tests.loops", "4bc867dd43aad13c"], + ["tests.timerWait", "02f66613e9490d15"], + ["tests.pulse", "04378653193a64e7"], + ["tests.pendingIo", "02f66613e9490d15"] ], "histories": [ ["functions", "66c0e9c56951c377"], @@ -111,6 +117,12 @@ ["laterSteps", "fb28f2575c00f223"], ["tornRows", "ed6866229ecc4969"], ["pauseEngages", "c93e2e40cd0c4b9f"], - ["warmReplaced", "ed6866229ecc4969"] + ["warmReplaced", "ed6866229ecc4969"], + ["hangMs", "f6e71aca47bcb3e2"], + ["tests.slowMs", "33494e17be1a5ba5"], + ["tests.loops", "4bc867dd43aad13c"], + ["tests.timerWait", "02f66613e9490d15"], + ["tests.pulse", "04378653193a64e7"], + ["tests.pendingIo", "02f66613e9490d15"] ] } diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 4321082e..c732e132 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -84,15 +84,13 @@ // [--ordinary] (also run ordinary test files, for information) // [--seed ] [--mutants ] [--out ] [--only ] // [--diff-base ] -// [--first-limit ] (a copy's first budget; default 2000) -// [--hang-limit ] (the full limit a cut copy gets again; default 10000) -// [--mutant-limit ] (the floor of each mutant's budget, which the -// runs that confirm its cut copies share; the -// budget is the larger of this and ten times -// its owner files' unmutated time; default 30000) -// [--test-timeout ] (default 15000; also bounds hooks and teardown. -// Each mutant run also has a wall-clock limit of -// 10 s plus three times its file's unmutated time.) +// [--hang-limit ] (the base of each copy's backstop, which adds +// 3 s per second of its file's unmutated time; +// default 60000) +// [--first-limit ] (retired: checked and noted, no effect) +// [--mutant-limit ] (retired: checked and noted, no effect) +// [--test-timeout ] (Vitest's per-test limit, also bounding hooks +// and teardown; default 65000) // [--sample ] (a fixed-seed sample of n mutants per source; // the report marks the run as partial) // [--contexts ] (warm contexts, one worker each; default diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index bae8a959..0f7761b4 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -52,9 +52,11 @@ export function change(result) { * and again in the summary. */ export function gapLine(result) { + // A timeout names the cut that ended it (classify.mjs `outcomeOf`). + const cut = result.outcome === `timeout` ? ` (cause: ${result.detail})` : `` return streamLine( `gap`, - `${kindOf(result)} L${result.line ?? `?`} ${result.id} ${change(result)}`, + `${kindOf(result)} L${result.line ?? `?`} ${result.id} ${change(result)}${cut}`, ) } diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index d85fb067..e006d8fc 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -60,6 +60,30 @@ export const shards = { `determinism: constructed history: a declaration fault: projects`, `determinism: constructed history: a package with no config`, `determinism: constructed history: an edit sequence through the daemon`, + `fixed seed 20261105: a slow owner's kill and another owner's kill`, + `random seed: a slow owner's kill and another owner's kill`, + `determinism: constructed history: a slow owner's kill and another owner's kill`, + `fixed seed 20261105: a slow kill after a 16 s wait`, + `random seed: a slow kill after a 16 s wait`, + `determinism: constructed history: a slow kill after a 16 s wait`, + `fixed seed 20261105: a copy waiting on a timer`, + `random seed: a copy waiting on a timer`, + `determinism: constructed history: a copy waiting on a timer`, + `fixed seed 20261105: a copy waiting on an unref'd timer`, + `random seed: a copy waiting on an unref'd timer`, + `determinism: constructed history: a copy waiting on an unref'd timer`, + `fixed seed 20261105: a copy waiting on a pipe it opened`, + `random seed: a copy waiting on a pipe it opened`, + `determinism: constructed history: a copy waiting on a pipe it opened`, + `fixed seed 20261105: a copy beside the runner's pending write`, + `random seed: a copy beside the runner's pending write`, + `determinism: constructed history: a copy beside the runner's pending write`, + `fixed seed 20261105: a kill past the work budget in a later owner`, + `random seed: a kill past the work budget in a later owner`, + `determinism: constructed history: a kill past the work budget in a later owner`, + `fixed seed 20261105: a backstop cut ends the copies ahead of a later kill`, + `random seed: a backstop cut ends the copies ahead of a later kill`, + `determinism: constructed history: a backstop cut ends the copies ahead of a later kill`, // The witness module's edit sequence (option M): 189 s here, measured // with the daemon fix of fix/daemon-current-inputs. `determinism: constructed history: an edit sequence over the witness module`, @@ -261,6 +285,7 @@ export const shards = { `calibration: the stdout drops a gap line`, `calibration: a stdout gap line shows another change`, `calibration: a stdout gap line names another kind`, + `calibration: a stdout gap line names another cause`, `calibration: a stdout gap line does not read as a gap`, `calibration: the stdout has a gap line for a mutant that is not a gap`, `calibration: a check through the daemon drops a gap line`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 24a91921..bf7db89f 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -574,7 +574,17 @@ export const partitions = { 'ordinary tests left out by default', 'a declaration fault: configNull', ]), + // The deterministic-cuts histories (the 16 s wait, the backstop, the work + // budget, the twin and the timer) are short, except the 16 s wait. 10: anyOf([ + "a copy waiting on an unref'd timer", + 'a copy waiting on a pipe it opened', + "a copy beside the runner's pending write", + "a slow owner's kill and another owner's kill", + 'a backstop cut ends the copies ahead of a later kill', + 'a kill past the work budget in a later owner', + 'a slow kill after a 16 s wait', + 'a copy waiting on a timer', 'an edit that changed the cold result after a hang replaced the only context', 'a test edits the helper it read, during a check through the daemon', 'an edit to data a test reads through fs (callback)', diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index e7d22ad1..c0dff580 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -21,6 +21,28 @@ import { drawn, histories } from '../oracle-gaps/gap-report-model-oracle.mjs' // campaign draws 2 histories, and those resolve as on main. const FIXED_SEED = 20261105 const DRAWS = 500 +// The constructed-only test fields (deterministic-cuts change, 2026-11-14): +// constants in every drawn test record, which no generated history sets. A +// test's record hashes without them, so its line is main's, and each one is +// recorded as its own line after the others, with its neutral value checked. +const CONSTRUCTED_TEST_FIELDS = [ + `slowMs`, + `loops`, + `timerWait`, + `pulse`, + `pendingIo`, +] +const NEUTRAL_TEST_VALUES: Record = { + slowMs: 3000, + loops: 0, + timerWait: null, + pulse: false, + pendingIo: null, +} +type Sample = Record + +const hashOf = (value: unknown): string => + createHash(`sha1`).update(JSON.stringify(value)).digest(`hex`).slice(0, 16) const fingerprints = ( arbitrary: fc.Arbitrary, @@ -29,14 +51,37 @@ const fingerprints = ( const samples = fc.sample(arbitrary, { seed: FIXED_SEED, numRuns: DRAWS, - }) as Array> - return Object.keys(samples[0] ?? {}).map((key) => [ - key, - createHash(`sha1`) - .update(JSON.stringify(samples.map((history) => history[key]))) - .digest(`hex`) - .slice(0, 16), - ]) + }) as Array + const testsOf = (history: Sample) => history.tests as Array + const withoutConstructed = (record: Sample) => + Object.fromEntries( + Object.entries(record).filter( + ([key]) => !CONSTRUCTED_TEST_FIELDS.includes(key), + ), + ) + const lines: Array<[string, string]> = Object.keys(samples[0] ?? {}).map( + (key) => [ + key, + hashOf( + samples.map((history) => + key === `tests` + ? testsOf(history).map(withoutConstructed) + : history[key], + ), + ), + ], + ) + for (const field of CONSTRUCTED_TEST_FIELDS) { + const values = samples.flatMap((history) => + testsOf(history).map((record) => record[field]), + ) + // Generated tests hold the neutral value: only a constructed history sets it. + expect(values.every((value) => value === NEUTRAL_TEST_VALUES[field])).toBe( + true, + ) + lines.push([`tests.${field}`, hashOf(values)]) + } + return lines } describe(`the gap oracle's history grammar`, () => {