diff --git a/packages/oracles/README.md b/packages/oracles/README.md index a2452c5..d5fac0f 100644 --- a/packages/oracles/README.md +++ b/packages/oracles/README.md @@ -14,3 +14,19 @@ Show me what oracle testing with @tanstack/oracles can find in this repository. 3. Read node_modules/@tanstack/oracles/skills/oracles-explore/SKILL.md in full, and follow it. Use wherever a skill asks for the commit to pin. ``` + +## Test hooks + +`ORACLE_GAPS_TEST_RETENTION` (a file path, unset in normal use) makes the gap CLI append one JSON line per mutant batch it runs, however the batch ends: settled, cut off, or rejected. It only observes: it never changes an outcome. The e2e oracle reads it to check bounded retention. The hook's code is in `oracle-gaps/retention-hook.mjs`, which the CLI loads only when this variable or `ORACLE_GAPS_TEST_REJECT_BATCH` is set, so a normal run never loads it. + +Under the hook, each check also writes `check.pid` (the id of the process that runs the check: the CLI's supervised child, or a daemon) into its `--out` directory, after the CLI creates that directory. Each line has: + +- `run`: the check's `--out` path. +- `pid`: the id of the process that wrote the line. The oracle compares it with `check.pid`. +- `batch`: the batch's own index within its check, from 0, taken when the batch starts. +- `before`: Vitest's and Vite's structure sizes before the batch, and the `stale*` counts. `staleIndexed` and `staleSpecs` count registry entries that name no live context. `staleOpen` counts contexts that left the live array and that Vitest has not started to close (its own close state). All three must be 0. +- `held`: how many of the entries the check gave its store are still reachable after a full garbage collection, which the hook runs before it writes the line. A cold check's store keeps none, so this must be 0 there. A daemon's store keeps its entries by design. +- For a settled or rejected batch: `after` (the sizes after its copies are retired), `copyNodes` (Vite module nodes that still name a copy, must be 0), and `nonCopyDup` (non-copy files with more than one module node in an environment, must be 0 beyond the measured plain-plus-`?raw` pair). +- For a cut batch: `cut: true` and no `after`. Its context is replaced, not retired. + +`ORACLE_GAPS_TEST_REJECT_BATCH` (a batch number from 1, unset in normal use) makes the gap CLI's n-th batch run reject, as a crash in Vitest would. It is a test-only fault: the batch's run completes, so its copies are in Vitest's and Vite's state, and then the run's promise rejects, which takes the same path a real crash takes. The run then ends as an internal error: exit 3, the done error line, and `error.log` in its output directory. The batch's copies are still retired and its retention line is still written. diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 2e548ec..6f1be39 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -612,6 +612,17 @@ export default async (env) => { let seedVariable = `ORACLE_SEED` const contextSpecs = new Map() let runCounter = 0 + // The retention test hook (retention-hook.mjs, README "Test hooks"): loaded + // only when one of its variables is set, so a normal run never loads it. + const hook = + process.env.ORACLE_GAPS_TEST_RETENTION || + process.env.ORACLE_GAPS_TEST_REJECT_BATCH + ? (await import(`./retention-hook.mjs`)).retentionHook({ + live: () => contexts, + indexOf, + contextSpecs, + }) + : undefined // A context loads the package's own config through its wrapper, so an // error there is the author's: the run refuses, naming the config file @@ -783,21 +794,19 @@ export default async (env) => { `Run the gap report on a package whose Vitest config has no test.projects.`, ) } - indexOf.set(context, index) lastServed.delete(index) return context } - async function globSpecs(context) { - // A warm context caches the test files it found; a file added since the - // last check, such as a new oracle file, appears only after the cache is - // cleared. + // A warm context caches the test files it found; a file added since the last check, such as a new oracle + // file, appears only after the cache is cleared. Returns the context's specs (registered by the caller). + async function specsOf(context) { context.clearSpecificationsCache?.() const map = new Map() for (const spec of await context.globTestSpecifications()) { if (isRunnerCopy(spec.moduleId)) continue map.set(spec.moduleId, [...(map.get(spec.moduleId) ?? []), spec]) } - contextSpecs.set(context, map) + return map } async function closeContexts() { await Promise.all( @@ -808,6 +817,12 @@ export default async (env) => { ]), ), ) + // The closed contexts leave the registry with the array: a context closed for a config change is no longer + // reachable through indexOf or contextSpecs (the daemon's edit sequence left them there before). + for (const context of contexts) { + indexOf.delete(context) + contextSpecs.delete(context) + } contexts = [] lastServed.clear() } @@ -1084,6 +1099,9 @@ export default async (env) => { let holdCounter = 0 async function runBatch(context, out, items, seed) { const capture = join(out, `run-${process.pid}-${++runCounter}.jsonl`) + // The retention hook's view of this batch: its own index, and the totals + // before it (retention-hook.mjs). + const observed = hook?.begin(context) // Each copy is named beside its file, so that its imports resolve as the // file's do, and the wrapper's plugin serves its text: nothing is // written there. @@ -1148,10 +1166,13 @@ export default async (env) => { writeFileSync(join(out, `state.json`), JSON.stringify(state)) const project = context.getRootProject() const started = Date.now() - const running = context.runTestSpecifications( + const run = context.runTestSpecifications( copies.map((copy) => project.createSpecification(copy)), false, ) + // Test-only fault (ORACLE_GAPS_TEST_REJECT_BATCH=): the n-th batch's + // run rejects once its copies are in Vitest's state, as a crash would. + const running = hook ? hook.fault(observed, run) : run // The watchdog: each copy writes a starting row when its setup runs and // a closing row when it finishes, so the copy that is running is the // last that started, or the one after it once that one closed. A copy @@ -1214,45 +1235,50 @@ export default async (env) => { } timer = setTimeout(poll, 50) }) - const result = await Promise.race([running, watchdog]) - clearTimeout(timer) - const seconds = (Date.now() - started) / 1000 - // Each copy's rows, and the copies that skipped their tests because an - // earlier copy in the batch killed their mutant. - const skipped = new Set() - // Each copy's closing row: the files of the modules it evaluated, and the - // files it read with their hashes. - const closings = new Map() - const rowsOf = (partial) => { - const rows = new Map(copies.map((copy) => [copy, []])) - closings.clear() - if (existsSync(capture)) - for (const line of captureLines(capture, { partial })) { - const row = JSON.parse(line) - const copy = copies.find( - (c) => - posix(row.key ?? ``).startsWith(`${c} > `) || - posix(row.file ?? ``) === c, - ) - if (copy && row.skipped) skipped.add(copy) - if (copy && row.closed) closings.set(copy, row) - if (copy && row.spin) spun.add(copy) - if (copy && !row.closed && !row.idle && !row.spin && !row.started) - rows.get(copy).push(row) - } - return rows - } - const skippedUnit = (item) => ({ file: item.file, skipped: true }) - // A copy that ran past its call budget, even one whose throw ended the - // loop and let it finish, is a timeout as it stands (a count, not a - // clock: no confirming run). - const spunUnit = (item) => ({ - file: item.file, - timedOut: true, - spin: true, - seconds, - }) + // The race sits inside the try below, so a rejected run (a crash, not a cut) still retires its + // copies and writes its retention line. `raced` tells that apart from a cut, which resolves undefined. + let result + let raced = false try { + result = await Promise.race([running, watchdog]) + raced = true + clearTimeout(timer) + const seconds = (Date.now() - started) / 1000 + // Each copy's rows, and the copies that skipped their tests because an + // earlier copy in the batch killed their mutant. + const skipped = new Set() + // Each copy's closing row: the files of the modules it evaluated, and the + // files it read with their hashes. + const closings = new Map() + const rowsOf = (partial) => { + const rows = new Map(copies.map((copy) => [copy, []])) + closings.clear() + if (existsSync(capture)) + for (const line of captureLines(capture, { partial })) { + const row = JSON.parse(line) + const copy = copies.find( + (c) => + posix(row.key ?? ``).startsWith(`${c} > `) || + posix(row.file ?? ``) === c, + ) + if (copy && row.skipped) skipped.add(copy) + if (copy && row.closed) closings.set(copy, row) + if (copy && row.spin) spun.add(copy) + if (copy && !row.closed && !row.idle && !row.spin && !row.started) + rows.get(copy).push(row) + } + return rows + } + const skippedUnit = (item) => ({ file: item.file, skipped: true }) + // A copy that ran past its call budget, even one whose throw ended the + // loop and let it finish, is a timeout as it stands (a count, not a + // clock: no confirming run). + const spunUnit = (item) => ({ + file: item.file, + timedOut: true, + spin: true, + seconds, + }) if (result === undefined) { // Cancelled without waiting; the caller replaces the context (see // runUnit). @@ -1359,10 +1385,106 @@ export default async (env) => { } } finally { for (const copy of copies) globalThis.__oracleGapsCopies.delete(copy) + // A run cut off by a hang is still running: its worker may still send + // updates, so its state is not touched here. Its context is replaced + // by the caller (replace), which closes it and drops its state. + clearTimeout(timer) + const cut = raced && result === undefined + if (!cut) retireCopies(context, copies) + await hook?.line(context, observed, copies, cut) rmSync(capture, { force: true }) } } + /** + * Removes a batch's copies from Vitest's state and from Vite's module + * graphs once their units are plain data (the register's C entry: the gap + * CLI keeps no Vitest or Vite state for a mutant copy after the copy's + * results are plain data). Each copy is a new file. Vitest keeps its path + * in `filesMap` and `pathsSet`, and the file and test tasks it ran in + * `idMap`, and Vite keeps a module node, with its transform, per copy. + * Vitest's `clearFiles` replaces the path's file entry only, and leaves + * `pathsSet` and the test tasks, so it is not used: each entry is removed. + * Every map touched is guarded, and a Vitest or Vite whose shape changes + * throws here rather than leaking quietly. During a run, Vitest holds the + * run's copies until it ends, so the in-run peak scales with the batch's + * share of copies, not with the suite. + */ + function retireCopies(context, copies) { + const { filesMap, idMap, pathsSet } = context.state ?? {} + if ( + !(filesMap instanceof Map) || + !(idMap instanceof Map) || + !(pathsSet instanceof Set) + ) + throw new Error( + `oracle-gaps: this Vitest keeps no filesMap, idMap, or pathsSet in its state, so a check cannot retire its copies`, + ) + const gone = new Set(copies) + for (const [id, task] of idMap) + if (gone.has(task.file?.filepath ?? task.filepath)) idMap.delete(id) + for (const copy of copies) { + filesMap.delete(copy) + pathsSet.delete(copy) + } + retireModules(context.getRootProject().vite, gone) + } + + /** + * Removes the copies' module nodes from every environment's Vite graph. + * Each node is invalidated through Vite's API (which drops its transform), + * then removed from the graph's id, url, etag, and file maps, and from its + * neighbours' import sets. The graph's map shapes are guarded. + */ + function retireModules(server, gone) { + const environments = viteEnvironments(server) + for (const environment of environments) { + const graph = environment.moduleGraph + const { + urlToModuleMap, + idToModuleMap, + etagToModuleMap, + fileToModulesMap, + } = graph ?? {} + if ( + !(urlToModuleMap instanceof Map) || + !(idToModuleMap instanceof Map) || + !(etagToModuleMap instanceof Map) || + !(fileToModulesMap instanceof Map) || + typeof graph.invalidateModule !== `function` + ) + throw new Error( + `oracle-gaps: this Vite's module graph has no id, url, etag, or file maps, so a check cannot retire its copies`, + ) + for (const copy of gone) { + for (const mod of fileToModulesMap.get(copy) ?? []) { + graph.invalidateModule(mod) + for (const imported of mod.importedModules) + imported.importers.delete(mod) + for (const importer of mod.importers) + importer.importedModules.delete(mod) + idToModuleMap.delete(mod.id) + urlToModuleMap.delete(mod.url) + // Vite keys the etag map by the etag, whose value is the node. + for (const [etag, node] of etagToModuleMap) + if (node === mod) etagToModuleMap.delete(etag) + } + fileToModulesMap.delete(copy) + } + } + } + + // Vite's environments, each with its module graph. A Vite that exposes none + // would make the module step a silent no-op, so it throws here. + function viteEnvironments(server) { + const environments = Object.values(server?.environments ?? {}) + if (environments.length === 0 || environments.some((e) => !e?.moduleGraph)) + throw new Error( + `oracle-gaps: this Vitest exposes no Vite environments with module graphs, so a check cannot retire its copies' module nodes`, + ) + return environments + } + async function check(options, store, log = console.error) { const authored = options.mutants ? loadAuthored(options.mutants) : [] seedVariable = `${options.envPrefix}_SEED` @@ -1375,6 +1497,7 @@ export default async (env) => { every: Number(process.env.ORACLE_GAPS_HEARTBEAT_MS ?? 30_000), } mkdirSync(options.out, { recursive: true }) + hook?.check(options.out) // An earlier run's internal error; a completed check writes the other // files of an ending over the earlier ones (ending.mjs). rmSync(join(options.out, `error.log`), { force: true }) @@ -1421,13 +1544,19 @@ export default async (env) => { (contexts.length > 0 && configHash() !== contextsConfig) ) { await closeContexts() - for (let i = 0; i < options.contexts; i += 1) + for (let i = 0; i < options.contexts; i += 1) { contexts.push(await warmContext(settings, i)) + indexOf.set(contexts.at(-1), i) + hook?.joined(contexts.at(-1)) + } contextsKey = JSON.stringify(settings) contextsConfig = configHash() } - while (contexts.length < options.contexts) + while (contexts.length < options.contexts) { contexts.push(await warmContext(settings, contexts.length)) + indexOf.set(contexts.at(-1), contexts.length - 1) + hook?.joined(contexts.at(-1)) + } endPhase(`warm contexts`, { contexts: contexts.length }) // The run names the config its contexts loaded, before any refusal that // the analysis or the baseline makes. @@ -1437,7 +1566,8 @@ export default async (env) => { settings.config ?? `none, Vitest's default discovery`, ), ) - for (const context of contexts) await globSpecs(context) + for (const context of contexts) + contextSpecs.set(context, await specsOf(context)) endPhase(`find test files`) // 1. The static half (analysis.mjs): instrumented sources, mutants, @@ -1555,7 +1685,10 @@ export default async (env) => { return stable ? inputs : undefined } const remember = (id, unit, inputs) => { - if (inputs) store.put(id, { inputs, unit }) + if (!inputs) return + const entry = { inputs, unit } + hook?.stored(entry) + store.put(id, entry) } // A batch run in a slot's context, with the inputs to store each item // under: the modules its file's unmutated run read, and those its copy @@ -1830,9 +1963,18 @@ export default async (env) => { closeContext(slot.context).catch(() => undefined), sleep(10_000), ]) - slot.context = await warmContext(settings, slot.index) - await globSpecs(slot.context) - contexts[slot.index] = slot.context + const replaced = slot.context + const next = await warmContext(settings, slot.index) + const specs = await specsOf(next) + // One synchronous step: the replaced context leaves every map keyed by it and the new one joins them, + // with the contexts array, so no reader (and no measurement) sees the registry and the array disagree. + indexOf.delete(replaced) + contextSpecs.delete(replaced) + indexOf.set(next, slot.index) + contextSpecs.set(next, specs) + contexts[slot.index] = next + slot.context = next + hook?.joined(next) span(`replace context`, started, { slot: slot.index }) } // `after(index)` is more work for context `index` once its files ran. diff --git a/packages/oracles/oracle-gaps/evidence.mjs b/packages/oracles/oracle-gaps/evidence.mjs index dd018c1..acf4600 100644 --- a/packages/oracles/oracle-gaps/evidence.mjs +++ b/packages/oracles/oracle-gaps/evidence.mjs @@ -57,14 +57,15 @@ const withVersion = (row, variant) => ), ].slice(0, VERSIONS) -/** A store that lives for one check. */ +/** + * The store of a cold check. It keeps no unit: a cold run serves no lookup + * (its evidence reports `reused` 0 on every klona run measured), so a unit + * kept for reuse would only be retained, per mutant copy, for the whole run. + * The retention law observes this through the entries' reachability + * (retention-hook.mjs), not through anything the store reports. + */ export function memoryStore() { - const rows = new Map() - return { - get: (id) => rows.get(id), - put: (id, variant) => - rows.set(id, { id, variants: withVersion(rows.get(id), variant) }), - } + return { get: () => undefined, put: () => {} } } /** diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index e1d4272..6dbe5bd 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -857,6 +857,9 @@ const drawn = fc.record({ // with a sync read (\`LOAD_PLUGIN\`, review of #103). Only a constructed // history sets it; \`histories\` takes it back as this constant. loadPlugin: fc.constant(false), + // The batch whose run rejects: a constructed-only field (0 is no fault). Only the constructed history at + // the end of the campaign sets it; the take-back below returns it to 0 (the generator rule, extended). + rejectBatch: fc.constant(0), }) // The edit sequence over the witness module, the one list of steps that a @@ -911,8 +914,9 @@ const histories = drawn.map( if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) // Take-back into the drawn language: a disarmed pause (pauseEngages false, - // only the constructed history sets it) comes back as engaged, so the - // arbitrary can shrink it. The disarm is therefore not kept by a shrink. + // only the constructed history sets it) comes back as engaged, and a + // rejected batch (rejectBatch, likewise) as none, so the arbitrary can + // shrink it. Neither is therefore kept by a shrink. const taken = history.tornRows === false && history.testRoot === `none` && @@ -923,12 +927,18 @@ const histories = drawn.map( laterEdits.includes(step), ) if (later.length === 0) - return { ...taken, pauseEngages: true, warmReplaced: false } + return { + ...taken, + pauseEngages: true, + warmReplaced: false, + rejectBatch: 0, + } const earlier = history.steps.filter((step) => !laterEdits.includes(step)) return { ...taken, pauseEngages: true, warmReplaced: false, + rejectBatch: 0, steps: earlier.length > 0 ? earlier : null, laterSteps: later, } @@ -2957,6 +2967,8 @@ function resolveHistory(history) { // the test that aborts when the driver asks. aborts: (history.stopRuns ?? []).some((stop) => stop.when === `abort`), ordinary: history.ordinary === true, + // The batch whose run rejects (a test-only fault; 0 is none). + rejectBatch: history.rejectBatch ?? 0, // An `owns` key in an assertion's arguments, in the ordinary file: its // assertion form and whether its list is a literal, or none. matcherOwns: history.matcherOwns ?? `none`, @@ -4593,6 +4605,8 @@ function model(plan) { const gaps = entries.map((e) => e.key) return { configLines, + // The batch whose run the model predicts rejects (a constructed fault): the run ends as an internal error. + rejectedBatch: plan.rejectBatch > 0 ? plan.rejectBatch : undefined, baseline: baselineOf(plan), blocking: blockingOf(plan).filter((f) => !f.ordinary), replayed: blockingOf(plan).find((f) => f.replayed)?.replayed, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index d7856d7..fa15adc 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -657,6 +657,116 @@ const configArgs = (plan) => ({ contexts: plan.contexts, }) +// ### Bounded retention: the law over one call's retention lines +// +// The retention hook (ORACLE_GAPS_TEST_RETENTION, retention-hook.mjs) writes +// one line per batch, however the batch ends (runner law L2): settled, cut +// off, or rejected. Each line names the check's output directory (`run`), the +// process that wrote it (`pid`), and the batch's own index within its check +// (`batch`, from 0, taken when the batch starts). The law over one CLI call's +// lines, independent of every counter the CLI keeps: +// +// - Ownership: a line is the call's own when its pid is the pid the call's +// check wrote (check.pid). The one other pid allowed is a daemon that the +// driver killed during the call and saw die (daemonChecks); its lines judge +// that process's batches up to its death. A line from any other pid, or +// with no pid, is a breach: two processes ran one call's check. +// - One line per batch: a check that ran to its report owns lines whose +// indices are exactly 0..n-1, each once. A check that ended before its +// report (a rejected batch, a stop) or a killed daemon may leave the +// batches it was running unlogged, so there each index appears at most +// once. A rejected batch's own line, with its state after retirement, is +// owed. +// - Retirement: after a settled batch, no Vite node names one of its copies +// (copyNodes), no non-copy file has more than one node beyond the measured +// plain-plus-?raw pair (nonCopyDup), and Vitest's state (filesMap, pathsSet, +// idMap, errorsSet, leakSet) has its size before the batch. Vite's totals +// (`vite.*`) are not compared: Vite caches non-copy modules legitimately +// (register: #61 retention decisions (3)). +// - No stale entry, before any batch: every indexOf entry names a live +// context at its own slot, every contextSpecs key a live context, and no +// context that left the live array is unclosed by Vitest's own state +// (staleIndexed, staleSpecs, staleOpen). +// - A cold check's store keeps nothing (register: #61 retention decisions +// (1)): of the entries the check gave its store, none is reachable after a +// full collection (`held`). A daemon's store keeps its entries by design, +// so `held` is judged on cold calls only. +// +// The breaches are returned as sorted names, so a passing call observes []. +function retentionBreaches( + lines, + { checkPid, killed, complete, cold, rejectBatch = 0 }, +) { + const breaches = new Set() + const own = lines.filter( + (line) => line.pid !== undefined && line.pid === checkPid, + ) + const dead = lines.filter( + (line) => killed !== undefined && line.pid === killed, + ) + if (own.length + dead.length < lines.length) breaches.add(`foreignLine`) + const indices = (group) => + group.map((line) => line.batch).sort((a, b) => a - b) + const once = (group) => + indices(group).every( + (index, i, all) => Number.isInteger(index) && index !== all[i - 1], + ) + const exact = (group) => indices(group).every((index, i) => index === i) + if (!(complete ? exact(own) : once(own))) breaches.add(`batchIndex`) + if (!once(dead)) breaches.add(`killedIndex`) + if ( + rejectBatch > 0 && + !own.some( + (line) => line.batch === rejectBatch - 1 && line.after !== undefined, + ) + ) + breaches.add(`rejectLine`) + for (const line of [...own, ...dead]) { + if ((line.copyNodes ?? 0) !== 0) breaches.add(`copyNodes`) + if ((line.nonCopyDup ?? 0) !== 0) breaches.add(`nonCopyDup`) + if (cold && line.held !== 0) breaches.add(`held`) + for (const key of Object.keys(line.before ?? {})) + if ( + !key.startsWith(`vite.`) && + ((key.startsWith(`stale`) && line.before[key] !== 0) || + (line.after !== undefined && line.before[key] !== line.after[key])) + ) + breaches.add(key) + } + return [...breaches].sort() +} + +// One hook file per package, in its own temporary directory (removed with the +// package, `dropHook`): a daemon keeps the environment of the call that +// started it, so every call for this package writes to the same file. +const hookFiles = new Map() +let runSeq = 0 +const hookOf = (dir) => { + if (!hookFiles.has(dir)) + hookFiles.set( + dir, + join(mkdtempSync(join(tmpdir(), `gap-retention-`)), `retention.jsonl`), + ) + return hookFiles.get(dir) +} +const dropHook = (dir) => { + const file = hookFiles.get(dir) + hookFiles.delete(dir) + if (file) rmSync(dirname(file), { recursive: true, force: true }) +} +// The pid of the process that ran a check, which the hook writes into the check's output directory. +const checkPidOf = (out) => + existsSync(join(out, `check.pid`)) + ? Number(readFileSync(join(out, `check.pid`), `utf8`)) + : undefined +const retentionLinesOf = (file) => + existsSync(file) + ? readFileSync(file, `utf8`) + .split(`\n`) + .filter(Boolean) + .map((line) => JSON.parse(line)) + : [] + /** * Writes the fixture's files and runs the CLI on it. A file of an earlier * state that this state lacks is removed. @@ -664,14 +774,28 @@ const configArgs = (plan) => ({ function runCli(dir, files, how = {}) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) - const { args, out } = cliArgs(dir, files, how) + const { args, out: outBase } = cliArgs(dir, files, how) + // Each call has its own output directory: its retention lines are then its own, even when a + // daemon serves several checks of one test. + const out = `${outBase}-${++runSeq}` + args[args.indexOf(`--out`) + 1] = out + // The retention hook (ORACLE_GAPS_TEST_RETENTION): one line per batch, + // written beside the run's own temporary files, not in the package. + // One hook file per package: a daemon keeps the environment of the call that started it, so every call + // for this package writes to the same file; each line names its run (below). + const hook = hookOf(dir) const child = spawnSync(process.execPath, args, { cwd: repo, encoding: `utf8`, timeout: 300_000, maxBuffer: 64 * 1024 * 1024, - env: { ...cleanEnv(), ...how.env }, + env: { ...cleanEnv(), ORACLE_GAPS_TEST_RETENTION: hook, ...how.env }, }) + // Each line names its check's output directory, so this call keeps every + // line for its own directory, whichever process wrote it (retentionBreaches + // judges the writer). + const checkPid = checkPidOf(out) + const retention = retentionLinesOf(hook).filter((line) => line.run === out) const run = readRun( dir, out, @@ -688,7 +812,13 @@ function runCli(dir, files, how = {}) { .split(`\n`) .find((line) => line.startsWith(label)) ?.slice(label.length) - return { ...run, strayWrites: strays(dir, before), replay } + return { + ...run, + strayWrites: strays(dir, before, relative(dir, out)), + replay, + retention, + checkPid, + } } /** @@ -698,12 +828,13 @@ function runCli(dir, files, how = {}) { * directory, `.oracle-gaps-/` (its capture setup and Vitest configs, * which resolve the package's modules from there), is its own too. */ -const strays = (dir, before, running = false) => +const strays = (dir, before, own, running = false) => filesUnder(dir).filter( (file) => !before.has(file) && !(running && /^\.oracle-gaps-\d+\//.test(file)) && - !OWN_WRITES.some((home) => file === home || file.startsWith(`${home}/`)), + // The one path a run may write under: the --out the driver gave it (cliArgs names it). + !(file === own || file.startsWith(`${own}/`)), ) /** @@ -752,13 +883,19 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { only: ids, pool: abort ? `threads` : pool, }) + // The retention hook is set on every stop, so a stopped run's completed batches are judged too. + // Every stop of a history writes under out-stop, so this run's lines are the ones appended after + // the lines already in the file. + const hook = hookOf(dir) + const earlierLines = retentionLinesOf(hook).length + const base = { ...cleanEnv(), ORACLE_GAPS_TEST_RETENTION: hook } const env = after - ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } + ? { ...base, ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } : warm - ? { ...cleanEnv(), ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`) } + ? { ...base, ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`) } : abort - ? { ...cleanEnv(), GAP_ABORT: `1` } - : { ...cleanEnv(), GAP_STOP_DIR: pids } + ? { ...base, GAP_ABORT: `1` } + : { ...base, GAP_STOP_DIR: pids } if (win) env.ORACLE_GAPS_TEST_STOP = stopFile const child = spawn(process.execPath, args, { cwd: repo, @@ -808,7 +945,12 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { : warm ? existsSync(join(pids, `warming`)) && !exited : readdirSync(pids).length >= 2 - const strayWhileRunning = strays(dir, before, true) + const strayWhileRunning = strays( + dir, + before, + relative(dir, join(dir, `out-stop`)), + true, + ) if (!abort) { if (win) writeFileSync(stopFile, ``) else child.kill(signal) @@ -846,6 +988,11 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { survivors: alive.length, strayWhileRunning, when: when ?? `run`, + // The batches this run completed, each with its retention line (the hook above). + retention: retentionLinesOf(hook) + .slice(earlierLines) + .filter((line) => line.run === join(dir, `out-stop`)), + checkPid: checkPidOf(join(dir, `out-stop`)), // Whether a refusal's block reached stderr: a stop is never a refusal. refused: stderr.split(`\n`).some((line) => line.startsWith(`refused: `)), exit: exitCode, @@ -868,7 +1015,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { /^\.oracle-gaps-\d+$/.test(name), ), // What else it left in the package. - left: strays(dir, before).filter( + left: strays(dir, before, relative(dir, join(dir, `out-stop`))).filter( (file) => !/^\.oracle-gaps-\d+\//.test(file), ), } @@ -938,17 +1085,24 @@ function filesUnder(root, within = ``) { // (maintainer decision, 2026-11-11: the gap check's working files stay out // of the repository), so a package, and a workspace dependency in it, keeps // nothing of a run. -const OWN_WRITES = [`out`, `out-daemon`, `out-stop`, `out-only`] -/** Runs the CLI several times at once on one state of the fixture. */ -async function runClisAtOnce(dir, files, hows) { +/** + * Runs the CLI several times at once on one state of the fixture. `outs` + * receives each call's output directory synchronously, before any call + * starts, so a driver can watch the directory its own call writes. + */ +async function runClisAtOnce(dir, files, hows, outs = []) { writeFixture(dir, files) return Promise.all( hows.map(async (how) => { - const { args, out } = cliArgs(dir, files, how) + const { args, out: outBase } = cliArgs(dir, files, how) + const out = `${outBase}-${++runSeq}` + outs.push(out) + args[args.indexOf(`--out`) + 1] = out + const hook = hookOf(dir) const child = spawn(process.execPath, args, { cwd: repo, - env: { ...cleanEnv(), ...how.env }, + env: { ...cleanEnv(), ...how.env, ORACLE_GAPS_TEST_RETENTION: hook }, stdio: [`ignore`, `pipe`, `pipe`], }) // Both streams, as runCli reads them: law 15 judges each run's lines, @@ -968,7 +1122,12 @@ async function runClisAtOnce(dir, files, hows) { resolvePromise(code) }) }) - return readRun(dir, out, status, stderr, how.cli ?? here, stdout) + const run = readRun(dir, out, status, stderr, how.cli ?? here, stdout) + return { + ...run, + retention: retentionLinesOf(hook).filter((line) => line.run === out), + checkPid: checkPidOf(out), + } }), ) } @@ -1430,7 +1589,26 @@ async function incremental(dir, plan, cli, copy, observe, states) { step === `recheck` && plan.tests.every((t) => !t.flaky) ? (daemon.evidence?.computed ?? 0) - (daemon.evidence?.cut ?? 0) : undefined - for (const [i, run] of [daemon, ...others].entries()) + for (const [i, run] of [daemon, ...others].entries()) { + // A daemon's call: the daemon the driver killed during it, if any, wrote + // the lines of its own batches (daemonChecks). + observe( + `retention:daemon`, + retentionBreaches(run.retention, { + checkPid: run.checkPid, + killed: run.killed, + complete: run.reported, + cold: false, + }), + ) + observe( + `retention:cold`, + retentionBreaches(coldRun.retention, { + checkPid: coldRun.checkPid, + complete: coldRun.reported, + cold: true, + }), + ) observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, markerStored, @@ -1485,6 +1663,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { stderr: run.stderr?.split(`\n`)[0], }, }) + } if (twice && step === `start`) states.add(`two checks started the daemon at once`) if ( @@ -1670,6 +1849,17 @@ async function incremental(dir, plan, cli, copy, observe, states) { path.endsWith(`/midrun.txt`), ) midrunChecks = { during, before: runCli(dir, before, how), landed } + // The check the edit overlapped, through the daemon, and the cold run of + // the state before the edit: each call's lines are judged as any call's. + for (const run of [during, midrunChecks.before]) + observe( + `retention:midrun`, + retentionBreaches(run.retention, { + checkPid: run.checkPid, + complete: run.reported, + cold: run === midrunChecks.before, + }), + ) return [next] } if (step !== `dieMidCheck` && step !== `resetBeforeRead`) @@ -1705,9 +1895,17 @@ async function incremental(dir, plan, cli, copy, observe, states) { // served mark holds (the witness for the warm label). let warm if (!mid) { - await runClisAtOnce(dir, files, [ + const [warmRun] = await runClisAtOnce(dir, files, [ { ...how, daemon: true, outName: `out-warm`, env }, ]) + observe( + `retention:warm`, + retentionBreaches(warmRun.retention, { + checkPid: warmRun.checkPid, + complete: warmRun.reported, + cold: false, + }), + ) warm = pidOf(servedMark) // The warm daemon is stopped before the pause is armed: an armed daemon // holds the stop's own connection unread, so the stop would never end. @@ -1748,6 +1946,10 @@ async function incremental(dir, plan, cli, copy, observe, states) { if (killed) process.kill(pid, `SIGKILL`) const gone = killed ? await waitGone(pid) : false const runs = await running + // The daemon the driver killed, once it is dead: its lines are its own. + if (killed && !isAlive(pid)) runs[0].killed = pid + if (runs[0].retention.some((line) => line.pid === runs[0].killed)) + states.add(`a killed daemon's retention lines were judged as its own`) rmSync(marks, { recursive: true, force: true }) // Each label needs its pause's observation: a kill the driver sent and saw // through (gone), and the CLI's report of the check. A label without a pause @@ -1859,6 +2061,8 @@ async function incremental(dir, plan, cli, copy, observe, states) { } async function driver(plan, observe) { + // Calls are counted within this history, so two runs of one history name the same directories. + runSeq = 0 // A package that resolves no TypeScript, has no config, or has its // config in its parent directory lives outside this repository // (`outsideRepository`); the last in a directory `pkg` of the scratch @@ -1897,14 +2101,17 @@ async function driver(plan, observe) { budget: plan.budget, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), + ...(plan.rejectBatch > 0 + ? { env: { ORACLE_GAPS_TEST_REJECT_BATCH: String(plan.rejectBatch) } } + : {}), } - const { replay, ...cli } = runCli( + const { replay, retention, checkPid, ...cli } = runCli( dir, files, plan.tornRows ? { ...how, - env: { ORACLE_GAPS_TEST_TORN_ROWS: tornPolls }, + env: { ...how.env, ORACLE_GAPS_TEST_TORN_ROWS: tornPolls }, } : how, ) @@ -1933,13 +2140,52 @@ async function driver(plan, observe) { ) const tornRead = reached > 0 // How many units were computed or reused says how the result was - // produced, not what it is, so it is not observed. - observe(`cli`, { ...cli, evidence: undefined }) + // produced, not what it is, so it is not observed. A rejected batch's run ends in an internal error: + // the baseline lines printed before its batch come in no fixed order, so it observes its exit, its + // last line and the laws that judge the run before retention (the register: the order of streamed + // lines is not part of the contract). + observe( + `cli`, + plan.rejectBatch > 0 + ? { + exit: cli.exit, + lastLine: cli.lastLine, + // The done line ends both streams (P10): stderr's last line. + stderrLast: cli.stderr?.trimEnd().split(`\n`).at(-1), + configLines: cli.configLines, + strayWrites: cli.strayWrites, + warnings: cli.warnings, + } + : { ...cli, evidence: undefined }, + ) + // Deterministic: the law's breaches (none), not the lines, whose batch + // splits vary with hang cuts. A rejected batch (a constructed fault) ends + // the check before its report, and owes its own line. + observe( + `retention`, + retentionBreaches(retention, { + checkPid, + complete: cli.reported, + cold: true, + rejectBatch: plan.rejectBatch, + }), + ) // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if ( + plan.rejectBatch > 0 && + retention.some( + (line) => + line.pid === checkPid && + line.batch === plan.rejectBatch - 1 && + line.after !== undefined, + ) + ) + states.add(`a rejected batch retired its copies and logged them`) + if (retention.length > 1) states.add(`a check that ran more than one batch`) if (tornRead && cli.reported) states.add(`a capture row read while the worker wrote it`) if (cli.exit === 0 && !plan.consoleOutput) @@ -1958,6 +2204,14 @@ async function driver(plan, observe) { only: ids, outName: `out-only`, }) + observe( + `retention:partial`, + retentionBreaches(partial.retention, { + checkPid: partial.checkPid, + complete: partial.reported, + cold: true, + }), + ) observe(`partial`, { function: chosen.function, source: chosen.source, @@ -1985,7 +2239,22 @@ async function driver(plan, observe) { // of them the previous stopped run had left. if (cli.exit === 0 && stopIds.length > 0) for (const stop of plan.stopRuns) { - const run = await stoppedRun(dir, files, how, stopIds, stop) + const { retention, checkPid, ...run } = await stoppedRun( + dir, + files, + how, + stopIds, + stop, + ) + // A stopped check ends before its report: each index at most once. + observe( + `retention:stop`, + retentionBreaches(retention, { + checkPid, + complete: false, + cold: true, + }), + ) const stopped = { ...run, sessions: { @@ -2386,16 +2655,39 @@ async function driver(plan, observe) { const moved = { ...files, [SOURCE]: edited(plan) } // The evidence counts are not observed, as for `cli`: they depend on // timing (oracles-followups.md G1). Verdicts and keys are. - observe(`stability`, { ...runCli(dir, moved, how), evidence: undefined }) + const { + retention: movedRetention, + checkPid: movedPid, + ...movedRun + } = runCli(dir, moved, how) + observe(`stability`, { ...movedRun, evidence: undefined }) + observe( + `retention:stability`, + retentionBreaches(movedRetention, { + checkPid: movedPid, + complete: movedRun.reported, + cold: true, + }), + ) states.add(`stability rerun`) } if (plan.steps.length > 0) - await incremental(dir, plan, cli, copy, observe, states) + // The first cold run keeps its retention lines and check pid: law 9's + // cold result of the first state is this run (incremental). + await incremental( + dir, + plan, + { ...cli, retention, checkPid }, + copy, + observe, + states, + ) for (const state of states) observe(`reach`, state) } finally { if (copy) rmSync(copy, { recursive: true, force: true }) removePath(`${dir}.link`) written.delete(dir) + dropHook(dir) try { rmSync(scratch, { recursive: true, force: true }) } catch { @@ -2553,7 +2845,8 @@ function check(plan, observations, expected) { // An internal error comes first: it also leaves the session directory, so // the namespace law below would name only its symptom (issue #83). The // message carries the run's error.log (readRun). - if (cli.exit === 3) + // A rejected batch (the constructed fault) is judged below, after retention. + if (cli.exit === 3 && expected.rejectedBatch === undefined) return violation( `cli`, `the run ended in an internal error: ${cli.stderr ?? `no stderr`}`, @@ -2584,8 +2877,44 @@ function check(plan, observations, expected) { `config`, { actual: cli.configLines, expected: expected.configLines }, ) - const stream = streamBreach(`cli`, cli, expected, plan) + // A rejected batch's run ends with the internal error's lines, which are not stream lines: its ending is + // judged below, after retention (the constructed fault only). + const stream = + expected.rejectedBatch === undefined + ? streamBreach(`cli`, cli, expected, plan) + : undefined if (stream) return stream + // Bounded retention (retentionBreaches): every call's lines are its own + // check's, or a daemon's that the driver killed, one per batch, and each + // settled batch retired its copies, with no stale entry and no unit kept by + // a cold store. + const breached = observations.filter( + (o) => o.checkpoint.startsWith(`retention`) && o.value.length > 0, + ) + if (breached.length > 0) + return violation( + `retention`, + `retention lines break the law: ${breached + .map((o) => `${o.checkpoint} (${o.value.join(`, `)})`) + .join(`; `)}`, + `bounded retention`, + { actual: breached.map((o) => [o.checkpoint, o.value]), expected: [] }, + ) + // A rejected batch (the constructed fault) ends the run as an internal error: exit 3 and its done line + // (ending.mjs). Retention was judged above, so only the ending is judged here, and the run is then done. + if (expected.rejectedBatch !== undefined) { + if ( + cli.exit !== 3 || + !cli.lastLine?.startsWith(`oracle-gaps: done error, log `) || + !cli.stderrLast?.startsWith(`oracle-gaps: done error, log `) + ) + return violation( + `cli`, + `the rejected batch owes exit 3 and an internal error's done line on both streams, got exit ${cli.exit} ending stdout with ${JSON.stringify(cli.lastLine)} and stderr with ${JSON.stringify(cli.stderrLast)}`, + `internal error`, + ) + return pass + } // Through the daemon, each check streams the same kinds of line, and a // completed one ends with its done line (law 15). The state of each step // differs, so its verdict and counts are judged by law 9's comparison. @@ -3800,6 +4129,8 @@ const incrementalStates = [ .map((edit) => `edit ${edit} changed the cold result`), `a check whose daemon died during it`, `a check whose daemon held it past the CLI's wait, then died`, + // The retention law's second writer: lines of a daemon the driver killed. + `a killed daemon's retention lines were judged as its own`, `a warm daemon's connection was reset`, `a check whose connection was reset`, `a check after a daemon crash`, @@ -3990,6 +4321,7 @@ const required = [ `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, `a lazy edit changed the cold result after a batch was cut off`, + `a rejected batch retired its copies and logged them`, ] // The faults the CLI analyzes anyway: a package's TypeScript is no input. const analyzedFaults = [`noTypescript`, `apiLessTypescript`] @@ -4075,6 +4407,7 @@ const separate = [ `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, `a lazy edit changed the cold result after a batch was cut off`, + `a rejected batch retired its copies and logged them`, ] // A stop by SIGTERM exists only where Node can deliver it: on win32 kill is // TerminateProcess, so there the stop law is judged by SIGINT (the test stop @@ -4191,7 +4524,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A line with any text after its change does not read and fails loudly: the \` (cause: )\` suffix that #99 adds to timeout lines is not read yet, so whichever of #99 and #115 lands second extends readGapLine. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin) and, in one constructed history, a plugin whose load reads each test file with a sync read (loadPlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a config's test.root only in four shapes, each in one constructed history of an ES module config that Vitest discovers, with one owner and one context, where the test files' globs and the setup file are relative to the root (issue #101): the directory that holds the tests, the directory above the package, a directory of the package that holds no test file, and the package's tests as a symbolic link to the directory that holds them (required on every platform, since Windows creates the link as the driver's other links); not under --config, a CommonJS config, test.projects, several contexts with different roots, or a root that is the filesystem's root; the runner serves each copy of a test file by its absolute path, through its pre plugin's load (which answers before the config's own plugins) and through its read of the copy's path for Vitest's pragma read (serveCopies), and histories run copies only in Vite's server environments (node and the pragma's ssr environment): jsdom and happy-dom, Vite's client environment, are not generated (the #103 merge-gate review ran jsdom by hand under all five root shapes, and the copies loaded in each); a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Windows idle gap (recorded, not witnessed): the state \`a hang cut off for idleness\` is required on every platform, and win32 reaches it (run 43683794816, job 132719872988), so it is not platform-scoped. The sampler (capture-setup.mjs) counts a copy idle only after 500 ms with no late tick, no non-quiet handle, utilization at most 5%, and no timer due, so \`every outcome class\` reaches the state only when the sampler sees that window. On win32 the check missed it once (run 43683794816, job 132719872940); the cause is not confirmed (a late tick on a shared runner, or a stdio handle outside \`quiet\` for this file's console output, are the candidates). Its witness, a test-only hold at the hang until the window is seen, is owed, and that is a timing dependency the determinism-under-load rule forbids; the maintainer decides whether it lands before the Windows job becomes blocking.`, + limits: `Law 12 judges refusals. The one internal error a history reaches is the constructed rejected-batch history (rejectBatch, a test-only fault that rejects a batch's run after its copies are in Vitest's and Vite's state). Its run is judged in this order: the output namespace, process warnings, and config lines, as for any run; then bounded retention, which owes the rejected batch's own line with its state after retirement; then exit 3 with an internal error's done line last on both streams. The stream law is not applied to it, and the stack in its error.log is not judged. Bounded retention reads the hook's lines (retention-hook.mjs). Its count of one line per batch (runner law L2) rests on the index the hook takes when a batch starts, so a CLI that skipped a batch's index and its line together, or that omitted only the line of a check's last batch, keeps the indices 0..n-1 and is not seen; a check that ended before its report (the rejected batch, a stopped run) and a daemon the driver killed are judged for at most one line per index, since the batches they were running at their end owe none. A line's process is told by check.pid, which the check writes, and the driver itself knows only the daemon it killed. The cold store's law (held) counts the entries given to the store's put that a full collection leaves reachable, so a store that kept only a part of an entry (its unit without the entry) is not seen; staleOpen reads Vitest's own close state (closingPromise, set when close starts), so a close that starts and never resolves is the runner oracle's (L8); and a context replaced after a check's last batch is observed only by a later check's first batch through the daemon. The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A line with any text after its change does not read and fails loudly: the \` (cause: )\` suffix that #99 adds to timeout lines is not read yet, so whichever of #99 and #115 lands second extends readGapLine. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin) and, in one constructed history, a plugin whose load reads each test file with a sync read (loadPlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a config's test.root only in four shapes, each in one constructed history of an ES module config that Vitest discovers, with one owner and one context, where the test files' globs and the setup file are relative to the root (issue #101): the directory that holds the tests, the directory above the package, a directory of the package that holds no test file, and the package's tests as a symbolic link to the directory that holds them (required on every platform, since Windows creates the link as the driver's other links); not under --config, a CommonJS config, test.projects, several contexts with different roots, or a root that is the filesystem's root; the runner serves each copy of a test file by its absolute path, through its pre plugin's load (which answers before the config's own plugins) and through its read of the copy's path for Vitest's pragma read (serveCopies), and histories run copies only in Vite's server environments (node and the pragma's ssr environment): jsdom and happy-dom, Vite's client environment, are not generated (the #103 merge-gate review ran jsdom by hand under all five root shapes, and the copies loaded in each); a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Windows idle gap (recorded, not witnessed): the state \`a hang cut off for idleness\` is required on every platform, and win32 reaches it (run 43683794816, job 132719872988), so it is not platform-scoped. The sampler (capture-setup.mjs) counts a copy idle only after 500 ms with no late tick, no non-quiet handle, utilization at most 5%, and no timer due, so \`every outcome class\` reaches the state only when the sampler sees that window. On win32 the check missed it once (run 43683794816, job 132719872940); the cause is not confirmed (a late tick on a shared runner, or a stdio handle outside \`quiet\` for this file's console output, are the candidates). Its witness, a test-only hold at the hang until the window is seen, is owed, and that is a timing dependency the determinism-under-load rule forbids; the maintainer decides whether it lands before the Windows job becomes blocking.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4206,6 +4539,8 @@ oracleProperty({ ...staticStates, `extra work in a generated function`, `killed by a work bound in a generated function`, + // A check whose runs have more than one batch: the retention law's multi-batch reach. + `a check that ran more than one batch`, ], runs: 2, fixedSeeds: [20261105], @@ -5611,6 +5946,35 @@ oracleProperty({ enters: [loadPluginState], build: (generated) => oneOwnerHistory(generated, { loadPlugin: true }), }, + { + // A batch whose run rejects (the test-only fault ORACLE_GAPS_TEST_REJECT_BATCH): the CLI ends with an + // internal error, exit 3 and its done error line (ending.mjs). The batch's copies are retired, and its + // retention line is written with its state after retirement (the bounded retention law). + name: `a batch whose run rejects`, + enters: [`a rejected batch retired its copies and logged them`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + ordinary: false, + steps: null, + packageType: `module`, + configFile: `vitest.config.ts`, + configAwait: false, + configRequire: false, + configPlace: `discovered`, + native: `none`, + rejectBatch: 1, + }), + }, ]), }, resolve: resolveHistory, diff --git a/packages/oracles/oracle-gaps/history-fingerprint.json b/packages/oracles/oracle-gaps/history-fingerprint.json index cfe3fa9..20fef65 100644 --- a/packages/oracles/oracle-gaps/history-fingerprint.json +++ b/packages/oracles/oracle-gaps/history-fingerprint.json @@ -56,7 +56,8 @@ ["pauseEngages", "c93e2e40cd0c4b9f"], ["warmReplaced", "ed6866229ecc4969"], ["testRoot", "e18db063421a75f1"], - ["loadPlugin", "ed6866229ecc4969"] + ["loadPlugin", "ed6866229ecc4969"], + ["rejectBatch", "e5386ea9402dd268"] ], "histories": [ ["functions", "66c0e9c56951c377"], @@ -115,6 +116,7 @@ ["pauseEngages", "c93e2e40cd0c4b9f"], ["warmReplaced", "ed6866229ecc4969"], ["testRoot", "e18db063421a75f1"], - ["loadPlugin", "ed6866229ecc4969"] + ["loadPlugin", "ed6866229ecc4969"], + ["rejectBatch", "e5386ea9402dd268"] ] } diff --git a/packages/oracles/oracle-gaps/retention-hook.mjs b/packages/oracles/oracle-gaps/retention-hook.mjs new file mode 100644 index 0000000..0061e63 --- /dev/null +++ b/packages/oracles/oracle-gaps/retention-hook.mjs @@ -0,0 +1,200 @@ +// The retention test hook of the gap CLI (the package README, "Test hooks"). +// check.mjs imports this module only when ORACLE_GAPS_TEST_RETENTION or +// ORACLE_GAPS_TEST_REJECT_BATCH is set, so a normal run never loads it. +// +// The hook observes; it never changes an outcome (register: test-only hooks). +// The one exception is the rejected-batch fault, which triggers the path a +// crash in Vitest takes. Each field reads the real objects, never bookkeeping +// kept for the hook (review of #61, F6): +// - Vitest's state and Vite's module graphs, by their sizes; +// - the registry (indexOf, contextSpecs), by identity against the live +// contexts; +// - each context that left the live array, by Vitest's own close state; +// - the entries a check gave its store, by whether a full garbage collection +// leaves them reachable. +import { appendFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { setFlagsFromString } from 'node:v8' +import { runInNewContext } from 'node:vm' + +/** + * The hook for one session (check.mjs). `session.live()` returns the live + * contexts array, `session.indexOf` and `session.contextSpecs` are the + * registry. + */ +export function retentionHook(session) { + const file = process.env.ORACLE_GAPS_TEST_RETENTION + // The 1-based batch whose run rejects (unset: none). + const rejected = Number(process.env.ORACLE_GAPS_TEST_REJECT_BATCH ?? 0) + // A full collection, without --expose-gc on the command line: the flag is + // set, and `gc` is read from a fresh context, which then exposes it. + setFlagsFromString(`--expose-gc`) + const gc = runInNewContext(`gc`) + // The output directory of the check being run, and the next batch's index + // within that check. A daemon runs one check at a time (its queue). + let run + let next = 0 + // The contexts that joined the live array and have not been seen closed. + const joined = new Set() + // A weak reference to each entry a check gave its store. + let stored = [] + + // A context is closed, or closing, once Vitest's own close has started: + // Vitest sets its closing promise when close() is called. A Vitest whose + // shape has no such field throws here rather than counting it open. + const closing = (context) => { + if (!(`closingPromise` in context)) + throw new Error( + `oracle-gaps: this Vitest has no closingPromise, so the retention hook cannot see whether a context closed`, + ) + return context.closingPromise !== undefined + } + + // Every structure a copy can leave state in, unfiltered: Vitest's state + // (filesMap, pathsSet, idMap, errorsSet, leakSet) and each Vite + // environment's module graph (its four maps, and the nodes that hold a + // transform). The stale fields read the registry and the contexts that left + // the live array. + function totals(context) { + const contexts = session.live() + const { filesMap, pathsSet, idMap, errorsSet, leakSet } = context.state + const totals = { + // A registry entry is stale unless it names a live context at its own + // slot (indexOf), or a live context (contextSpecs). + staleIndexed: [...session.indexOf].filter(([c, i]) => contexts[i] !== c) + .length, + staleSpecs: [...session.contextSpecs.keys()].filter( + (c) => !contexts.includes(c), + ).length, + // A context that left the live array and that Vitest has not started to + // close. + staleOpen: [...joined].filter((c) => !contexts.includes(c) && !closing(c)) + .length, + filesMap: filesMap.size, + pathsSet: pathsSet.size, + idMap: idMap.size, + errorsSet: errorsSet.size, + leakSet: leakSet.size, + } + // A context that left and is closing is done: it is no longer watched. + for (const c of joined) + if (!contexts.includes(c) && closing(c)) joined.delete(c) + for (const [name, environment] of Object.entries( + context.getRootProject().vite?.environments ?? {}, + )) { + const graph = environment.moduleGraph + // Vite's keys are named `vite..`, so the law can tell + // them from Vitest's state. + totals[`vite.${name}.url`] = graph.urlToModuleMap.size + totals[`vite.${name}.id`] = graph.idToModuleMap.size + totals[`vite.${name}.etag`] = graph.etagToModuleMap.size + totals[`vite.${name}.file`] = graph.fileToModulesMap.size + totals[`vite.${name}.transformed`] = [ + ...graph.idToModuleMap.values(), + ].filter((node) => node.transformResult != null).length + } + return totals + } + + // Files, without their query, with more than one node in an environment's + // graph, outside the copies: a non-copy file is cached once, so each count + // must be 0 (the law's bound on originals). The one measured allowance is a + // source file's plain node and its ?raw node (the oracle reads source text + // with ?raw): one pair per file, not per batch. + function nonCopyDuplicates(context, owned) { + const dup = [] + for (const environment of Object.values( + context.getRootProject().vite?.environments ?? {}, + )) { + const perFile = new Map() + for (const node of environment.moduleGraph.idToModuleMap.values()) { + const path = node.id.split(`?`)[0] + if (owned.has(path)) continue + perFile.set(path, [...(perFile.get(path) ?? []), node.id]) + } + for (const [path, ids] of perFile) + if ( + ids.length > 1 && + !( + ids.length === 2 && + ids.includes(path) && + ids.includes(`${path}?raw`) + ) + ) + dup.push(`${environment.name ?? `env`} ${path}`) + } + return dup + } + + // The entries the store was given that a full collection leaves reachable. + // The collection runs after a macrotask turn: a weak reference keeps its + // target alive until the turn that created or read it ends. + async function held() { + await new Promise((resolvePromise) => setImmediate(resolvePromise)) + gc() + stored = stored.filter((ref) => ref.deref() !== undefined) + return stored.length + } + + return { + // A check starts: its batches are counted from 0, and the process that + // runs it is named in its output directory (the driver reads check.pid). + check(out) { + run = out + next = 0 + if (file) writeFileSync(join(out, `check.pid`), String(process.pid)) + }, + // A context joined the live array. + joined(context) { + joined.add(context) + }, + // An entry a check gave its store. + stored(entry) { + stored.push(new WeakRef(entry)) + }, + // A batch starts: its own index, and the totals before it. + begin(context) { + return { index: next++, before: file ? totals(context) : undefined } + }, + // The rejected-batch fault: the batch's run completes, so its copies are + // in Vitest's and Vite's state, and then rejects, as a crash in Vitest + // would. Any other batch's run is returned unchanged. + fault(batch, running) { + return batch.index + 1 === rejected + ? running.then(() => { + throw new Error( + `oracle-gaps test fault: batch ${batch.index + 1} rejected`, + ) + }) + : running + }, + // A batch ended (settled, cut, or rejected): its one line. A cut batch is + // not retired (its context is replaced), so its line has no after. + async line(context, batch, copies, cut) { + if (!file) return + const owned = new Set(copies) + let copyNodes = 0 + for (const environment of Object.values( + context.getRootProject().vite?.environments ?? {}, + )) + for (const copy of owned) + copyNodes += + environment.moduleGraph.fileToModulesMap.get(copy)?.size ?? 0 + const line = { + run, + pid: process.pid, + batch: batch.index, + before: batch.before, + ...(cut + ? { cut: true } + : { + after: totals(context), + copyNodes, + nonCopyDup: nonCopyDuplicates(context, owned).length, + }), + held: await held(), + } + appendFileSync(file, `${JSON.stringify(line)}\n`) + }, + } +} diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 4a95cb9..50b8a6b 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -171,6 +171,9 @@ export const shards = { `fixed seed 20261105: a stale authored mutant and a reformat`, `fixed seed 20261105: an awaiting config in a package with no module type`, `fixed seed 20261105: every outcome class`, + `fixed seed 20261105: a batch whose run rejects`, + `random seed: a batch whose run rejects`, + `determinism: constructed history: a batch whose run rejects`, `random seed: a .ts config in a package with no module type that calls require`, `random seed: a declaration fault: configThrows`, `random seed: a declaration fault: noTypescript`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index f030ffa..5267061 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -531,6 +531,7 @@ export const partitions = { 4: anyOf(['every outcome class, the warm and abort stops']), 5: anyOf(['an edit sequence through the daemon']), 6: anyOf([ + 'a batch whose run rejects', 'a second daemon check of the budget witness', 'a declaration fault: missingSource', 'a config fault in a run that owns nothing',