Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
5c957ce
ci(oracles): run the e2e oracle's random and determinism units on eve…
KyleAMathews Nov 15, 2026
10f6937
ci(oracles): exclude the six determinism units that fail on main, eac…
KyleAMathews Nov 15, 2026
dee496c
fix(oracles): judge heartbeats for shape only, never by their set
KyleAMathews Nov 15, 2026
8e1e820
fix(oracles): keep heartbeats out of every observed run, and judge ev…
KyleAMathews Nov 15, 2026
78bddab
fix(oracles): stop observing the heartbeat-derived reach state
KyleAMathews Nov 15, 2026
3cdb8a2
fix(oracles): stop observing the timing-derived timed-out reuse state
KyleAMathews Nov 15, 2026
d77b63e
fix(oracles): give the witness history an empty enters list
KyleAMathews Nov 15, 2026
082991d
ci(oracles): run the determinism units, excluding only the daemon not…
KyleAMathews Nov 15, 2026
2c5e0e7
test(oracles): canary for the constructed-history order
KyleAMathews Nov 15, 2026
7df4a50
ci(oracles): update the e2e shard plan for the heartbeat calibrations
KyleAMathews Nov 15, 2026
947e936
Merge origin/main (#73) into the e2e determinism and canary branch
KyleAMathews Nov 15, 2026
4bc4eb1
fix(oracles): a gap line's place in the stream is not judged
KyleAMathews Nov 15, 2026
cd8e0f9
fix(oracles): exclude the intermittent daemon-pause unit (#105); gate…
KyleAMathews Nov 15, 2026
4bb5c08
fix(oracles): run Vitest's own CLI for the canary's listing, not pnpm
KyleAMathews Nov 15, 2026
7baf1d2
ci(oracles): run the witness determinism unit in a non-blocking job (…
KyleAMathews Nov 15, 2026
7acfeda
measure: random shards under pinned seeds (scratch, not for merge)
KyleAMathews Nov 16, 2026
f39a98d
measure: narrow the scratch workflow to the random seed jobs (scratch…
KyleAMathews Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
475 changes: 8 additions & 467 deletions .github/workflows/pr.yml

Large diffs are not rendered by default.

55 changes: 55 additions & 0 deletions packages/oracles/oracle-gaps/constructed-order.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{
"constructed": [
"every outcome class",
"a second daemon check of the budget witness",
"a load failure beside a hang",
"a stale authored mutant and a reformat",
"an ambiguous authored mutant",
"an edit sequence through the daemon",
"two authored mutants",
"ordinary tests left out by default",
"a config that --config names",
"a package with no config",
"an awaiting config in a package with no module type",
"a .cjs config that calls require",
"a .ts config in a package with no module type that calls require",
"a run with eleven contexts",
"a package whose parent directory has its config",
"a failing owner under its parent's config",
"test.projects in a parent's config",
"a failing owner under a config that --config names",
"a config fault in a run that owns nothing",
"a declaration fault: badEntry",
"a declaration fault: missingSource",
"a declaration fault: nonLiteral",
"a declaration fault: brokenOwner",
"a declaration fault: projects",
"a declaration fault: outsideSource",
"a declaration fault: brokenVitest",
"a declaration fault: configThrows",
"a declaration fault: configNoDefault",
"a declaration fault: configNull",
"a declaration fault: configSyntax",
"a declaration fault: setupThrows",
"a declaration fault: noTypescript",
"a declaration fault: apiLessTypescript",
"an edit sequence over the witness module",
"an edit that changed the cold result after a hang replaced the only context",
"an edit during a check through the daemon",
"an unchanged check through the daemon under a config's pre plugin",
"an unchanged check through the daemon while a test reads a source through fs",
"a test edits the helper it read, during a check through the daemon",
"a test edits the data it read, during a check through the daemon",
"an edit to data a test reads through fs (sync)",
"an edit to data a test reads through fs (promise)",
"an edit to data a test reads through fs (callback)",
"an edit to data a test reads through fs (statDir)",
"an edit to a module that only mutated runs load",
"a lazy module loaded before a hang in the same batch",
"a capture row read while the worker wrote it",
"two batches each read a capture row while its worker wrote it",
"a daemon pause that cannot engage",
"every outcome class, the warm and abort stops",
"a reset check after its warm daemon was replaced"
]
}
22 changes: 0 additions & 22 deletions packages/oracles/oracle-gaps/gap-report-model-oracle.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4115,18 +4115,6 @@ function filesRun(runs) {
return ran
}

/**
* The owner files in which a slow test waits under a mutant, among the
* files its run reaches: each copy of such a file is busy for SLOW_MS, so
* the CLI owes a heartbeat for it.
*/
function slowedFiles(plan, tests, sites, active) {
const slowed = new Set()
for (const [file, own] of filesRun(runsOf(plan, tests, sites, active)))
if (own.some((r) => r.slowed)) slowed.add(file)
return slowed
}

/**
* A mutant's kind from its owner tests' runs. An operator mutant's
* confirming runs share its budget (`budgeted`); an authored mutant's runs
Expand Down Expand Up @@ -4522,15 +4510,6 @@ function model(plan) {
ordinal: site.ordinal,
probe: site.probe,
kind,
// A mutant whose run reaches a slow owner test keeps a copy busy for
// SLOW_MS, so the CLI must name it in a heartbeat while it runs.
heartbeat:
kind !== undefined &&
![`unowned`, `unreached`, `setup-error`, `indeterminate`].includes(
kind,
) &&
slowedFiles(plan, ownerTests(site.fnId), sites, { site: site.id })
.size > 0,
ordinaryOutcome:
plan.ordinary && survivorKinds.has(kind) && ordinaryReaches(site),
}
Expand Down Expand Up @@ -5910,7 +5889,6 @@ export {
ownersOf,
interpret,
judge,
slowedFiles,
MUTANT_LIMIT_MS,
HEARTBEAT_MS,
FULL_LIMIT_MS,
Expand Down
175 changes: 108 additions & 67 deletions packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs

Large diffs are not rendered by default.

9 changes: 6 additions & 3 deletions packages/oracles/scripts/e2e-shards.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ const suite = `oracle-gaps.report`
// cost follows its seed, so a shard's time varies from run to run. Option M
// of the e2e speed plan bounds the random units' tail: a generated history
// with the witness module takes no steps, and the one edit sequence over it
// is a constructed history. CI keeps its one fixed-seed job until the
// maintainer decides which units run on a pull request.
// is a constructed history. A pull request runs the fixed-seed, random, and
// determinism units as the partitions of pr-shards.mjs; these shards are
// the full-suite plan, which this file's check keeps partitioned.
export const shards = {
// 522 s in the measured run, about 221 s idle.
'e2e-1': [
Expand Down Expand Up @@ -205,7 +206,9 @@ export const shards = {
// Added with the gap CLI's runtime fixes (not yet measured).
`fixed seed 20261105: a second daemon check of the budget witness`,
`calibration: a stopped run leaves a process it started running`,
`calibration: a busy mutant is named in no heartbeat`,
// The heartbeat calibrations after the shape-only change (not yet measured).
`calibration: a heartbeat names a mutant whose gap line came first`,
`calibration: a heartbeat names no mutant of its run`,
// Law 15's calibrations (the greppable stream), measured at ab18c60:
// 79 s and 70 s; the daemon-notes one, bound to its history, 2 s (it
// took 433 s searching for one).
Expand Down
154 changes: 148 additions & 6 deletions packages/oracles/scripts/pr-shards.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
// shard of its partition takes. The check lists the suite's tests with
// `vitest list` and fails unless every selected test is in exactly one
// shard, no other test is in any shard, and every test that a shard names
// exists. So the jobs together run exactly the tests that one job ran
// before: a test in no shard would leave its law out of CI, and a test in
// two would run twice. A new test lands in the `rest` shard, so the jobs'
// balance can drift, but no test is left out.
// exists. A partition's `excluded` names are the exception: each is in no
// shard on purpose, with its own issue, and must exist. So the jobs together
// run exactly the tests that one job ran before: a test in no shard would
// leave its law out of CI, and a test in two would run twice. A new test
// lands in the `rest` shard, so the jobs' balance can drift, but no test is
// left out.
//
// node scripts/pr-shards.mjs check checks every
// partition
Expand Down Expand Up @@ -320,6 +322,131 @@ export const partitions = {
3: rest,
},
},
// The end-to-end gap oracle's random units, which every pull request runs
// (maintainer decision CI-1): each run draws a fresh campaign seed, and a
// counterexample prints its replay line. 38 units, measured on main at
// 35422e3 on one Arclet (8 vCPU Linux, one Vitest worker): 624 s in all.
// CI ran the random shards 1.7 times slower than the Arclet (9 min for a
// 312 s shard, in #90's first run), and a random unit's cost follows its
// seed. So the shards are three of about 208 s here, which is about 6 min
// in CI. The witness module's edit sequence alone is 178 s.
'oracle-gaps-random': {
args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`],
suite: `oracle-gaps.report`,
select: `random seed: `,
// Every name is checked against the listed tests (partitionErrors).
exactNames: true,
shards: {
// 208 s here.
1: anyOf([
`an edit sequence over the witness module`,
`a declaration fault: noTypescript`,
`a lazy module loaded before a hang in the same batch`,
`a load failure beside a hang`,
`an ambiguous authored mutant`,
`a .ts config in a package with no module type that calls require`,
`an awaiting config in a package with no module type`,
`a declaration fault: configNull`,
`a declaration fault: configSyntax`,
`a declaration fault: missingSource`,
`a config fault in a run that owns nothing`,
]),
// 208 s here.
2: anyOf([
`every outcome class`,
`an edit sequence through the daemon`,
`a declaration fault: brokenVitest`,
`an edit that changed the cold result after a hang replaced the only context`,
`an edit to a module that only mutated runs load`,
`a failing owner under a config that --config names`,
`a failing owner under its parent's config`,
`a package whose parent directory has its config`,
`a package with no config`,
`ordinary tests left out by default`,
`a declaration fault: configNoDefault`,
`a declaration fault: badEntry`,
`a declaration fault: projects`,
`a declaration fault: outsideSource`,
]),
// The rest: 208 s here, and any unit added later.
3: rest,
},
},
// The end-to-end gap oracle's determinism units, which pull requests run
// except the one excluded below. Each unit runs one history twice and
// compares the two traces: results and fixed lines, not the heartbeats,
// which are progress (maintainer decision, 2026-11-14). 51 units; measured
// on Arclets at d77b63e, one Vitest worker: 1,305 s for the 50 units that
// are not excluded. The witness module's edit sequence is 344 s alone, so
// it is a shard by itself; the other 49 units are packed longest first into
// three shards of 240 s, and the rest shard takes the fourth bin and any
// unit added later. The excluded unit, the daemon's test-output note (#92),
// has a cause that is not heartbeats, so it stays out until #92 is fixed.
'oracle-gaps-determinism': {
args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`],
suite: `oracle-gaps.report`,
select: `determinism: `,
// Every name is checked against the listed tests (partitionErrors).
exactNames: true,
excluded: [
// The daemon's test-output note (#92).
`constructed history: an edit sequence through the daemon`,
// A daemon check's trace differs between two runs (#105): intermittent.
`constructed history: a daemon pause that cannot engage`,
],
shards: {
// 344 s: the witness module's edit sequence, alone.
1: anyOf([
`constructed history: an edit sequence over the witness module`,
]),
// 240 s here.
2: anyOf([
`constructed history: every outcome class`,
`constructed history: an edit to data a test reads through fs (statDir)`,
`constructed history: a declaration fault: apiLessTypescript`,
`constructed history: a failing owner under its parent's config`,
`constructed history: a package with no config`,
`constructed history: a .ts config in a package with no module type that calls require`,
`constructed history: a declaration fault: configNoDefault`,
`constructed history: a declaration fault: badEntry`,
`constructed history: a declaration fault: outsideSource`,
]),
// 240 s here.
3: anyOf([
`generated 2`,
`constructed history: an edit during a check through the daemon`,
`constructed history: an edit to data a test reads through fs (sync)`,
`constructed history: an unchanged check through the daemon while a test reads a source through fs`,
`constructed history: two authored mutants`,
`constructed history: a capture row read while the worker wrote it`,
`constructed history: two batches each read a capture row while its worker wrote it`,
`constructed history: a package whose parent directory has its config`,
`constructed history: a declaration fault: configNull`,
`constructed history: a declaration fault: missingSource`,
`constructed history: a config fault in a run that owns nothing`,
]),
// 240 s here.
4: anyOf([
`constructed history: a declaration fault: noTypescript`,
`constructed history: a declaration fault: brokenVitest`,
`constructed history: a stale authored mutant and a reformat`,
`constructed history: an edit that changed the cold result after a hang replaced the only context`,
`constructed history: a test edits the helper it read, during a check through the daemon`,
`generated 1`,
`constructed history: an edit to data a test reads through fs (callback)`,
`constructed history: an unchanged check through the daemon under a config's pre plugin`,
`constructed history: an awaiting config in a package with no module type`,
`constructed history: a run with eleven contexts`,
`constructed history: a .cjs config that calls require`,
`constructed history: ordinary tests left out by default`,
`constructed history: a declaration fault: setupThrows`,
`constructed history: a declaration fault: nonLiteral`,
`generated 0`,
]),
// The rest: 240 s here, and any unit added later.
5: rest,
},
},
'oracle-gaps-static-windows': {
args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-static`],
suite: `oracle-gaps.report-static`,
Expand Down Expand Up @@ -591,11 +718,18 @@ export const partitions = {
},
}

/** A partition's excluded names, relative to its select: tests in no shard. */
const excludedOf = (partition) => new Set(partition.excluded ?? [])

/** A shard's pattern over the name that follows the suite's. */
function bodyOf(partition, shard) {
const own = partition.shards[shard]
if (own !== rest) return `${escape(partition.select)}(?:${own})`
const others = Object.values(partition.shards).filter((body) => body !== rest)
// A rest shard takes what no other shard and no exclusion takes.
const others = [
...Object.values(partition.shards).filter((body) => body !== rest),
...[...excludedOf(partition)].map(escape),
]
return `(?!${escape(partition.select)}(?:${others.join(`|`)})$)${escape(partition.select)}.*`
}

Expand Down Expand Up @@ -643,7 +777,12 @@ export function partitionErrors(name, listed) {
joined.every((text) => owners[0].pattern.test(text))
const own = full.startsWith(prefix) ? full.slice(prefix.length) : undefined
const selected = own !== undefined && own.startsWith(partition.select)
if (selected && !whole)
// An excluded test is in no shard on purpose; its issue is the record.
const excluded =
selected && excludedOf(partition).has(own.slice(partition.select.length))
if (excluded && owners.length > 0)
errors.push(`${JSON.stringify(full)} is excluded, but is in a shard`)
if (selected && !whole && !excluded)
errors.push(
`${JSON.stringify(full)} is in ${owners.length === 0 ? `no shard` : owners.length === 1 ? `shard ${owners[0].shard} under only one of Vitest's name joins` : `shards ${owners.map(({ shard }) => shard).join(`, `)}`}`,
)
Expand All @@ -661,6 +800,9 @@ export function partitionErrors(name, listed) {
for (const exception of partition.exceptions ?? [])
if (!names.has(`${prefix}${exception}`))
errors.push(`exception ${JSON.stringify(exception)} names no test`)
for (const exclusion of excludedOf(partition))
if (!names.has(`${prefix}${partition.select}${exclusion}`))
errors.push(`excluded ${JSON.stringify(exclusion)} names no test`)
for (const [shard, body] of Object.entries(partition.shards))
if (
body !== rest &&
Expand Down
72 changes: 71 additions & 1 deletion packages/oracles/tests/gap-report-history-fingerprint.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
import { spawnSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import fc from 'fast-check'
import { describe, expect, test } from 'vitest'
import { version as vitestVersion } from 'vitest/package.json'
import { drawn, histories } from '../oracle-gaps/gap-report-model-oracle.mjs'

// The generator rule (coordinator decision, 2026-11-13): a new field of the
Expand Down Expand Up @@ -59,4 +65,68 @@ describe(`the gap oracle's history grammar`, () => {
)
}
})

// The constructed histories of the end-to-end gap oracle, in the order the
// oracle registers them: each one is built from the fixed seed and its index
// (`seed + index`), so a history inserted before a recorded one changes every
// later history's draw. The register's generator rule (2026-11-14) says new
// constructed histories are appended at the end. The recorded names are
// the list the oracle registered on main; the current list must begin with
// them, so a new name can only follow.
// The gap oracle loads under Vitest 4; the Vitest 5 job runs no gap oracle
// (its listing finds no suite in this file), so the canary runs there only
// where the file loads.
const gapOracleLoads = Number(vitestVersion.split(`.`)[0]) < 5
test.skipIf(!gapOracleLoads)(
`appends new constructed histories, and never inserts one before a recorded one`,
() => {
const recorded = JSON.parse(
readFileSync(
new URL(`../oracle-gaps/constructed-order.json`, import.meta.url),
`utf8`,
),
) as { constructed: Array<string> }
// Vitest writes the listing to a file, so no package manager's stdout
// line can reach the JSON (as pr-shards.mjs does).
const dir = mkdtempSync(join(tmpdir(), `constructed-order-`))
let tests: Array<{ name: string }>
try {
const file = join(dir, `list.json`)
const packageDir = dirname(
createRequire(import.meta.url).resolve(`vitest/package.json`),
)
const listing = spawnSync(
process.execPath,
[
join(packageDir, `vitest.mjs`),
`list`,
`--json=${file}`,
`--config`,
`oracle-gaps/vitest.config.mjs`,
`gap-report-oracle`,
],
{
cwd: fileURLToPath(new URL(`..`, import.meta.url)),
encoding: `utf8`,
maxBuffer: 64 * 1024 * 1024,
},
)
expect(listing.status, listing.stderr).toBe(0)
tests = JSON.parse(readFileSync(file, `utf8`)) as Array<{
name: string
}>
} finally {
rmSync(dir, { recursive: true, force: true })
}
const unit = `oracle-gaps.report > fixed seed ${FIXED_SEED}: `
const current = tests
.map(({ name }) => name)
.filter((name) => name.startsWith(unit))
.map((name) => name.slice(unit.length))
.filter((name) => name !== `generated`)
expect(current.slice(0, recorded.constructed.length)).toEqual(
recorded.constructed,
)
},
)
})
Loading