Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
5c957ce
ci(oracles): run the e2e oracle's random and determinism units on eve…
KyleAMathews Nov 15, 2026
10f6937
ci(oracles): exclude the six determinism units that fail on main, eac…
KyleAMathews Nov 15, 2026
dee496c
fix(oracles): judge heartbeats for shape only, never by their set
KyleAMathews Nov 15, 2026
8e1e820
fix(oracles): keep heartbeats out of every observed run, and judge ev…
KyleAMathews Nov 15, 2026
78bddab
fix(oracles): stop observing the heartbeat-derived reach state
KyleAMathews Nov 15, 2026
3cdb8a2
fix(oracles): stop observing the timing-derived timed-out reuse state
KyleAMathews Nov 15, 2026
d77b63e
fix(oracles): give the witness history an empty enters list
KyleAMathews Nov 15, 2026
082991d
ci(oracles): run the determinism units, excluding only the daemon not…
KyleAMathews Nov 15, 2026
2c5e0e7
test(oracles): canary for the constructed-history order
KyleAMathews Nov 15, 2026
7df4a50
ci(oracles): update the e2e shard plan for the heartbeat calibrations
KyleAMathews Nov 15, 2026
947e936
Merge origin/main (#73) into the e2e determinism and canary branch
KyleAMathews Nov 15, 2026
4bc4eb1
fix(oracles): a gap line's place in the stream is not judged
KyleAMathews Nov 15, 2026
cd8e0f9
fix(oracles): exclude the intermittent daemon-pause unit (#105); gate…
KyleAMathews Nov 15, 2026
4bb5c08
fix(oracles): run Vitest's own CLI for the canary's listing, not pnpm
KyleAMathews Nov 15, 2026
7baf1d2
ci(oracles): run the witness determinism unit in a non-blocking job (…
KyleAMathews Nov 15, 2026
e558496
ci(oracles): split the random units into five weighted shards (P12)
KyleAMathews Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 65 additions & 1 deletion .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,73 @@ jobs:
- name: Setup Tools
uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3
# The end-to-end oracle's fixed-seed units, in three shards. Its random
# units, calibrations, and determinism units run locally.
# units and determinism units run in their own jobs, and its
# calibrations run locally.
- run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")"
working-directory: packages/oracles
env:
SHARD: ${{ matrix.shard }}
test-oracle-gaps-random:
name: Gap report end-to-end oracle (random seed, ${{ matrix.shard }}/5)
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5]
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Setup Tools
uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3
# The end-to-end oracle's random units, in five weighted shards. Each run draws a
# fresh campaign seed (the ORACLE_SEED line in the log), and a failing
# unit prints its Replay line. A counterexample is filed as an issue.
- run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-random "$SHARD")"
working-directory: packages/oracles
env:
SHARD: ${{ matrix.shard }}
test-oracle-gaps-determinism:
name: Gap report end-to-end oracle (determinism, ${{ matrix.shard }}/5)
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
# Shard 1 is the witness module's edit sequence, which runs in its own
# non-blocking job below (#106).
shard: [2, 3, 4, 5]
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Setup Tools
uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3
# The end-to-end oracle's determinism units, in shards 2 to 5 (shard 1 is
# below), except the one excluded in pr-shards.mjs (#92). Each unit runs a
# history twice and compares the results and fixed lines; heartbeats are
# checked for shape.
- run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism "$SHARD")"
working-directory: packages/oracles
env:
SHARD: ${{ matrix.shard }}
# The witness module's edit sequence alone takes 10 to 11 minutes here,
# over P12's 10-minute budget. It is non-blocking and not a dependency of
# preview, so it still runs on every pull request until its cost is cut (#106).
test-oracle-gaps-witness:
name: Gap report end-to-end oracle (determinism, witness module; non-blocking, #106)
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Setup Tools
uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3
- run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-determinism 1)"
working-directory: packages/oracles
test-vitest-5:
name: Test on Vitest 5 (${{ matrix.suite }})
# The package's vitest peer range includes 5; this job is the evidence.
Expand Down Expand Up @@ -167,6 +229,8 @@ jobs:
- test-oracle-framework
- test-oracle-gaps-fast
- test-oracle-gaps-end-to-end
- test-oracle-gaps-random
- test-oracle-gaps-determinism
- test-vitest-5
# The blocking Windows jobs (the non-blocking end-to-end job is not a
# dependency until it is green).
Expand Down
55 changes: 55 additions & 0 deletions packages/oracles/oracle-gaps/constructed-order.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{
"constructed": [
"every outcome class",
"a second daemon check of the budget witness",
"a load failure beside a hang",
"a stale authored mutant and a reformat",
"an ambiguous authored mutant",
"an edit sequence through the daemon",
"two authored mutants",
"ordinary tests left out by default",
"a config that --config names",
"a package with no config",
"an awaiting config in a package with no module type",
"a .cjs config that calls require",
"a .ts config in a package with no module type that calls require",
"a run with eleven contexts",
"a package whose parent directory has its config",
"a failing owner under its parent's config",
"test.projects in a parent's config",
"a failing owner under a config that --config names",
"a config fault in a run that owns nothing",
"a declaration fault: badEntry",
"a declaration fault: missingSource",
"a declaration fault: nonLiteral",
"a declaration fault: brokenOwner",
"a declaration fault: projects",
"a declaration fault: outsideSource",
"a declaration fault: brokenVitest",
"a declaration fault: configThrows",
"a declaration fault: configNoDefault",
"a declaration fault: configNull",
"a declaration fault: configSyntax",
"a declaration fault: setupThrows",
"a declaration fault: noTypescript",
"a declaration fault: apiLessTypescript",
"an edit sequence over the witness module",
"an edit that changed the cold result after a hang replaced the only context",
"an edit during a check through the daemon",
"an unchanged check through the daemon under a config's pre plugin",
"an unchanged check through the daemon while a test reads a source through fs",
"a test edits the helper it read, during a check through the daemon",
"a test edits the data it read, during a check through the daemon",
"an edit to data a test reads through fs (sync)",
"an edit to data a test reads through fs (promise)",
"an edit to data a test reads through fs (callback)",
"an edit to data a test reads through fs (statDir)",
"an edit to a module that only mutated runs load",
"a lazy module loaded before a hang in the same batch",
"a capture row read while the worker wrote it",
"two batches each read a capture row while its worker wrote it",
"a daemon pause that cannot engage",
"every outcome class, the warm and abort stops",
"a reset check after its warm daemon was replaced"
]
}
22 changes: 0 additions & 22 deletions packages/oracles/oracle-gaps/gap-report-model-oracle.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4115,18 +4115,6 @@ function filesRun(runs) {
return ran
}

/**
* The owner files in which a slow test waits under a mutant, among the
* files its run reaches: each copy of such a file is busy for SLOW_MS, so
* the CLI owes a heartbeat for it.
*/
function slowedFiles(plan, tests, sites, active) {
const slowed = new Set()
for (const [file, own] of filesRun(runsOf(plan, tests, sites, active)))
if (own.some((r) => r.slowed)) slowed.add(file)
return slowed
}

/**
* A mutant's kind from its owner tests' runs. An operator mutant's
* confirming runs share its budget (`budgeted`); an authored mutant's runs
Expand Down Expand Up @@ -4522,15 +4510,6 @@ function model(plan) {
ordinal: site.ordinal,
probe: site.probe,
kind,
// A mutant whose run reaches a slow owner test keeps a copy busy for
// SLOW_MS, so the CLI must name it in a heartbeat while it runs.
heartbeat:
kind !== undefined &&
![`unowned`, `unreached`, `setup-error`, `indeterminate`].includes(
kind,
) &&
slowedFiles(plan, ownerTests(site.fnId), sites, { site: site.id })
.size > 0,
ordinaryOutcome:
plan.ordinary && survivorKinds.has(kind) && ordinaryReaches(site),
}
Expand Down Expand Up @@ -5910,7 +5889,6 @@ export {
ownersOf,
interpret,
judge,
slowedFiles,
MUTANT_LIMIT_MS,
HEARTBEAT_MS,
FULL_LIMIT_MS,
Expand Down
Loading
Loading