diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index edf59139..db4b17ba 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -2,275 +2,22 @@ name: PR on: pull_request: - push: - branches: [main] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.number || github.ref }} - # Only a newer push to the same pull request cancels a run. Every commit on - # main keeps its run, so every commit gets a preview. - cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read jobs: - # The test suites run as parallel jobs. The workflow's wall time is then - # its slowest job, about 6 to 7 minutes, not the sum of every suite, about - # 21 minutes when they ran as one job. Each job should stay well under the - # 10-minute limit; pr-shards.mjs records how each suite is split. - test: - name: Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm run build - - run: pnpm run lint:check - - run: pnpm run format:check - - run: pnpm run typecheck - - run: pnpm --filter @tanstack/oracles run test - # Checks the partition of the end-to-end gap oracle into CI shards - # (packages/oracles/scripts/e2e-shards.mjs) ahead of the shard jobs, - # which CI does not run yet: every test is in exactly one shard. - - run: pnpm --filter @tanstack/oracles run test:oracle-gaps:shards - # The framework self-oracle, the gap report's static oracle, and the - # end-to-end oracle's fixed seed each run as several jobs. Together - # they run exactly the tests one job ran - # (packages/oracles/scripts/pr-shards.mjs). - - run: pnpm --filter @tanstack/oracles run test:pr-shards - - run: pnpm run test:publint - - run: pnpm run test:sherif - test-eslint-plugin: - name: Test eslint-plugin-oracles - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The plugin's tests import @tanstack/oracles, which resolves to its - # build output. - - run: pnpm run build - - run: pnpm --filter @tanstack/eslint-plugin-oracles run test - test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/8) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in eight - # shards. - - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-fast: - name: Gap report oracles (${{ matrix.shard }}/3) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The gap report's own oracles also run real Vitest runs on fixture - # packages (test:oracle-gaps:fast): the static oracle in two shards, - # and the classify, probe-form, and mutant-validity oracles in a third. - - run: | - if [ "$SHARD" = 3 ]; then - exec pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling - fi - pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-end-to-end: - name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's fixed-seed units, in three shards. Its random - # units, calibrations, and determinism units run locally. - - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-vitest-5: - name: Test on Vitest 5 (${{ matrix.suite }}) - # The package's vitest peer range includes 5; this job is the evidence. - # The unit tests and each self-oracle shard are their own jobs. - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - suite: - - test - - oracle-framework 1 - - oracle-framework 2 - - oracle-framework 3 - - oracle-framework 4 - - oracle-framework 5 - - oracle-framework 6 - - oracle-framework 7 - - oracle-framework 8 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles add --save-dev vitest@^5.0.5 - - run: pnpm --filter @tanstack/oracles exec vitest --version - - run: | - if [ "$SUITE" = test ]; then pnpm run test; exit; fi - pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern $SUITE)" - working-directory: packages/oracles - env: - SUITE: ${{ matrix.suite }} - preview: - name: Publish preview - # Only a commit that passes every check gets an installable preview. - needs: - - test - - test-eslint-plugin - - test-oracle-framework - - test-oracle-gaps-fast - - test-oracle-gaps-end-to-end - - test-vitest-5 - # The blocking Windows jobs (the non-blocking end-to-end job is not a - # dependency until it is green). - - windows-unit - - windows-gap-classify - - windows-gap-oracles - - windows-gap-cli - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm run build - # Consumers install a commit as - # https://pkg.pr.new/TanStack/oracles/@tanstack/oracles@ - # --compact needs the package on npm, so it is added after the first - # npm release. pkg.pr.new authenticates through its GitHub App, not an - # npm token, so npm trusted publishing does not apply. - - run: pnpm dlx pkg-pr-new@0.0.92 publish --pnpm './packages/*' # zizmor: ignore[use-trusted-publishing] - zizmor: - name: zizmor + # MEASUREMENT ONLY (scratch, not for merge): the random shards under pinned + # campaign seeds, nothing else. + measure-random-seeds: + name: measure random ${{ matrix.shard }}/3 seed ${{ matrix.seed }} runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 - with: - advanced-security: false - annotations: true - # Windows is a required part of CI: the framework, the eslint plugin, and - # the gap CLI run on windows-latest, split the way the Linux jobs split - # them. Each job stays under 10 minutes; each has a timeout backstop. - windows-unit: - name: Windows unit suites - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - run: pnpm --filter @tanstack/oracles run test - # Runs even when the framework tests fail, so each suite reports. - - if: always() - run: pnpm --filter @tanstack/eslint-plugin-oracles run test - windows-gap-oracles: - name: Windows gap report static oracle (${{ matrix.shard }}/6) - runs-on: windows-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - # The same partition of the static oracle that Linux runs, by pattern. - - name: Gap report static oracle - shell: bash - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static-windows "$SHARD")" - windows-gap-classify: - name: Windows gap report classify, probe-form, and mutant-validity oracles - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Gap report oracles - working-directory: packages/oracles - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling - windows-gap-end-to-end: - name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/10) - runs-on: windows-latest - # Not blocking yet: these suites fail on Windows, tracked by class in - # https://github.com/TanStack/oracles/issues/72. Remove this line when they - # are green. continue-on-error: true - timeout-minutes: 20 strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + shard: [1, 2, 3] + seed: [-972199262, 333199054, -1269403528, -1659558018, 1835238112, -330614923] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -278,214 +25,8 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Gap report end-to-end oracle - shell: bash + - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-random "$SHARD")" working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} - # The verbose reporter prints each test with its duration, which sizes the shards. - run: pnpm exec vitest --run --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" - windows-gap-cli: - name: Windows gap CLI on a fixture - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Write gap CLI fixture - shell: bash - run: | - set -euo pipefail - dir="$GITHUB_WORKSPACE/.ci-fixture" - mkdir -p "$dir/src" "$dir/tests" - cat > "$dir/package.json" <<'EOT' - {"name": "ci-fixture", "private": true, "type": "module"} - EOT - cat > "$dir/src/clamp.ts" <<'EOT' - export function clamp(x: number): number { - return x < 0 ? 0 : x > 10 ? 10 : x - } - EOT - cat > "$dir/tests/clamp.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - import { clamp } from '../src/clamp' - - oracleProperty({ - owns: ['src/clamp.ts#clamp'], - id: `fixture.clamp`, - law: { - statement: `clamp returns its argument within 0 and 10.`, - authority: `the CI fixture for the Windows gap-CLI run`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 50, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => (x < 0 ? 0 : x > 10 ? 10 : x), - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, clamp(x)) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => { - const seen = trace.find(({ checkpoint }) => checkpoint === `result`)?.value - return differs(`result`, `clamp`, seen, expected) ?? pass - }, - }) - EOT - cat > "$dir/vitest.config.mjs" <<'EOT' - export default { - test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } }, - } - EOT - # A failing property for the replay witness: every history fails, so a - # random unit prints its replay lines. Written in bash, as the fixture is. - - name: Write replay fixture - shell: bash - run: | - set -euo pipefail - dir="$GITHUB_WORKSPACE/.ci-replay" - mkdir -p "$dir/tests" - cat > "$dir/tests/fail.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture ’ & | < > ( ) ^ replay`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - cat > "$dir/tests/hostile.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture Q"1 replay`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - cat > "$dir/tests/percent.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture 100% done!`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - printf '%s\n' "export default { test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } } }" > "$dir/vitest.config.mjs" - # The user-facing path: PowerShell, the default shell, as a user runs it. - # Each run's stdout and stderr are kept apart, and the done line is - # checked on both streams against the exit code (P10). - - name: Gap CLI on the fixture - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs > cli-out.txt 2> cli-err.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out.txt cli-err.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - - name: Gap CLI on the fixture, discovered config - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts > cli-out2.txt 2> cli-err2.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out2.txt cli-err2.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - # The daemon path: start it, check through it, then stop it. - - name: Gap CLI through the daemon - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs --daemon > cli-out3.txt 2> cli-err3.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out3.txt cli-err3.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - - name: Stop the gap CLI daemon - if: always() - run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture - # A package with no package.json is refused, with exit 2 and a done line - # that says so. - - name: Gap CLI refuses a package with no package.json - run: | - New-Item -ItemType Directory -Force .ci-no-manifest | Out-Null - node packages/oracles/oracle-gaps/index.mjs --package .ci-no-manifest --source src/x.ts > nm-out.txt 2> nm-err.txt - $code = $LASTEXITCODE - if ($code -ne 2) { throw "want a refusal with exit 2, got $code" } - if (-not (Select-String -Path nm-out.txt, nm-err.txt -Pattern 'has no package.json' -Quiet)) { throw "the refusal does not name package.json" } - node packages/oracles/scripts/windows-witness.mjs done nm-out.txt nm-err.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - # One early stop: SIGINT once the CLI has printed its first line. The - # run must end with the stop's done line on both streams, or with no - # output at all (the all-or-nothing law, #57). - - name: Early stop of the gap CLI - run: node packages/oracles/scripts/windows-witness.mjs stop .ci-fixture --package . --source src/clamp.ts --config vitest.config.mjs - # The per-shell replay lines: a failing property's PowerShell and cmd - # lines each reproduce its failure in their own shell (RED: the POSIX - # line does not, in PowerShell). - - name: Replay lines reproduce the failure in PowerShell and cmd - if: always() - run: | - node "$env:GITHUB_WORKSPACE\packages\oracles\node_modules\vitest\vitest.mjs" run --root .ci-replay > replay-out.txt 2>&1 - Get-Content replay-out.txt - node packages/oracles/scripts/windows-witness.mjs replay replay-out.txt "$env:GITHUB_WORKSPACE\.ci-replay" - if ($LASTEXITCODE -ne 0) { throw "the replay witness failed" } + ORACLE_SEED: ${{ matrix.seed }} diff --git a/packages/oracles/oracle-gaps/constructed-order.json b/packages/oracles/oracle-gaps/constructed-order.json new file mode 100644 index 00000000..90f9f734 --- /dev/null +++ b/packages/oracles/oracle-gaps/constructed-order.json @@ -0,0 +1,55 @@ +{ + "constructed": [ + "every outcome class", + "a second daemon check of the budget witness", + "a load failure beside a hang", + "a stale authored mutant and a reformat", + "an ambiguous authored mutant", + "an edit sequence through the daemon", + "two authored mutants", + "ordinary tests left out by default", + "a config that --config names", + "a package with no config", + "an awaiting config in a package with no module type", + "a .cjs config that calls require", + "a .ts config in a package with no module type that calls require", + "a run with eleven contexts", + "a package whose parent directory has its config", + "a failing owner under its parent's config", + "test.projects in a parent's config", + "a failing owner under a config that --config names", + "a config fault in a run that owns nothing", + "a declaration fault: badEntry", + "a declaration fault: missingSource", + "a declaration fault: nonLiteral", + "a declaration fault: brokenOwner", + "a declaration fault: projects", + "a declaration fault: outsideSource", + "a declaration fault: brokenVitest", + "a declaration fault: configThrows", + "a declaration fault: configNoDefault", + "a declaration fault: configNull", + "a declaration fault: configSyntax", + "a declaration fault: setupThrows", + "a declaration fault: noTypescript", + "a declaration fault: apiLessTypescript", + "an edit sequence over the witness module", + "an edit that changed the cold result after a hang replaced the only context", + "an edit during a check through the daemon", + "an unchanged check through the daemon under a config's pre plugin", + "an unchanged check through the daemon while a test reads a source through fs", + "a test edits the helper it read, during a check through the daemon", + "a test edits the data it read, during a check through the daemon", + "an edit to data a test reads through fs (sync)", + "an edit to data a test reads through fs (promise)", + "an edit to data a test reads through fs (callback)", + "an edit to data a test reads through fs (statDir)", + "an edit to a module that only mutated runs load", + "a lazy module loaded before a hang in the same batch", + "a capture row read while the worker wrote it", + "two batches each read a capture row while its worker wrote it", + "a daemon pause that cannot engage", + "every outcome class, the warm and abort stops", + "a reset check after its warm daemon was replaced" + ] +} diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 343634c3..b489eb74 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4115,18 +4115,6 @@ function filesRun(runs) { return ran } -/** - * The owner files in which a slow test waits under a mutant, among the - * files its run reaches: each copy of such a file is busy for SLOW_MS, so - * the CLI owes a heartbeat for it. - */ -function slowedFiles(plan, tests, sites, active) { - const slowed = new Set() - for (const [file, own] of filesRun(runsOf(plan, tests, sites, active))) - if (own.some((r) => r.slowed)) slowed.add(file) - return slowed -} - /** * A mutant's kind from its owner tests' runs. An operator mutant's * confirming runs share its budget (`budgeted`); an authored mutant's runs @@ -4522,15 +4510,6 @@ function model(plan) { ordinal: site.ordinal, probe: site.probe, kind, - // A mutant whose run reaches a slow owner test keeps a copy busy for - // SLOW_MS, so the CLI must name it in a heartbeat while it runs. - heartbeat: - kind !== undefined && - ![`unowned`, `unreached`, `setup-error`, `indeterminate`].includes( - kind, - ) && - slowedFiles(plan, ownerTests(site.fnId), sites, { site: site.id }) - .size > 0, ordinaryOutcome: plan.ordinary && survivorKinds.has(kind) && ordinaryReaches(site), } @@ -5910,7 +5889,6 @@ export { ownersOf, interpret, judge, - slowedFiles, MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 54652d4b..ed40d626 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1053,6 +1053,45 @@ function streamOf(lines, name) { } } +// A run as the trace observes it: its heartbeats and their positions are +// progress (see heartbeatBreaches), and its evidence counts are timing. +function unbeaten(run) { + return { + ...run, + evidence: undefined, + beats: undefined, + heartbeats: undefined, + } +} + +/** + * Law 11, shape only (maintainer decision, 2026-11-14). A heartbeat names the + * baseline or a mutant of this run, in a file that runs it, and a CLI prints + * it only while its copy runs, so a beat after its mutant's own streamed gap + * line names a mutant that was not running. Returns the breaches as text, + * none in a correct run. The beats' set and count depend on wall-clock timing, + * so they are progress, not outcome: no trace keeps them. + */ +function heartbeatBreaches(cli) { + const byId = new Map((cli.mutants ?? []).map((m) => [m.id, m])) + return (cli.beats ?? []).flatMap(({ label, file, index, finishedAt }) => { + if (label === `baseline`) + return /^tests\/[abc]\.test\.ts$/.test(file) + ? [] + : [`baseline in ${file}: not a baseline file`] + const m = byId.get(label) + if (m === undefined) return [`${label} in ${file}: no mutant of this run`] + if (finishedAt !== undefined && finishedAt < index) + return [ + `${label} in ${file}: its gap line came first, so it was not running`, + ] + const runs = m.authored + ? /^tests\/\w+\.test\.ts$/.test(file) && file !== `tests/c.test.ts` + : [...(m.oracleFiles ?? []), ...(m.ordinaryFiles ?? [])].includes(file) + return runs ? [] : [`${label} in ${file}: the file does not run it`] + }) +} + function readRun(dir, out, status, stderr, cli, stdout) { // The output names the CLI directory that ran and the package, as scratch // directories do, before it is cut to its last characters. @@ -1098,6 +1137,27 @@ function readRun(dir, out, status, stderr, cli, stdout) { ] .sort() .map((beat) => JSON.parse(beat)) + // Each beat in stream order, with the stream index of its mutant's own + // streamed gap line when that came first (law 11's shape). The driver + // judges these with heartbeatBreaches; the trace observes only the breaches. + const streamedAt = new Map() + lines.forEach((line, index) => { + const id = line.match(streamedGap)?.[1] + if (id !== undefined && !streamedAt.has(id)) streamedAt.set(id, index) + }) + const beats = lines.flatMap((line, index) => { + const beat = line.match(/^oracle-gaps: busy (\S+) in (\S+) for \d+s$/) + return beat + ? [ + { + label: beat[1], + file: beat[2], + index, + finishedAt: streamedAt.get(beat[1]), + }, + ] + : [] + }) // The stdout summary's lead, up to its first blank line (law 14), its // last line, and the files the baseline lines on stderr name. Since the // stream has no blank line (law 15), the lead is the whole stdout, whose @@ -1113,13 +1173,17 @@ function readRun(dir, out, status, stderr, cli, stdout) { ), lastLine: printed.at(-1) && namedScratch(printed.at(-1), scratch), // The stdout's own lines of the stream's kinds: the gap lines under - // their functions, as the stream prints them (law 15). + // their functions. A context's gap line is printed when its copies finish, + // so the order is timing: the stream is recorded as relations (law 15), and + // a gap line's place in the output is not judged. stdoutGaps: printed .filter((line) => streamedGap.test(line)) - .map((line) => namedScratch(line, scratch)), + .map((line) => namedScratch(line, scratch)) + .sort(), stderrGaps: lines .filter((line) => streamedGap.test(line)) - .map((line) => namedScratch(line, scratch)), + .map((line) => namedScratch(line, scratch)) + .sort(), baselineFiles: lines.flatMap((line) => { const file = line.match( /^oracle-gaps: baseline (?:pass|fail) (\S+)(?: \(ordinary\))?: /, @@ -1127,6 +1191,7 @@ function readRun(dir, out, status, stderr, cli, stdout) { return file === undefined ? [] : [file] }), heartbeats, + beats, // The config the run says it used (law 3), and the warnings Node printed // for the process. configLines: lines.filter((line) => @@ -1531,18 +1596,6 @@ async function incremental(dir, plan, cli, copy, observe, states) { `an unchanged check through the daemon computed nothing while a test read a source through fs`, ) } - // The witness module's hangs are cut off in every cold run. A daemon - // check after an edit that cut off fewer runs than the cold run of the - // same state reused a timed-out run, so law 9 judged a timeout that came - // from the store. - if ( - plan.witness !== false && - step !== `start` && - !daemonEvents.has(step) && - reused > 0 && - (daemon.evidence?.cut ?? 0) < (coldRun.evidence?.cut ?? 0) - ) - states.add(`a timed-out run reused after an edit`) // Undoing an edit to a file returns to a state the daemon has seen, so // it reruns nothing. if ( @@ -1914,7 +1967,11 @@ async function driver(plan, observe) { const tornRead = reached > 0 // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. - observe(`cli`, { ...cli, evidence: undefined }) + // Heartbeats are progress, not outcome (maintainer decision, 2026-11-14): + // their set depends on wall-clock timing, so the trace keeps none of them. + // Their shape breaches are observed as `beats`, which is empty in a correct run. + observe(`cli`, unbeaten(cli)) + observe(`beats`, { breaches: heartbeatBreaches(cli) }) // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) observe(`replay`, { command: replay, ran: replayed(dir, replay) }) @@ -2226,8 +2283,6 @@ async function driver(plan, observe) { states.add(`a hang in one owner and a survivor in another timed out`) if (m.function === `w_slowpair` && m.outcome === `timeout`) states.add(`confirmations past a mutant's budget timed out`) - if (cli.heartbeats.some((beat) => beat.label === m.id)) - states.add(`a heartbeat named a busy mutant`) const named = { killed: `killed`, 'random-only-kill': `random-only kill`, @@ -2362,7 +2417,9 @@ async function driver(plan, observe) { const moved = { ...files, [SOURCE]: edited(plan) } // The evidence counts are not observed, as for `cli`: they depend on // timing (oracles-followups.md G1). Verdicts and keys are. - observe(`stability`, { ...runCli(dir, moved, how), evidence: undefined }) + const again = runCli(dir, moved, how) + observe(`beats`, { breaches: heartbeatBreaches(again) }) + observe(`stability`, unbeaten(again)) states.add(`stability rerun`) } if (plan.steps.length > 0) @@ -2703,38 +2760,15 @@ function check(plan, observations, expected) { `streaming`, { actual: cli.streamed, expected: cli.gaps }, ) - // A heartbeat names what its busy copy runs: a mutant, and a test file - // that runs it (an owner file whose tests reached it, an ordinary file for - // the ordinary tier, any oracle file for an authored mutant), or the - // baseline and a test file. Each mutant whose run reaches a slow owner - // test keeps a copy busy past the interval, so a heartbeat names it. - const byId = new Map(cli.mutants.map((m) => [m.id, m])) - for (const beat of cli.heartbeats) { - const m = byId.get(beat.label) - const runsIt = - beat.label === `baseline` - ? /^tests\/[abc]\.test\.ts$/.test(beat.file) - : m?.authored - ? /^tests\/\w+\.test\.ts$/.test(beat.file) && - beat.file !== `tests/c.test.ts` - : [...(m?.oracleFiles ?? []), ...(m?.ordinaryFiles ?? [])].includes( - beat.file, - ) - if (!runsIt) - return violation( - `cli`, - `a heartbeat names ${beat.label} in ${beat.file}, which does not run it`, - `streaming`, - { actual: beat }, - ) - } - for (const [owed, got] of pairs) - if (owed.heartbeat && !cli.heartbeats.some((beat) => beat.label === got.id)) + // Law 11, shape only (maintainer decision, 2026-11-14): the driver observes + // the heartbeats that break the shape, and a breach is a streaming failure. + for (const { value } of observations.filter((o) => o.checkpoint === `beats`)) + if (value.breaches.length > 0) return violation( `cli`, - `${site(got)}: no heartbeat named it while its slow owner test ran`, + `a heartbeat breaks its shape: ${value.breaches[0]}`, `streaming`, - { actual: cli.heartbeats, expected: got.id }, + { actual: value.breaches }, ) for (const [owed, got] of pairs) if (!owed.probe && got.ordinaryOutcome !== owed.ordinaryOutcome) @@ -3161,6 +3195,16 @@ const corruptCli = (change) => (trace) => { if (value === undefined) return undefined return trace.map((o, i) => (i === index ? { ...o, value } : o)) } +// A checker calibration's wrong beat observation: one breach of the shape law, +// set in a run that completed (a run that did not complete has no beats). +const corruptBeats = (breach) => (trace) => { + const cli = trace.find((o) => o.checkpoint === `cli`)?.value + const index = trace.findIndex((o) => o.checkpoint === `beats`) + if (cli?.exit !== 0 || index < 0) return undefined + return trace.map((o, i) => + i === index ? { ...o, value: { breaches: [breach] } } : o, + ) +} const firstWhere = (predicate, change) => corruptCli((cli) => { if (cli.exit !== 0) return undefined @@ -3390,25 +3434,25 @@ calibrations.push( })), }, { - name: `a busy mutant is named in no heartbeat`, + name: `a heartbeat names a file that does not run its mutant`, law: `streaming`, - corrupt: corruptCli((cli) => - cli.exit !== 0 || cli.heartbeats.length === 0 - ? undefined - : { ...cli, heartbeats: [] }, + corrupt: corruptBeats( + `sut.ts:M1 in tests/elsewhere.test.ts: the file does not run it`, ), }, { - name: `a heartbeat names a file that does not run its mutant`, + name: `a heartbeat names a mutant whose gap line came first`, law: `streaming`, - corrupt: corruptCli((cli) => { - const beat = cli.heartbeats.find((b) => b.label !== `baseline`) - if (cli.exit !== 0 || !beat) return undefined - return { - ...cli, - heartbeats: [{ ...beat, file: `tests/elsewhere.test.ts` }], - } - }), + corrupt: corruptBeats( + `sut.ts:M1 in tests/a.test.ts: its gap line came first, so it was not running`, + ), + }, + { + name: `a heartbeat names no mutant of its run`, + law: `streaming`, + corrupt: corruptBeats( + `sut.ts:M999 in tests/a.test.ts: no mutant of this run`, + ), }, // Law 15: a line without the stream's prefix, the decisive baseline line // after a mutant's line, a completed run that ends with the old @@ -3652,7 +3696,6 @@ const required = [ `a hang in one owner and a kill in another killed`, `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, - `a heartbeat named a busy mutant`, `a run stopped by SIGINT while a worker spun`, `a stop signal after the completed line`, `a stop signal while the config warms`, @@ -3707,7 +3750,6 @@ const required = [ `a config fault in a run that owns nothing`, ...incrementalStates, ...ownedIncrementalStates, - `a timed-out run reused after an edit`, `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, `a lazy edit changed the cold result after a batch was cut off`, @@ -3790,7 +3832,6 @@ const separate = [ `a failing owner replayed under a config that --config names`, ...incrementalStates, ...ownedIncrementalStates, - `a timed-out run reused after an edit`, `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, `a lazy edit changed the cold result after a batch was cut off`, @@ -3860,7 +3901,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Windows idle gap (recorded, not witnessed): the state \`a hang cut off for idleness\` is required on every platform, and win32 reaches it (run 43683794816, job 132719872988), so it is not platform-scoped. The sampler (capture-setup.mjs) counts a copy idle only after 500 ms with no late tick, no non-quiet handle, utilization at most 5%, and no timer due, so \`every outcome class\` reaches the state only when the sampler sees that window. On win32 the check missed it once (run 43683794816, job 132719872940); the cause is not confirmed (a late tick on a shared runner, or a stdio handle outside \`quiet\` for this file's console output, are the candidates). Its witness, a test-only hold at the hang until the window is seen, is owed, and that is a timing dependency the determinism-under-load rule forbids; the maintainer decides whether it lands before the Windows job becomes blocking.`, + limits: `The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats; beats are checked for shape only (maintainer decision, 2026-11-14): each names a mutant of the run and a file that runs it, and a beat that follows its mutant's gap line is a breach, while their set and count, which depend on timing, are not compared and no beat's presence is required; so the reach state "a heartbeat named a busy mutant" is no longer observed or required (it was progress, derived from a beat's timing; #84 owns restoring its coverage through a deterministic pause point); and "a timed-out run reused after an edit" is no longer observed or required, because its predicate compares evidence cut counts, which depend on timing (the evidence counts are not observed, as for cli), so the witness module's edit sequence no longer enters it; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units at main's bases; the disarmed-pause history's base, appended at its index, does reach it, so that history pins its generated inputs to ones that declare no bound, and the model predicts no work verdict for a generated test, a model gap for the maintainer to decide whether to model work counts), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Windows idle gap (recorded, not witnessed): the state \`a hang cut off for idleness\` is required on every platform, and win32 reaches it (run 43683794816, job 132719872988), so it is not platform-scoped. The sampler (capture-setup.mjs) counts a copy idle only after 500 ms with no late tick, no non-quiet handle, utilization at most 5%, and no timer due, so \`every outcome class\` reaches the state only when the sampler sees that window. On win32 the check missed it once (run 43683794816, job 132719872940); the cause is not confirmed (a late tick on a shared runner, or a stdio handle outside \`quiet\` for this file's console output, are the candidates). Its witness, a test-only hold at the hang until the window is seen, is owed, and that is a timing dependency the determinism-under-load rule forbids; the maintainer decides whether it lands before the Windows job becomes blocking.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4384,7 +4425,7 @@ oracleProperty({ // cheapest legal setting, so that the history costs about two cold // runs and four daemon checks of the witness module. name: `an edit sequence over the witness module`, - enters: [`a timed-out run reused after an edit`], + enters: [], build: (generated) => ({ ...generated, functions: [ diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 77f72982..0996123b 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -36,8 +36,9 @@ const suite = `oracle-gaps.report` // cost follows its seed, so a shard's time varies from run to run. Option M // of the e2e speed plan bounds the random units' tail: a generated history // with the witness module takes no steps, and the one edit sequence over it -// is a constructed history. CI keeps its one fixed-seed job until the -// maintainer decides which units run on a pull request. +// is a constructed history. A pull request runs the fixed-seed, random, and +// determinism units as the partitions of pr-shards.mjs; these shards are +// the full-suite plan, which this file's check keeps partitioned. export const shards = { // 522 s in the measured run, about 221 s idle. 'e2e-1': [ @@ -205,7 +206,9 @@ export const shards = { // Added with the gap CLI's runtime fixes (not yet measured). `fixed seed 20261105: a second daemon check of the budget witness`, `calibration: a stopped run leaves a process it started running`, - `calibration: a busy mutant is named in no heartbeat`, + // The heartbeat calibrations after the shape-only change (not yet measured). + `calibration: a heartbeat names a mutant whose gap line came first`, + `calibration: a heartbeat names no mutant of its run`, // Law 15's calibrations (the greppable stream), measured at ab18c60: // 79 s and 70 s; the daemon-notes one, bound to its history, 2 s (it // took 433 s searching for one). diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 24a91921..1cf8051b 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -5,10 +5,12 @@ // shard of its partition takes. The check lists the suite's tests with // `vitest list` and fails unless every selected test is in exactly one // shard, no other test is in any shard, and every test that a shard names -// exists. So the jobs together run exactly the tests that one job ran -// before: a test in no shard would leave its law out of CI, and a test in -// two would run twice. A new test lands in the `rest` shard, so the jobs' -// balance can drift, but no test is left out. +// exists. A partition's `excluded` names are the exception: each is in no +// shard on purpose, with its own issue, and must exist. So the jobs together +// run exactly the tests that one job ran before: a test in no shard would +// leave its law out of CI, and a test in two would run twice. A new test +// lands in the `rest` shard, so the jobs' balance can drift, but no test is +// left out. // // node scripts/pr-shards.mjs check checks every // partition @@ -320,6 +322,131 @@ export const partitions = { 3: rest, }, }, + // The end-to-end gap oracle's random units, which every pull request runs + // (maintainer decision CI-1): each run draws a fresh campaign seed, and a + // counterexample prints its replay line. 38 units, measured on main at + // 35422e3 on one Arclet (8 vCPU Linux, one Vitest worker): 624 s in all. + // CI ran the random shards 1.7 times slower than the Arclet (9 min for a + // 312 s shard, in #90's first run), and a random unit's cost follows its + // seed. So the shards are three of about 208 s here, which is about 6 min + // in CI. The witness module's edit sequence alone is 178 s. + 'oracle-gaps-random': { + args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], + suite: `oracle-gaps.report`, + select: `random seed: `, + // Every name is checked against the listed tests (partitionErrors). + exactNames: true, + shards: { + // 208 s here. + 1: anyOf([ + `an edit sequence over the witness module`, + `a declaration fault: noTypescript`, + `a lazy module loaded before a hang in the same batch`, + `a load failure beside a hang`, + `an ambiguous authored mutant`, + `a .ts config in a package with no module type that calls require`, + `an awaiting config in a package with no module type`, + `a declaration fault: configNull`, + `a declaration fault: configSyntax`, + `a declaration fault: missingSource`, + `a config fault in a run that owns nothing`, + ]), + // 208 s here. + 2: anyOf([ + `every outcome class`, + `an edit sequence through the daemon`, + `a declaration fault: brokenVitest`, + `an edit that changed the cold result after a hang replaced the only context`, + `an edit to a module that only mutated runs load`, + `a failing owner under a config that --config names`, + `a failing owner under its parent's config`, + `a package whose parent directory has its config`, + `a package with no config`, + `ordinary tests left out by default`, + `a declaration fault: configNoDefault`, + `a declaration fault: badEntry`, + `a declaration fault: projects`, + `a declaration fault: outsideSource`, + ]), + // The rest: 208 s here, and any unit added later. + 3: rest, + }, + }, + // The end-to-end gap oracle's determinism units, which pull requests run + // except the one excluded below. Each unit runs one history twice and + // compares the two traces: results and fixed lines, not the heartbeats, + // which are progress (maintainer decision, 2026-11-14). 51 units; measured + // on Arclets at d77b63e, one Vitest worker: 1,305 s for the 50 units that + // are not excluded. The witness module's edit sequence is 344 s alone, so + // it is a shard by itself; the other 49 units are packed longest first into + // three shards of 240 s, and the rest shard takes the fourth bin and any + // unit added later. The excluded unit, the daemon's test-output note (#92), + // has a cause that is not heartbeats, so it stays out until #92 is fixed. + 'oracle-gaps-determinism': { + args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], + suite: `oracle-gaps.report`, + select: `determinism: `, + // Every name is checked against the listed tests (partitionErrors). + exactNames: true, + excluded: [ + // The daemon's test-output note (#92). + `constructed history: an edit sequence through the daemon`, + // A daemon check's trace differs between two runs (#105): intermittent. + `constructed history: a daemon pause that cannot engage`, + ], + shards: { + // 344 s: the witness module's edit sequence, alone. + 1: anyOf([ + `constructed history: an edit sequence over the witness module`, + ]), + // 240 s here. + 2: anyOf([ + `constructed history: every outcome class`, + `constructed history: an edit to data a test reads through fs (statDir)`, + `constructed history: a declaration fault: apiLessTypescript`, + `constructed history: a failing owner under its parent's config`, + `constructed history: a package with no config`, + `constructed history: a .ts config in a package with no module type that calls require`, + `constructed history: a declaration fault: configNoDefault`, + `constructed history: a declaration fault: badEntry`, + `constructed history: a declaration fault: outsideSource`, + ]), + // 240 s here. + 3: anyOf([ + `generated 2`, + `constructed history: an edit during a check through the daemon`, + `constructed history: an edit to data a test reads through fs (sync)`, + `constructed history: an unchanged check through the daemon while a test reads a source through fs`, + `constructed history: two authored mutants`, + `constructed history: a capture row read while the worker wrote it`, + `constructed history: two batches each read a capture row while its worker wrote it`, + `constructed history: a package whose parent directory has its config`, + `constructed history: a declaration fault: configNull`, + `constructed history: a declaration fault: missingSource`, + `constructed history: a config fault in a run that owns nothing`, + ]), + // 240 s here. + 4: anyOf([ + `constructed history: a declaration fault: noTypescript`, + `constructed history: a declaration fault: brokenVitest`, + `constructed history: a stale authored mutant and a reformat`, + `constructed history: an edit that changed the cold result after a hang replaced the only context`, + `constructed history: a test edits the helper it read, during a check through the daemon`, + `generated 1`, + `constructed history: an edit to data a test reads through fs (callback)`, + `constructed history: an unchanged check through the daemon under a config's pre plugin`, + `constructed history: an awaiting config in a package with no module type`, + `constructed history: a run with eleven contexts`, + `constructed history: a .cjs config that calls require`, + `constructed history: ordinary tests left out by default`, + `constructed history: a declaration fault: setupThrows`, + `constructed history: a declaration fault: nonLiteral`, + `generated 0`, + ]), + // The rest: 240 s here, and any unit added later. + 5: rest, + }, + }, 'oracle-gaps-static-windows': { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-static`], suite: `oracle-gaps.report-static`, @@ -591,11 +718,18 @@ export const partitions = { }, } +/** A partition's excluded names, relative to its select: tests in no shard. */ +const excludedOf = (partition) => new Set(partition.excluded ?? []) + /** A shard's pattern over the name that follows the suite's. */ function bodyOf(partition, shard) { const own = partition.shards[shard] if (own !== rest) return `${escape(partition.select)}(?:${own})` - const others = Object.values(partition.shards).filter((body) => body !== rest) + // A rest shard takes what no other shard and no exclusion takes. + const others = [ + ...Object.values(partition.shards).filter((body) => body !== rest), + ...[...excludedOf(partition)].map(escape), + ] return `(?!${escape(partition.select)}(?:${others.join(`|`)})$)${escape(partition.select)}.*` } @@ -643,7 +777,12 @@ export function partitionErrors(name, listed) { joined.every((text) => owners[0].pattern.test(text)) const own = full.startsWith(prefix) ? full.slice(prefix.length) : undefined const selected = own !== undefined && own.startsWith(partition.select) - if (selected && !whole) + // An excluded test is in no shard on purpose; its issue is the record. + const excluded = + selected && excludedOf(partition).has(own.slice(partition.select.length)) + if (excluded && owners.length > 0) + errors.push(`${JSON.stringify(full)} is excluded, but is in a shard`) + if (selected && !whole && !excluded) errors.push( `${JSON.stringify(full)} is in ${owners.length === 0 ? `no shard` : owners.length === 1 ? `shard ${owners[0].shard} under only one of Vitest's name joins` : `shards ${owners.map(({ shard }) => shard).join(`, `)}`}`, ) @@ -661,6 +800,9 @@ export function partitionErrors(name, listed) { for (const exception of partition.exceptions ?? []) if (!names.has(`${prefix}${exception}`)) errors.push(`exception ${JSON.stringify(exception)} names no test`) + for (const exclusion of excludedOf(partition)) + if (!names.has(`${prefix}${partition.select}${exclusion}`)) + errors.push(`excluded ${JSON.stringify(exclusion)} names no test`) for (const [shard, body] of Object.entries(partition.shards)) if ( body !== rest && diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index e7d22ad1..15a520ac 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -1,7 +1,13 @@ +import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { readFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' import fc from 'fast-check' import { describe, expect, test } from 'vitest' +import { version as vitestVersion } from 'vitest/package.json' import { drawn, histories } from '../oracle-gaps/gap-report-model-oracle.mjs' // The generator rule (coordinator decision, 2026-11-13): a new field of the @@ -59,4 +65,68 @@ describe(`the gap oracle's history grammar`, () => { ) } }) + + // The constructed histories of the end-to-end gap oracle, in the order the + // oracle registers them: each one is built from the fixed seed and its index + // (`seed + index`), so a history inserted before a recorded one changes every + // later history's draw. The register's generator rule (2026-11-14) says new + // constructed histories are appended at the end. The recorded names are + // the list the oracle registered on main; the current list must begin with + // them, so a new name can only follow. + // The gap oracle loads under Vitest 4; the Vitest 5 job runs no gap oracle + // (its listing finds no suite in this file), so the canary runs there only + // where the file loads. + const gapOracleLoads = Number(vitestVersion.split(`.`)[0]) < 5 + test.skipIf(!gapOracleLoads)( + `appends new constructed histories, and never inserts one before a recorded one`, + () => { + const recorded = JSON.parse( + readFileSync( + new URL(`../oracle-gaps/constructed-order.json`, import.meta.url), + `utf8`, + ), + ) as { constructed: Array } + // Vitest writes the listing to a file, so no package manager's stdout + // line can reach the JSON (as pr-shards.mjs does). + const dir = mkdtempSync(join(tmpdir(), `constructed-order-`)) + let tests: Array<{ name: string }> + try { + const file = join(dir, `list.json`) + const packageDir = dirname( + createRequire(import.meta.url).resolve(`vitest/package.json`), + ) + const listing = spawnSync( + process.execPath, + [ + join(packageDir, `vitest.mjs`), + `list`, + `--json=${file}`, + `--config`, + `oracle-gaps/vitest.config.mjs`, + `gap-report-oracle`, + ], + { + cwd: fileURLToPath(new URL(`..`, import.meta.url)), + encoding: `utf8`, + maxBuffer: 64 * 1024 * 1024, + }, + ) + expect(listing.status, listing.stderr).toBe(0) + tests = JSON.parse(readFileSync(file, `utf8`)) as Array<{ + name: string + }> + } finally { + rmSync(dir, { recursive: true, force: true }) + } + const unit = `oracle-gaps.report > fixed seed ${FIXED_SEED}: ` + const current = tests + .map(({ name }) => name) + .filter((name) => name.startsWith(unit)) + .map((name) => name.slice(unit.length)) + .filter((name) => name !== `generated`) + expect(current.slice(0, recorded.constructed.length)).toEqual( + recorded.constructed, + ) + }, + ) })