From 30d6dc655652251572f58f77255759ad3ce4d07f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 13:57:45 +0000 Subject: [PATCH 1/2] scratch: #73 class 3 mutant (the backslash path given to invalidateFile) Co-authored-by: Isaac --- packages/oracles/oracle-gaps/check.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 6f1be39..69ffa21 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -844,7 +844,7 @@ export default async (env) => { for (const path of paths) { const hash = servedHash(path, index) if (served.get(path) === hash) continue - context.invalidateFile(posix(path)) + context.invalidateFile(path) // Its next transform records the text it then serves. globalThis.__oracleGapsLoads[index]?.delete(path) served.set(path, hash) @@ -1675,7 +1675,7 @@ export default async (env) => { const read = served.get(path) ?? hash // The context may hold other text than the path's text now. if (read !== now || hash !== now) { - context.invalidateFile(posix(path)) + context.invalidateFile(path) loads?.delete(path) } if (hash !== now) stable = false From 7b731a69a58d183967243a9814d57033846b2828 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 13:58:41 +0000 Subject: [PATCH 2/2] scratch: run only the Windows gap end-to-end job Co-authored-by: Isaac --- .github/workflows/pr.yml | 447 --------------------------------------- 1 file changed, 447 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index edf5913..48c48ae 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -15,250 +15,6 @@ permissions: contents: read jobs: - # The test suites run as parallel jobs. The workflow's wall time is then - # its slowest job, about 6 to 7 minutes, not the sum of every suite, about - # 21 minutes when they ran as one job. Each job should stay well under the - # 10-minute limit; pr-shards.mjs records how each suite is split. - test: - name: Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm run build - - run: pnpm run lint:check - - run: pnpm run format:check - - run: pnpm run typecheck - - run: pnpm --filter @tanstack/oracles run test - # Checks the partition of the end-to-end gap oracle into CI shards - # (packages/oracles/scripts/e2e-shards.mjs) ahead of the shard jobs, - # which CI does not run yet: every test is in exactly one shard. - - run: pnpm --filter @tanstack/oracles run test:oracle-gaps:shards - # The framework self-oracle, the gap report's static oracle, and the - # end-to-end oracle's fixed seed each run as several jobs. Together - # they run exactly the tests one job ran - # (packages/oracles/scripts/pr-shards.mjs). - - run: pnpm --filter @tanstack/oracles run test:pr-shards - - run: pnpm run test:publint - - run: pnpm run test:sherif - test-eslint-plugin: - name: Test eslint-plugin-oracles - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The plugin's tests import @tanstack/oracles, which resolves to its - # build output. - - run: pnpm run build - - run: pnpm --filter @tanstack/eslint-plugin-oracles run test - test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/8) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in eight - # shards. - - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-fast: - name: Gap report oracles (${{ matrix.shard }}/3) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The gap report's own oracles also run real Vitest runs on fixture - # packages (test:oracle-gaps:fast): the static oracle in two shards, - # and the classify, probe-form, and mutant-validity oracles in a third. - - run: | - if [ "$SHARD" = 3 ]; then - exec pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling - fi - pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-oracle-gaps-end-to-end: - name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's fixed-seed units, in three shards. Its random - # units, calibrations, and determinism units run locally. - - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - test-vitest-5: - name: Test on Vitest 5 (${{ matrix.suite }}) - # The package's vitest peer range includes 5; this job is the evidence. - # The unit tests and each self-oracle shard are their own jobs. - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - suite: - - test - - oracle-framework 1 - - oracle-framework 2 - - oracle-framework 3 - - oracle-framework 4 - - oracle-framework 5 - - oracle-framework 6 - - oracle-framework 7 - - oracle-framework 8 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles add --save-dev vitest@^5.0.5 - - run: pnpm --filter @tanstack/oracles exec vitest --version - - run: | - if [ "$SUITE" = test ]; then pnpm run test; exit; fi - pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern $SUITE)" - working-directory: packages/oracles - env: - SUITE: ${{ matrix.suite }} - preview: - name: Publish preview - # Only a commit that passes every check gets an installable preview. - needs: - - test - - test-eslint-plugin - - test-oracle-framework - - test-oracle-gaps-fast - - test-oracle-gaps-end-to-end - - test-vitest-5 - # The blocking Windows jobs (the non-blocking end-to-end job is not a - # dependency until it is green). - - windows-unit - - windows-gap-classify - - windows-gap-oracles - - windows-gap-cli - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm run build - # Consumers install a commit as - # https://pkg.pr.new/TanStack/oracles/@tanstack/oracles@ - # --compact needs the package on npm, so it is added after the first - # npm release. pkg.pr.new authenticates through its GitHub App, not an - # npm token, so npm trusted publishing does not apply. - - run: pnpm dlx pkg-pr-new@0.0.92 publish --pnpm './packages/*' # zizmor: ignore[use-trusted-publishing] - zizmor: - name: zizmor - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 - with: - advanced-security: false - annotations: true - # Windows is a required part of CI: the framework, the eslint plugin, and - # the gap CLI run on windows-latest, split the way the Linux jobs split - # them. Each job stays under 10 minutes; each has a timeout backstop. - windows-unit: - name: Windows unit suites - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - run: pnpm --filter @tanstack/oracles run test - # Runs even when the framework tests fail, so each suite reports. - - if: always() - run: pnpm --filter @tanstack/eslint-plugin-oracles run test - windows-gap-oracles: - name: Windows gap report static oracle (${{ matrix.shard }}/6) - runs-on: windows-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - # The same partition of the static oracle that Linux runs, by pattern. - - name: Gap report static oracle - shell: bash - working-directory: packages/oracles - env: - SHARD: ${{ matrix.shard }} - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static-windows "$SHARD")" - windows-gap-classify: - name: Windows gap report classify, probe-form, and mutant-validity oracles - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Gap report oracles - working-directory: packages/oracles - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling windows-gap-end-to-end: name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/10) runs-on: windows-latest @@ -286,206 +42,3 @@ jobs: SHARD: ${{ matrix.shard }} # The verbose reporter prints each test with its duration, which sizes the shards. run: pnpm exec vitest --run --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" - windows-gap-cli: - name: Windows gap CLI on a fixture - runs-on: windows-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Write gap CLI fixture - shell: bash - run: | - set -euo pipefail - dir="$GITHUB_WORKSPACE/.ci-fixture" - mkdir -p "$dir/src" "$dir/tests" - cat > "$dir/package.json" <<'EOT' - {"name": "ci-fixture", "private": true, "type": "module"} - EOT - cat > "$dir/src/clamp.ts" <<'EOT' - export function clamp(x: number): number { - return x < 0 ? 0 : x > 10 ? 10 : x - } - EOT - cat > "$dir/tests/clamp.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - import { clamp } from '../src/clamp' - - oracleProperty({ - owns: ['src/clamp.ts#clamp'], - id: `fixture.clamp`, - law: { - statement: `clamp returns its argument within 0 and 10.`, - authority: `the CI fixture for the Windows gap-CLI run`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 50, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => (x < 0 ? 0 : x > 10 ? 10 : x), - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, clamp(x)) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => { - const seen = trace.find(({ checkpoint }) => checkpoint === `result`)?.value - return differs(`result`, `clamp`, seen, expected) ?? pass - }, - }) - EOT - cat > "$dir/vitest.config.mjs" <<'EOT' - export default { - test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } }, - } - EOT - # A failing property for the replay witness: every history fails, so a - # random unit prints its replay lines. Written in bash, as the fixture is. - - name: Write replay fixture - shell: bash - run: | - set -euo pipefail - dir="$GITHUB_WORKSPACE/.ci-replay" - mkdir -p "$dir/tests" - cat > "$dir/tests/fail.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture ’ & | < > ( ) ^ replay`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - cat > "$dir/tests/hostile.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture Q"1 replay`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - cat > "$dir/tests/percent.property.test.ts" <<'EOT' - import fc from 'fast-check' - import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' - - oracleProperty({ - id: `fixture 100% done!`, - law: { - statement: `a fixture that always fails, so its replay lines are printed.`, - authority: `the CI witness for the per-shell replay lines`, - limits: `integers from -20 to 20`, - }, - histories: { - arbitrary: fc.integer({ min: -20, max: 20 }), - runs: 20, - fixedSeeds: [1], - states: [], - measured: [], - }, - timeout: 5_000, - model: (x: number) => x, - driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { - observe(`result`, x + 1) - }, - check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => - differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, - }) - EOT - printf '%s\n' "export default { test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } } }" > "$dir/vitest.config.mjs" - # The user-facing path: PowerShell, the default shell, as a user runs it. - # Each run's stdout and stderr are kept apart, and the done line is - # checked on both streams against the exit code (P10). - - name: Gap CLI on the fixture - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs > cli-out.txt 2> cli-err.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out.txt cli-err.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - - name: Gap CLI on the fixture, discovered config - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts > cli-out2.txt 2> cli-err2.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out2.txt cli-err2.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - # The daemon path: start it, check through it, then stop it. - - name: Gap CLI through the daemon - run: | - node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs --daemon > cli-out3.txt 2> cli-err3.txt - $code = $LASTEXITCODE - node packages/oracles/scripts/windows-witness.mjs done cli-out3.txt cli-err3.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - - name: Stop the gap CLI daemon - if: always() - run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture - # A package with no package.json is refused, with exit 2 and a done line - # that says so. - - name: Gap CLI refuses a package with no package.json - run: | - New-Item -ItemType Directory -Force .ci-no-manifest | Out-Null - node packages/oracles/oracle-gaps/index.mjs --package .ci-no-manifest --source src/x.ts > nm-out.txt 2> nm-err.txt - $code = $LASTEXITCODE - if ($code -ne 2) { throw "want a refusal with exit 2, got $code" } - if (-not (Select-String -Path nm-out.txt, nm-err.txt -Pattern 'has no package.json' -Quiet)) { throw "the refusal does not name package.json" } - node packages/oracles/scripts/windows-witness.mjs done nm-out.txt nm-err.txt $code - if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - # One early stop: SIGINT once the CLI has printed its first line. The - # run must end with the stop's done line on both streams, or with no - # output at all (the all-or-nothing law, #57). - - name: Early stop of the gap CLI - run: node packages/oracles/scripts/windows-witness.mjs stop .ci-fixture --package . --source src/clamp.ts --config vitest.config.mjs - # The per-shell replay lines: a failing property's PowerShell and cmd - # lines each reproduce its failure in their own shell (RED: the POSIX - # line does not, in PowerShell). - - name: Replay lines reproduce the failure in PowerShell and cmd - if: always() - run: | - node "$env:GITHUB_WORKSPACE\packages\oracles\node_modules\vitest\vitest.mjs" run --root .ci-replay > replay-out.txt 2>&1 - Get-Content replay-out.txt - node packages/oracles/scripts/windows-witness.mjs replay replay-out.txt "$env:GITHUB_WORKSPACE\.ci-replay" - if ($LASTEXITCODE -ne 0) { throw "the replay witness failed" }