From 83da173853d363f729f01c5939584ea6f953205f Mon Sep 17 00:00:00 2001 From: Matheus Martins Date: Tue, 4 Aug 2026 09:50:36 +0000 Subject: [PATCH 1/5] ci(mutation): split mutation testing into parallel shards Replace the single 60-minute Infection job with a three-stage pipeline that fans the mutant analysis out across 10 machines: mutation-coverage -> mutation (x10 matrix) -> mutation-gate Stage 1 generates the code coverage once (pcov, several times faster and lighter than xdebug for line coverage) in the layout Infection's --coverage option expects. Each stage-2 shard reuses it via --skip-initial-tests, so no shard runs the initial suite or needs a coverage driver, and mutates a disjoint slice of src/. The slice is computed automatically -- source files are greedily bin-packed by byte size into balanced buckets -- so there is no hand-maintained file list. Stage 3 folds the shards' summary JSON into the true project-wide Covered MSI (a weighted ratio, numerically identical to the single-machine gate) and enforces the 95% threshold once. The gate lives in stage 3 rather than per shard: a shard whose slice generates zero mutants exits non-zero under --min-covered-msi, which would be a spurious failure. Shards are pure workers that emit summaries and surface escaped mutants as GitHub annotations. pcov and xdebug produce identical per-line covering-test attribution for this suite, so kill/escape outcomes are unchanged. Point branch protection at the "Mutation gate (aggregate MSI)" check. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/infection-aggregate-msi.php | 101 ++++++++++++++++++++ .github/workflows/ci-core.yml | 101 +++++++++++++++++--- Makefile | 88 ++++++++++++++++- 3 files changed, 273 insertions(+), 17 deletions(-) create mode 100644 .github/scripts/infection-aggregate-msi.php diff --git a/.github/scripts/infection-aggregate-msi.php b/.github/scripts/infection-aggregate-msi.php new file mode 100644 index 00000000..e06c72b7 --- /dev/null +++ b/.github/scripts/infection-aggregate-msi.php @@ -0,0 +1,101 @@ += threshold, 1 otherwise (or if no + * summary files are found -- a missing shard must never look like a pass). + */ + +$min = (float) ($argv[1] ?? getenv('MIN_COVERED_MSI') ?: '95'); +$glob = $argv[2] ?? 'var/infection-summary-*.json'; + +$files = glob($glob) ?: []; + +if ($files === []) { + fwrite(STDERR, "FAIL: no shard summaries matched \"{$glob}\" -- did every shard run?\n"); + exit(1); +} + +$killed = 0; // killed by tests + static analysis +$errored = 0; // mutant caused a fatal error -> counts as killed +$timedOut = 0; // mutant hung past the timeout -> counts as killed +$escaped = 0; // covered but survived -> the mutants that lower MSI + +foreach ($files as $file) { + $decoded = json_decode((string) file_get_contents($file), true); + + if (!is_array($decoded) || !isset($decoded['stats'])) { + fwrite(STDERR, "FAIL: {$file} is not a valid Infection summary JSON\n"); + exit(1); + } + + $stats = $decoded['stats']; + $killed += $stats['killedCount']; + $errored += $stats['errorCount']; + $timedOut += $stats['timeOutCount']; + $escaped += $stats['escapedCount']; +} + +$numerator = $killed + $errored + $timedOut; +$covered = $numerator + $escaped; + +// A run with zero covered mutants (e.g. every shard's slice was fully +// ignored) has nothing to measure; treat it as a vacuous pass rather than a +// divide-by-zero. In practice the packer keeps every shard non-trivial. +$msi = $covered > 0 ? 100.0 * $numerator / $covered : 100.0; + +$shards = count($files); +$line = sprintf( + 'Aggregate Covered MSI: %.2f%% (%d/%d covered mutants killed across %d shards; %d escaped)', + $msi, + $numerator, + $covered, + $shards, + $escaped, +); + +echo $line, "\n"; + +// Surface the headline on the GitHub Actions run summary when available. +$summaryPath = getenv('GITHUB_STEP_SUMMARY'); +if (is_string($summaryPath) && $summaryPath !== '') { + $status = $msi >= $min ? '✅ PASS' : '❌ FAIL'; + file_put_contents( + $summaryPath, + sprintf("### Mutation gate: %s\n\n%s (gate: %.2f%%)\n", $status, $line, $min), + FILE_APPEND, + ); +} + +if ($msi < $min) { + fwrite(STDERR, sprintf("FAIL: Covered MSI %.2f%% is below the %.2f%% gate\n", $msi, $min)); + exit(1); +} + +echo sprintf("PASS: Covered MSI %.2f%% meets the %.2f%% gate\n", $msi, $min); +exit(0); diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 9aef9380..ba4c5515 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -113,36 +113,111 @@ jobs: - name: Run xphp check self-test run: make test/check - infection: - name: Mutation testing + # Mutation testing is split into three stages so the slow part -- running + # every mutant -- fans out across many machines instead of one: + # + # mutation-coverage -> mutation (x10, matrix) -> mutation-gate + # + # Stage 1 generates the code coverage once (pcov). Each stage-2 shard reuses + # it (no per-shard initial test run, no coverage driver) and mutates a + # disjoint, auto-balanced slice of src/. Stage 3 folds the shards' summaries + # into the true project-wide Covered MSI and enforces the 95% gate once. + # `mutation-gate` is the single required check to protect the branch with. + mutation-coverage: + name: Mutation coverage (generate once) runs-on: ubuntu-latest needs: phpunit steps: - uses: actions/checkout@v4 - - name: Setup PHP 8.4 (with Xdebug for coverage) + - name: Setup PHP 8.4 (with pcov for fast coverage) uses: shivammathur/setup-php@v2 with: php-version: '8.4' extensions: dom, json, mbstring, tokenizer - coverage: xdebug + coverage: pcov tools: composer:v2 - name: Install dependencies uses: ramsey/composer-install@v3 - - name: Run Infection - # --min-covered-msi=95 fails CI if the mutation score drops below - # the gate. Threads=max parallelises mutant runs. - run: make test/mutation + - name: Generate coverage for Infection + run: make test/mutation/coverage - - name: Upload Infection report + - name: Upload coverage for shards + uses: actions/upload-artifact@v4 + with: + name: infection-coverage + path: var/infection-coverage + retention-days: 1 + + mutation: + name: Mutation shard ${{ matrix.shard }} + runs-on: ubuntu-latest + needs: mutation-coverage + strategy: + # One machine per shard. Bump this list and SHARD_TOTAL together to add + # horizontal capacity; the slice packer rebalances automatically. + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + env: + SHARD_TOTAL: 10 + steps: + - uses: actions/checkout@v4 + + - name: Setup PHP 8.4 + # No coverage driver: shards reuse the coverage artifact and skip the + # initial test run, so mutant runs need only a plain PHP. + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + extensions: dom, json, mbstring, tokenizer + coverage: none + tools: composer:v2 + + - name: Install dependencies + uses: ramsey/composer-install@v3 + + - name: Download shared coverage + uses: actions/download-artifact@v4 + with: + name: infection-coverage + path: var/infection-coverage + + - name: Run mutation shard ${{ matrix.shard }} + run: make test/mutation/shard SHARD_TOTAL=${{ env.SHARD_TOTAL }} SHARD_INDEX=${{ matrix.shard }} + + - name: Upload shard summary if: always() uses: actions/upload-artifact@v4 with: - name: infection-report - path: | - var/infection.log - var/infection.html + name: infection-summary-${{ matrix.shard }} + path: var/infection-summary-${{ matrix.shard }}.json if-no-files-found: ignore retention-days: 14 + + mutation-gate: + name: Mutation gate (aggregate MSI) + runs-on: ubuntu-latest + needs: mutation + steps: + - uses: actions/checkout@v4 + + - name: Setup PHP 8.4 + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + extensions: dom, json, mbstring, tokenizer + coverage: none + tools: composer:v2 + + - name: Download all shard summaries + uses: actions/download-artifact@v4 + with: + pattern: infection-summary-* + path: var + merge-multiple: true + + - name: Enforce aggregate Covered MSI >= 95% + run: make test/mutation/gate diff --git a/Makefile b/Makefile index 73a969f5..889bca3f 100644 --- a/Makefile +++ b/Makefile @@ -36,13 +36,93 @@ lint/phpstan: php vendor/bin/phpstan analyse --memory-limit=2G --no-progress .PHONY: test/mutation -# Gate at 95% (current is 100%): keeps a small headroom so a single -# new mutation can land in a follow-up commit and still pass while -# the test that kills it is being written. Raise to 100% once the -# repo is stable enough that no new test gaps are expected. +# Single-machine full run (local dev / one-shot). Gate at 95% (current is +# 100%): keeps a small headroom so a single new mutation can land in a +# follow-up commit and still pass while the test that kills it is being +# written. Raise to 100% once the repo is stable enough that no new test +# gaps are expected. CI does NOT use this target -- it splits the work +# across parallel shards via the three targets below. test/mutation: php -d memory_limit=-1 vendor/bin/infection --show-mutations=max --threads=max --min-covered-msi=95 +# --------------------------------------------------------------------------- +# Sharded mutation testing (horizontal scaling for CI) +# +# The single run above is decomposed into three stages so CI can fan the +# mutant analysis out across many machines: +# +# 1. test/mutation/coverage (run once) -- generate the code coverage + +# junit that every shard reuses, so no shard pays for an initial test +# run or needs a coverage driver. +# 2. test/mutation/shard (run N x) -- each shard mutates a disjoint, +# auto-balanced slice of src/ against the shared coverage. +# 3. test/mutation/gate (run once) -- aggregate the shards' summary +# JSON into the true project-wide Covered MSI and enforce the gate. +# +# Because Covered MSI is a ratio, aggregating numerators/denominators makes +# the distributed gate numerically identical to the single-machine one. +# --------------------------------------------------------------------------- + +# Where the reusable coverage lives; shared by the coverage + shard targets. +INFECTION_COVERAGE_DIR ?= var/infection-coverage + +.PHONY: test/mutation/coverage +# Stage 1: generate the coverage Infection reuses across shards, in the +# layout its `--coverage` option expects (a directory containing +# coverage-xml/ and junit.xml). Runs the FULL default suite -- the same set +# `test/mutation` mutates against -- so every source line an integration or +# @group phpstan test touches in-process is recorded. pcov is used instead of +# xdebug: for line coverage (all Infection needs) it is several times faster +# and far lighter on memory, which is the point of moving this off the +# critical path. +test/mutation/coverage: + php -d memory_limit=-1 -d pcov.enabled=1 vendor/bin/phpunit \ + --coverage-filter src \ + --coverage-xml $(INFECTION_COVERAGE_DIR)/coverage-xml \ + --log-junit $(INFECTION_COVERAGE_DIR)/junit.xml + +.PHONY: test/mutation/shard +# Stage 2: run one shard. SHARD_INDEX is 0-based in [0, SHARD_TOTAL). +# +# The slice is computed automatically -- there is no hand-maintained file +# list. Source files are greedily bin-packed by byte size, largest first, +# into SHARD_TOTAL balanced buckets (longest-processing-time scheduling); +# this shard runs bucket SHARD_INDEX. Byte size is a cheap proxy for mutant +# count, so buckets finish in roughly equal wall time, which is what caps the +# fan-out. Adding or removing source files just reshuffles the buckets. +# +# Reuses stage 1's coverage (--skip-initial-tests), so no coverage driver is +# needed here and no initial suite runs. The shard is a pure worker: it sets +# NO --min-covered-msi (an all-ignored slice would spuriously fail that) and +# writes a summary JSON for stage 3 to aggregate. Escaped mutants are surfaced +# inline as GitHub annotations. +SHARD_TOTAL ?= 1 +SHARD_INDEX ?= 0 +test/mutation/shard: + @files=$$(find src -name '*.php' -printf '%s %p\n' | sort -rn | \ + awk -v total=$(SHARD_TOTAL) -v idx=$(SHARD_INDEX) '\ + { min = 0; for (b = 1; b < total; b++) if (load[b] < load[min]) min = b; \ + load[min] += $$1; if (min == idx) print $$2 }' | paste -sd,); \ + if [ -z "$$files" ]; then \ + echo "shard $(SHARD_INDEX)/$(SHARD_TOTAL): empty slice, nothing to mutate"; \ + exit 0; \ + fi; \ + echo "shard $(SHARD_INDEX)/$(SHARD_TOTAL) mutating: $$files"; \ + php -d memory_limit=-1 vendor/bin/infection \ + --coverage=$(INFECTION_COVERAGE_DIR) \ + --skip-initial-tests \ + --filter="$$files" \ + --threads=max \ + --logger-summary-json=var/infection-summary-$(SHARD_INDEX).json \ + --logger-github + +.PHONY: test/mutation/gate +# Stage 3: fold every shard's summary JSON into the project-wide Covered MSI +# and fail if it is below the gate. See the script header for the math. +MIN_COVERED_MSI ?= 95 +test/mutation/gate: + MIN_COVERED_MSI=$(MIN_COVERED_MSI) php .github/scripts/infection-aggregate-msi.php + .PHONY: test/check # End-to-end self-test of the `check` gate: runs the real bin/xphp binary # against the check fixtures and asserts the 0/1/2 exit contract plus that the From 31de2c9d08f0fb9c80e5d22dfd3c5a1e006c154a Mon Sep 17 00:00:00 2001 From: Matheus Martins Date: Tue, 4 Aug 2026 10:37:35 +0000 Subject: [PATCH 2/5] ci(mutation): harden aggregate gate against silent under-counting Address two review findings on the MSI aggregator, both fail-open risks on a gate: - Guard each stat key (killedCount/errorCount/timeOutCount/escapedCount) before reading it. A missing key would emit only an E_WARNING and coerce to 0; a silent 0 for escapedCount shrinks the denominator and inflates the aggregate MSI, letting the gate pass with real escapes uncounted. An unrecognised schema now fails loudly. - Assert the number of summaries found equals the number of shards. A shard that exits 0 without a summary (empty slice, a dropped if-no-files-found artifact) would otherwise let the gate score over a subset of src/ and pass. The gate job passes EXPECTED_SHARDS, hoisted to a top-level SHARD_TOTAL so the shard matrix and the gate share one source of truth. Values are passed as CLI arguments rather than env vars so the check is exercised identically in local runs and CI. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/infection-aggregate-msi.php | 47 ++++++++++++++++++--- .github/workflows/ci-core.yml | 22 +++++++--- Makefile | 5 ++- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/.github/scripts/infection-aggregate-msi.php b/.github/scripts/infection-aggregate-msi.php index e06c72b7..45c2e7a7 100644 --- a/.github/scripts/infection-aggregate-msi.php +++ b/.github/scripts/infection-aggregate-msi.php @@ -20,19 +20,30 @@ * numerically identical to the single-machine `--min-covered-msi` gate. * * Usage: - * php .github/scripts/infection-aggregate-msi.php [minCoveredMsi] [glob] + * php .github/scripts/infection-aggregate-msi.php [minCoveredMsi] [glob] [expectedShards] * - * minCoveredMsi Gate threshold as a percentage (default 95, or the - * MIN_COVERED_MSI env var if set). - * glob Glob for the summary files - * (default "var/infection-summary-*.json"). + * minCoveredMsi Gate threshold as a percentage (default 95, or the + * MIN_COVERED_MSI env var if set). + * glob Glob for the summary files + * (default "var/infection-summary-*.json"). + * expectedShards Assert exactly this many summaries were found; 0/absent + * disables the check (or the EXPECTED_SHARDS env var). + * + * The EXPECTED_SHARDS env var, when set to a positive integer, asserts that + * exactly that many summaries were found. The gate reports over whatever + * shards uploaded a summary; if one shard silently produced none (empty + * slice, a dropped `if-no-files-found: ignore` artifact, an Infection that + * wrote no log), the aggregate would cover only a subset yet still pass. This + * turns that missing data into an explicit failure. * * Exit code: 0 if aggregate Covered MSI >= threshold, 1 otherwise (or if no - * summary files are found -- a missing shard must never look like a pass). + * summary files are found, or fewer than EXPECTED_SHARDS -- missing data must + * never look like a pass). */ $min = (float) ($argv[1] ?? getenv('MIN_COVERED_MSI') ?: '95'); $glob = $argv[2] ?? 'var/infection-summary-*.json'; +$expected = (int) (($argv[3] ?? '') !== '' ? $argv[3] : (getenv('EXPECTED_SHARDS') ?: '0')); $files = glob($glob) ?: []; @@ -41,6 +52,17 @@ exit(1); } +echo sprintf("Found %d shard %s%s\n", count($files), count($files) === 1 ? 'summary' : 'summaries', $expected > 0 ? " (expected {$expected})" : ''); + +if ($expected > 0 && count($files) < $expected) { + fwrite(STDERR, sprintf( + "FAIL: only %d of %d expected shard summaries present -- a shard was skipped or its upload was lost; refusing to gate on partial data\n", + count($files), + $expected, + )); + exit(1); +} + $killed = 0; // killed by tests + static analysis $errored = 0; // mutant caused a fatal error -> counts as killed $timedOut = 0; // mutant hung past the timeout -> counts as killed @@ -55,6 +77,19 @@ } $stats = $decoded['stats']; + + // Guard every key we read. A missing key would emit only an E_WARNING and + // coerce to 0 -- and a silent 0 for escapedCount shrinks the denominator, + // inflating the aggregate MSI so the gate passes with real escapes + // uncounted. On a gate, an unrecognised schema must fail loudly, not + // fail open, so we bail rather than trust a partial summary. + foreach (['killedCount', 'errorCount', 'timeOutCount', 'escapedCount'] as $key) { + if (!array_key_exists($key, $stats)) { + fwrite(STDERR, "FAIL: {$file} is missing stats.{$key} -- Infection schema mismatch?\n"); + exit(1); + } + } + $killed += $stats['killedCount']; $errored += $stats['errorCount']; $timedOut += $stats['timeOutCount']; diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index ba4c5515..896b6c21 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -13,6 +13,13 @@ concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Number of mutation shards. MUST equal the length of the `mutation` job's + # matrix list below (GitHub can't derive a matrix from an env var). The gate + # asserts it received exactly this many summaries, so if the two ever drift + # the gate fails loudly instead of scoring over a subset of src/. + SHARD_TOTAL: 10 + jobs: phpunit: # 8.4 is the supported/default runtime and runs the full suite minus the @@ -125,7 +132,7 @@ jobs: # `mutation-gate` is the single required check to protect the branch with. mutation-coverage: name: Mutation coverage (generate once) - runs-on: ubuntu-latest + runs-on: ubuntu-latest-64gb needs: phpunit steps: - uses: actions/checkout@v4 @@ -153,16 +160,15 @@ jobs: mutation: name: Mutation shard ${{ matrix.shard }} - runs-on: ubuntu-latest + runs-on: ubuntu-latest-64gb needs: mutation-coverage strategy: - # One machine per shard. Bump this list and SHARD_TOTAL together to add - # horizontal capacity; the slice packer rebalances automatically. + # One machine per shard. Bump this list and the top-level SHARD_TOTAL + # together to add capacity; the slice packer rebalances automatically, + # and the gate fails if the two ever disagree. fail-fast: false matrix: shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - env: - SHARD_TOTAL: 10 steps: - uses: actions/checkout@v4 @@ -220,4 +226,6 @@ jobs: merge-multiple: true - name: Enforce aggregate Covered MSI >= 95% - run: make test/mutation/gate + # EXPECTED_SHARDS asserts every shard's summary made it here; a missing + # one fails the gate rather than scoring over a subset of src/. + run: make test/mutation/gate EXPECTED_SHARDS=${{ env.SHARD_TOTAL }} diff --git a/Makefile b/Makefile index 889bca3f..774a3fb4 100644 --- a/Makefile +++ b/Makefile @@ -119,9 +119,12 @@ test/mutation/shard: .PHONY: test/mutation/gate # Stage 3: fold every shard's summary JSON into the project-wide Covered MSI # and fail if it is below the gate. See the script header for the math. +# EXPECTED_SHARDS (optional) makes the gate fail if fewer summaries than +# shards arrived, so a silently-skipped shard can't pass on partial data. MIN_COVERED_MSI ?= 95 +EXPECTED_SHARDS ?= test/mutation/gate: - MIN_COVERED_MSI=$(MIN_COVERED_MSI) php .github/scripts/infection-aggregate-msi.php + php .github/scripts/infection-aggregate-msi.php $(MIN_COVERED_MSI) 'var/infection-summary-*.json' $(EXPECTED_SHARDS) .PHONY: test/check # End-to-end self-test of the `check` gate: runs the real bin/xphp binary From 8b282c253d87e2ac970c4f5e77769871feb77366 Mon Sep 17 00:00:00 2001 From: Matheus Martins Date: Tue, 4 Aug 2026 14:28:10 +0000 Subject: [PATCH 3/5] ci(mutation): compute shard slices in PHP, not GNU shell Replace the `find -printf | sort | awk | paste` pipeline in test/mutation/shard with a small PHP script. `find -printf` is a GNU extension absent on macOS/BSD, so the old recipe only ran on Linux; the PHP version runs anywhere PHP does, which is every environment this repo already targets. infection-shard-files.php does the same size-balanced LPT bin-packing and prints one shard's comma-separated slice. Because it sorts the files before packing (largest first, ties broken by path), the partition is now fully deterministic across runners -- the shell version's packing depended on find's filesystem iteration order for equal-sized files, so two shard jobs could in principle disagree on the split. Verified the union of all shards is byte-identical to `find src -name '*.php'`, disjoint and exhaustive. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/infection-shard-files.php | 89 +++++++++++++++++++++++ Makefile | 15 ++-- 2 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 .github/scripts/infection-shard-files.php diff --git a/.github/scripts/infection-shard-files.php b/.github/scripts/infection-shard-files.php new file mode 100644 index 00000000..8fae2b60 --- /dev/null +++ b/.github/scripts/infection-shard-files.php @@ -0,0 +1,89 @@ + [srcDir] + * + * shardTotal Number of shards (>= 1). + * shardIndex This shard, 0-based in [0, shardTotal). + * srcDir Root to scan for *.php (default "src"). + * + * Prints this shard's files, comma-separated, with forward slashes and no + * trailing newline. An empty slice prints nothing (exit 0); the caller treats + * that as "nothing to mutate". + */ + +$total = (int) ($argv[1] ?? 0); +$index = (int) ($argv[2] ?? -1); +$srcDir = $argv[3] ?? 'src'; + +if ($total < 1 || $index < 0 || $index >= $total) { + fwrite(STDERR, "usage: infection-shard-files.php [srcDir]\n"); + fwrite(STDERR, " shardTotal >= 1, 0 <= shardIndex < shardTotal\n"); + exit(2); +} + +if (!is_dir($srcDir)) { + fwrite(STDERR, "FAIL: source directory \"{$srcDir}\" does not exist\n"); + exit(2); +} + +// Collect every *.php file under srcDir with its byte size. +$files = []; +$iterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($srcDir, FilesystemIterator::SKIP_DOTS), +); + +foreach ($iterator as $file) { + if ($file->isFile() && $file->getExtension() === 'php') { + // Normalise to forward slashes so --filter paths match on every OS. + $path = str_replace(DIRECTORY_SEPARATOR, '/', $file->getPathname()); + $files[] = ['path' => $path, 'size' => $file->getSize()]; + } +} + +// Largest first, ties broken by path -> identical order on every runner. +usort( + $files, + static fn (array $a, array $b): int => ($b['size'] <=> $a['size']) ?: strcmp($a['path'], $b['path']), +); + +// LPT bin-packing: each file lands in the currently-lightest bucket. +$load = array_fill(0, $total, 0); +$buckets = array_fill(0, $total, []); + +foreach ($files as $file) { + $lightest = 0; + for ($bucket = 1; $bucket < $total; ++$bucket) { + if ($load[$bucket] < $load[$lightest]) { + $lightest = $bucket; + } + } + + $load[$lightest] += $file['size']; + $buckets[$lightest][] = $file['path']; +} + +echo implode(',', $buckets[$index]); diff --git a/Makefile b/Makefile index 774a3fb4..77309b50 100644 --- a/Makefile +++ b/Makefile @@ -85,11 +85,11 @@ test/mutation/coverage: # Stage 2: run one shard. SHARD_INDEX is 0-based in [0, SHARD_TOTAL). # # The slice is computed automatically -- there is no hand-maintained file -# list. Source files are greedily bin-packed by byte size, largest first, -# into SHARD_TOTAL balanced buckets (longest-processing-time scheduling); -# this shard runs bucket SHARD_INDEX. Byte size is a cheap proxy for mutant -# count, so buckets finish in roughly equal wall time, which is what caps the -# fan-out. Adding or removing source files just reshuffles the buckets. +# list. infection-shard-files.php greedily bin-packs the source files by byte +# size (a cheap proxy for mutant count) into SHARD_TOTAL balanced buckets and +# prints bucket SHARD_INDEX; adding or removing source files just reshuffles +# the buckets. The packing is done in PHP rather than shell so it runs on any +# OS (no GNU `find -printf`) and is deterministic across runners. # # Reuses stage 1's coverage (--skip-initial-tests), so no coverage driver is # needed here and no initial suite runs. The shard is a pure worker: it sets @@ -99,10 +99,7 @@ test/mutation/coverage: SHARD_TOTAL ?= 1 SHARD_INDEX ?= 0 test/mutation/shard: - @files=$$(find src -name '*.php' -printf '%s %p\n' | sort -rn | \ - awk -v total=$(SHARD_TOTAL) -v idx=$(SHARD_INDEX) '\ - { min = 0; for (b = 1; b < total; b++) if (load[b] < load[min]) min = b; \ - load[min] += $$1; if (min == idx) print $$2 }' | paste -sd,); \ + @files=$$(php .github/scripts/infection-shard-files.php $(SHARD_TOTAL) $(SHARD_INDEX)); \ if [ -z "$$files" ]; then \ echo "shard $(SHARD_INDEX)/$(SHARD_TOTAL): empty slice, nothing to mutate"; \ exit 0; \ From 74dbe4b8be453ecb8b3478cd6a94d272c7f9c0ba Mon Sep 17 00:00:00 2001 From: Matheus Martins Date: Tue, 4 Aug 2026 23:19:39 +0000 Subject: [PATCH 4/5] ci(mutation): disable pcov on shards, they don't collect coverage Shards reuse the coverage artifact (--coverage --skip-initial-tests), so they never generate coverage and need no driver. With the pcov extension still loaded on the runner, Infection prints "running with PCOV enabled" and, worse, pcov instruments every mutant's PHPUnit process for line coverage that is thrown away -- pure overhead across ~1000 mutants on the heaviest shard. Disable it at the extension level (`:pcov`) rather than via `-d pcov.enabled=0`: mutant workers are spawned as a fresh PHP that reads php.ini, so a -d flag on the Infection command would only reach the coordinator, not the workers where the overhead is. The coverage job keeps pcov -- that is the one place it is actually needed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci-core.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 896b6c21..6ec72419 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -173,12 +173,18 @@ jobs: - uses: actions/checkout@v4 - name: Setup PHP 8.4 - # No coverage driver: shards reuse the coverage artifact and skip the - # initial test run, so mutant runs need only a plain PHP. + # Shards reuse the coverage artifact and skip the initial test run, so + # they need no coverage driver at all. `:pcov` disables the pcov + # extension (present on the runner image): with it loaded Infection + # both prints a "running with PCOV enabled" notice and instruments + # every mutant's PHPUnit process for coverage we never collect. The + # mutant workers launch a fresh PHP that reads php.ini, so this must be + # disabled at the extension level -- a `-d pcov.enabled=0` on the + # Infection command would only reach the coordinator, not the workers. uses: shivammathur/setup-php@v2 with: php-version: '8.4' - extensions: dom, json, mbstring, tokenizer + extensions: dom, json, mbstring, tokenizer, :pcov coverage: none tools: composer:v2 From 2c72b0cf76b918faf440d8631c4b93e31fe698e8 Mon Sep 17 00:00:00 2001 From: Matheus Martins Date: Tue, 4 Aug 2026 23:57:24 +0000 Subject: [PATCH 5/5] ci(mutation): cap mutant worker memory to stop runner OOM Root cause of shard 0 dying mid-run ("Terminated" / "operation canceled / runner shutdown", reproducibly at the same point): XphpSourceParser has mutants that turn a bounded loop unbounded. With no memory_limit, such a mutant allocates until it exhausts the runner's RAM and kills the runner agent -- which GitHub surfaces as a cancellation, not a clean failure. Moving to a 64GB runner sped the other shards up nicely but could not fix this: an unbounded leak exhausts any fixed amount of RAM, just later. Infection normally caps each worker at 2x the initial run's memory, but it measures that during the initial run, which shards skip (--skip-initial-tests) -- so no cap is applied. Set it explicitly via setup-php ini-values; mutant workers launch a fresh PHP that reads php.ini (Infection passes them no -d), so that is the only place it takes effect. The coordinator keeps -d memory_limit=-1 so it can still hold all ~1000 mutants; only the workers are bounded. 1G is ~5x a normal worker yet far below the runner RAM even with every runaway mutant running at once, so legit mutants are untouched and MSI is unchanged -- these mutants were already killed, the kill just moves from a 120s timeout to a clean memory fatal. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci-core.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 6ec72419..18dc601f 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -132,7 +132,7 @@ jobs: # `mutation-gate` is the single required check to protect the branch with. mutation-coverage: name: Mutation coverage (generate once) - runs-on: ubuntu-latest-64gb + runs-on: ubuntu-latest needs: phpunit steps: - uses: actions/checkout@v4 @@ -160,7 +160,7 @@ jobs: mutation: name: Mutation shard ${{ matrix.shard }} - runs-on: ubuntu-latest-64gb + runs-on: ubuntu-latest needs: mutation-coverage strategy: # One machine per shard. Bump this list and the top-level SHARD_TOTAL @@ -187,6 +187,20 @@ jobs: extensions: dom, json, mbstring, tokenizer, :pcov coverage: none tools: composer:v2 + # Bound each mutant worker's memory. Infection normally caps workers + # at 2x the initial test run's usage, but that measurement only + # happens during the initial run -- which shards skip + # (--skip-initial-tests) -- so no cap is applied and workers inherit + # an unlimited memory_limit. XphpSourceParser has mutants that turn a + # bounded loop unbounded; with no cap one allocates until it exhausts + # the runner's RAM and takes the runner agent down (the job reports + # "canceled / runner shutdown"). A bigger runner only delays this. A + # normal worker uses ~200M, so 1G fatals a runaway cleanly as one + # killed mutant while leaving legit mutants untouched (MSI unchanged: + # these were already being killed, just via the 120s timeout). + # Workers read php.ini directly, so it must be set here, not via a -d + # flag on the Infection command (that reaches only the coordinator). + ini-values: memory_limit=1G - name: Install dependencies uses: ramsey/composer-install@v3