diff --git a/.github/scripts/infection-aggregate-msi.php b/.github/scripts/infection-aggregate-msi.php new file mode 100644 index 00000000..45c2e7a7 --- /dev/null +++ b/.github/scripts/infection-aggregate-msi.php @@ -0,0 +1,136 @@ += threshold, 1 otherwise (or if no + * summary files are found, or fewer than EXPECTED_SHARDS -- missing data must + * never look like a pass). + */ + +$min = (float) ($argv[1] ?? getenv('MIN_COVERED_MSI') ?: '95'); +$glob = $argv[2] ?? 'var/infection-summary-*.json'; +$expected = (int) (($argv[3] ?? '') !== '' ? $argv[3] : (getenv('EXPECTED_SHARDS') ?: '0')); + +$files = glob($glob) ?: []; + +if ($files === []) { + fwrite(STDERR, "FAIL: no shard summaries matched \"{$glob}\" -- did every shard run?\n"); + exit(1); +} + +echo sprintf("Found %d shard %s%s\n", count($files), count($files) === 1 ? 'summary' : 'summaries', $expected > 0 ? " (expected {$expected})" : ''); + +if ($expected > 0 && count($files) < $expected) { + fwrite(STDERR, sprintf( + "FAIL: only %d of %d expected shard summaries present -- a shard was skipped or its upload was lost; refusing to gate on partial data\n", + count($files), + $expected, + )); + exit(1); +} + +$killed = 0; // killed by tests + static analysis +$errored = 0; // mutant caused a fatal error -> counts as killed +$timedOut = 0; // mutant hung past the timeout -> counts as killed +$escaped = 0; // covered but survived -> the mutants that lower MSI + +foreach ($files as $file) { + $decoded = json_decode((string) file_get_contents($file), true); + + if (!is_array($decoded) || !isset($decoded['stats'])) { + fwrite(STDERR, "FAIL: {$file} is not a valid Infection summary JSON\n"); + exit(1); + } + + $stats = $decoded['stats']; + + // Guard every key we read. A missing key would emit only an E_WARNING and + // coerce to 0 -- and a silent 0 for escapedCount shrinks the denominator, + // inflating the aggregate MSI so the gate passes with real escapes + // uncounted. On a gate, an unrecognised schema must fail loudly, not + // fail open, so we bail rather than trust a partial summary. + foreach (['killedCount', 'errorCount', 'timeOutCount', 'escapedCount'] as $key) { + if (!array_key_exists($key, $stats)) { + fwrite(STDERR, "FAIL: {$file} is missing stats.{$key} -- Infection schema mismatch?\n"); + exit(1); + } + } + + $killed += $stats['killedCount']; + $errored += $stats['errorCount']; + $timedOut += $stats['timeOutCount']; + $escaped += $stats['escapedCount']; +} + +$numerator = $killed + $errored + $timedOut; +$covered = $numerator + $escaped; + +// A run with zero covered mutants (e.g. every shard's slice was fully +// ignored) has nothing to measure; treat it as a vacuous pass rather than a +// divide-by-zero. In practice the packer keeps every shard non-trivial. +$msi = $covered > 0 ? 100.0 * $numerator / $covered : 100.0; + +$shards = count($files); +$line = sprintf( + 'Aggregate Covered MSI: %.2f%% (%d/%d covered mutants killed across %d shards; %d escaped)', + $msi, + $numerator, + $covered, + $shards, + $escaped, +); + +echo $line, "\n"; + +// Surface the headline on the GitHub Actions run summary when available. +$summaryPath = getenv('GITHUB_STEP_SUMMARY'); +if (is_string($summaryPath) && $summaryPath !== '') { + $status = $msi >= $min ? '✅ PASS' : '❌ FAIL'; + file_put_contents( + $summaryPath, + sprintf("### Mutation gate: %s\n\n%s (gate: %.2f%%)\n", $status, $line, $min), + FILE_APPEND, + ); +} + +if ($msi < $min) { + fwrite(STDERR, sprintf("FAIL: Covered MSI %.2f%% is below the %.2f%% gate\n", $msi, $min)); + exit(1); +} + +echo sprintf("PASS: Covered MSI %.2f%% meets the %.2f%% gate\n", $msi, $min); +exit(0); diff --git a/.github/scripts/infection-shard-files.php b/.github/scripts/infection-shard-files.php new file mode 100644 index 00000000..8fae2b60 --- /dev/null +++ b/.github/scripts/infection-shard-files.php @@ -0,0 +1,89 @@ + [srcDir] + * + * shardTotal Number of shards (>= 1). + * shardIndex This shard, 0-based in [0, shardTotal). + * srcDir Root to scan for *.php (default "src"). + * + * Prints this shard's files, comma-separated, with forward slashes and no + * trailing newline. An empty slice prints nothing (exit 0); the caller treats + * that as "nothing to mutate". + */ + +$total = (int) ($argv[1] ?? 0); +$index = (int) ($argv[2] ?? -1); +$srcDir = $argv[3] ?? 'src'; + +if ($total < 1 || $index < 0 || $index >= $total) { + fwrite(STDERR, "usage: infection-shard-files.php [srcDir]\n"); + fwrite(STDERR, " shardTotal >= 1, 0 <= shardIndex < shardTotal\n"); + exit(2); +} + +if (!is_dir($srcDir)) { + fwrite(STDERR, "FAIL: source directory \"{$srcDir}\" does not exist\n"); + exit(2); +} + +// Collect every *.php file under srcDir with its byte size. +$files = []; +$iterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($srcDir, FilesystemIterator::SKIP_DOTS), +); + +foreach ($iterator as $file) { + if ($file->isFile() && $file->getExtension() === 'php') { + // Normalise to forward slashes so --filter paths match on every OS. + $path = str_replace(DIRECTORY_SEPARATOR, '/', $file->getPathname()); + $files[] = ['path' => $path, 'size' => $file->getSize()]; + } +} + +// Largest first, ties broken by path -> identical order on every runner. +usort( + $files, + static fn (array $a, array $b): int => ($b['size'] <=> $a['size']) ?: strcmp($a['path'], $b['path']), +); + +// LPT bin-packing: each file lands in the currently-lightest bucket. +$load = array_fill(0, $total, 0); +$buckets = array_fill(0, $total, []); + +foreach ($files as $file) { + $lightest = 0; + for ($bucket = 1; $bucket < $total; ++$bucket) { + if ($load[$bucket] < $load[$lightest]) { + $lightest = $bucket; + } + } + + $load[$lightest] += $file['size']; + $buckets[$lightest][] = $file['path']; +} + +echo implode(',', $buckets[$index]); diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 9aef9380..18dc601f 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -13,6 +13,13 @@ concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Number of mutation shards. MUST equal the length of the `mutation` job's + # matrix list below (GitHub can't derive a matrix from an env var). The gate + # asserts it received exactly this many summaries, so if the two ever drift + # the gate fails loudly instead of scoring over a subset of src/. + SHARD_TOTAL: 10 + jobs: phpunit: # 8.4 is the supported/default runtime and runs the full suite minus the @@ -113,36 +120,132 @@ jobs: - name: Run xphp check self-test run: make test/check - infection: - name: Mutation testing + # Mutation testing is split into three stages so the slow part -- running + # every mutant -- fans out across many machines instead of one: + # + # mutation-coverage -> mutation (x10, matrix) -> mutation-gate + # + # Stage 1 generates the code coverage once (pcov). Each stage-2 shard reuses + # it (no per-shard initial test run, no coverage driver) and mutates a + # disjoint, auto-balanced slice of src/. Stage 3 folds the shards' summaries + # into the true project-wide Covered MSI and enforces the 95% gate once. + # `mutation-gate` is the single required check to protect the branch with. + mutation-coverage: + name: Mutation coverage (generate once) runs-on: ubuntu-latest needs: phpunit steps: - uses: actions/checkout@v4 - - name: Setup PHP 8.4 (with Xdebug for coverage) + - name: Setup PHP 8.4 (with pcov for fast coverage) uses: shivammathur/setup-php@v2 with: php-version: '8.4' extensions: dom, json, mbstring, tokenizer - coverage: xdebug + coverage: pcov + tools: composer:v2 + + - name: Install dependencies + uses: ramsey/composer-install@v3 + + - name: Generate coverage for Infection + run: make test/mutation/coverage + + - name: Upload coverage for shards + uses: actions/upload-artifact@v4 + with: + name: infection-coverage + path: var/infection-coverage + retention-days: 1 + + mutation: + name: Mutation shard ${{ matrix.shard }} + runs-on: ubuntu-latest + needs: mutation-coverage + strategy: + # One machine per shard. Bump this list and the top-level SHARD_TOTAL + # together to add capacity; the slice packer rebalances automatically, + # and the gate fails if the two ever disagree. + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - uses: actions/checkout@v4 + + - name: Setup PHP 8.4 + # Shards reuse the coverage artifact and skip the initial test run, so + # they need no coverage driver at all. `:pcov` disables the pcov + # extension (present on the runner image): with it loaded Infection + # both prints a "running with PCOV enabled" notice and instruments + # every mutant's PHPUnit process for coverage we never collect. The + # mutant workers launch a fresh PHP that reads php.ini, so this must be + # disabled at the extension level -- a `-d pcov.enabled=0` on the + # Infection command would only reach the coordinator, not the workers. + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + extensions: dom, json, mbstring, tokenizer, :pcov + coverage: none tools: composer:v2 + # Bound each mutant worker's memory. Infection normally caps workers + # at 2x the initial test run's usage, but that measurement only + # happens during the initial run -- which shards skip + # (--skip-initial-tests) -- so no cap is applied and workers inherit + # an unlimited memory_limit. XphpSourceParser has mutants that turn a + # bounded loop unbounded; with no cap one allocates until it exhausts + # the runner's RAM and takes the runner agent down (the job reports + # "canceled / runner shutdown"). A bigger runner only delays this. A + # normal worker uses ~200M, so 1G fatals a runaway cleanly as one + # killed mutant while leaving legit mutants untouched (MSI unchanged: + # these were already being killed, just via the 120s timeout). + # Workers read php.ini directly, so it must be set here, not via a -d + # flag on the Infection command (that reaches only the coordinator). + ini-values: memory_limit=1G - name: Install dependencies uses: ramsey/composer-install@v3 - - name: Run Infection - # --min-covered-msi=95 fails CI if the mutation score drops below - # the gate. Threads=max parallelises mutant runs. - run: make test/mutation + - name: Download shared coverage + uses: actions/download-artifact@v4 + with: + name: infection-coverage + path: var/infection-coverage + + - name: Run mutation shard ${{ matrix.shard }} + run: make test/mutation/shard SHARD_TOTAL=${{ env.SHARD_TOTAL }} SHARD_INDEX=${{ matrix.shard }} - - name: Upload Infection report + - name: Upload shard summary if: always() uses: actions/upload-artifact@v4 with: - name: infection-report - path: | - var/infection.log - var/infection.html + name: infection-summary-${{ matrix.shard }} + path: var/infection-summary-${{ matrix.shard }}.json if-no-files-found: ignore retention-days: 14 + + mutation-gate: + name: Mutation gate (aggregate MSI) + runs-on: ubuntu-latest + needs: mutation + steps: + - uses: actions/checkout@v4 + + - name: Setup PHP 8.4 + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + extensions: dom, json, mbstring, tokenizer + coverage: none + tools: composer:v2 + + - name: Download all shard summaries + uses: actions/download-artifact@v4 + with: + pattern: infection-summary-* + path: var + merge-multiple: true + + - name: Enforce aggregate Covered MSI >= 95% + # EXPECTED_SHARDS asserts every shard's summary made it here; a missing + # one fails the gate rather than scoring over a subset of src/. + run: make test/mutation/gate EXPECTED_SHARDS=${{ env.SHARD_TOTAL }} diff --git a/Makefile b/Makefile index 73a969f5..77309b50 100644 --- a/Makefile +++ b/Makefile @@ -36,13 +36,93 @@ lint/phpstan: php vendor/bin/phpstan analyse --memory-limit=2G --no-progress .PHONY: test/mutation -# Gate at 95% (current is 100%): keeps a small headroom so a single -# new mutation can land in a follow-up commit and still pass while -# the test that kills it is being written. Raise to 100% once the -# repo is stable enough that no new test gaps are expected. +# Single-machine full run (local dev / one-shot). Gate at 95% (current is +# 100%): keeps a small headroom so a single new mutation can land in a +# follow-up commit and still pass while the test that kills it is being +# written. Raise to 100% once the repo is stable enough that no new test +# gaps are expected. CI does NOT use this target -- it splits the work +# across parallel shards via the three targets below. test/mutation: php -d memory_limit=-1 vendor/bin/infection --show-mutations=max --threads=max --min-covered-msi=95 +# --------------------------------------------------------------------------- +# Sharded mutation testing (horizontal scaling for CI) +# +# The single run above is decomposed into three stages so CI can fan the +# mutant analysis out across many machines: +# +# 1. test/mutation/coverage (run once) -- generate the code coverage + +# junit that every shard reuses, so no shard pays for an initial test +# run or needs a coverage driver. +# 2. test/mutation/shard (run N x) -- each shard mutates a disjoint, +# auto-balanced slice of src/ against the shared coverage. +# 3. test/mutation/gate (run once) -- aggregate the shards' summary +# JSON into the true project-wide Covered MSI and enforce the gate. +# +# Because Covered MSI is a ratio, aggregating numerators/denominators makes +# the distributed gate numerically identical to the single-machine one. +# --------------------------------------------------------------------------- + +# Where the reusable coverage lives; shared by the coverage + shard targets. +INFECTION_COVERAGE_DIR ?= var/infection-coverage + +.PHONY: test/mutation/coverage +# Stage 1: generate the coverage Infection reuses across shards, in the +# layout its `--coverage` option expects (a directory containing +# coverage-xml/ and junit.xml). Runs the FULL default suite -- the same set +# `test/mutation` mutates against -- so every source line an integration or +# @group phpstan test touches in-process is recorded. pcov is used instead of +# xdebug: for line coverage (all Infection needs) it is several times faster +# and far lighter on memory, which is the point of moving this off the +# critical path. +test/mutation/coverage: + php -d memory_limit=-1 -d pcov.enabled=1 vendor/bin/phpunit \ + --coverage-filter src \ + --coverage-xml $(INFECTION_COVERAGE_DIR)/coverage-xml \ + --log-junit $(INFECTION_COVERAGE_DIR)/junit.xml + +.PHONY: test/mutation/shard +# Stage 2: run one shard. SHARD_INDEX is 0-based in [0, SHARD_TOTAL). +# +# The slice is computed automatically -- there is no hand-maintained file +# list. infection-shard-files.php greedily bin-packs the source files by byte +# size (a cheap proxy for mutant count) into SHARD_TOTAL balanced buckets and +# prints bucket SHARD_INDEX; adding or removing source files just reshuffles +# the buckets. The packing is done in PHP rather than shell so it runs on any +# OS (no GNU `find -printf`) and is deterministic across runners. +# +# Reuses stage 1's coverage (--skip-initial-tests), so no coverage driver is +# needed here and no initial suite runs. The shard is a pure worker: it sets +# NO --min-covered-msi (an all-ignored slice would spuriously fail that) and +# writes a summary JSON for stage 3 to aggregate. Escaped mutants are surfaced +# inline as GitHub annotations. +SHARD_TOTAL ?= 1 +SHARD_INDEX ?= 0 +test/mutation/shard: + @files=$$(php .github/scripts/infection-shard-files.php $(SHARD_TOTAL) $(SHARD_INDEX)); \ + if [ -z "$$files" ]; then \ + echo "shard $(SHARD_INDEX)/$(SHARD_TOTAL): empty slice, nothing to mutate"; \ + exit 0; \ + fi; \ + echo "shard $(SHARD_INDEX)/$(SHARD_TOTAL) mutating: $$files"; \ + php -d memory_limit=-1 vendor/bin/infection \ + --coverage=$(INFECTION_COVERAGE_DIR) \ + --skip-initial-tests \ + --filter="$$files" \ + --threads=max \ + --logger-summary-json=var/infection-summary-$(SHARD_INDEX).json \ + --logger-github + +.PHONY: test/mutation/gate +# Stage 3: fold every shard's summary JSON into the project-wide Covered MSI +# and fail if it is below the gate. See the script header for the math. +# EXPECTED_SHARDS (optional) makes the gate fail if fewer summaries than +# shards arrived, so a silently-skipped shard can't pass on partial data. +MIN_COVERED_MSI ?= 95 +EXPECTED_SHARDS ?= +test/mutation/gate: + php .github/scripts/infection-aggregate-msi.php $(MIN_COVERED_MSI) 'var/infection-summary-*.json' $(EXPECTED_SHARDS) + .PHONY: test/check # End-to-end self-test of the `check` gate: runs the real bin/xphp binary # against the check fixtures and asserts the 0/1/2 exit contract plus that the