From 66f149c5526fe7b7095bbe85a3e459440473daec Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Wed, 18 Nov 2026 03:44:00 +0100 Subject: [PATCH] Document a CI job and say why a run tested nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CI step that pipes strain's output, such as `strain run … | tee log`, passes whatever strain returns unless the step says `shell: bash`, because GitHub Actions only adds -o pipefail then. The README gains a CI section: - what the exit status means; - why `shell: bash` matters; - a GitHub Actions job that tests a pull request's changes with --since, appends summary.md to the job summary and uploads the report, both even when strain fails. tests/test_ci.py checks the documented job keeps those properties. With --since, a run without mutants passes, but its summary used to say only "No mutants were generated", which reads like a clean run. Now summary.md and index.html start with "Nothing was mutation tested" and say why: no source line changed since the ref, or none of the changed lines is executed and has something to mutate. The console summary says the same. The Reports section also lists index.html and summary.md. Fixes #45. Co-Authored-By: Claude Opus 5.5 --- README.md | 67 ++++++++++++++++++++++++++++++++++++++++ strain/html_report.py | 6 ++-- strain/report.py | 12 +++++-- strain/runner.py | 24 ++++++++++++-- tests/test_ci.py | 32 +++++++++++++++++++ tests/test_end_to_end.py | 43 ++++++++++++++++++++++++++ tests/test_report.py | 10 ++++-- 7 files changed, 182 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 545cdef..09c004b 100644 --- a/README.md +++ b/README.md @@ -220,10 +220,77 @@ To suppress mutants in source, add `// mutate-off` to the line. Written to `strain-report/` : +- `index.html`: a browsable report, with each source annotated +- `summary.md`: the score, the files and the survivors, for a CI job summary or a pull request comment - `report.json`: full results, in the mutation-testing-elements schema - `events.jsonl`: every event of the run - compiler-style warnings on the console (`file:line:col: warning: survived [ror]`), clickable in editors +## CI + +strain's exit status is what a CI job should fail on: + +- **Non-zero when nothing was tested:** without `--since`, strain exits non-zero when no mutant was generated, because + a run that mutated nothing tested nothing. +- **Non-zero below `--threshold`.** +- **`--since` with nothing to mutate passes:** a change that touches no line the tests run has nothing to test. The + summary then starts with **Nothing was mutation tested** and gives the reason, so the result can't be mistaken for a + clean one. + +**Pipes hide the exit status unless the step says `shell: bash`.** Without a `shell:` key, GitHub Actions runs +`bash -e`, without `pipefail`, so `strain run … | tee strain.log` returns the status of `tee` and the step passes +whatever strain found. With `shell: bash`, GitHub adds `-o pipefail`. + +This job tests the lines a pull request changes, puts the summary on the run's page and keeps the full report: + +```yaml +name: Mutation testing +on: pull_request + +jobs: + strain: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 # --since needs the base branch + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install + run: | + sudo apt-get update && sudo apt-get install -y ninja-build cppcheck + pip install "git+https://github.com/cpp-best-practices/mutant_strain" + - name: Configure a build for strain + run: | + cmake -S . -B build-strain -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_CXX_FLAGS="--coverage -fsanitize=address,undefined -fno-omit-frame-pointer -D_GLIBCXX_ASSERTIONS -ftrivial-auto-var-init=pattern" \ + -DCMAKE_EXE_LINKER_FLAGS="--coverage -fsanitize=address,undefined" + cmake --build build-strain + - name: Mutation test + shell: bash # adds -o pipefail, so the step fails when strain does, even through tee + run: | + strain run --build-dir build-strain --ui log --since "origin/${{ github.base_ref }}" --threshold 80 \ + | tee strain.log + - name: Job summary + if: always() + shell: bash + run: | + if [ -f strain-report/summary.md ]; then cat strain-report/summary.md >> "$GITHUB_STEP_SUMMARY"; fi + - uses: actions/upload-artifact@v4 + if: always() + with: + name: strain-report + path: | + strain-report/ + strain.log +``` + +To test everything instead, on a schedule or on pushes to `main`, drop `--since`: the job then also fails when no +mutant was generated. + ## Pluggable backends | Part | Options | diff --git a/strain/html_report.py b/strain/html_report.py index 892b445..c36c904 100644 --- a/strain/html_report.py +++ b/strain/html_report.py @@ -4,7 +4,7 @@ from html import escape from .console import HIGH_SCORE, LOW_SCORE -from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint +from .report import NO_MUTANTS, TAB_WIDTH, FileScore, file_scores, format_score, hint from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score KILLED_CLASS = "killed" @@ -145,7 +145,7 @@ def _counts(combined: Sequence[CombinedResult]) -> str: return ", ".join(f"{escape(state)} {states.count(state)}" for state in MutantState if state in states) -def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> str: +def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str], no_mutants: str = NO_MUTANTS) -> str: score = mutation_score(entry.state for entry in combined) body = ( [ @@ -155,7 +155,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) *_file_sections(combined, sources), ] if combined - else ["

No mutants were generated.

"] + else [f"

Nothing was mutation tested. {escape(no_mutants)}

"] ) page = [ "", diff --git a/strain/report.py b/strain/report.py index 89c735b..8ca4420 100644 --- a/strain/report.py +++ b/strain/report.py @@ -199,11 +199,17 @@ def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]: ] -def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None) -> str: - """A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment.""" +NO_MUTANTS = "No mutants were generated." + + +def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None, no_mutants: str = NO_MUTANTS) -> str: + """A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment. + + `no_mutants` says why there is nothing to show, so an empty run doesn't read like a clean one. + """ heading = ["## strain mutation testing", ""] if not combined: - return "\n".join([*heading, "No mutants were generated."]) + "\n" + return "\n".join([*heading, f"**Nothing was mutation tested.** {no_mutants}"]) + "\n" score = mutation_score(entry.state for entry in combined) lines = [ *heading, diff --git a/strain/runner.py b/strain/runner.py index c4f7afd..640c5ec 100644 --- a/strain/runner.py +++ b/strain/runner.py @@ -27,7 +27,7 @@ ) from .build_cache import BuildFailureCache, cache_path from .config import CONFIG_FILE, Config, load_config -from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, create_console, plural, score_style +from .console import BOLD, DIM, RED, STATE_STYLES, YELLOW, UiMode, create_console, plural, score_style from .coverage import FileCoverage, GcovCoverage from .journal import Journal from .mutators import ( @@ -44,6 +44,7 @@ from .operator_sites import EngineError, OperatorSite from .html_report import html_report from .report import ( + NO_MUTANTS, EventLog, FileScore, action_items, @@ -582,8 +583,23 @@ def write_reports(self, combined: Sequence[CombinedResult], sources: Mapping[str (report_dir / "report.json").write_text( json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False), encoding="utf-8" ) - (report_dir / "index.html").write_text(html_report(combined, sources), encoding="utf-8") - (report_dir / "summary.md").write_text(markdown_summary(combined, self.options.threshold), encoding="utf-8") + no_mutants = self.why_no_mutants() + (report_dir / "index.html").write_text(html_report(combined, sources, no_mutants), encoding="utf-8") + (report_dir / "summary.md").write_text( + markdown_summary(combined, self.options.threshold, no_mutants), encoding="utf-8" + ) + + def why_no_mutants(self) -> str: + """Why a run may have no mutants; with --since, an empty run passes, so it must say why.""" + if self.since is None: + return NO_MUTANTS + changed = sum(len(lines) for lines in self.since.values()) + if changed == 0: + return f"No source line changed since {self.options.since}, so there was nothing to mutate." + return ( + f"No mutant was generated on the {plural(changed, 'line')} changed since {self.options.since}: " + "the selected tests don't execute them, or they hold nothing to mutate." + ) def show_survivors(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None: survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED] @@ -631,6 +647,8 @@ def show_summary(self, combined: Sequence[CombinedResult], score: float | None) if state in states ] console.field("mutants", f"{len(combined)} {' '.join(counts)}" if counts else "none", 8) + if not combined: + console.line(f" {console.paint(self.why_no_mutants(), BOLD + YELLOW)}") percent = None if score is None else score * 100 console.line( f" {console.paint('mutation score', BOLD)} {console.score(percent)}{self.threshold_note(percent)}" diff --git a/tests/test_ci.py b/tests/test_ci.py index 55099da..13f5726 100644 --- a/tests/test_ci.py +++ b/tests/test_ci.py @@ -4,6 +4,7 @@ ROOT = Path(__file__).resolve().parent.parent WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +README = ROOT / "README.md" def workflow_text() -> str: @@ -20,6 +21,37 @@ def test_ci_runs_the_validator_on_this_project(self) -> None: self.assertRegex(workflow_text(), r"python3? -m project_validator\b[^\n]*--exclude tests/fixtures") +def readme_ci_job() -> str: + section = README.read_text(encoding="utf-8").partition("\n## CI\n")[2].partition("\n## ")[0] + match = re.search(r"```yaml\n(.*?)```", section, re.S) + assert match is not None, "the README's CI section has no yaml job" + return match.group(1) + + +def steps(job: str) -> list[str]: + return re.split(r"\n(?= - )", job.partition(" steps:\n")[2]) + + +class ReadmeCiRecipeTest(unittest.TestCase): + """The documented job must fail when strain does, even though its output goes through a pipe.""" + + def test_the_strain_step_pipes_its_output_with_pipefail(self) -> None: + [step] = [step for step in steps(readme_ci_job()) if "strain run" in step] + self.assertIn("| tee", step) + self.assertRegex(step, r"(?m)^ shell: bash\b") + + def test_the_reports_are_kept_when_strain_fails(self) -> None: + job = readme_ci_job() + for marker in ("GITHUB_STEP_SUMMARY", "upload-artifact"): + [step] = [step for step in steps(job) if marker in step] + self.assertIn("if: always()", step, marker) + + def test_since_has_the_base_branch_to_compare_with(self) -> None: + job = readme_ci_job() + self.assertIn("--since", job) + self.assertIn("fetch-depth: 0", job) + + class ValidatorStandardsTest(unittest.TestCase): def test_ci_runs_the_tests_with_warnings_as_errors(self) -> None: self.assertRegex(workflow_text(), r"python -W error -m pytest") diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index e668dd4..776beb8 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -503,6 +503,49 @@ def test_interrupted_run_still_writes_the_reports(self) -> None: self.assertEqual(console.finished_runs, 0) self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source) + def run_since(self, ref: str) -> tuple[int, str, str]: + return run_strain( + "--build-dir", + str(self.build_dir), + "--report-dir", + str(self.report_dir), + "--since", + ref, + "--threshold", + "80", + ) + + def test_since_without_changes_says_nothing_was_tested(self) -> None: + configure_project(self.root, self.build_dir) + + exit_code, stdout, _ = self.run_since("HEAD") + + # A change that touches no source is not a failure, but it must not read like a clean run. + self.assertEqual(exit_code, 0, stdout) + reason = "No source line changed since HEAD, so there was nothing to mutate." + self.assertIn(reason, stdout) + self.assertEqual( + (self.report_dir / "summary.md").read_text(), + f"## strain mutation testing\n\n**Nothing was mutation tested.** {reason}\n", + ) + + def test_since_with_only_unexecuted_changes_says_nothing_was_tested(self) -> None: + configure_project(self.root, self.build_dir) + source = self.root / "src" / "sample.cpp" + source.write_text("// a comment\n" + source.read_text()) + run_checked( + "git", "-c", "user.name=s", "-c", "user.email=s@example.com", "commit", "-qam", "comment", cwd=self.root + ) + + exit_code, stdout, _ = self.run_since("HEAD~1") + + self.assertEqual(exit_code, 0, stdout) + self.assertIn( + "**Nothing was mutation tested.** No mutant was generated on the 1 line changed since HEAD~1: " + "the selected tests don't execute them, or they hold nothing to mutate.", + (self.report_dir / "summary.md").read_text(), + ) + def test_dirty_working_tree_is_refused(self) -> None: configure_project(self.root, self.build_dir) (self.root / "src" / "sample.cpp").write_text("// local edit\n") diff --git a/tests/test_report.py b/tests/test_report.py index 29ac1a2..3ae45eb 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -198,7 +198,11 @@ def test_summary_without_survivors_meets_the_threshold(self) -> None: self.assertIn("**Mutation score: 100.0%** ✅ meets the 90% threshold\n", markdown) self.assertNotIn("Survivors", markdown) - def test_summary_without_mutants(self) -> None: + def test_summary_without_mutants_says_nothing_was_tested_and_why(self) -> None: + self.assertEqual( + markdown_summary([], threshold=None, no_mutants="No source line changed since main."), + "## strain mutation testing\n\n**Nothing was mutation tested.** No source line changed since main.\n", + ) self.assertIn("No mutants were generated.", markdown_summary([], threshold=None)) def test_pipes_in_file_names_do_not_break_the_table(self) -> None: @@ -229,9 +233,9 @@ def test_report_shows_escaped_source_with_its_mutants(self) -> None: self.assertNotIn(" None: - page = html_report([], {}) + page = html_report([], {}, "No source line changed since astrain mutation report: n/a", page) - self.assertIn("No mutants were generated", page) + self.assertIn("Nothing was mutation tested. No source line changed since a<b.", page) class MutationTestingElementsTest(unittest.TestCase):