diff --git a/README.md b/README.md index 545cdef..09c004b 100644 --- a/README.md +++ b/README.md @@ -220,10 +220,77 @@ To suppress mutants in source, add `// mutate-off` to the line. Written to `strain-report/` : +- `index.html`: a browsable report, with each source annotated +- `summary.md`: the score, the files and the survivors, for a CI job summary or a pull request comment - `report.json`: full results, in the mutation-testing-elements schema - `events.jsonl`: every event of the run - compiler-style warnings on the console (`file:line:col: warning: survived [ror]`), clickable in editors +## CI + +strain's exit status is what a CI job should fail on: + +- **Non-zero when nothing was tested:** without `--since`, strain exits non-zero when no mutant was generated, because + a run that mutated nothing tested nothing. +- **Non-zero below `--threshold`.** +- **`--since` with nothing to mutate passes:** a change that touches no line the tests run has nothing to test. The + summary then starts with **Nothing was mutation tested** and gives the reason, so the result can't be mistaken for a + clean one. + +**Pipes hide the exit status unless the step says `shell: bash`.** Without a `shell:` key, GitHub Actions runs +`bash -e`, without `pipefail`, so `strain run … | tee strain.log` returns the status of `tee` and the step passes +whatever strain found. With `shell: bash`, GitHub adds `-o pipefail`. + +This job tests the lines a pull request changes, puts the summary on the run's page and keeps the full report: + +```yaml +name: Mutation testing +on: pull_request + +jobs: + strain: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 # --since needs the base branch + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install + run: | + sudo apt-get update && sudo apt-get install -y ninja-build cppcheck + pip install "git+https://github.com/cpp-best-practices/mutant_strain" + - name: Configure a build for strain + run: | + cmake -S . -B build-strain -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_CXX_FLAGS="--coverage -fsanitize=address,undefined -fno-omit-frame-pointer -D_GLIBCXX_ASSERTIONS -ftrivial-auto-var-init=pattern" \ + -DCMAKE_EXE_LINKER_FLAGS="--coverage -fsanitize=address,undefined" + cmake --build build-strain + - name: Mutation test + shell: bash # adds -o pipefail, so the step fails when strain does, even through tee + run: | + strain run --build-dir build-strain --ui log --since "origin/${{ github.base_ref }}" --threshold 80 \ + | tee strain.log + - name: Job summary + if: always() + shell: bash + run: | + if [ -f strain-report/summary.md ]; then cat strain-report/summary.md >> "$GITHUB_STEP_SUMMARY"; fi + - uses: actions/upload-artifact@v4 + if: always() + with: + name: strain-report + path: | + strain-report/ + strain.log +``` + +To test everything instead, on a schedule or on pushes to `main`, drop `--since`: the job then also fails when no +mutant was generated. + ## Pluggable backends | Part | Options | diff --git a/strain/html_report.py b/strain/html_report.py index 892b445..c36c904 100644 --- a/strain/html_report.py +++ b/strain/html_report.py @@ -4,7 +4,7 @@ from html import escape from .console import HIGH_SCORE, LOW_SCORE -from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint +from .report import NO_MUTANTS, TAB_WIDTH, FileScore, file_scores, format_score, hint from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score KILLED_CLASS = "killed" @@ -145,7 +145,7 @@ def _counts(combined: Sequence[CombinedResult]) -> str: return ", ".join(f"{escape(state)} {states.count(state)}" for state in MutantState if state in states) -def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> str: +def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str], no_mutants: str = NO_MUTANTS) -> str: score = mutation_score(entry.state for entry in combined) body = ( [ @@ -155,7 +155,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) *_file_sections(combined, sources), ] if combined - else ["

No mutants were generated.

"] + else [f"

Nothing was mutation tested. {escape(no_mutants)}

"] ) page = [ "", diff --git a/strain/report.py b/strain/report.py index 89c735b..8ca4420 100644 --- a/strain/report.py +++ b/strain/report.py @@ -199,11 +199,17 @@ def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]: ] -def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None) -> str: - """A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment.""" +NO_MUTANTS = "No mutants were generated." + + +def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None, no_mutants: str = NO_MUTANTS) -> str: + """A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment. + + `no_mutants` says why there is nothing to show, so an empty run doesn't read like a clean one. + """ heading = ["## strain mutation testing", ""] if not combined: - return "\n".join([*heading, "No mutants were generated."]) + "\n" + return "\n".join([*heading, f"**Nothing was mutation tested.** {no_mutants}"]) + "\n" score = mutation_score(entry.state for entry in combined) lines = [ *heading, diff --git a/strain/runner.py b/strain/runner.py index c4f7afd..640c5ec 100644 --- a/strain/runner.py +++ b/strain/runner.py @@ -27,7 +27,7 @@ ) from .build_cache import BuildFailureCache, cache_path from .config import CONFIG_FILE, Config, load_config -from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, create_console, plural, score_style +from .console import BOLD, DIM, RED, STATE_STYLES, YELLOW, UiMode, create_console, plural, score_style from .coverage import FileCoverage, GcovCoverage from .journal import Journal from .mutators import ( @@ -44,6 +44,7 @@ from .operator_sites import EngineError, OperatorSite from .html_report import html_report from .report import ( + NO_MUTANTS, EventLog, FileScore, action_items, @@ -582,8 +583,23 @@ def write_reports(self, combined: Sequence[CombinedResult], sources: Mapping[str (report_dir / "report.json").write_text( json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False), encoding="utf-8" ) - (report_dir / "index.html").write_text(html_report(combined, sources), encoding="utf-8") - (report_dir / "summary.md").write_text(markdown_summary(combined, self.options.threshold), encoding="utf-8") + no_mutants = self.why_no_mutants() + (report_dir / "index.html").write_text(html_report(combined, sources, no_mutants), encoding="utf-8") + (report_dir / "summary.md").write_text( + markdown_summary(combined, self.options.threshold, no_mutants), encoding="utf-8" + ) + + def why_no_mutants(self) -> str: + """Why a run may have no mutants; with --since, an empty run passes, so it must say why.""" + if self.since is None: + return NO_MUTANTS + changed = sum(len(lines) for lines in self.since.values()) + if changed == 0: + return f"No source line changed since {self.options.since}, so there was nothing to mutate." + return ( + f"No mutant was generated on the {plural(changed, 'line')} changed since {self.options.since}: " + "the selected tests don't execute them, or they hold nothing to mutate." + ) def show_survivors(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None: survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED] @@ -631,6 +647,8 @@ def show_summary(self, combined: Sequence[CombinedResult], score: float | None) if state in states ] console.field("mutants", f"{len(combined)} {' '.join(counts)}" if counts else "none", 8) + if not combined: + console.line(f" {console.paint(self.why_no_mutants(), BOLD + YELLOW)}") percent = None if score is None else score * 100 console.line( f" {console.paint('mutation score', BOLD)} {console.score(percent)}{self.threshold_note(percent)}" diff --git a/tests/test_ci.py b/tests/test_ci.py index 55099da..13f5726 100644 --- a/tests/test_ci.py +++ b/tests/test_ci.py @@ -4,6 +4,7 @@ ROOT = Path(__file__).resolve().parent.parent WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +README = ROOT / "README.md" def workflow_text() -> str: @@ -20,6 +21,37 @@ def test_ci_runs_the_validator_on_this_project(self) -> None: self.assertRegex(workflow_text(), r"python3? -m project_validator\b[^\n]*--exclude tests/fixtures") +def readme_ci_job() -> str: + section = README.read_text(encoding="utf-8").partition("\n## CI\n")[2].partition("\n## ")[0] + match = re.search(r"```yaml\n(.*?)```", section, re.S) + assert match is not None, "the README's CI section has no yaml job" + return match.group(1) + + +def steps(job: str) -> list[str]: + return re.split(r"\n(?= - )", job.partition(" steps:\n")[2]) + + +class ReadmeCiRecipeTest(unittest.TestCase): + """The documented job must fail when strain does, even though its output goes through a pipe.""" + + def test_the_strain_step_pipes_its_output_with_pipefail(self) -> None: + [step] = [step for step in steps(readme_ci_job()) if "strain run" in step] + self.assertIn("| tee", step) + self.assertRegex(step, r"(?m)^ shell: bash\b") + + def test_the_reports_are_kept_when_strain_fails(self) -> None: + job = readme_ci_job() + for marker in ("GITHUB_STEP_SUMMARY", "upload-artifact"): + [step] = [step for step in steps(job) if marker in step] + self.assertIn("if: always()", step, marker) + + def test_since_has_the_base_branch_to_compare_with(self) -> None: + job = readme_ci_job() + self.assertIn("--since", job) + self.assertIn("fetch-depth: 0", job) + + class ValidatorStandardsTest(unittest.TestCase): def test_ci_runs_the_tests_with_warnings_as_errors(self) -> None: self.assertRegex(workflow_text(), r"python -W error -m pytest") diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index e668dd4..776beb8 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -503,6 +503,49 @@ def test_interrupted_run_still_writes_the_reports(self) -> None: self.assertEqual(console.finished_runs, 0) self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source) + def run_since(self, ref: str) -> tuple[int, str, str]: + return run_strain( + "--build-dir", + str(self.build_dir), + "--report-dir", + str(self.report_dir), + "--since", + ref, + "--threshold", + "80", + ) + + def test_since_without_changes_says_nothing_was_tested(self) -> None: + configure_project(self.root, self.build_dir) + + exit_code, stdout, _ = self.run_since("HEAD") + + # A change that touches no source is not a failure, but it must not read like a clean run. + self.assertEqual(exit_code, 0, stdout) + reason = "No source line changed since HEAD, so there was nothing to mutate." + self.assertIn(reason, stdout) + self.assertEqual( + (self.report_dir / "summary.md").read_text(), + f"## strain mutation testing\n\n**Nothing was mutation tested.** {reason}\n", + ) + + def test_since_with_only_unexecuted_changes_says_nothing_was_tested(self) -> None: + configure_project(self.root, self.build_dir) + source = self.root / "src" / "sample.cpp" + source.write_text("// a comment\n" + source.read_text()) + run_checked( + "git", "-c", "user.name=s", "-c", "user.email=s@example.com", "commit", "-qam", "comment", cwd=self.root + ) + + exit_code, stdout, _ = self.run_since("HEAD~1") + + self.assertEqual(exit_code, 0, stdout) + self.assertIn( + "**Nothing was mutation tested.** No mutant was generated on the 1 line changed since HEAD~1: " + "the selected tests don't execute them, or they hold nothing to mutate.", + (self.report_dir / "summary.md").read_text(), + ) + def test_dirty_working_tree_is_refused(self) -> None: configure_project(self.root, self.build_dir) (self.root / "src" / "sample.cpp").write_text("// local edit\n") diff --git a/tests/test_report.py b/tests/test_report.py index 29ac1a2..3ae45eb 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -198,7 +198,11 @@ def test_summary_without_survivors_meets_the_threshold(self) -> None: self.assertIn("**Mutation score: 100.0%** ✅ meets the 90% threshold\n", markdown) self.assertNotIn("Survivors", markdown) - def test_summary_without_mutants(self) -> None: + def test_summary_without_mutants_says_nothing_was_tested_and_why(self) -> None: + self.assertEqual( + markdown_summary([], threshold=None, no_mutants="No source line changed since main."), + "## strain mutation testing\n\n**Nothing was mutation tested.** No source line changed since main.\n", + ) self.assertIn("No mutants were generated.", markdown_summary([], threshold=None)) def test_pipes_in_file_names_do_not_break_the_table(self) -> None: @@ -229,9 +233,9 @@ def test_report_shows_escaped_source_with_its_mutants(self) -> None: self.assertNotIn(" None: - page = html_report([], {}) + page = html_report([], {}, "No source line changed since astrain mutation report: n/a", page) - self.assertIn("No mutants were generated", page) + self.assertIn("Nothing was mutation tested. No source line changed since a<b.", page) class MutationTestingElementsTest(unittest.TestCase):