Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,77 @@ To suppress mutants in source, add `// mutate-off` to the line.

Written to `strain-report/` <!-- TODO(claude): confirm path -->:

- `index.html`: a browsable report, with each source annotated
- `summary.md`: the score, the files and the survivors, for a CI job summary or a pull request comment
- `report.json`: full results, in the mutation-testing-elements schema
- `events.jsonl`: every event of the run
- compiler-style warnings on the console (`file:line:col: warning: survived [ror]`), clickable in editors

## CI

strain's exit status is what a CI job should fail on:

- **Non-zero when nothing was tested:** without `--since`, strain exits non-zero when no mutant was generated, because
a run that mutated nothing tested nothing.
- **Non-zero below `--threshold`.**
- **`--since` with nothing to mutate passes:** a change that touches no line the tests run has nothing to test. The
summary then starts with **Nothing was mutation tested** and gives the reason, so the result can't be mistaken for a
clean one.

**Pipes hide the exit status unless the step says `shell: bash`.** Without a `shell:` key, GitHub Actions runs
`bash -e`, without `pipefail`, so `strain run … | tee strain.log` returns the status of `tee` and the step passes
whatever strain found. With `shell: bash`, GitHub adds `-o pipefail`.

This job tests the lines a pull request changes, puts the summary on the run's page and keeps the full report:

```yaml
name: Mutation testing
on: pull_request

jobs:
strain:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0 # --since needs the base branch
- uses: actions/setup-python@v6
with:
python-version: "3.13"
- name: Install
run: |
sudo apt-get update && sudo apt-get install -y ninja-build cppcheck
pip install "git+https://github.com/cpp-best-practices/mutant_strain"
- name: Configure a build for strain
run: |
cmake -S . -B build-strain -G Ninja \
-DCMAKE_BUILD_TYPE=Debug \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DCMAKE_CXX_FLAGS="--coverage -fsanitize=address,undefined -fno-omit-frame-pointer -D_GLIBCXX_ASSERTIONS -ftrivial-auto-var-init=pattern" \
-DCMAKE_EXE_LINKER_FLAGS="--coverage -fsanitize=address,undefined"
cmake --build build-strain
- name: Mutation test
shell: bash # adds -o pipefail, so the step fails when strain does, even through tee
run: |
strain run --build-dir build-strain --ui log --since "origin/${{ github.base_ref }}" --threshold 80 \
| tee strain.log
- name: Job summary
if: always()
shell: bash
run: |
if [ -f strain-report/summary.md ]; then cat strain-report/summary.md >> "$GITHUB_STEP_SUMMARY"; fi
- uses: actions/upload-artifact@v4
if: always()
with:
name: strain-report
path: |
strain-report/
strain.log
```

To test everything instead, on a schedule or on pushes to `main`, drop `--since`: the job then also fails when no
mutant was generated.

## Pluggable backends

| Part | Options |
Expand Down
6 changes: 3 additions & 3 deletions strain/html_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from html import escape

from .console import HIGH_SCORE, LOW_SCORE
from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint
from .report import NO_MUTANTS, TAB_WIDTH, FileScore, file_scores, format_score, hint
from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score

KILLED_CLASS = "killed"
Expand Down Expand Up @@ -145,7 +145,7 @@ def _counts(combined: Sequence[CombinedResult]) -> str:
return ", ".join(f"{escape(state)} {states.count(state)}" for state in MutantState if state in states)


def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> str:
def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str], no_mutants: str = NO_MUTANTS) -> str:
score = mutation_score(entry.state for entry in combined)
body = (
[
Expand All @@ -155,7 +155,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str])
*_file_sections(combined, sources),
]
if combined
else ["<p>No mutants were generated.</p>"]
else [f"<p><strong>Nothing was mutation tested.</strong> {escape(no_mutants)}</p>"]
)
page = [
"<!DOCTYPE html>",
Expand Down
12 changes: 9 additions & 3 deletions strain/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,17 @@ def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]:
]


def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None) -> str:
"""A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment."""
NO_MUTANTS = "No mutants were generated."


def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None, no_mutants: str = NO_MUTANTS) -> str:
"""A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment.

`no_mutants` says why there is nothing to show, so an empty run doesn't read like a clean one.
"""
heading = ["## strain mutation testing", ""]
if not combined:
return "\n".join([*heading, "No mutants were generated."]) + "\n"
return "\n".join([*heading, f"**Nothing was mutation tested.** {no_mutants}"]) + "\n"
score = mutation_score(entry.state for entry in combined)
lines = [
*heading,
Expand Down
24 changes: 21 additions & 3 deletions strain/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
)
from .build_cache import BuildFailureCache, cache_path
from .config import CONFIG_FILE, Config, load_config
from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, create_console, plural, score_style
from .console import BOLD, DIM, RED, STATE_STYLES, YELLOW, UiMode, create_console, plural, score_style
from .coverage import FileCoverage, GcovCoverage
from .journal import Journal
from .mutators import (
Expand All @@ -44,6 +44,7 @@
from .operator_sites import EngineError, OperatorSite
from .html_report import html_report
from .report import (
NO_MUTANTS,
EventLog,
FileScore,
action_items,
Expand Down Expand Up @@ -582,8 +583,23 @@ def write_reports(self, combined: Sequence[CombinedResult], sources: Mapping[str
(report_dir / "report.json").write_text(
json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False), encoding="utf-8"
)
(report_dir / "index.html").write_text(html_report(combined, sources), encoding="utf-8")
(report_dir / "summary.md").write_text(markdown_summary(combined, self.options.threshold), encoding="utf-8")
no_mutants = self.why_no_mutants()
(report_dir / "index.html").write_text(html_report(combined, sources, no_mutants), encoding="utf-8")
(report_dir / "summary.md").write_text(
markdown_summary(combined, self.options.threshold, no_mutants), encoding="utf-8"
)

def why_no_mutants(self) -> str:
"""Why a run may have no mutants; with --since, an empty run passes, so it must say why."""
if self.since is None:
return NO_MUTANTS
changed = sum(len(lines) for lines in self.since.values())
if changed == 0:
return f"No source line changed since {self.options.since}, so there was nothing to mutate."
return (
f"No mutant was generated on the {plural(changed, 'line')} changed since {self.options.since}: "
"the selected tests don't execute them, or they hold nothing to mutate."
)

def show_survivors(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None:
survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED]
Expand Down Expand Up @@ -631,6 +647,8 @@ def show_summary(self, combined: Sequence[CombinedResult], score: float | None)
if state in states
]
console.field("mutants", f"{len(combined)} {' '.join(counts)}" if counts else "none", 8)
if not combined:
console.line(f" {console.paint(self.why_no_mutants(), BOLD + YELLOW)}")
percent = None if score is None else score * 100
console.line(
f" {console.paint('mutation score', BOLD)} {console.score(percent)}{self.threshold_note(percent)}"
Expand Down
32 changes: 32 additions & 0 deletions tests/test_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

ROOT = Path(__file__).resolve().parent.parent
WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml"
README = ROOT / "README.md"


def workflow_text() -> str:
Expand All @@ -20,6 +21,37 @@ def test_ci_runs_the_validator_on_this_project(self) -> None:
self.assertRegex(workflow_text(), r"python3? -m project_validator\b[^\n]*--exclude tests/fixtures")


def readme_ci_job() -> str:
section = README.read_text(encoding="utf-8").partition("\n## CI\n")[2].partition("\n## ")[0]
match = re.search(r"```yaml\n(.*?)```", section, re.S)
assert match is not None, "the README's CI section has no yaml job"
return match.group(1)


def steps(job: str) -> list[str]:
return re.split(r"\n(?= - )", job.partition(" steps:\n")[2])


class ReadmeCiRecipeTest(unittest.TestCase):
"""The documented job must fail when strain does, even though its output goes through a pipe."""

def test_the_strain_step_pipes_its_output_with_pipefail(self) -> None:
[step] = [step for step in steps(readme_ci_job()) if "strain run" in step]
self.assertIn("| tee", step)
self.assertRegex(step, r"(?m)^ shell: bash\b")

def test_the_reports_are_kept_when_strain_fails(self) -> None:
job = readme_ci_job()
for marker in ("GITHUB_STEP_SUMMARY", "upload-artifact"):
[step] = [step for step in steps(job) if marker in step]
self.assertIn("if: always()", step, marker)

def test_since_has_the_base_branch_to_compare_with(self) -> None:
job = readme_ci_job()
self.assertIn("--since", job)
self.assertIn("fetch-depth: 0", job)


class ValidatorStandardsTest(unittest.TestCase):
def test_ci_runs_the_tests_with_warnings_as_errors(self) -> None:
self.assertRegex(workflow_text(), r"python -W error -m pytest")
Expand Down
43 changes: 43 additions & 0 deletions tests/test_end_to_end.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,49 @@ def test_interrupted_run_still_writes_the_reports(self) -> None:
self.assertEqual(console.finished_runs, 0)
self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source)

def run_since(self, ref: str) -> tuple[int, str, str]:
return run_strain(
"--build-dir",
str(self.build_dir),
"--report-dir",
str(self.report_dir),
"--since",
ref,
"--threshold",
"80",
)

def test_since_without_changes_says_nothing_was_tested(self) -> None:
configure_project(self.root, self.build_dir)

exit_code, stdout, _ = self.run_since("HEAD")

# A change that touches no source is not a failure, but it must not read like a clean run.
self.assertEqual(exit_code, 0, stdout)
reason = "No source line changed since HEAD, so there was nothing to mutate."
self.assertIn(reason, stdout)
self.assertEqual(
(self.report_dir / "summary.md").read_text(),
f"## strain mutation testing\n\n**Nothing was mutation tested.** {reason}\n",
)

def test_since_with_only_unexecuted_changes_says_nothing_was_tested(self) -> None:
configure_project(self.root, self.build_dir)
source = self.root / "src" / "sample.cpp"
source.write_text("// a comment\n" + source.read_text())
run_checked(
"git", "-c", "user.name=s", "-c", "user.email=s@example.com", "commit", "-qam", "comment", cwd=self.root
)

exit_code, stdout, _ = self.run_since("HEAD~1")

self.assertEqual(exit_code, 0, stdout)
self.assertIn(
"**Nothing was mutation tested.** No mutant was generated on the 1 line changed since HEAD~1: "
"the selected tests don't execute them, or they hold nothing to mutate.",
(self.report_dir / "summary.md").read_text(),
)

def test_dirty_working_tree_is_refused(self) -> None:
configure_project(self.root, self.build_dir)
(self.root / "src" / "sample.cpp").write_text("// local edit\n")
Expand Down
10 changes: 7 additions & 3 deletions tests/test_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,11 @@ def test_summary_without_survivors_meets_the_threshold(self) -> None:
self.assertIn("**Mutation score: 100.0%** ✅ meets the 90% threshold\n", markdown)
self.assertNotIn("Survivors", markdown)

def test_summary_without_mutants(self) -> None:
def test_summary_without_mutants_says_nothing_was_tested_and_why(self) -> None:
self.assertEqual(
markdown_summary([], threshold=None, no_mutants="No source line changed since main."),
"## strain mutation testing\n\n**Nothing was mutation tested.** No source line changed since main.\n",
)
self.assertIn("No mutants were generated.", markdown_summary([], threshold=None))

def test_pipes_in_file_names_do_not_break_the_table(self) -> None:
Expand Down Expand Up @@ -229,9 +233,9 @@ def test_report_shows_escaped_source_with_its_mutants(self) -> None:
self.assertNotIn("<script", page)

def test_report_without_mutants(self) -> None:
page = html_report([], {})
page = html_report([], {}, "No source line changed since a<b.")
self.assertIn("<title>strain mutation report: n/a</title>", page)
self.assertIn("No mutants were generated", page)
self.assertIn("<strong>Nothing was mutation tested.</strong> No source line changed since a&lt;b.", page)


class MutationTestingElementsTest(unittest.TestCase):
Expand Down
Loading