diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f1de68..353ec6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,8 +73,67 @@ jobs: - name: nothing is left unchecked in a full clone run: | deadtrigger --default-branch main > report.txt + # Whole line, and before the absence check, because the absence + # check says nothing about how much was read. The no-workflows case + # is already covered -- deadtrigger exits 2 and `bash -e` kills the + # step at the redirect -- but a second workflow file in this + # repository would read as `2 workflows checked, clean` and the + # guard would go on passing without a word about its subject having + # changed. -x so the count has to be re-read rather than grown into. + grep -qxF '1 workflow checked, clean' report.txt if grep -q '^not checked:' report.txt; then cat report.txt echo "a check declined to run in a full clone" exit 1 fi + + - name: the guard above fires when a check really is declined + # The guard is an absence check and nothing here established that + # `not checked:` can be printed at all, so it would have gone on + # passing if the skips stopped working (logbook#64). The witness + # cannot come from this repository, because the guard requires the + # string's absence here -- it has to come from a corpus where the + # line does appear. + # + # One control per cause. `core.check` declines in exactly two places + # and a witness for one says nothing about the other; the test in + # tests/test_dogfood.py fails if a third one is added without a + # control here. + # + # Both corpora are built here rather than inherited. Plain + # actions/checkout leaves refs/remotes/origin/HEAD unset, which would + # witness the second cause for free from the runner and stop doing it, + # silently, the day that changes. + run: | + root=/tmp/control + mkdir -p "$root/.github/workflows" + cat > "$root/.github/workflows/live.yml" <<'YAML' + name: Live + on: + push: + branches: [main] + jobs: + a: + runs-on: ubuntu-latest + steps: [{run: 'true'}] + YAML + git -C "$root" init -q -b main + git -C "$root" add -A + git -C "$root" -c user.name=ci -c user.email=ci@example.invalid \ + commit -q -m control + + # A fresh `git init` has no origin, so there is no default branch to + # find and the default-branch checks decline. + deadtrigger "$root" > no-default-branch.txt + cat no-default-branch.txt + grep -q '^not checked: default-branch-only triggers --' no-default-branch.txt + + # `git clone` does set origin/HEAD, so in the shallow copy the only + # check that declines is the one about branch and tag lists. + git clone -q --depth 1 "file://$root" /tmp/control-shallow + deadtrigger /tmp/control-shallow --default-branch main > shallow.txt + cat shallow.txt + grep -q '^not checked: branch and tag filters --' shallow.txt + + # Both controls exit 0. A declined check does not change the exit + # code, which is the whole reason the guard above reads stdout. diff --git a/tests/test_dogfood.py b/tests/test_dogfood.py new file mode 100644 index 0000000..97dd6e7 --- /dev/null +++ b/tests/test_dogfood.py @@ -0,0 +1,82 @@ +"""The dogfood job's own assertions, where they can be tested at all. + +`.github/workflows/ci.yml` ends with a step that reads the tool's report of +this repository and fails if anything in it was declined: + + deadtrigger --default-branch main > report.txt + if grep -q '^not checked:' report.txt; then ... exit 1; fi + +`deadassert` reported that as `unwitnessed` (logbook#64): it fails only when +its pattern appears, and nothing in those workflows established that the +pattern can appear at all. If the two skips stopped working -- if +`is_shallow` always answered False, say -- the step would have gone on +passing, and so would the step above it that exists to prove the +`fetch-depth: 0` advice is needed. + +The fix is in ci.yml rather than here. A witness has to come from a corpus +where `not checked:` really is printed, and it cannot be this repository, +because the guard requires the string's absence here. The job now builds one +control per cause and greps each for the line it should produce. + +That leaves one thing a control cannot tell you, and it is what this file +is for: whether the controls still cover every check the tool can decline. +Add a third `report.skip(...)` to `core.check` and the guard quietly goes +back to being part-witnessed -- two causes proven, one not, output +indistinguishable. + +The behaviour of the skips themselves is tested properly elsewhere and +belongs there: `TestSkippedChecks.test_shallow_clone_declines_the_branch_check`, +`test_check_is_named_as_skipped_when_the_default_branch_is_unknown` and +`TestHealthy.test_nothing_is_skipped_in_a_full_clone_with_a_default_branch`, +all in test_cli.py. This file does not repeat them. + +Reading source text in a test is a cost, and it is deliberate: the thing +being checked is a relationship between a Python file and a YAML file, and +nothing at runtime holds both. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CORE = ROOT / "src" / "deadtrigger" / "core.py" +CI = ROOT / ".github" / "workflows" / "ci.yml" + +#: Every check `core.check` can decline, by the name the CLI prints for it. +#: One control in the dogfood job per entry. +DECLINABLE = ("branch and tag filters", "default-branch-only triggers") + + +def skip_calls() -> tuple[int, list[str]]: + """How many times `core.check` declines, and the names it declines under. + + The two numbers are returned together on purpose. A `report.skip(reason)` + built from a variable would be invisible to the second, and the test + below notices because it is no longer invisible to the first. + """ + source = CORE.read_text(encoding="utf-8") + calls = re.findall(r"report\.skip\(", source) + named = re.findall(r'report\.skip\(\s*"([^"]+)"', source) + return len(calls), named + + +def test_every_declinable_check_names_itself_with_a_literal() -> None: + calls, named = skip_calls() + assert calls == len(named), ( + f"{calls} skip call(s) in core.py, {len(named)} of them with a literal " + "name. The rest cannot be matched up with a control in the dogfood " + "job; give the check a literal name or witness it some other way." + ) + assert sorted(named) == sorted(DECLINABLE) + + +def test_every_declinable_check_has_a_witness_in_the_dogfood_job() -> None: + ci = CI.read_text(encoding="utf-8") + missing = [name for name in DECLINABLE if f"^not checked: {name} --" not in ci] + assert not missing, ( + f"declined without a control in the dogfood job: {missing}. The " + "`^not checked:` guard in ci.yml passes when nothing is declined, so " + "each cause needs a corpus where it fires and a grep that names it." + )