From 5027e5215a238153948b31f894181115477ed718 Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:38:48 +0000 Subject: [PATCH 1/3] feat: count the assertion-shaped constructs this does not read `deadassert` reads matchers. A `run:` block can assert with a Python `assert`, a `diff` or a `jq -e`, and logbook#69 is the case where that mattered: stockbox's dogfood job asserted `loose > tight * 5` in Python -- the same runner-image-dependent defect deadassert reported in a grep two steps up in the same file -- and the run said `2 findings` and `32 assertions in 20 scripts` with nothing to say one of those 20 scripts asserted something it had not read. `unread.py` finds those constructs and counts them. It evaluates nothing: the point is that a clean run say what it covered rather than imply it covered everything, the same move already made by `not decided:` and by the skipped non-POSIX shells. `test "$s" -eq N` is deliberately not counted, per the caveat on the issue -- it is the commonest assertion in the org and almost always sound, so counting it would put a non-zero number on nearly every file and the number would stop being read. Reuses `shell.command_head` for the fiddly parts (`!`, `VAR=` prefixes, a path to the program) rather than re-deriving them, which is why that function is now parameterised by program name. --- src/deadassert/analyse.py | 18 ++- src/deadassert/report.py | 97 +++++++++++++--- src/deadassert/shell.py | 32 +++++- src/deadassert/unread.py | 230 ++++++++++++++++++++++++++++++++++++++ tests/test_unread.py | 170 ++++++++++++++++++++++++++++ 5 files changed, 525 insertions(+), 22 deletions(-) create mode 100644 src/deadassert/unread.py create mode 100644 tests/test_unread.py diff --git a/src/deadassert/analyse.py b/src/deadassert/analyse.py index 1a33355..7363f4e 100644 --- a/src/deadassert/analyse.py +++ b/src/deadassert/analyse.py @@ -39,7 +39,7 @@ import re from dataclasses import dataclass, field -from . import shell, workflows +from . import shell, unread, workflows from .shell import Assertion from .workflows import Script, WorkflowError @@ -90,6 +90,11 @@ class Result: against: str | None = None #: Patterns `--against` showed matching nothing in the output. absent: list[tuple[str, str]] = field(default_factory=list) + #: Assertion-shaped constructs not read at all, as `(where, kind, text)`. + #: Never findings, and never part of the exit code: see `unread.py`. + unread: list[tuple[str, str, str]] = field(default_factory=list) + #: How many `run:` scripts hold at least one of those. + unread_scripts: int = 0 @property def live(self) -> list[tuple[Script, Assertion]]: @@ -282,6 +287,17 @@ def analyse( result.foreign.append((f"{os.path.basename(rel)} {label}", shell_name)) for script in workflow.scripts: result.scripts += 1 + others = unread.scan(script.body) + if others: + result.unread_scripts += 1 + for item in others: + result.unread.append( + ( + f"{os.path.basename(rel)}:{script.line_of(item.line)}", + item.kind, + item.text, + ) + ) for assertion in shell.scan(script.body): where = f"{os.path.basename(rel)}:{script.line_of(assertion.line)}" if assertion.polarity == "undecided": diff --git a/src/deadassert/report.py b/src/deadassert/report.py index 9309fdf..fb32577 100644 --- a/src/deadassert/report.py +++ b/src/deadassert/report.py @@ -24,6 +24,32 @@ def _wrap(text: str, indent: str) -> str: return textwrap.fill(text, width=92, initial_indent=indent, subsequent_indent=indent) +def _unread_tally(result: Result) -> str: + """`2 diff, 1 assert`, commonest kind first.""" + kinds: dict[str, int] = {} + for _, kind, _ in result.unread: + kinds[kind] = kinds.get(kind, 0) + 1 + ordered = sorted(kinds.items(), key=lambda item: (-item[1], item[0])) + return ", ".join(f"{count} {kind}" for kind, count in ordered) + + +def _coverage(result: Result) -> str: + """The sentence that stops a clean run reading as a complete one. + + logbook#69. Everything above a denominator is about the assertions this + tool *read*, and until this sentence existed there was nothing to say + that a script could also assert in a way it had not read -- which is how + a sweep for dead assertions walked past one in the repository it was + auditing. Not a finding and not in the exit code; `unread.py` evaluates + nothing. It is the size of the gap, stated. + """ + return ( + f"Not counted above: {_plural(len(result.unread), 'assertion')} this does " + f"not read, in {result.unread_scripts} of those scripts " + f"({_unread_tally(result)}). Listed below." + ) + + def _denominator(result: Result) -> str: """What the verdict was measured over. @@ -31,21 +57,33 @@ def _denominator(result: Result) -> str: apart from a verdict over nothing, and this tool is itself about assertions that look like they are checking something and are not. """ - return ( + line = ( f"{_plural(len(result.assertions), 'assertion')} in " f"{_plural(result.scripts, 'script')} across " f"{_plural(len(result.files), 'workflow file')}, " f"{result.negated} of them negated." ) + if result.unread: + return f"{line} {_coverage(result)}" + return line def nothing_to_check(result: Result) -> str: - """Said instead of an all-clear when there were no assertions at all.""" - return ( + """Said instead of an all-clear when there were no assertions at all. + + The `unread` clause matters more here than anywhere else. A repository + that asserts entirely with `diff` -- which this project's own README + tells people to prefer -- read `Nothing to check.` before logbook#69, + and that was the broadest false statement the tool made. + """ + read = ( f"No assertions here. Read {_plural(result.scripts, 'run: script')} in " f"{_plural(len(result.files), 'workflow file')} and found no command whose " - "exit status decides a step. Nothing to check." + "exit status decides a step." ) + if result.unread: + return f"{read} But {_coverage(result)}" + return f"{read} Nothing to check." def error_lines(result: Result) -> list[str]: @@ -58,20 +96,26 @@ def error_lines(result: Result) -> list[str]: return lines +def _tail(result: Result) -> list[str]: + """The blocks under every verdict: what was not matched, decided or read.""" + out: list[str] = [] + for block in (_absent_block, _undecided_block, _unread_block): + lines = block(result) + if lines: + out.append("") + out.extend(lines) + return out + + def text(result: Result, quiet: bool = False) -> str: if result.nothing_to_check: - return nothing_to_check(result) + return "\n".join([nothing_to_check(result), *_tail(result)]) out: list[str] = [] if not result.findings: out.append("Every assertion here fails when the thing it names stops being true.") out.append(_denominator(result)) - if result.absent: - out.append("") - out.extend(_absent_block(result)) - if result.undecided: - out.append("") - out.extend(_undecided_block(result)) + out.extend(_tail(result)) return "\n".join(out) if quiet: @@ -99,13 +143,7 @@ def text(result: Result, quiet: bool = False) -> str: f"({breakdown})." ) out.append(_denominator(result)) - - if result.absent: - out.append("") - out.extend(_absent_block(result)) - if result.undecided: - out.append("") - out.extend(_undecided_block(result)) + out.extend(_tail(result)) return "\n".join(out) @@ -117,6 +155,8 @@ def _absent_block(result: Result) -> list[str]: given. Worth printing either way, because it says the comparison is not measuring what the reader thinks. """ + if not result.absent: + return [] lines = [f"not matched in {result.against}:"] for where, pattern in result.absent: lines.append(f" {where} {pattern!r}") @@ -124,12 +164,31 @@ def _absent_block(result: Result) -> list[str]: def _undecided_block(result: Result) -> list[str]: + if not result.undecided: + return [] lines = ["not decided:"] for where, why in result.undecided: lines.append(f" {where} {why}") return lines +def _unread_block(result: Result) -> list[str]: + """Assertion-shaped constructs this tool did not read. See `unread.py`. + + Separate from `not decided:` on purpose. That block is for a matcher + `deadassert` read and could not judge; this one is for an assertion it + never read at all, which is a different admission and a wider one. + """ + if not result.unread: + return [] + lines = ["not read:"] + place = max(len(where) for where, _, _ in result.unread) + width = max(len(kind) for _, kind, _ in result.unread) + for where, kind, command in result.unread: + lines.append(f" {where:<{place}} {kind:<{width}} {command}") + return lines + + def as_json(result: Result) -> str: payload = { "files": result.files, @@ -153,6 +212,8 @@ def as_json(result: Result) -> str: } for finding in result.findings ], + "unread_scripts": result.unread_scripts, + "unread": [{"where": w, "kind": k, "command": c} for w, k, c in result.unread], "undecided": [{"where": w, "why": y} for w, y in result.undecided], "foreign": [{"where": w, "shell": s} for w, s in result.foreign], "unreadable": [{"path": p, "why": y} for p, y in result.unreadable], diff --git a/src/deadassert/shell.py b/src/deadassert/shell.py index 9c8ff81..f3d0d97 100644 --- a/src/deadassert/shell.py +++ b/src/deadassert/shell.py @@ -215,8 +215,14 @@ def pipeline_parts(text: str) -> list[tuple[int, str]]: return parts -def _matcher_head(segment: str) -> tuple[str, bool] | None: - """The matcher command in a simple command, and whether `!`-negated.""" +def command_head(segment: str, programs: tuple[str, ...] = MATCHERS) -> tuple[str, bool] | None: + """A simple command whose program is one of `programs`, and whether `!`-negated. + + `programs` is a parameter because `unread` asks the same question of + `diff` and `jq` that this module asks of `grep`, and the fiddly part -- + leading `VAR=value` assignments, `env`/`command` prefixes, a path to the + program, and a `!` that belongs to the pipeline -- is the same either way. + """ text = segment.strip() negated = False while text.startswith("!"): @@ -231,11 +237,31 @@ def _matcher_head(segment: str) -> tuple[str, bool] | None: match = re.match(r"([A-Za-z_][\w./-]*)", text) if not match: return None - if match.group(1).split("/")[-1] not in MATCHERS: + if match.group(1).split("/")[-1] not in programs: return None return text, negated +def _matcher_head(segment: str) -> tuple[str, bool] | None: + """The matcher command in a simple command, and whether `!`-negated.""" + return command_head(segment, MATCHERS) + + +def swallows(text: str) -> bool: + """Whether `text` discards a command's failure with `|| true` or `|| :`.""" + return bool(_SWALLOW.search(text)) + + +def in_substitution(text: str, offset: int) -> bool: + """Whether `offset` in `text` falls inside `$(...)` or backticks.""" + return _in_substitution(text, offset) + + +def split(text: str) -> list[str] | None: + """`shlex.split`, or None when the text does not tokenise.""" + return _split(text) + + def parse_matcher(words: list[str]) -> tuple[str | None, bool, bool, bool, bool]: """`(pattern, fixed, anchored, quiet, producing)` from a matcher argv.""" pattern: str | None = None diff --git a/src/deadassert/unread.py b/src/deadassert/unread.py new file mode 100644 index 0000000..76e54af --- /dev/null +++ b/src/deadassert/unread.py @@ -0,0 +1,230 @@ +"""Assertion-shaped constructs in a `run:` script that this tool does not read. + +`deadassert` reads matchers -- `grep`, `rg` -- and says which of them cannot +fail. A step can assert in other ways, and the ones it cannot see fail in +exactly the shapes it exists to report. From `logbook#69`: + + stockbox roots -c loose.toml --json > loose.json + python3 -c " + import json + tight = json.load(open('trees.json'))['shapes'][0]['candidates'] + loose = json.load(open('loose.json'))['shapes'][0]['candidates'] + assert loose > tight * 5, (loose, tight) + " + +That is an assertion whose truth depends on the runner image rather than on +the thing it names -- the same defect `deadassert` reported two steps up in +the same file -- and it was invisible, because it is a Python `assert` and +not a grep. The run said `2 findings` and `32 assertions in 20 scripts`, with +nothing to say that one of those 20 scripts asserted something it had not +read. The sweep that exists to find dead assertions walked past one. + +**This module does not evaluate anything.** It finds the constructs and +counts them, so that a clean run says what it covered instead of implying it +covered everything -- the same move `deadassert` already makes with `not +decided:` and with the skipped non-POSIX shells. Deciding whether a `diff` +or a `jq -e` can fail is a different question, and a bigger one; the count is +the part that stops a clean run overclaiming. + +Three shapes, chosen by counting what the organisation's 38 workflow files +actually use: + +* `assert` -- a language's own assert, almost always Python via `python -c` + or a heredoc. The one from `logbook#69`. +* `diff` -- and `cmp`. This is the shape this repository's own README + recommends over `grep`, which is exactly why it needs counting: a file + that takes the advice ends up with a tool that reads almost none of its + assertions and does not currently admit it. +* `jq -e` -- the one jq invocation whose exit status is a verdict. + +`test "$s" -eq 0` and `[ -f x ]` are deliberately **not** counted, per the +caveat on `logbook#69`. They are the most common assertion in the org and +almost always sound -- the status *is* the assertion, with no pattern to rot +-- so counting them would put a non-zero number on nearly every file in the +org on day one and the number would stop being read. The cost of that choice +is real and it is written down in the README: a `test` whose captured status +came from the wrong command is not covered here either. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from . import shell + +#: Programs whose exit status is a verdict about two files being equal. +DIFFERS = ("diff", "cmp") + +#: Interpreters whose `-c` or heredoc body can hold an `assert`. +INTERPRETERS = ("python", "python3", "node", "ruby", "perl") + +#: A line that *is* an assert statement, in an embedded program. `assert` is +#: not a shell command, so a `run:` line beginning with it is inside a +#: heredoc or a quoted `-c` body by construction. Requiring whitespace or a +#: bracket after the word keeps `assert_called_once()` and a bare mention of +#: `assertions` out. +_ASSERT_LINE = re.compile(r"^assert[\s(]") + +#: An interpreter and an `assert` on one physical line: `python3 -c 'assert x'`. +#: `\bassert\b` rather than a substring so that `import deadassert` -- and the +#: name of this very package -- does not match. +_ASSERT_INLINE = re.compile(rf"\b(?:{'|'.join(INTERPRETERS)})\b.*\bassert\b") + +#: Comment openers to skip. `#` covers both shell and Python; `//` covers an +#: embedded node program. +_COMMENT = ("#", "//") + +#: Shell keywords that can sit in front of a simple command. `if diff -q a b; +#: then` is a `diff` assertion, and without this the clause's first word is +#: `if` and the construct is missed -- which is how the first version of this +#: module read `logbook#69`'s sibling shape as absent. +_KEYWORD = re.compile(r"^\s*(?:if|elif|while|until|then|else|do|done|fi|!|\{|\()\s+") + +#: Operators that end one simple command and begin the next. +_OPERATORS = ("||", "&&", ";;", ";", "|", "&") + + +@dataclass +class Unread: + """One assertion-shaped construct this tool did not read.""" + + #: 1-based, within the script. + line: int + #: One of `assert`, `diff`, `jq -e`. + kind: str + #: The line it was found in, stripped, as one line. + text: str + + +def _asserts(script: str) -> list[Unread]: + """`assert` statements, found on physical lines. + + Physical rather than logical lines on purpose. `shell.logical_lines` + models a shell script, and an embedded Python program is not one: the + `python3 -c "` that opens it leaves a quote hanging that the shell + tokeniser would have to track across the whole body to get right. The + line-shape test below needs none of that, and it cannot be fooled into + reporting the wrong line number. + """ + found: list[Unread] = [] + for number, raw in enumerate(script.split("\n"), 1): + text = raw.strip() + if not text or text.startswith(_COMMENT): + continue + if _ASSERT_LINE.match(text) or _ASSERT_INLINE.search(text): + found.append(Unread(line=number, kind="assert", text=text)) + return found + + +def _jq_asserts(words: list[str]) -> bool: + """Whether a `jq` argv asks for `--exit-status`. + + Plain `jq -r .x` is extraction and its status says only that the JSON + parsed. `-e` is the one that makes it a verdict about the value. + """ + for word in words[1:]: + if word == "--exit-status": + return True + if word.startswith("--"): + continue + if word.startswith("-") and len(word) > 1 and "e" in word[1:]: + return True + return False + + +def _simple_commands(text: str) -> list[tuple[int, str]]: + """Every simple command in a logical line, with its offset in that line. + + `shell.pipeline_parts` splits on `|` alone, which is the right split for + deciding whether a matcher's exit status survives its position. This + module declines to decide that (see `_commands`) and wants something + different: every place a command *starts*, so that the `diff` in + `if diff -q a b; then ...` is found at all. So it splits on every + operator and then steps over any shell keyword in front of the command. + """ + out: list[tuple[int, str]] = [] + begin = 0 + quote = None + index = 0 + + def emit(start: int, end: int) -> None: + chunk = text[start:end] + while True: + match = _KEYWORD.match(chunk) + if not match: + break + start += match.end() + chunk = text[start:end] + if chunk.strip(): + out.append((start, chunk)) + + while index < len(text): + ch = text[index] + if quote: + if ch == quote: + quote = None + index += 1 + continue + if ch in "'\"": + quote = ch + index += 1 + continue + for operator in _OPERATORS: + if text.startswith(operator, index): + emit(begin, index) + index += len(operator) + begin = index + break + else: + index += 1 + emit(begin, len(text)) + return out + + +def _commands(script: str) -> list[Unread]: + """`diff`, `cmp` and `jq -e` commands whose failure is not discarded. + + Two exclusions, and only two, because this is a disclosure and not a + verdict: + + * inside `$(...)` or backticks -- a capture, feeding a variable, exactly + as in `shell.scan`; + * `|| true` on the logical line -- the failure is thrown away, so there + is no assertion here to have missed. + + Everything else is counted, including a `diff` partway down a pipeline. + Working out whether such a status reaches the step is the analysis this + module is explicitly not doing, and guessing it away would reintroduce + the silence the module exists to break. + """ + found: list[Unread] = [] + programs = DIFFERS + ("jq",) + for lineno, text in shell.logical_lines(script): + if not any(name in text for name in programs): + continue + if shell.swallows(text): + continue + for offset, segment in _simple_commands(text): + head = shell.command_head(segment, programs) + if head is None: + continue + words = shell.split(head[0]) + if words is None: + continue + if shell.in_substitution(text, offset): + continue + if words[0].split("/")[-1] == "jq": + if _jq_asserts(words): + found.append(Unread(line=lineno, kind="jq -e", text=text)) + else: + found.append(Unread(line=lineno, kind="diff", text=text)) + return found + + +def scan(script: str) -> list[Unread]: + """Every assertion-shaped construct in `script` that `shell.scan` ignores. + + Sorted by line, so the report reads in file order. + """ + return sorted(_asserts(script) + _commands(script), key=lambda item: item.line) diff --git a/tests/test_unread.py b/tests/test_unread.py new file mode 100644 index 0000000..090f338 --- /dev/null +++ b/tests/test_unread.py @@ -0,0 +1,170 @@ +"""What counts as an assertion this tool does not read, and what does not. + +Every negative test in here is a false positive the first version produced or +would have produced. A count that is wrong in the generous direction is worse +than no count at all: the whole point of `logbook#69` is that the number be +trustworthy enough that a non-zero one sends you to look. +""" + +from __future__ import annotations + +from deadassert import unread + + +def kinds(script: str) -> list[str]: + return [item.kind for item in unread.scan(script)] + + +def one(script: str) -> unread.Unread: + found = unread.scan(script) + assert len(found) == 1, f"expected one construct, got {found}" + return found[0] + + +class TestPythonAsserts: + def test_the_logbook_69_instance(self): + # Verbatim from stockbox's dogfood job, which is what this exists for: + # an assertion whose truth depends on the runner image, in the same + # repository where deadassert reported that defect in a grep, and + # invisible to it. + script = """ +stockbox roots -c loose.toml --json > loose.json +python3 -c " +import json +tight = json.load(open('trees.json'))['shapes'][0]['candidates'] +loose = json.load(open('loose.json'))['shapes'][0]['candidates'] +assert loose > tight * 5, (loose, tight) +" +""" + found = one(script) + assert found.kind == "assert" + assert found.text == "assert loose > tight * 5, (loose, tight)" + assert found.line == 7 + + def test_a_heredoc_body(self): + script = """ +set -euo pipefail +python - <<'PY' +import json +d = json.load(open("out.json")) +assert d["assertions"] >= 2, f"only {d['assertions']} seen" +assert d["findings"] == [], d["findings"] +PY +""" + assert kinds(script) == ["assert", "assert"] + + def test_an_indented_assert_inside_a_function(self): + assert kinds("python3 - <<'PY'\ndef check(d):\n assert d, d\ncheck({})\nPY") == ["assert"] + + def test_a_one_liner(self): + assert kinds("python3 -c 'assert 1 == 1'") == ["assert"] + + def test_a_node_assert(self): + assert kinds("node -e 'assert.ok(ran)'") == ["assert"] + + def test_a_bracketed_assert(self): + assert kinds("python3 - <<'PY'\nassert(x)\nPY") == ["assert"] + + +class TestNotAnAssert: + def test_a_commented_out_assert(self): + assert kinds("python3 - <<'PY'\n# assert total > 0\nprint(1)\nPY") == [] + + def test_a_commented_out_assert_in_node(self): + assert kinds("node -e 'x'\n// assert.ok(x)") == [] + + def test_a_method_whose_name_merely_starts_with_assert(self): + # `assert_called_once()` is a call, not a statement, and a tool that + # counted it would put a number on every mock-using repo in the org. + assert kinds("python3 - <<'PY'\nassert_called_once()\nPY") == [] + + def test_the_name_of_this_package_is_not_an_assert(self): + # `\\bassert\\b` and not a substring: `import deadassert` sits on a + # line with an interpreter on it and must not match. + assert kinds("python3 -c 'import deadassert; print(deadassert.__file__)'") == [] + + def test_the_word_assertions_in_prose(self): + assert kinds("python3 -m pytest && echo 'all assertions ran'") == [] + + def test_a_shell_line_mentioning_assert(self): + assert kinds("echo 'assert something' > note.txt") == [] + + +class TestDiff: + def test_a_plain_diff(self): + found = one("mytool . > got.txt\ndiff -u expected.txt got.txt") + assert found.kind == "diff" + assert found.line == 2 + + def test_cmp_counts_too(self): + assert kinds("cmp -s a.bin b.bin") == ["diff"] + + def test_a_diff_in_an_if_condition(self): + assert kinds("if diff -q a b > /dev/null; then echo same; fi") == ["diff"] + + def test_a_redirected_diff_still_counts(self): + # Under `bash -e` the status reaches the step whatever happens to + # stdout, so this is an assertion and not extraction. + assert kinds("diff -u a.txt b.txt > changes.diff") == ["diff"] + + def test_a_swallowed_diff_does_not_count(self): + # The failure is thrown away, so there is no assertion here to have + # missed reading. + assert kinds("diff -u a.txt b.txt || true") == [] + + def test_a_captured_diff_does_not_count(self): + # Feeding a variable, exactly as `shell.scan` treats a captured grep. + assert kinds('changed=$(diff -u a.txt b.txt); echo "$changed"') == [] + + def test_a_filename_containing_diff_does_not_count(self): + assert kinds("cat expected.diff") == [] + + +class TestJq: + def test_jq_e_counts(self): + found = one("curl -sS $url | jq -e '.ok' > /dev/null") + assert found.kind == "jq -e" + + def test_a_combined_short_flag(self): + assert kinds("jq -er '.permalink' < resp.json") == ["jq -e"] + + def test_the_long_flag(self): + assert kinds("jq --exit-status '.ok' resp.json") == ["jq -e"] + + def test_plain_jq_is_extraction(self): + # Without -e the status says only that the JSON parsed, which is not + # a verdict about the value and not what this is counting. + assert kinds("jq -r '.items[].name' resp.json") == [] + + def test_an_arg_named_e_is_not_the_flag(self): + assert kinds("jq --arg e 1 '.x' resp.json") == [] + + +class TestDeliberateOmissions: + def test_test_is_not_counted(self): + # The caveat on logbook#69: `test "$s" -eq N` is the commonest + # assertion in the org and almost always sound, so counting it would + # put a number on nearly every file and the number would stop being + # read. Documented in the README rather than silently dropped. + assert kinds('mytool .; s=$?\ntest "$s" -eq 1') == [] + + def test_bracket_is_not_counted(self): + assert kinds('[ -f out.txt ] || exit 1') == [] + + def test_a_grep_is_not_counted_here(self): + # `shell.scan` reads these, so counting them again would double-count + # the only assertions the tool does understand. + assert kinds("grep -qxF 'All clear.' out.txt") == [] + + +class TestOrdering: + def test_constructs_come_back_in_line_order(self): + script = """ +diff -u a.txt b.txt +python3 - <<'PY' +assert 1 +PY +jq -e '.ok' r.json +""" + assert [item.line for item in unread.scan(script)] == [2, 4, 6] + assert kinds(script) == ["diff", "assert", "jq -e"] From 6d44b14e7369c68adf3bb2dc20b2bd9769a087bf Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:42:39 +0000 Subject: [PATCH 2/3] feat: say what was not read in the report, and in --json Three places the verdict could overclaim, in rising order of how wrong it was: * the denominator, which now carries `Not counted above: N assertions in M of those scripts`; * `--json`, which gains `unread` and `unread_scripts`; * `No assertions here ... Nothing to check.`, which was the broadest false statement the tool made. `deadassert uncollected` said it over a CI file holding nine Python asserts, and `whichhook` and `unapplied` got the same answer. That path now says what it does not read instead. Not findings and not in the exit code: a repository asserting entirely in `diff` is doing the thing this README recommends, and the count is coverage, not a complaint. `-q` is unchanged, since it is a list of findings for another program to read. The worked example gains the two shapes so the shipped output demonstrates the block, and this repo's dogfood job now asserts the count -- it cannot be vacuous, because the lines it describes are in the file it reads. --- .github/workflows/ci.yml | 22 ++++++++- examples/.github/workflows/ci.yml | 20 ++++++++ examples/expected-against.txt | 6 ++- src/deadassert/report.py | 22 ++++----- tests/test_cli.py | 76 +++++++++++++++++++++++++++++++ tests/test_unread.py | 6 ++- 6 files changed, 135 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cc3414..69561a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,16 +159,34 @@ jobs: # The richer corpus is `examples/`, which holds one of every kind and is # checked by the `example` job above. What gives *this* step its teeth # is the mutation below, not the size of the number. - - name: and it was measured over something + # + # logbook#69 is the other half of that admission, and it is now the + # tool's own job rather than this comment's: almost every real + # assertion in this file is a Python `assert` or a `diff`, deadassert + # reads neither, and the `unread` count is it saying so. The assertions + # below cannot be vacuous -- the lines they describe are in this file. + - name: and it was measured over something, and it says what it missed run: | set -euo pipefail deadassert . --json > out.json python - <<'PY' import json d = json.load(open("out.json")) - print(f"{d['assertions']} assertions in {d['scripts']} scripts") + print(f"{d['assertions']} assertions in {d['scripts']} scripts, " + f"{len(d['unread'])} not read in {d['unread_scripts']} of them") assert d["assertions"] >= 2, f"only {d['assertions']} assertions seen" assert d["findings"] == [], d["findings"] + + kinds = {x["kind"] for x in d["unread"]} + assert "assert" in kinds, kinds + assert "diff" in kinds, kinds + assert d["unread_scripts"] >= 4, d["unread_scripts"] + + # Anchored, not a substring: this is a repository about assertions + # that match less than they look like. + commands = [x["command"] for x in d["unread"]] + wanted = "diff -u examples/expected-quiet.txt" + assert any(c.startswith(wanted) for c in commands), commands PY # The assertion above is only worth anything if it can go red. Break diff --git a/examples/.github/workflows/ci.yml b/examples/.github/workflows/ci.yml index d0f5dfb..9601df4 100644 --- a/examples/.github/workflows/ci.yml +++ b/examples/.github/workflows/ci.yml @@ -77,3 +77,23 @@ jobs: countdead examples/ > /dev/null; test $? -eq 1 countdead /nonexistent > /dev/null 2>&1; test $? -eq 2 countdead examples/ -q | grep -qxF 'examples/a.txt:3: never-read' + + # The two assertions deadassert does not read, and the reason the run + # ends with a `not read:` block instead of implying the four findings + # above are the whole story (logbook#69). + # + # The `assert` decides the step and can rot exactly as a grep can, and + # a tool that reads matchers sees nothing here at all. The `diff` is + # the shape this project's own README recommends -- counting it is not + # a complaint, it is the tool saying how much of this file its four + # findings were actually measured over. + - name: the json mode and the porcelain mode agree + run: | + countdead examples/ --json > out.json || test $? -le 1 + python3 -c " + import json + d = json.load(open('out.json')) + assert d['dead'] == 2, d['dead'] + " + countdead examples/ --porcelain > got.txt || test $? -le 1 + diff -u examples/expected-porcelain.txt got.txt diff --git a/examples/expected-against.txt b/examples/expected-against.txt index 38fd49a..45839fe 100644 --- a/examples/expected-against.txt +++ b/examples/expected-against.txt @@ -22,4 +22,8 @@ ci.yml:67 decorative 'skipped' if grep -qF 'skipped' seen.txt; then 4 assertions here do not check what they read as checking (1 unwitnessed, 1 prefix, 1 swallowed, 1 decorative). -7 assertions in 6 scripts across 1 workflow file, 1 of them negated. +7 assertions in 7 scripts across 1 workflow file, 1 of them negated. Not counted above: 2 assertions in 1 of those scripts (1 assert, 1 diff), listed below. + +not read: + ci.yml:96 assert assert d['dead'] == 2, d['dead'] + ci.yml:99 diff diff -u examples/expected-porcelain.txt got.txt diff --git a/src/deadassert/report.py b/src/deadassert/report.py index fb32577..73ebf84 100644 --- a/src/deadassert/report.py +++ b/src/deadassert/report.py @@ -33,20 +33,18 @@ def _unread_tally(result: Result) -> str: return ", ".join(f"{count} {kind}" for kind, count in ordered) -def _coverage(result: Result) -> str: - """The sentence that stops a clean run reading as a complete one. +def _gap(result: Result) -> str: + """`9 assertions in 1 of those scripts (9 assert)` -- the size of the gap. logbook#69. Everything above a denominator is about the assertions this - tool *read*, and until this sentence existed there was nothing to say - that a script could also assert in a way it had not read -- which is how - a sweep for dead assertions walked past one in the repository it was - auditing. Not a finding and not in the exit code; `unread.py` evaluates - nothing. It is the size of the gap, stated. + tool *read*, and until this existed there was nothing to say that a + script could also assert in a way it had not read -- which is how a sweep + for dead assertions walked past one in the repository it was auditing. + Not a finding and not in the exit code; `unread.py` evaluates nothing. """ return ( - f"Not counted above: {_plural(len(result.unread), 'assertion')} this does " - f"not read, in {result.unread_scripts} of those scripts " - f"({_unread_tally(result)}). Listed below." + f"{_plural(len(result.unread), 'assertion')} in " + f"{result.unread_scripts} of those scripts ({_unread_tally(result)})" ) @@ -64,7 +62,7 @@ def _denominator(result: Result) -> str: f"{result.negated} of them negated." ) if result.unread: - return f"{line} {_coverage(result)}" + return f"{line} Not counted above: {_gap(result)}, listed below." return line @@ -82,7 +80,7 @@ def nothing_to_check(result: Result) -> str: "exit status decides a step." ) if result.unread: - return f"{read} But {_coverage(result)}" + return f"{read} What it does not read: {_gap(result)}, listed below." return f"{read} Nothing to check." diff --git a/tests/test_cli.py b/tests/test_cli.py index ace9e9d..e894316 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -84,6 +84,82 @@ def test_a_workflow_with_no_assertions_does_not_claim_all_clear(self, capsys, tm assert "Every assertion here fails" not in out +class TestNotRead: + """logbook#69: a clean run over the greps is not a clean run.""" + + DIFFS = "mytool . > got.txt\ndiff -u expected.txt got.txt" + + def test_a_repo_that_asserts_only_with_diff_no_longer_claims_nothing_to_check( + self, capsys, tmp_path + ): + # The broadest false statement the tool made. `diff` is what this + # project's own README tells people to prefer over `grep`, so a repo + # that took the advice got `Nothing to check.` over a file full of + # assertions. + root = repo(tmp_path, self.DIFFS) + code, out, _ = run(capsys, root) + assert "No assertions here." in out + assert "Nothing to check." not in out + assert "What it does not read: 1 assertion in 1 of those scripts (1 diff)" in out + assert "diff -u expected.txt got.txt" in out + # Still not a finding, so still not a failure. + assert code == 0 + + def test_the_clean_verdict_says_what_it_did_not_read(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt", self.DIFFS) + code, out, _ = run(capsys, root) + assert code == 0 + assert "Every assertion here fails when the thing it names stops being true." in out + assert "1 assertion in 2 scripts across 1 workflow file, 0 of them negated." in out + assert "Not counted above: 1 assertion in 1 of those scripts (1 diff)" in out + + def test_the_logbook_69_shape_is_named_in_a_run_that_has_findings(self, capsys, tmp_path): + root = repo( + tmp_path, + "if grep -qF 'kind-x' out.txt; then\n exit 1\nfi", + "python3 -c \"\nimport json\nassert json.load(open('o.json'))['n'] > 5\n\"", + ) + code, out, _ = run(capsys, root) + assert code == 1 + assert "unwitnessed" in out + assert "not read:" in out + assert "assert json.load(open('o.json'))['n'] > 5" in out + + def test_a_repo_with_nothing_unread_says_nothing_about_it(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt") + _, out, _ = run(capsys, root) + assert "not read:" not in out + assert "Not counted above" not in out + + def test_quiet_stays_one_line_per_finding(self, capsys, tmp_path): + # `not read:` is a coverage note, not a finding, and -q is for + # feeding a list of findings to something else. + root = repo(tmp_path, "if grep -qF 'kind-x' o.txt; then\n exit 1\nfi", self.DIFFS) + _, out, _ = run(capsys, root, "-q") + assert out.strip().splitlines() == [".github/workflows/ci.yml:7: unwitnessed: 'kind-x'"] + + def test_json_carries_the_constructs_and_the_script_count(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt", self.DIFFS) + code, out, _ = run(capsys, root, "--json") + payload = json.loads(out) + assert code == 0 + assert payload["findings"] == [] + assert payload["unread_scripts"] == 1 + assert payload["unread"] == [ + { + "where": "ci.yml:10", + "kind": "diff", + "command": "diff -u expected.txt got.txt", + } + ] + + def test_the_line_number_points_at_the_construct(self, capsys, tmp_path): + root = repo(tmp_path, "echo one\necho two\ndiff -u a.txt b.txt") + _, out, _ = run(capsys, root, "--json") + # `run: |` on line 6, so the script's own lines start at 7. + assert json.loads(out)["unread"][0]["where"] == "ci.yml:9" + + class TestDenominator: """logbook#59 and logbook#57: a verdict needs what it was measured over.""" diff --git a/tests/test_unread.py b/tests/test_unread.py index 090f338..ed27796 100644 --- a/tests/test_unread.py +++ b/tests/test_unread.py @@ -54,7 +54,9 @@ def test_a_heredoc_body(self): assert kinds(script) == ["assert", "assert"] def test_an_indented_assert_inside_a_function(self): - assert kinds("python3 - <<'PY'\ndef check(d):\n assert d, d\ncheck({})\nPY") == ["assert"] + assert kinds("python3 - <<'PY'\ndef check(d):\n assert d, d\ncheck({})\nPY") == [ + "assert" + ] def test_a_one_liner(self): assert kinds("python3 -c 'assert 1 == 1'") == ["assert"] @@ -149,7 +151,7 @@ def test_test_is_not_counted(self): assert kinds('mytool .; s=$?\ntest "$s" -eq 1') == [] def test_bracket_is_not_counted(self): - assert kinds('[ -f out.txt ] || exit 1') == [] + assert kinds("[ -f out.txt ] || exit 1") == [] def test_a_grep_is_not_counted_here(self): # `shell.scan` reads these, so counting them again would double-count From 909395027cd91bb14df7e42228126aee57ea4b3b Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:44:00 +0000 Subject: [PATCH 3/3] docs: the README said invisible, and it is now counted The "What it will not tell you" list already admitted that a verdict reached by `diff`, `test` or a language's own assert was invisible to this tool, and recommended preferring those -- advice that quietly shrank what its own clean runs meant. logbook#69 is that admission turning into a number. Re-measured over the org: 170 assertions read against 74 not read, 9 findings down from 33 after the sweep. The not-read row is the one that changes how the others should be read, so it says so. --- README.md | 129 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 8d8a53d..0982c47 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ stopped saying that, this tells you. It reads the `run:` scripts in `.github/workflows`, works out which way round each assertion fails, and reports the ones that cannot fail at all. +It also counts the assertions it did *not* read — a `diff`, a `jq -e`, a +Python `assert` — so that a clean run says what it covered rather than +implying it covered everything. + ## Install ```sh @@ -90,9 +94,17 @@ ci.yml:67 decorative 'skipped' if grep -qF 'skipped' seen.txt; then 4 assertions here do not check what they read as checking (1 unwitnessed, 1 prefix, 1 swallowed, 1 decorative). -7 assertions in 6 scripts across 1 workflow file, 1 of them negated. +7 assertions in 7 scripts across 1 workflow file, 1 of them negated. Not counted above: 2 assertions in 1 of those scripts (1 assert, 1 diff), listed below. + +not read: + ci.yml:96 assert assert d['dead'] == 2, d['dead'] + ci.yml:99 diff diff -u examples/expected-porcelain.txt got.txt ``` +The last two lines are not findings — they are the two assertions in that +file `deadassert` never read, and the reason the four above them are a +verdict about the greps rather than about the file. See `not read:` below. + ## The kinds | kind | what it means | @@ -103,6 +115,11 @@ ci.yml:67 decorative 'skipped' | `masked` | its exit status is thrown away by its position in a pipeline | | `decorative` | it picks a branch, and neither branch can fail the step | +`not read` is not in that table on purpose. The five kinds are verdicts on +an assertion; `not read` is the admission that there was an assertion there +and no verdict was reached. It is never a finding and never changes the exit +code. + ## `--against`, and the one guess in here `prefix` is the only kind that rests on what somebody *meant*. `grep -qF 'x'` @@ -144,13 +161,80 @@ stops being reported. is a real assertion, but the characters in the file are not the characters grep gets, so it goes in `not decided:` and is left out of the counts. - **It only knows about matchers.** `grep`, `egrep`, `fgrep`, `rg`. A - verdict reached by `diff`, `test` or a language's own assert is invisible - to it — which is a reason to prefer those, not a gap to fill. See below. + verdict reached by `diff`, `test` or a language's own assert is not read — + which is a reason to prefer those, not a gap to fill. See below. It is no + longer *invisible*, though: those get counted, and the count is printed. - **`unwitnessed` is a smell, not a proof.** It says nothing in *these workflows* establishes the string can appear. The proof may well live in a unit test, where it is arguably better placed. What the finding is really telling you is that the CI step, read on its own, does not stand up. +## `not read:` — the assertions it did not read + +A clean run used to read as *the assertions in this file can fail*. It only +ever meant *the greps in this file can fail*, and the difference is not +academic. On the night `deadassert` was swept across this organisation +looking for dead assertions, it walked past one — in the repository it was +auditing, two steps below a finding it did report: + +```sh +python3 -c " +loose = json.load(open('loose.json'))['shapes'][0]['candidates'] +assert loose > tight * 5, (loose, tight) +" +``` + +An assertion whose truth depends on the runner image rather than on the +thing it names. Same defect, different punctuation, and the run said `2 +findings` over `32 assertions in 20 scripts` without a word to say that one +of those 20 scripts asserted something it had not read. + +So it counts them now: + +```console +$ deadassert . +Every assertion here fails when the thing it names stops being true. +2 assertions in 15 scripts across 1 workflow file, 0 of them negated. Not counted above: 11 assertions in 5 of those scripts (9 assert, 2 diff), listed below. + +not read: + ci.yml:37 assert assert total >= 110, f"only {total} tests collected" + ci.yml:85 diff diff -u examples/expected-quiet.txt got.txt + ... +``` + +Three shapes, picked by counting what the workflow files in this +organisation actually use: a language's own `assert` (nearly always Python, +via `-c` or a heredoc), `diff` and `cmp`, and `jq -e`. + +**This decides nothing.** It does not judge whether a `diff` can fail; it +reports that one is there and was not read. The same move the tool already +makes with `not decided:` and with the non-POSIX shells it skips — and for +the same reason, which is that a tool about assertions that quietly check +nothing has no business quietly covering nothing. It is coverage, not a +complaint: `diff` is what the section below tells you to prefer, and a +repository full of them gets a count, not a finding. Nothing here affects +the exit code, and `-q` still prints findings only. + +The sharpest case was the verdict over no greps at all. `deadassert +uncollected` used to print: + +``` +No assertions here. Read 12 run: scripts in 1 workflow file and found no +command whose exit status decides a step. Nothing to check. +``` + +over a CI file holding nine Python asserts. Two other repositories in this +organisation got the same answer. + +**`test` is deliberately not counted.** `test "$s" -eq 0` and `[ -f out ]` +are the most common assertions in these files and are usually the soundest +thing in them — the status *is* the assertion, with no pattern to rot. A +count that included them would be non-zero nearly everywhere on day one and +would stop being read, which is the one failure mode that would make this +worse than silence. The cost is real and this is where it is written down: a +`test` on a status captured from the wrong command is not covered here +either. + ## `grep` is a poor way to assert a whole output Having written this, the conclusion it kept pushing me towards is that most @@ -170,22 +254,37 @@ is small on purpose, and it is printed rather than hidden. ## Measured on something real -Run across the 30 repositories of the organisation this was written in — 38 -workflow files, 337 `run:` scripts: +Run across the 31 repositories of the organisation this was written in — 37 +workflow files, 360 `run:` scripts: | | count | | --- | --- | -| assertions found | 157 | -| negated (`needs-no-match`) | 22 | -| `unwitnessed` | 12 | -| `prefix` | 21 | +| assertions read | 170 | +| negated (`needs-no-match`) | 24 | +| `unwitnessed` | 6 | +| `prefix` | 3 | | `swallowed` / `masked` / `decorative` | 0 | - -Twelve of the thirty repositories have at least one finding. The three -can't-fail kinds scoring zero is the result I did not expect: every `|| true` -in that organisation is on a *tool invocation*, deliberately allowing a -non-zero findings exit before the output is asserted, which is correct usage. -An early version of this tool reported all of them and was wrong to. +| not read | 74, in 24 scripts | + +Seven of the thirty-one repositories have at least one finding. The first +run of this tool found 33 — 12 `unwitnessed` and 21 `prefix`, across 12 +repositories — and the 24 that have gone are a repo-at-a-time sweep, not a +`sed`; the remaining 9 are the tail of it. + +The three can't-fail kinds scoring zero is the result I did not expect: +every `|| true` in that organisation is on a *tool invocation*, deliberately +allowing a non-zero findings exit before the output is asserted, which is +correct usage. An early version of this tool reported all of them and was +wrong to. + +The last row is the one that changes how you should read the others. **74 +assertions in these files are not read at all** — 67 Python asserts, 5 +`diff`, 2 `jq -e` — against 170 that are, and they are concentrated: 15 +repositories have at least one, and 3 have nothing *but* unread assertions, +so the verdict on them used to be `Nothing to check.` The 9 findings above +are a verdict about the greps in this organisation, which is a smaller thing +than it looks, and the point of printing the number is that you no longer +have to take my word for how much smaller. `shellcheck` 0.9.0 reports none of these at default settings. With `--enable=all` it finds the two pipeline cases as `SC2312` (info) and