diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cc3414..69561a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,16 +159,34 @@ jobs: # The richer corpus is `examples/`, which holds one of every kind and is # checked by the `example` job above. What gives *this* step its teeth # is the mutation below, not the size of the number. - - name: and it was measured over something + # + # logbook#69 is the other half of that admission, and it is now the + # tool's own job rather than this comment's: almost every real + # assertion in this file is a Python `assert` or a `diff`, deadassert + # reads neither, and the `unread` count is it saying so. The assertions + # below cannot be vacuous -- the lines they describe are in this file. + - name: and it was measured over something, and it says what it missed run: | set -euo pipefail deadassert . --json > out.json python - <<'PY' import json d = json.load(open("out.json")) - print(f"{d['assertions']} assertions in {d['scripts']} scripts") + print(f"{d['assertions']} assertions in {d['scripts']} scripts, " + f"{len(d['unread'])} not read in {d['unread_scripts']} of them") assert d["assertions"] >= 2, f"only {d['assertions']} assertions seen" assert d["findings"] == [], d["findings"] + + kinds = {x["kind"] for x in d["unread"]} + assert "assert" in kinds, kinds + assert "diff" in kinds, kinds + assert d["unread_scripts"] >= 4, d["unread_scripts"] + + # Anchored, not a substring: this is a repository about assertions + # that match less than they look like. + commands = [x["command"] for x in d["unread"]] + wanted = "diff -u examples/expected-quiet.txt" + assert any(c.startswith(wanted) for c in commands), commands PY # The assertion above is only worth anything if it can go red. Break diff --git a/README.md b/README.md index 8d8a53d..0982c47 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ stopped saying that, this tells you. It reads the `run:` scripts in `.github/workflows`, works out which way round each assertion fails, and reports the ones that cannot fail at all. +It also counts the assertions it did *not* read — a `diff`, a `jq -e`, a +Python `assert` — so that a clean run says what it covered rather than +implying it covered everything. + ## Install ```sh @@ -90,9 +94,17 @@ ci.yml:67 decorative 'skipped' if grep -qF 'skipped' seen.txt; then 4 assertions here do not check what they read as checking (1 unwitnessed, 1 prefix, 1 swallowed, 1 decorative). -7 assertions in 6 scripts across 1 workflow file, 1 of them negated. +7 assertions in 7 scripts across 1 workflow file, 1 of them negated. Not counted above: 2 assertions in 1 of those scripts (1 assert, 1 diff), listed below. + +not read: + ci.yml:96 assert assert d['dead'] == 2, d['dead'] + ci.yml:99 diff diff -u examples/expected-porcelain.txt got.txt ``` +The last two lines are not findings — they are the two assertions in that +file `deadassert` never read, and the reason the four above them are a +verdict about the greps rather than about the file. See `not read:` below. + ## The kinds | kind | what it means | @@ -103,6 +115,11 @@ ci.yml:67 decorative 'skipped' | `masked` | its exit status is thrown away by its position in a pipeline | | `decorative` | it picks a branch, and neither branch can fail the step | +`not read` is not in that table on purpose. The five kinds are verdicts on +an assertion; `not read` is the admission that there was an assertion there +and no verdict was reached. It is never a finding and never changes the exit +code. + ## `--against`, and the one guess in here `prefix` is the only kind that rests on what somebody *meant*. `grep -qF 'x'` @@ -144,13 +161,80 @@ stops being reported. is a real assertion, but the characters in the file are not the characters grep gets, so it goes in `not decided:` and is left out of the counts. - **It only knows about matchers.** `grep`, `egrep`, `fgrep`, `rg`. A - verdict reached by `diff`, `test` or a language's own assert is invisible - to it — which is a reason to prefer those, not a gap to fill. See below. + verdict reached by `diff`, `test` or a language's own assert is not read — + which is a reason to prefer those, not a gap to fill. See below. It is no + longer *invisible*, though: those get counted, and the count is printed. - **`unwitnessed` is a smell, not a proof.** It says nothing in *these workflows* establishes the string can appear. The proof may well live in a unit test, where it is arguably better placed. What the finding is really telling you is that the CI step, read on its own, does not stand up. +## `not read:` — the assertions it did not read + +A clean run used to read as *the assertions in this file can fail*. It only +ever meant *the greps in this file can fail*, and the difference is not +academic. On the night `deadassert` was swept across this organisation +looking for dead assertions, it walked past one — in the repository it was +auditing, two steps below a finding it did report: + +```sh +python3 -c " +loose = json.load(open('loose.json'))['shapes'][0]['candidates'] +assert loose > tight * 5, (loose, tight) +" +``` + +An assertion whose truth depends on the runner image rather than on the +thing it names. Same defect, different punctuation, and the run said `2 +findings` over `32 assertions in 20 scripts` without a word to say that one +of those 20 scripts asserted something it had not read. + +So it counts them now: + +```console +$ deadassert . +Every assertion here fails when the thing it names stops being true. +2 assertions in 15 scripts across 1 workflow file, 0 of them negated. Not counted above: 11 assertions in 5 of those scripts (9 assert, 2 diff), listed below. + +not read: + ci.yml:37 assert assert total >= 110, f"only {total} tests collected" + ci.yml:85 diff diff -u examples/expected-quiet.txt got.txt + ... +``` + +Three shapes, picked by counting what the workflow files in this +organisation actually use: a language's own `assert` (nearly always Python, +via `-c` or a heredoc), `diff` and `cmp`, and `jq -e`. + +**This decides nothing.** It does not judge whether a `diff` can fail; it +reports that one is there and was not read. The same move the tool already +makes with `not decided:` and with the non-POSIX shells it skips — and for +the same reason, which is that a tool about assertions that quietly check +nothing has no business quietly covering nothing. It is coverage, not a +complaint: `diff` is what the section below tells you to prefer, and a +repository full of them gets a count, not a finding. Nothing here affects +the exit code, and `-q` still prints findings only. + +The sharpest case was the verdict over no greps at all. `deadassert +uncollected` used to print: + +``` +No assertions here. Read 12 run: scripts in 1 workflow file and found no +command whose exit status decides a step. Nothing to check. +``` + +over a CI file holding nine Python asserts. Two other repositories in this +organisation got the same answer. + +**`test` is deliberately not counted.** `test "$s" -eq 0` and `[ -f out ]` +are the most common assertions in these files and are usually the soundest +thing in them — the status *is* the assertion, with no pattern to rot. A +count that included them would be non-zero nearly everywhere on day one and +would stop being read, which is the one failure mode that would make this +worse than silence. The cost is real and this is where it is written down: a +`test` on a status captured from the wrong command is not covered here +either. + ## `grep` is a poor way to assert a whole output Having written this, the conclusion it kept pushing me towards is that most @@ -170,22 +254,37 @@ is small on purpose, and it is printed rather than hidden. ## Measured on something real -Run across the 30 repositories of the organisation this was written in — 38 -workflow files, 337 `run:` scripts: +Run across the 31 repositories of the organisation this was written in — 37 +workflow files, 360 `run:` scripts: | | count | | --- | --- | -| assertions found | 157 | -| negated (`needs-no-match`) | 22 | -| `unwitnessed` | 12 | -| `prefix` | 21 | +| assertions read | 170 | +| negated (`needs-no-match`) | 24 | +| `unwitnessed` | 6 | +| `prefix` | 3 | | `swallowed` / `masked` / `decorative` | 0 | - -Twelve of the thirty repositories have at least one finding. The three -can't-fail kinds scoring zero is the result I did not expect: every `|| true` -in that organisation is on a *tool invocation*, deliberately allowing a -non-zero findings exit before the output is asserted, which is correct usage. -An early version of this tool reported all of them and was wrong to. +| not read | 74, in 24 scripts | + +Seven of the thirty-one repositories have at least one finding. The first +run of this tool found 33 — 12 `unwitnessed` and 21 `prefix`, across 12 +repositories — and the 24 that have gone are a repo-at-a-time sweep, not a +`sed`; the remaining 9 are the tail of it. + +The three can't-fail kinds scoring zero is the result I did not expect: +every `|| true` in that organisation is on a *tool invocation*, deliberately +allowing a non-zero findings exit before the output is asserted, which is +correct usage. An early version of this tool reported all of them and was +wrong to. + +The last row is the one that changes how you should read the others. **74 +assertions in these files are not read at all** — 67 Python asserts, 5 +`diff`, 2 `jq -e` — against 170 that are, and they are concentrated: 15 +repositories have at least one, and 3 have nothing *but* unread assertions, +so the verdict on them used to be `Nothing to check.` The 9 findings above +are a verdict about the greps in this organisation, which is a smaller thing +than it looks, and the point of printing the number is that you no longer +have to take my word for how much smaller. `shellcheck` 0.9.0 reports none of these at default settings. With `--enable=all` it finds the two pipeline cases as `SC2312` (info) and diff --git a/examples/.github/workflows/ci.yml b/examples/.github/workflows/ci.yml index d0f5dfb..9601df4 100644 --- a/examples/.github/workflows/ci.yml +++ b/examples/.github/workflows/ci.yml @@ -77,3 +77,23 @@ jobs: countdead examples/ > /dev/null; test $? -eq 1 countdead /nonexistent > /dev/null 2>&1; test $? -eq 2 countdead examples/ -q | grep -qxF 'examples/a.txt:3: never-read' + + # The two assertions deadassert does not read, and the reason the run + # ends with a `not read:` block instead of implying the four findings + # above are the whole story (logbook#69). + # + # The `assert` decides the step and can rot exactly as a grep can, and + # a tool that reads matchers sees nothing here at all. The `diff` is + # the shape this project's own README recommends -- counting it is not + # a complaint, it is the tool saying how much of this file its four + # findings were actually measured over. + - name: the json mode and the porcelain mode agree + run: | + countdead examples/ --json > out.json || test $? -le 1 + python3 -c " + import json + d = json.load(open('out.json')) + assert d['dead'] == 2, d['dead'] + " + countdead examples/ --porcelain > got.txt || test $? -le 1 + diff -u examples/expected-porcelain.txt got.txt diff --git a/examples/expected-against.txt b/examples/expected-against.txt index 38fd49a..45839fe 100644 --- a/examples/expected-against.txt +++ b/examples/expected-against.txt @@ -22,4 +22,8 @@ ci.yml:67 decorative 'skipped' if grep -qF 'skipped' seen.txt; then 4 assertions here do not check what they read as checking (1 unwitnessed, 1 prefix, 1 swallowed, 1 decorative). -7 assertions in 6 scripts across 1 workflow file, 1 of them negated. +7 assertions in 7 scripts across 1 workflow file, 1 of them negated. Not counted above: 2 assertions in 1 of those scripts (1 assert, 1 diff), listed below. + +not read: + ci.yml:96 assert assert d['dead'] == 2, d['dead'] + ci.yml:99 diff diff -u examples/expected-porcelain.txt got.txt diff --git a/src/deadassert/analyse.py b/src/deadassert/analyse.py index 1a33355..7363f4e 100644 --- a/src/deadassert/analyse.py +++ b/src/deadassert/analyse.py @@ -39,7 +39,7 @@ import re from dataclasses import dataclass, field -from . import shell, workflows +from . import shell, unread, workflows from .shell import Assertion from .workflows import Script, WorkflowError @@ -90,6 +90,11 @@ class Result: against: str | None = None #: Patterns `--against` showed matching nothing in the output. absent: list[tuple[str, str]] = field(default_factory=list) + #: Assertion-shaped constructs not read at all, as `(where, kind, text)`. + #: Never findings, and never part of the exit code: see `unread.py`. + unread: list[tuple[str, str, str]] = field(default_factory=list) + #: How many `run:` scripts hold at least one of those. + unread_scripts: int = 0 @property def live(self) -> list[tuple[Script, Assertion]]: @@ -282,6 +287,17 @@ def analyse( result.foreign.append((f"{os.path.basename(rel)} {label}", shell_name)) for script in workflow.scripts: result.scripts += 1 + others = unread.scan(script.body) + if others: + result.unread_scripts += 1 + for item in others: + result.unread.append( + ( + f"{os.path.basename(rel)}:{script.line_of(item.line)}", + item.kind, + item.text, + ) + ) for assertion in shell.scan(script.body): where = f"{os.path.basename(rel)}:{script.line_of(assertion.line)}" if assertion.polarity == "undecided": diff --git a/src/deadassert/report.py b/src/deadassert/report.py index 9309fdf..73ebf84 100644 --- a/src/deadassert/report.py +++ b/src/deadassert/report.py @@ -24,6 +24,30 @@ def _wrap(text: str, indent: str) -> str: return textwrap.fill(text, width=92, initial_indent=indent, subsequent_indent=indent) +def _unread_tally(result: Result) -> str: + """`2 diff, 1 assert`, commonest kind first.""" + kinds: dict[str, int] = {} + for _, kind, _ in result.unread: + kinds[kind] = kinds.get(kind, 0) + 1 + ordered = sorted(kinds.items(), key=lambda item: (-item[1], item[0])) + return ", ".join(f"{count} {kind}" for kind, count in ordered) + + +def _gap(result: Result) -> str: + """`9 assertions in 1 of those scripts (9 assert)` -- the size of the gap. + + logbook#69. Everything above a denominator is about the assertions this + tool *read*, and until this existed there was nothing to say that a + script could also assert in a way it had not read -- which is how a sweep + for dead assertions walked past one in the repository it was auditing. + Not a finding and not in the exit code; `unread.py` evaluates nothing. + """ + return ( + f"{_plural(len(result.unread), 'assertion')} in " + f"{result.unread_scripts} of those scripts ({_unread_tally(result)})" + ) + + def _denominator(result: Result) -> str: """What the verdict was measured over. @@ -31,21 +55,33 @@ def _denominator(result: Result) -> str: apart from a verdict over nothing, and this tool is itself about assertions that look like they are checking something and are not. """ - return ( + line = ( f"{_plural(len(result.assertions), 'assertion')} in " f"{_plural(result.scripts, 'script')} across " f"{_plural(len(result.files), 'workflow file')}, " f"{result.negated} of them negated." ) + if result.unread: + return f"{line} Not counted above: {_gap(result)}, listed below." + return line def nothing_to_check(result: Result) -> str: - """Said instead of an all-clear when there were no assertions at all.""" - return ( + """Said instead of an all-clear when there were no assertions at all. + + The `unread` clause matters more here than anywhere else. A repository + that asserts entirely with `diff` -- which this project's own README + tells people to prefer -- read `Nothing to check.` before logbook#69, + and that was the broadest false statement the tool made. + """ + read = ( f"No assertions here. Read {_plural(result.scripts, 'run: script')} in " f"{_plural(len(result.files), 'workflow file')} and found no command whose " - "exit status decides a step. Nothing to check." + "exit status decides a step." ) + if result.unread: + return f"{read} What it does not read: {_gap(result)}, listed below." + return f"{read} Nothing to check." def error_lines(result: Result) -> list[str]: @@ -58,20 +94,26 @@ def error_lines(result: Result) -> list[str]: return lines +def _tail(result: Result) -> list[str]: + """The blocks under every verdict: what was not matched, decided or read.""" + out: list[str] = [] + for block in (_absent_block, _undecided_block, _unread_block): + lines = block(result) + if lines: + out.append("") + out.extend(lines) + return out + + def text(result: Result, quiet: bool = False) -> str: if result.nothing_to_check: - return nothing_to_check(result) + return "\n".join([nothing_to_check(result), *_tail(result)]) out: list[str] = [] if not result.findings: out.append("Every assertion here fails when the thing it names stops being true.") out.append(_denominator(result)) - if result.absent: - out.append("") - out.extend(_absent_block(result)) - if result.undecided: - out.append("") - out.extend(_undecided_block(result)) + out.extend(_tail(result)) return "\n".join(out) if quiet: @@ -99,13 +141,7 @@ def text(result: Result, quiet: bool = False) -> str: f"({breakdown})." ) out.append(_denominator(result)) - - if result.absent: - out.append("") - out.extend(_absent_block(result)) - if result.undecided: - out.append("") - out.extend(_undecided_block(result)) + out.extend(_tail(result)) return "\n".join(out) @@ -117,6 +153,8 @@ def _absent_block(result: Result) -> list[str]: given. Worth printing either way, because it says the comparison is not measuring what the reader thinks. """ + if not result.absent: + return [] lines = [f"not matched in {result.against}:"] for where, pattern in result.absent: lines.append(f" {where} {pattern!r}") @@ -124,12 +162,31 @@ def _absent_block(result: Result) -> list[str]: def _undecided_block(result: Result) -> list[str]: + if not result.undecided: + return [] lines = ["not decided:"] for where, why in result.undecided: lines.append(f" {where} {why}") return lines +def _unread_block(result: Result) -> list[str]: + """Assertion-shaped constructs this tool did not read. See `unread.py`. + + Separate from `not decided:` on purpose. That block is for a matcher + `deadassert` read and could not judge; this one is for an assertion it + never read at all, which is a different admission and a wider one. + """ + if not result.unread: + return [] + lines = ["not read:"] + place = max(len(where) for where, _, _ in result.unread) + width = max(len(kind) for _, kind, _ in result.unread) + for where, kind, command in result.unread: + lines.append(f" {where:<{place}} {kind:<{width}} {command}") + return lines + + def as_json(result: Result) -> str: payload = { "files": result.files, @@ -153,6 +210,8 @@ def as_json(result: Result) -> str: } for finding in result.findings ], + "unread_scripts": result.unread_scripts, + "unread": [{"where": w, "kind": k, "command": c} for w, k, c in result.unread], "undecided": [{"where": w, "why": y} for w, y in result.undecided], "foreign": [{"where": w, "shell": s} for w, s in result.foreign], "unreadable": [{"path": p, "why": y} for p, y in result.unreadable], diff --git a/src/deadassert/shell.py b/src/deadassert/shell.py index 9c8ff81..f3d0d97 100644 --- a/src/deadassert/shell.py +++ b/src/deadassert/shell.py @@ -215,8 +215,14 @@ def pipeline_parts(text: str) -> list[tuple[int, str]]: return parts -def _matcher_head(segment: str) -> tuple[str, bool] | None: - """The matcher command in a simple command, and whether `!`-negated.""" +def command_head(segment: str, programs: tuple[str, ...] = MATCHERS) -> tuple[str, bool] | None: + """A simple command whose program is one of `programs`, and whether `!`-negated. + + `programs` is a parameter because `unread` asks the same question of + `diff` and `jq` that this module asks of `grep`, and the fiddly part -- + leading `VAR=value` assignments, `env`/`command` prefixes, a path to the + program, and a `!` that belongs to the pipeline -- is the same either way. + """ text = segment.strip() negated = False while text.startswith("!"): @@ -231,11 +237,31 @@ def _matcher_head(segment: str) -> tuple[str, bool] | None: match = re.match(r"([A-Za-z_][\w./-]*)", text) if not match: return None - if match.group(1).split("/")[-1] not in MATCHERS: + if match.group(1).split("/")[-1] not in programs: return None return text, negated +def _matcher_head(segment: str) -> tuple[str, bool] | None: + """The matcher command in a simple command, and whether `!`-negated.""" + return command_head(segment, MATCHERS) + + +def swallows(text: str) -> bool: + """Whether `text` discards a command's failure with `|| true` or `|| :`.""" + return bool(_SWALLOW.search(text)) + + +def in_substitution(text: str, offset: int) -> bool: + """Whether `offset` in `text` falls inside `$(...)` or backticks.""" + return _in_substitution(text, offset) + + +def split(text: str) -> list[str] | None: + """`shlex.split`, or None when the text does not tokenise.""" + return _split(text) + + def parse_matcher(words: list[str]) -> tuple[str | None, bool, bool, bool, bool]: """`(pattern, fixed, anchored, quiet, producing)` from a matcher argv.""" pattern: str | None = None diff --git a/src/deadassert/unread.py b/src/deadassert/unread.py new file mode 100644 index 0000000..76e54af --- /dev/null +++ b/src/deadassert/unread.py @@ -0,0 +1,230 @@ +"""Assertion-shaped constructs in a `run:` script that this tool does not read. + +`deadassert` reads matchers -- `grep`, `rg` -- and says which of them cannot +fail. A step can assert in other ways, and the ones it cannot see fail in +exactly the shapes it exists to report. From `logbook#69`: + + stockbox roots -c loose.toml --json > loose.json + python3 -c " + import json + tight = json.load(open('trees.json'))['shapes'][0]['candidates'] + loose = json.load(open('loose.json'))['shapes'][0]['candidates'] + assert loose > tight * 5, (loose, tight) + " + +That is an assertion whose truth depends on the runner image rather than on +the thing it names -- the same defect `deadassert` reported two steps up in +the same file -- and it was invisible, because it is a Python `assert` and +not a grep. The run said `2 findings` and `32 assertions in 20 scripts`, with +nothing to say that one of those 20 scripts asserted something it had not +read. The sweep that exists to find dead assertions walked past one. + +**This module does not evaluate anything.** It finds the constructs and +counts them, so that a clean run says what it covered instead of implying it +covered everything -- the same move `deadassert` already makes with `not +decided:` and with the skipped non-POSIX shells. Deciding whether a `diff` +or a `jq -e` can fail is a different question, and a bigger one; the count is +the part that stops a clean run overclaiming. + +Three shapes, chosen by counting what the organisation's 38 workflow files +actually use: + +* `assert` -- a language's own assert, almost always Python via `python -c` + or a heredoc. The one from `logbook#69`. +* `diff` -- and `cmp`. This is the shape this repository's own README + recommends over `grep`, which is exactly why it needs counting: a file + that takes the advice ends up with a tool that reads almost none of its + assertions and does not currently admit it. +* `jq -e` -- the one jq invocation whose exit status is a verdict. + +`test "$s" -eq 0` and `[ -f x ]` are deliberately **not** counted, per the +caveat on `logbook#69`. They are the most common assertion in the org and +almost always sound -- the status *is* the assertion, with no pattern to rot +-- so counting them would put a non-zero number on nearly every file in the +org on day one and the number would stop being read. The cost of that choice +is real and it is written down in the README: a `test` whose captured status +came from the wrong command is not covered here either. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from . import shell + +#: Programs whose exit status is a verdict about two files being equal. +DIFFERS = ("diff", "cmp") + +#: Interpreters whose `-c` or heredoc body can hold an `assert`. +INTERPRETERS = ("python", "python3", "node", "ruby", "perl") + +#: A line that *is* an assert statement, in an embedded program. `assert` is +#: not a shell command, so a `run:` line beginning with it is inside a +#: heredoc or a quoted `-c` body by construction. Requiring whitespace or a +#: bracket after the word keeps `assert_called_once()` and a bare mention of +#: `assertions` out. +_ASSERT_LINE = re.compile(r"^assert[\s(]") + +#: An interpreter and an `assert` on one physical line: `python3 -c 'assert x'`. +#: `\bassert\b` rather than a substring so that `import deadassert` -- and the +#: name of this very package -- does not match. +_ASSERT_INLINE = re.compile(rf"\b(?:{'|'.join(INTERPRETERS)})\b.*\bassert\b") + +#: Comment openers to skip. `#` covers both shell and Python; `//` covers an +#: embedded node program. +_COMMENT = ("#", "//") + +#: Shell keywords that can sit in front of a simple command. `if diff -q a b; +#: then` is a `diff` assertion, and without this the clause's first word is +#: `if` and the construct is missed -- which is how the first version of this +#: module read `logbook#69`'s sibling shape as absent. +_KEYWORD = re.compile(r"^\s*(?:if|elif|while|until|then|else|do|done|fi|!|\{|\()\s+") + +#: Operators that end one simple command and begin the next. +_OPERATORS = ("||", "&&", ";;", ";", "|", "&") + + +@dataclass +class Unread: + """One assertion-shaped construct this tool did not read.""" + + #: 1-based, within the script. + line: int + #: One of `assert`, `diff`, `jq -e`. + kind: str + #: The line it was found in, stripped, as one line. + text: str + + +def _asserts(script: str) -> list[Unread]: + """`assert` statements, found on physical lines. + + Physical rather than logical lines on purpose. `shell.logical_lines` + models a shell script, and an embedded Python program is not one: the + `python3 -c "` that opens it leaves a quote hanging that the shell + tokeniser would have to track across the whole body to get right. The + line-shape test below needs none of that, and it cannot be fooled into + reporting the wrong line number. + """ + found: list[Unread] = [] + for number, raw in enumerate(script.split("\n"), 1): + text = raw.strip() + if not text or text.startswith(_COMMENT): + continue + if _ASSERT_LINE.match(text) or _ASSERT_INLINE.search(text): + found.append(Unread(line=number, kind="assert", text=text)) + return found + + +def _jq_asserts(words: list[str]) -> bool: + """Whether a `jq` argv asks for `--exit-status`. + + Plain `jq -r .x` is extraction and its status says only that the JSON + parsed. `-e` is the one that makes it a verdict about the value. + """ + for word in words[1:]: + if word == "--exit-status": + return True + if word.startswith("--"): + continue + if word.startswith("-") and len(word) > 1 and "e" in word[1:]: + return True + return False + + +def _simple_commands(text: str) -> list[tuple[int, str]]: + """Every simple command in a logical line, with its offset in that line. + + `shell.pipeline_parts` splits on `|` alone, which is the right split for + deciding whether a matcher's exit status survives its position. This + module declines to decide that (see `_commands`) and wants something + different: every place a command *starts*, so that the `diff` in + `if diff -q a b; then ...` is found at all. So it splits on every + operator and then steps over any shell keyword in front of the command. + """ + out: list[tuple[int, str]] = [] + begin = 0 + quote = None + index = 0 + + def emit(start: int, end: int) -> None: + chunk = text[start:end] + while True: + match = _KEYWORD.match(chunk) + if not match: + break + start += match.end() + chunk = text[start:end] + if chunk.strip(): + out.append((start, chunk)) + + while index < len(text): + ch = text[index] + if quote: + if ch == quote: + quote = None + index += 1 + continue + if ch in "'\"": + quote = ch + index += 1 + continue + for operator in _OPERATORS: + if text.startswith(operator, index): + emit(begin, index) + index += len(operator) + begin = index + break + else: + index += 1 + emit(begin, len(text)) + return out + + +def _commands(script: str) -> list[Unread]: + """`diff`, `cmp` and `jq -e` commands whose failure is not discarded. + + Two exclusions, and only two, because this is a disclosure and not a + verdict: + + * inside `$(...)` or backticks -- a capture, feeding a variable, exactly + as in `shell.scan`; + * `|| true` on the logical line -- the failure is thrown away, so there + is no assertion here to have missed. + + Everything else is counted, including a `diff` partway down a pipeline. + Working out whether such a status reaches the step is the analysis this + module is explicitly not doing, and guessing it away would reintroduce + the silence the module exists to break. + """ + found: list[Unread] = [] + programs = DIFFERS + ("jq",) + for lineno, text in shell.logical_lines(script): + if not any(name in text for name in programs): + continue + if shell.swallows(text): + continue + for offset, segment in _simple_commands(text): + head = shell.command_head(segment, programs) + if head is None: + continue + words = shell.split(head[0]) + if words is None: + continue + if shell.in_substitution(text, offset): + continue + if words[0].split("/")[-1] == "jq": + if _jq_asserts(words): + found.append(Unread(line=lineno, kind="jq -e", text=text)) + else: + found.append(Unread(line=lineno, kind="diff", text=text)) + return found + + +def scan(script: str) -> list[Unread]: + """Every assertion-shaped construct in `script` that `shell.scan` ignores. + + Sorted by line, so the report reads in file order. + """ + return sorted(_asserts(script) + _commands(script), key=lambda item: item.line) diff --git a/tests/test_cli.py b/tests/test_cli.py index ace9e9d..e894316 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -84,6 +84,82 @@ def test_a_workflow_with_no_assertions_does_not_claim_all_clear(self, capsys, tm assert "Every assertion here fails" not in out +class TestNotRead: + """logbook#69: a clean run over the greps is not a clean run.""" + + DIFFS = "mytool . > got.txt\ndiff -u expected.txt got.txt" + + def test_a_repo_that_asserts_only_with_diff_no_longer_claims_nothing_to_check( + self, capsys, tmp_path + ): + # The broadest false statement the tool made. `diff` is what this + # project's own README tells people to prefer over `grep`, so a repo + # that took the advice got `Nothing to check.` over a file full of + # assertions. + root = repo(tmp_path, self.DIFFS) + code, out, _ = run(capsys, root) + assert "No assertions here." in out + assert "Nothing to check." not in out + assert "What it does not read: 1 assertion in 1 of those scripts (1 diff)" in out + assert "diff -u expected.txt got.txt" in out + # Still not a finding, so still not a failure. + assert code == 0 + + def test_the_clean_verdict_says_what_it_did_not_read(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt", self.DIFFS) + code, out, _ = run(capsys, root) + assert code == 0 + assert "Every assertion here fails when the thing it names stops being true." in out + assert "1 assertion in 2 scripts across 1 workflow file, 0 of them negated." in out + assert "Not counted above: 1 assertion in 1 of those scripts (1 diff)" in out + + def test_the_logbook_69_shape_is_named_in_a_run_that_has_findings(self, capsys, tmp_path): + root = repo( + tmp_path, + "if grep -qF 'kind-x' out.txt; then\n exit 1\nfi", + "python3 -c \"\nimport json\nassert json.load(open('o.json'))['n'] > 5\n\"", + ) + code, out, _ = run(capsys, root) + assert code == 1 + assert "unwitnessed" in out + assert "not read:" in out + assert "assert json.load(open('o.json'))['n'] > 5" in out + + def test_a_repo_with_nothing_unread_says_nothing_about_it(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt") + _, out, _ = run(capsys, root) + assert "not read:" not in out + assert "Not counted above" not in out + + def test_quiet_stays_one_line_per_finding(self, capsys, tmp_path): + # `not read:` is a coverage note, not a finding, and -q is for + # feeding a list of findings to something else. + root = repo(tmp_path, "if grep -qF 'kind-x' o.txt; then\n exit 1\nfi", self.DIFFS) + _, out, _ = run(capsys, root, "-q") + assert out.strip().splitlines() == [".github/workflows/ci.yml:7: unwitnessed: 'kind-x'"] + + def test_json_carries_the_constructs_and_the_script_count(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt", self.DIFFS) + code, out, _ = run(capsys, root, "--json") + payload = json.loads(out) + assert code == 0 + assert payload["findings"] == [] + assert payload["unread_scripts"] == 1 + assert payload["unread"] == [ + { + "where": "ci.yml:10", + "kind": "diff", + "command": "diff -u expected.txt got.txt", + } + ] + + def test_the_line_number_points_at_the_construct(self, capsys, tmp_path): + root = repo(tmp_path, "echo one\necho two\ndiff -u a.txt b.txt") + _, out, _ = run(capsys, root, "--json") + # `run: |` on line 6, so the script's own lines start at 7. + assert json.loads(out)["unread"][0]["where"] == "ci.yml:9" + + class TestDenominator: """logbook#59 and logbook#57: a verdict needs what it was measured over.""" diff --git a/tests/test_unread.py b/tests/test_unread.py new file mode 100644 index 0000000..ed27796 --- /dev/null +++ b/tests/test_unread.py @@ -0,0 +1,172 @@ +"""What counts as an assertion this tool does not read, and what does not. + +Every negative test in here is a false positive the first version produced or +would have produced. A count that is wrong in the generous direction is worse +than no count at all: the whole point of `logbook#69` is that the number be +trustworthy enough that a non-zero one sends you to look. +""" + +from __future__ import annotations + +from deadassert import unread + + +def kinds(script: str) -> list[str]: + return [item.kind for item in unread.scan(script)] + + +def one(script: str) -> unread.Unread: + found = unread.scan(script) + assert len(found) == 1, f"expected one construct, got {found}" + return found[0] + + +class TestPythonAsserts: + def test_the_logbook_69_instance(self): + # Verbatim from stockbox's dogfood job, which is what this exists for: + # an assertion whose truth depends on the runner image, in the same + # repository where deadassert reported that defect in a grep, and + # invisible to it. + script = """ +stockbox roots -c loose.toml --json > loose.json +python3 -c " +import json +tight = json.load(open('trees.json'))['shapes'][0]['candidates'] +loose = json.load(open('loose.json'))['shapes'][0]['candidates'] +assert loose > tight * 5, (loose, tight) +" +""" + found = one(script) + assert found.kind == "assert" + assert found.text == "assert loose > tight * 5, (loose, tight)" + assert found.line == 7 + + def test_a_heredoc_body(self): + script = """ +set -euo pipefail +python - <<'PY' +import json +d = json.load(open("out.json")) +assert d["assertions"] >= 2, f"only {d['assertions']} seen" +assert d["findings"] == [], d["findings"] +PY +""" + assert kinds(script) == ["assert", "assert"] + + def test_an_indented_assert_inside_a_function(self): + assert kinds("python3 - <<'PY'\ndef check(d):\n assert d, d\ncheck({})\nPY") == [ + "assert" + ] + + def test_a_one_liner(self): + assert kinds("python3 -c 'assert 1 == 1'") == ["assert"] + + def test_a_node_assert(self): + assert kinds("node -e 'assert.ok(ran)'") == ["assert"] + + def test_a_bracketed_assert(self): + assert kinds("python3 - <<'PY'\nassert(x)\nPY") == ["assert"] + + +class TestNotAnAssert: + def test_a_commented_out_assert(self): + assert kinds("python3 - <<'PY'\n# assert total > 0\nprint(1)\nPY") == [] + + def test_a_commented_out_assert_in_node(self): + assert kinds("node -e 'x'\n// assert.ok(x)") == [] + + def test_a_method_whose_name_merely_starts_with_assert(self): + # `assert_called_once()` is a call, not a statement, and a tool that + # counted it would put a number on every mock-using repo in the org. + assert kinds("python3 - <<'PY'\nassert_called_once()\nPY") == [] + + def test_the_name_of_this_package_is_not_an_assert(self): + # `\\bassert\\b` and not a substring: `import deadassert` sits on a + # line with an interpreter on it and must not match. + assert kinds("python3 -c 'import deadassert; print(deadassert.__file__)'") == [] + + def test_the_word_assertions_in_prose(self): + assert kinds("python3 -m pytest && echo 'all assertions ran'") == [] + + def test_a_shell_line_mentioning_assert(self): + assert kinds("echo 'assert something' > note.txt") == [] + + +class TestDiff: + def test_a_plain_diff(self): + found = one("mytool . > got.txt\ndiff -u expected.txt got.txt") + assert found.kind == "diff" + assert found.line == 2 + + def test_cmp_counts_too(self): + assert kinds("cmp -s a.bin b.bin") == ["diff"] + + def test_a_diff_in_an_if_condition(self): + assert kinds("if diff -q a b > /dev/null; then echo same; fi") == ["diff"] + + def test_a_redirected_diff_still_counts(self): + # Under `bash -e` the status reaches the step whatever happens to + # stdout, so this is an assertion and not extraction. + assert kinds("diff -u a.txt b.txt > changes.diff") == ["diff"] + + def test_a_swallowed_diff_does_not_count(self): + # The failure is thrown away, so there is no assertion here to have + # missed reading. + assert kinds("diff -u a.txt b.txt || true") == [] + + def test_a_captured_diff_does_not_count(self): + # Feeding a variable, exactly as `shell.scan` treats a captured grep. + assert kinds('changed=$(diff -u a.txt b.txt); echo "$changed"') == [] + + def test_a_filename_containing_diff_does_not_count(self): + assert kinds("cat expected.diff") == [] + + +class TestJq: + def test_jq_e_counts(self): + found = one("curl -sS $url | jq -e '.ok' > /dev/null") + assert found.kind == "jq -e" + + def test_a_combined_short_flag(self): + assert kinds("jq -er '.permalink' < resp.json") == ["jq -e"] + + def test_the_long_flag(self): + assert kinds("jq --exit-status '.ok' resp.json") == ["jq -e"] + + def test_plain_jq_is_extraction(self): + # Without -e the status says only that the JSON parsed, which is not + # a verdict about the value and not what this is counting. + assert kinds("jq -r '.items[].name' resp.json") == [] + + def test_an_arg_named_e_is_not_the_flag(self): + assert kinds("jq --arg e 1 '.x' resp.json") == [] + + +class TestDeliberateOmissions: + def test_test_is_not_counted(self): + # The caveat on logbook#69: `test "$s" -eq N` is the commonest + # assertion in the org and almost always sound, so counting it would + # put a number on nearly every file and the number would stop being + # read. Documented in the README rather than silently dropped. + assert kinds('mytool .; s=$?\ntest "$s" -eq 1') == [] + + def test_bracket_is_not_counted(self): + assert kinds("[ -f out.txt ] || exit 1") == [] + + def test_a_grep_is_not_counted_here(self): + # `shell.scan` reads these, so counting them again would double-count + # the only assertions the tool does understand. + assert kinds("grep -qxF 'All clear.' out.txt") == [] + + +class TestOrdering: + def test_constructs_come_back_in_line_order(self): + script = """ +diff -u a.txt b.txt +python3 - <<'PY' +assert 1 +PY +jq -e '.ok' r.json +""" + assert [item.line for item in unread.scan(script)] == [2, 4, 6] + assert kinds(script) == ["diff", "assert", "jq -e"]