From bf26559586b4d2a34d0a4824feeeab2721a5dc35 Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:40:32 +0000 Subject: [PATCH 1/5] feat: read workflow run: scripts with a line number for each The five sibling tools that read workflow files all report at the level of a step, where the step name is enough to find the thing again. A finding here is one command inside a script that may be forty lines long, so the file is read twice: safe_load for the values, compose for the node marks. PyYAML marks a block scalar on its | and a plain scalar on its content, so the first content line differs by one between the two. Getting that wrong would put every finding in the file off by one. --- pyproject.toml | 49 +++++++ src/deadassert/workflows.py | 250 ++++++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100644 pyproject.toml create mode 100644 src/deadassert/workflows.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9084c8c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "deadassert" +version = "0.1.0" +description = "Say which of your CI assertions pass whether or not the thing they name is true." +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.10" +keywords = ["ci", "github-actions", "workflow", "assertions", "lint", "cli"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Quality Assurance", + "Topic :: Software Development :: Testing", + "Topic :: Utilities", +] +dependencies = ["PyYAML>=5.1"] + +[project.optional-dependencies] +dev = ["pytest>=7.0", "ruff>=0.5"] + +[project.scripts] +deadassert = "deadassert.cli:main" + +[project.urls] +Source = "https://github.com/committed-nightly/deadassert" + +[tool.hatch.build.targets.wheel] +packages = ["src/deadassert"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +# No -q. logbook#52: a -q here plus a -q on the command line is -qq, which +# drops the pass count, and the next person to read this output cannot tell +# a run of 90 tests from a run of none. +addopts = "" + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +# Pyflakes only, matching the rest of the org (see logbook#36). +select = ["F"] diff --git a/src/deadassert/workflows.py b/src/deadassert/workflows.py new file mode 100644 index 0000000..22ce0dc --- /dev/null +++ b/src/deadassert/workflows.py @@ -0,0 +1,250 @@ +"""Reading workflow files into the `run:` scripts an assertion can live in. + +The one thing to know before touching this file, and it is the same thing in +every tool in this family: in YAML 1.1, which is what PyYAML implements, the +bare word `on` is a **boolean**. The key every workflow file on GitHub starts +with does not come back as `"on"`, it comes back as `True`. + +What this module adds over the equivalent in `blankout`, `coldcache`, +`groupclash`, `deadtrigger` and `ungranted` is a line number for every `run:` +script. Those five all report at the level of a step, so a step's name is +enough to find the thing again. A finding here is one command inside a script +that may be forty lines long, and `ci.yml` on its own is not a location. So +the file is read twice: `safe_load` for the values, and `compose` for the node +marks, which is the only place PyYAML will tell you where a scalar started. + +The marks need one correction. For a block scalar (`run: |`), PyYAML's +`start_mark` is on the `|` itself, so the script's first line is the line +*after* the mark. For a plain or quoted scalar the mark is on the content, so +the first line is the mark's own line. `_script_first_line` is that +distinction and nothing else; getting it wrong puts every finding in the file +off by one, which is worse than having no line number at all. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any + +import yaml + +WORKFLOW_DIR = ".github/workflows" +WORKFLOW_SUFFIXES = (".yml", ".yaml") + +#: `runs-on` label prefixes whose default shell is pwsh rather than bash. +#: GitHub picks the shell from the runner when a step does not say. +WINDOWS_PREFIXES = ("windows", "win-") + +#: Shells whose scripts this tool will read. A `run:` under pwsh or python is +#: a script in another language; it is skipped and counted, never guessed at. +POSIX_SHELLS = ("bash", "sh", "dash", "zsh", "ksh") + + +class WorkflowError(ValueError): + """A file that could not be read as a workflow.""" + + +@dataclass +class Script: + """One `run:` script, and where it came from.""" + + workflow: str + job: str + step_index: int + step_name: str | None + shell: str + body: str + #: 1-based line in the workflow file holding the script's own first line. + first_line: int + + @property + def label(self) -> str: + return self.step_name or f"steps[{self.step_index}]" + + @property + def posix(self) -> bool: + """Whether the script is in a shell whose syntax this tool reads. + + A `shell:` of `bash -e {0}` or `/bin/bash --noprofile` counts: GitHub + lets you give a whole command line there, and the first word is the + interpreter. + """ + word = self.shell.strip().split()[0] if self.shell.strip() else "bash" + return os.path.basename(word) in POSIX_SHELLS + + def line_of(self, script_line: int) -> int: + """Turn a 1-based line within the script into one in the workflow file.""" + return self.first_line + script_line - 1 + + +@dataclass +class Workflow: + """One workflow file, flattened to the scripts it runs.""" + + path: str + name: str + scripts: list[Script] = field(default_factory=list) + #: Scripts skipped because their shell is not a POSIX one, as + #: `(step label, shell)`. Reported, so a skip never reads as a pass. + foreign: list[tuple[str, str]] = field(default_factory=list) + + +def is_workflow_path(path: str) -> bool: + return path.endswith(WORKFLOW_SUFFIXES) + + +def find_workflows(root: str) -> list[str]: + """Every workflow file under `root`, sorted, by path relative to it.""" + directory = os.path.join(root, WORKFLOW_DIR) + if not os.path.isdir(directory): + return [] + found = [] + for entry in sorted(os.listdir(directory)): + full = os.path.join(directory, entry) + if os.path.isfile(full) and is_workflow_path(entry): + found.append(os.path.join(WORKFLOW_DIR, entry)) + return found + + +def _defaults_shell(block: Any) -> Any: + if isinstance(block, dict): + run = block.get("run") + if isinstance(run, dict): + return run.get("shell") + return None + + +def _windows(runs_on: Any) -> bool: + labels = runs_on + if isinstance(labels, str): + labels = [labels] + elif isinstance(labels, dict): + labels = labels.get("labels") or [] + elif not isinstance(labels, list): + return False + return any(isinstance(x, str) and x.lower().startswith(WINDOWS_PREFIXES) for x in labels) + + +def _shell_for(step_shell: Any, job_defaults: Any, wf_defaults: Any, windows: bool) -> str: + """The shell a step's script actually runs under. + + A step's own `shell:` wins, then the job's `defaults.run.shell`, then the + workflow's, then the runner's default -- which is bash everywhere except + Windows, where it is pwsh. + """ + for candidate in (step_shell, job_defaults, wf_defaults): + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + return "pwsh" if windows else "bash" + + +def _script_first_line(node: yaml.Node) -> int: + """The 1-based file line holding the first line of a scalar's content. + + PyYAML marks a block scalar at its `|` or `>` indicator, with the content + starting on the next line; a plain or quoted scalar is marked at the + content itself. `node.style` is `|` or `>` for the former and `None`, + `'`, or `"` for the latter. + """ + line = node.start_mark.line + 1 + if node.style in ("|", ">"): + return line + 1 + return line + + +def _run_marks(text: str) -> dict[tuple[str, int], int]: + """`(job id, step index) -> first content line` for every `run:` scalar. + + Walked over the node graph rather than the loaded document because the + loaded document has thrown the marks away. Keyed on position so it can be + joined to the `safe_load` pass without depending on the two walks + producing the same objects. + """ + marks: dict[tuple[str, int], int] = {} + try: + root = yaml.compose(text) + except yaml.YAMLError: + return marks + if not isinstance(root, yaml.MappingNode): + return marks + + def value_for(mapping: yaml.Node, key: str) -> yaml.Node | None: + if not isinstance(mapping, yaml.MappingNode): + return None + for k, v in mapping.value: + if isinstance(k, yaml.ScalarNode) and k.value == key: + return v + return None + + jobs = value_for(root, "jobs") + if not isinstance(jobs, yaml.MappingNode): + return marks + for job_key, job_body in jobs.value: + if not isinstance(job_key, yaml.ScalarNode): + continue + steps = value_for(job_body, "steps") + if not isinstance(steps, yaml.SequenceNode): + continue + for index, step in enumerate(steps.value): + run = value_for(step, "run") + if isinstance(run, yaml.ScalarNode): + marks[(job_key.value, index)] = _script_first_line(run) + return marks + + +def load(root: str, relative_path: str) -> Workflow: + """Read one workflow file. Raises `WorkflowError` on anything unreadable.""" + full = os.path.join(root, relative_path) + try: + with open(full, encoding="utf-8") as handle: + text = handle.read() + except OSError as exc: + raise WorkflowError(f"{relative_path}: {exc}") from exc + try: + doc = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise WorkflowError(f"{relative_path}: not valid YAML: {exc}") from exc + + if doc is None: + raise WorkflowError(f"{relative_path}: empty file") + if not isinstance(doc, dict): + raise WorkflowError(f"{relative_path}: top level is not a mapping") + + workflow = Workflow(path=relative_path, name=str(doc.get("name") or relative_path)) + marks = _run_marks(text) + wf_shell = _defaults_shell(doc.get("defaults")) + + jobs = doc.get("jobs") + if not isinstance(jobs, dict): + return workflow + + for job_id, body in jobs.items(): + if not isinstance(body, dict): + continue + job_shell = _defaults_shell(body.get("defaults")) + windows = _windows(body.get("runs-on")) + steps = body.get("steps") + if not isinstance(steps, list): + continue + for index, raw in enumerate(steps): + if not isinstance(raw, dict): + continue + run = raw.get("run") + if not isinstance(run, str): + continue + script = Script( + workflow=relative_path, + job=str(job_id), + step_index=index, + step_name=raw.get("name") if isinstance(raw.get("name"), str) else None, + shell=_shell_for(raw.get("shell"), job_shell, wf_shell, windows), + body=run, + first_line=marks.get((str(job_id), index), 0), + ) + if script.posix: + workflow.scripts.append(script) + else: + workflow.foreign.append((script.label, script.shell)) + + return workflow From dcaa54c7664aa2ba64f9778872fd4c6591be5350 Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:40:41 +0000 Subject: [PATCH 2/5] feat: find shell assertions and say which way round each one fails Polarity is the spine of the tool. A needs-match assertion fails when its pattern goes away, so it rots loudly and somebody fixes it. A needs-no-match assertion passes when its pattern goes away, so it rots in silence. Three rules here each come from a false positive against this org's real workflows: a leading ! negates the whole pipeline rather than the command it precedes; a bare grep in an else branch does fail the step under bash -e, so such a branch is not decorative; and a grep inside $(...) or writing to a file is extraction, not an assertion. --- src/deadassert/shell.py | 450 ++++++++++++++++++++++++++++++++++++++++ tests/test_shell.py | 189 +++++++++++++++++ 2 files changed, 639 insertions(+) create mode 100644 src/deadassert/shell.py create mode 100644 tests/test_shell.py diff --git a/src/deadassert/shell.py b/src/deadassert/shell.py new file mode 100644 index 0000000..9c8ff81 --- /dev/null +++ b/src/deadassert/shell.py @@ -0,0 +1,450 @@ +"""Find the assertions in a shell script, and say which way round each fails. + +An assertion, here, is a command whose **exit status** decides whether the CI +step passes. `grep -q 'all clear' out.txt` is one. `grep -c 'x' out.txt > n` +is not -- it is extraction, and its status is incidental to what it is for. +Telling those two apart is most of this file, and getting it wrong in the +generous direction is how a tool like this becomes something people turn off. + +The distinction that the rest of `deadassert` is built on is **polarity**: + +* `needs-match` -- the step fails when the pattern is *absent*. If the output + it names changes, CI goes red and somebody fixes it. It rots loudly. +* `needs-no-match` -- the step fails when the pattern is *present*. If the + output it names changes, the assertion keeps passing, forever, while no + longer checking anything. It rots silently. + +Three things in here exist because the first version got them wrong against +real workflow files, and each one was a false positive rather than a miss: + +1. A leading `!` negates the whole **pipeline**, not the command it sits in + front of. `! printf '%s' "$out" | grep -q X` fails the step when grep + *does* match, and the `!` is five words away from the grep. +2. A branch can fail the step without an `exit`. Under `bash -e`, which is + what GitHub Actions runs, a bare `grep -q X f` in an `else` is itself an + assertion and fails the step on its own. A branch holding only that is not + a branch that does nothing. +3. `line=$(whichrc -q | grep '\\.bash_login:') || true` is a capture, not a + swallowed assertion. The `|| true` is correct there -- no match is an + expected answer, and the test comes later, on the variable. +""" + +from __future__ import annotations + +import re +import shlex +from dataclasses import dataclass + +#: Programs whose exit status answers "did this appear?". +MATCHERS = ("grep", "egrep", "fgrep", "rg") + +#: Flags that mean the command is being run for its output, not its verdict. +PRODUCING = ("o", "c", "l", "L") + +#: Flags that mean the command is being run for its verdict only. +QUIET = ("q", "s") + +#: Polarities for which the step's pass/fail actually depends on the match. +LIVE = ("needs-match", "needs-no-match") + +_FAILING = re.compile(r"(?:^|[;&|{(]\s*)(?:exit\s+[1-9]|exit\s*$|false\b|return\s+[1-9])") +_SWALLOW = re.compile(r"\|\|\s*(?:true\b|:\s*(?:;|$))") +_LONG_PRODUCING = ("--only-matching", "--count", "--files-with-matches", "--files-without-match") + + +@dataclass +class Assertion: + """One matcher command whose exit status reaches the step's exit status.""" + + #: 1-based, within the script. + line: int + #: The logical line it was found in, comments stripped, as one line. + text: str + program: str + pattern: str | None + polarity: str + #: Fixed-string matching: `-F`, `fgrep`, or `rg -F`. + fixed: bool + #: Whole-line matching: `-x`. + anchored: bool + #: Set when the shape was recognised but not decided. Never a finding. + undecided: str | None = None + + @property + def live(self) -> bool: + return self.polarity in LIVE + + +def _strip_comment(line: str) -> str: + """Drop a trailing `#` comment, respecting quotes. + + Naive splitting on `#` would cut `grep -qF '^not checked: #3'` in half and + silently change the pattern being reported. + """ + out: list[str] = [] + quote = None + for index, ch in enumerate(line): + if quote: + out.append(ch) + if ch == quote: + quote = None + continue + if ch in "'\"": + quote = ch + out.append(ch) + continue + if ch == "#" and (index == 0 or line[index - 1] in " \t"): + break + out.append(ch) + return "".join(out) + + +def logical_lines(script: str) -> list[tuple[int, str]]: + """The script as logical lines, each with the 1-based line it starts on. + + Joins backslash continuations and lines left open by a trailing `|`, `||`, + `&&`, `;` or `{`, because an assertion is routinely written across two + physical lines and its polarity lives on the second one. + """ + out: list[tuple[int, str]] = [] + buffer = "" + start = 0 + for number, raw in enumerate(script.split("\n"), 1): + body = _strip_comment(raw).rstrip() + stripped = body.strip() + if not buffer: + if not stripped: + continue + start = number + if body.endswith("\\"): + buffer += body[:-1].strip() + " " + continue + buffer += stripped + if re.search(r"(?:\|\||&&|&|\||;|\{)$", buffer): + buffer += " " + continue + out.append((start, buffer.strip())) + buffer = "" + if buffer.strip(): + out.append((start, buffer.strip())) + return out + + +def _substitution_spans(text: str) -> list[tuple[int, int]]: + """Character spans covered by `$(...)` or backticks. + + A matcher inside one of these is feeding a variable, not deciding the + step. `<(...)` is deliberately not included: a process substitution is an + argument, and the matcher outside it is still the assertion. + """ + spans: list[tuple[int, int]] = [] + quote = None + stack: list[int] = [] + index = 0 + while index < len(text): + ch = text[index] + if quote == "'": + if ch == "'": + quote = None + index += 1 + continue + if ch == "'" and quote is None: + quote = "'" + index += 1 + continue + if ch == '"' and quote in (None, '"'): + quote = None if quote == '"' else '"' + index += 1 + continue + if ch == "`": + if stack: + spans.append((stack.pop(), index)) + else: + stack.append(index) + index += 1 + continue + if ch == "$" and index + 1 < len(text) and text[index + 1] == "(": + stack.append(index) + index += 2 + continue + if ch == ")" and stack: + spans.append((stack.pop(), index)) + index += 1 + continue + index += 1 + return spans + + +def _in_substitution(text: str, offset: int) -> bool: + return any(a <= offset <= b for a, b in _substitution_spans(text)) + + +def pipeline_parts(text: str) -> list[tuple[int, str]]: + """Split on a `|` that is neither `||` nor inside quotes. Offsets kept.""" + parts: list[tuple[int, str]] = [] + buffer = "" + begin = 0 + quote = None + index = 0 + while index < len(text): + ch = text[index] + if quote: + buffer += ch + if ch == quote: + quote = None + index += 1 + continue + if ch in "'\"": + quote = ch + buffer += ch + index += 1 + continue + if ch == "|": + if index + 1 < len(text) and text[index + 1] == "|": + buffer += "||" + index += 2 + continue + parts.append((begin, buffer)) + buffer = "" + index += 1 + begin = index + continue + buffer += ch + index += 1 + parts.append((begin, buffer)) + return parts + + +def _matcher_head(segment: str) -> tuple[str, bool] | None: + """The matcher command in a simple command, and whether `!`-negated.""" + text = segment.strip() + negated = False + while text.startswith("!"): + negated = True + text = text[1:].strip() + # Step over leading `VAR=value` assignments and `command`/`env` prefixes. + while True: + match = re.match(r"(?:[A-Za-z_]\w*=\S*|command|env)\s+", text) + if not match: + break + text = text[match.end() :] + match = re.match(r"([A-Za-z_][\w./-]*)", text) + if not match: + return None + if match.group(1).split("/")[-1] not in MATCHERS: + return None + return text, negated + + +def parse_matcher(words: list[str]) -> tuple[str | None, bool, bool, bool, bool]: + """`(pattern, fixed, anchored, quiet, producing)` from a matcher argv.""" + pattern: str | None = None + fixed = words[0].split("/")[-1] == "fgrep" + anchored = quiet = producing = False + expecting = False + index = 1 + while index < len(words): + word = words[index] + if expecting: + pattern = word if pattern is None else pattern + expecting = False + index += 1 + continue + if word == "--": + index += 1 + if pattern is None and index < len(words): + pattern = words[index] + index += 1 + continue + if word.startswith("--"): + name = word.split("=", 1)[0] + if name in ("--fixed-strings",): + fixed = True + elif name in ("--line-regexp",): + anchored = True + elif name in ("--quiet", "--silent"): + quiet = True + elif name in _LONG_PRODUCING: + producing = True + elif name == "--regexp": + if "=" in word: + pattern = word.split("=", 1)[1] if pattern is None else pattern + else: + expecting = True + index += 1 + continue + if word.startswith("-") and len(word) > 1: + letters = word[1:] + if "F" in letters: + fixed = True + if "x" in letters: + anchored = True + if any(flag in letters for flag in QUIET): + quiet = True + if any(flag in letters for flag in PRODUCING): + producing = True + if "e" in letters: + tail = letters[letters.index("e") + 1 :] + if tail: + pattern = tail if pattern is None else pattern + else: + expecting = True + index += 1 + continue + if pattern is None: + pattern = word + index += 1 + return pattern, fixed, anchored, quiet, producing + + +def _redirects_stdout(segment: str) -> bool: + """Whether a simple command sends its stdout somewhere other than a pipe.""" + return bool(re.search(r"(?])>{1,2}\s*\S", segment)) + + +def _branch_polarity(lines: list[tuple[int, str]], index: int) -> tuple[str | None, str | None]: + """For an `if` at `index`, which branch can fail the step. + + A branch "can fail" if it holds an explicit `exit N`/`false`/`return N`, + or a bare matcher assertion, which under `bash -e` fails the step by + itself. Rule 2 in this module's docstring: leaving the second case out + made a real `if`/`else` in `whichrc` look decorative. + """ + depth = 0 + branch = "then" + fails = {"then": False, "else": False} + for _, text in lines[index:]: + for token in re.findall(r"(?:^|[;\s])(if|fi)(?=[;\s]|$)", text): + depth += 1 if token == "if" else -1 + if depth == 1 and (re.match(r"^(?:else|elif)\b", text) or re.search(r";\s*else\b", text)): + branch = "else" + body = text + head = re.match(r"^(?:el)?if\s+(.*?)(?:;\s*then\b|$)", text) + if head and text.startswith(("if", "elif")): + # The condition is not the branch; a matcher in it is the subject. + body = text[head.end(1) :] + if _FAILING.search(body) or _bare_assertion(body): + fails[branch] = True + if depth <= 0: + break + if depth > 0: + return None, "the `if` has no matching `fi` in this script" + if fails["then"] and not fails["else"]: + return "needs-no-match", None + if fails["else"] and not fails["then"]: + return "needs-match", None + if fails["then"] and fails["else"]: + return None, "either branch of the `if` can fail the step" + return "decorative", None + + +def _bare_assertion(text: str) -> bool: + """Whether `text` holds a matcher run for its verdict, under `set -e`.""" + for _, segment in pipeline_parts(text): + head = _matcher_head(segment) + if head is None: + continue + words = _split(head[0]) + if words is None: + continue + _, _, _, quiet, producing = parse_matcher(words) + if quiet and not producing: + return True + return False + + +def _split(text: str) -> list[str] | None: + try: + return shlex.split(text) + except ValueError: + return None + + +def scan(script: str) -> list[Assertion]: + """Every matcher command in `script` whose status reaches the step.""" + lines = logical_lines(script) + found: list[Assertion] = [] + + for index, (lineno, text) in enumerate(lines): + if not any(name in text for name in MATCHERS): + continue + + polarity: str | None = None + undecided: str | None = None + condition = text + + head = re.match(r"^(?:el)?if\s+(.*?)(?:;\s*then\b|$)", text) + if head and text.startswith(("if", "elif")): + condition = head.group(1) + polarity, undecided = _branch_polarity(lines, index) + elif re.match(r"^(?:while|until)\b", text): + undecided = "the matcher is a loop condition, not a verdict" + polarity = "undecided" + + if polarity is None: + parts = re.split(r"\s(\|\||&&)\s", condition, maxsplit=1) + if len(parts) == 3: + condition, operator, consequence = parts + if operator == "||": + if _SWALLOW.search(f"|| {consequence}"): + polarity = "swallowed" + elif _FAILING.search(consequence) or _bare_assertion(consequence): + polarity = "needs-match" + else: + polarity = "decorative" + elif _FAILING.search(consequence) or _bare_assertion(consequence): + polarity = "needs-no-match" + else: + polarity = "decorative" + else: + # A bare command under `bash -e`: a non-zero status ends the step. + polarity = "needs-match" + + negated_pipeline = False + condition = condition.strip() + while condition.startswith("!"): + negated_pipeline = not negated_pipeline + condition = condition[1:].strip() + + parts = pipeline_parts(condition) + for position, (offset, segment) in enumerate(parts): + head_match = _matcher_head(segment) + if head_match is None: + continue + command, negated_command = head_match + words = _split(command) + if words is None: + continue + pattern, fixed, anchored, quiet, producing = parse_matcher(words) + + # Rule 1 and 3: not every matcher is an assertion. + if producing: + continue + if _in_substitution(text, offset + segment.index(command.split()[0])): + continue + if not quiet and (_redirects_stdout(segment) or position != len(parts) - 1): + continue + + this = polarity + if negated_command != negated_pipeline: + this = {"needs-match": "needs-no-match", "needs-no-match": "needs-match"}.get( + this, this + ) + why = undecided + if position != len(parts) - 1 and this in LIVE: + this = "masked" + why = "its exit status is discarded by the rest of the pipeline" + + found.append( + Assertion( + line=lineno, + text=text, + program=words[0].split("/")[-1], + pattern=pattern, + polarity=this or "undecided", + fixed=fixed, + anchored=anchored, + undecided=why, + ) + ) + return found diff --git a/tests/test_shell.py b/tests/test_shell.py new file mode 100644 index 0000000..1804d3d --- /dev/null +++ b/tests/test_shell.py @@ -0,0 +1,189 @@ +"""Polarity, and the three shapes that made the first version wrong.""" + +from __future__ import annotations + +import pytest + +from deadassert import shell + + +def one(script: str) -> shell.Assertion: + found = shell.scan(script) + assert len(found) == 1, f"expected one assertion, got {found}" + return found[0] + + +def polarity(script: str) -> str: + return one(script).polarity + + +class TestPolarity: + def test_bare_command_needs_a_match(self): + # Under `bash -e`, which is what Actions runs, a non-zero status here + # ends the step. + assert polarity("grep -qF 'all clear' out.txt") == "needs-match" + + def test_bang_needs_no_match(self): + assert polarity("! grep -qF 'error' out.txt") == "needs-no-match" + + def test_or_exit_needs_a_match(self): + assert polarity("grep -qF 'x' out.txt || { echo no; exit 1; }") == "needs-match" + + def test_and_exit_needs_no_match(self): + assert polarity("grep -qF 'x' out.txt && { echo bad; exit 1; }") == "needs-no-match" + + def test_if_then_exit_needs_no_match(self): + script = "if grep -qF 'oops' out.txt; then\n echo bad >&2\n exit 1\nfi" + assert polarity(script) == "needs-no-match" + + def test_if_not_then_exit_needs_a_match(self): + script = "if ! grep -qF 'good' out.txt; then\n echo missing\n exit 1\nfi" + assert polarity(script) == "needs-match" + + def test_else_exit_needs_a_match(self): + script = "if grep -qF 'good' out.txt; then\n :\nelse\n exit 1\nfi" + assert polarity(script) == "needs-match" + + def test_or_true_is_swallowed(self): + assert polarity("grep -qF 'x' out.txt || true") == "swallowed" + + def test_or_colon_is_swallowed(self): + assert polarity("grep -qF 'x' out.txt || :") == "swallowed" + + def test_if_with_no_failing_branch_is_decorative(self): + script = "if grep -qF 'x' out.txt; then\n echo found\nelse\n echo absent\nfi" + assert polarity(script) == "decorative" + + def test_status_discarded_by_pipeline_is_masked(self): + assert polarity("grep -qF 'x' out.txt | tee -a log") == "masked" + + +class TestPipelineNegation: + """The `!` binds to the pipeline, not to the command it precedes. + + `! printf '%s' "$out" | grep -q X` fails the step when grep *does* match, + and the `!` is nowhere near the grep. Reading it as unnegated classified + three real assertions in this organisation's workflows backwards. + """ + + def test_bang_in_front_of_a_pipeline_reaches_the_last_command(self): + script = "if ! printf '%s\\n' \"$out\" | grep -qx 'done.'; then\n exit 1\nfi" + assert polarity(script) == "needs-match" + + def test_without_the_bang_the_same_pipeline_is_the_other_way(self): + script = "if printf '%s\\n' \"$out\" | grep -qx 'done.'; then\n exit 1\nfi" + assert polarity(script) == "needs-no-match" + + def test_bang_on_the_grep_itself_still_works(self): + assert polarity("cat out.txt | ! grep -qF 'x'") == "needs-no-match" + + +class TestNotAnAssertion: + """A matcher run for its output is not a matcher run for its verdict.""" + + def test_output_flag_is_extraction(self): + script = "man githooks | col -b | grep -oE '^ [a-z-]*$' | sort > /tmp/from-man" + assert shell.scan(script) == [] + + def test_count_flag_is_extraction(self): + assert shell.scan("grep -c 'x' out.txt > n.txt") == [] + + def test_files_with_matches_is_extraction(self): + assert shell.scan("grep -rl 'TODO' src > files.txt") == [] + + def test_command_substitution_is_a_capture(self): + # The `|| true` here is correct: no match is an expected answer and + # the test comes later, on the variable. + script = "line=$(whichrc --home \"$HOME\" -q | grep '\\.bash_login:') || true" + assert shell.scan(script) == [] + + def test_backtick_substitution_is_a_capture(self): + assert shell.scan("n=`grep 'x' out.txt`") == [] + + def test_unquiet_grep_redirected_to_a_file_is_extraction(self): + assert shell.scan("grep 'warning' log.txt > warnings.txt") == [] + + def test_quiet_grep_redirected_to_devnull_is_still_an_assertion(self): + # -q already means "status only"; the redirect is belt and braces. + assert polarity("grep -q 'x' out.txt > /dev/null") == "needs-match" + + def test_process_substitution_is_not_a_capture(self): + # `<(...)` is an argument. The grep outside it is still the assertion. + script = "if grep -q 'unread' <(whichrc -q); then\n exit 1\nfi" + assert polarity(script) == "needs-no-match" + + +class TestBareAssertionCanFailABranch: + """A branch can fail the step without an `exit`. + + Under `bash -e` a bare `grep -q X f` in an `else` is itself an assertion. + Looking only for `exit`/`false`/`return` called a real `if`/`else` in + `whichrc` decorative when its else branch asserts. + """ + + def test_else_branch_holding_only_a_bare_grep(self): + script = ( + "if grep -q '^ ssh *nothing$' rows.txt; then\n" + " echo 'this bash does not source it'\n" + "else\n" + " grep -q '^ ssh *\\.bashrc$' rows.txt\n" + "fi" + ) + found = shell.scan(script) + assert [a.polarity for a in found] == ["needs-match", "needs-match"] + + +class TestFlags: + @pytest.mark.parametrize( + "command,fixed,anchored", + [ + ("grep -qF 'x' f", True, False), + ("grep -qxF 'x' f", True, True), + ("grep -qFx 'x' f", True, True), + ("grep -q -F -x 'x' f", True, True), + ("grep -q --fixed-strings --line-regexp 'x' f", True, True), + ("fgrep -q 'x' f", True, False), + ("grep -q 'x' f", False, False), + ("grep -qE 'a|b' f", False, False), + ], + ) + def test_fixed_and_anchored(self, command, fixed, anchored): + found = one(command) + assert (found.fixed, found.anchored) == (fixed, anchored) + + @pytest.mark.parametrize( + "command,pattern", + [ + ("grep -qF 'needle' f", "needle"), + ("grep -q -e 'needle' f", "needle"), + ("grep -qF --regexp=needle f", "needle"), + ("grep -qF -- '-leading-dash' f", "-leading-dash"), + ("grep -qF 'two words here' f", "two words here"), + ], + ) + def test_pattern(self, command, pattern): + assert one(command).pattern == pattern + + def test_pattern_keeps_a_trailing_space(self): + # `'unbuilt: '` is a real assertion in this org and the space is the point. + assert one("grep -qF 'unbuilt: ' out.txt").pattern == "unbuilt: " + + +class TestLogicalLines: + def test_backslash_continuation_is_one_assertion(self): + script = "grep -qF 'x' out.txt \\\n || { echo no; exit 1; }" + assert polarity(script) == "needs-match" + + def test_trailing_pipe_continuation(self): + script = "cat out.txt |\n grep -qF 'x'" + assert polarity(script) == "needs-match" + + def test_comment_is_stripped(self): + assert shell.scan("# grep -qF 'x' out.txt") == [] + + def test_hash_inside_a_pattern_is_not_a_comment(self): + assert one("grep -qF 'issue #3 open' out.txt").pattern == "issue #3 open" + + def test_line_number_is_within_the_script(self): + script = "set -e\n\ncat out.txt\ngrep -qF 'x' out.txt" + assert one(script).line == 4 From c050dd5e52cfe7ba2c8aef3e64339cbe0dbc98a8 Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:40:41 +0000 Subject: [PATCH 3/5] feat: four finding kinds, and --against to decide prefix by comparison unwitnessed is the headline: a negated assertion that nothing else proves can ever match. The witness rule is deliberately generous, since a false positive costs more than a miss. prefix is the logbook#63 shape. It is the one judgement about intent rather than semantics, so --against replaces it with a comparison against real output -- and the case worth finding is a pattern that is the *opening* of a longer line, meaning the append already happened and the assertion did not notice. A pattern holding a shell expansion is never judged; its value is not in the file. --- src/deadassert/__init__.py | 3 + src/deadassert/__main__.py | 5 + src/deadassert/analyse.py | 355 +++++++++++++++++++++++++++++++++++++ src/deadassert/cli.py | 88 +++++++++ src/deadassert/report.py | 161 +++++++++++++++++ tests/test_analyse.py | 260 +++++++++++++++++++++++++++ 6 files changed, 872 insertions(+) create mode 100644 src/deadassert/__init__.py create mode 100644 src/deadassert/__main__.py create mode 100644 src/deadassert/analyse.py create mode 100644 src/deadassert/cli.py create mode 100644 src/deadassert/report.py create mode 100644 tests/test_analyse.py diff --git a/src/deadassert/__init__.py b/src/deadassert/__init__.py new file mode 100644 index 0000000..71576c3 --- /dev/null +++ b/src/deadassert/__init__.py @@ -0,0 +1,3 @@ +"""Say which of your CI assertions pass whether or not the thing they name is true.""" + +__version__ = "0.1.0" diff --git a/src/deadassert/__main__.py b/src/deadassert/__main__.py new file mode 100644 index 0000000..dd8a8c9 --- /dev/null +++ b/src/deadassert/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/src/deadassert/analyse.py b/src/deadassert/analyse.py new file mode 100644 index 0000000..1a33355 --- /dev/null +++ b/src/deadassert/analyse.py @@ -0,0 +1,355 @@ +"""Turning assertions into findings. + +Four kinds, and the first one is why this tool exists. + +`unwitnessed` + A `needs-no-match` assertion -- one that fails the step when its pattern + *appears* -- where nothing else in the workflows establishes that the + pattern can appear at all. `if grep -q 'merge-upstream' findings.txt; + then exit 1; fi` is a promise that a particular finding kind stays out of + the output. Rename that finding kind in the tool and the assertion keeps + passing, forever, protecting nothing. Its positive twin would have gone + red the same afternoon. + +`prefix` + A `needs-match` assertion matching a whole sentence as a *substring*. + `grep -qF 'No rules here.' out.txt` keeps passing when the line becomes + `No rules here. Read 5 files.` -- which is exactly the edit people make + when they add a count to a summary line. This is the only kind that rests + on a guess about intent, and `--against` replaces the guess with the + output. + +`swallowed`, `masked`, `decorative` + The assertion's exit status never reaches the step's: discarded by + `|| true`, thrown away by its position in a pipeline, or used only to + pick which branch prints something. + +The witness rule is deliberately generous, because a false positive here +costs more than a miss. A pattern counts as witnessed if any `needs-match` +assertion anywhere in the files scanned asserts that pattern, *or* asserts a +longer string containing it -- asserting `no rules here, so nothing to check` +proves `nothing to check` can appear. The scope is every file scanned rather +than the job, since the vocabulary of a tool's output is a property of the +tool and not of the job that ran it. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field + +from . import shell, workflows +from .shell import Assertion +from .workflows import Script, WorkflowError + + +KIND_ORDER = ("unwitnessed", "prefix", "swallowed", "masked", "decorative") + + +class CorpusError(ValueError): + """Nowhere to read workflows from.""" + + +@dataclass +class Finding: + kind: str + script: Script + assertion: Assertion + #: The sentence explaining this finding, built where the facts are. + detail: str + + @property + def file(self) -> str: + return self.script.workflow + + @property + def line(self) -> int: + return self.script.line_of(self.assertion.line) + + @property + def where(self) -> str: + return f"{os.path.basename(self.file)}:{self.line}" + + +@dataclass +class Result: + root: str + #: Workflow files read. + files: list[str] = field(default_factory=list) + scripts: int = 0 + assertions: list[tuple[Script, Assertion]] = field(default_factory=list) + findings: list[Finding] = field(default_factory=list) + #: Assertions recognised but not decided, as `(where, why)`. + undecided: list[tuple[str, str]] = field(default_factory=list) + #: `run:` scripts skipped for their shell, as `(where, shell)`. + foreign: list[tuple[str, str]] = field(default_factory=list) + #: Files that would not parse, as `(path, why)`. + unreadable: list[tuple[str, str]] = field(default_factory=list) + #: The `--against` file, when one was given. + against: str | None = None + #: Patterns `--against` showed matching nothing in the output. + absent: list[tuple[str, str]] = field(default_factory=list) + + @property + def live(self) -> list[tuple[Script, Assertion]]: + return [(s, a) for s, a in self.assertions if a.live] + + @property + def negated(self) -> int: + return sum(1 for _, a in self.assertions if a.polarity == "needs-no-match") + + @property + def nothing_to_check(self) -> bool: + return not self.assertions + + +def _is_expansion(pattern: str | None) -> bool: + """Whether a pattern's value depends on something the shell fills in. + + `$var`, `${var}`, `$(cmd)` and backticks all mean the characters in the + workflow file are not the characters grep is given, so nothing here can + say what the assertion matches. A literal `\\$` is not an expansion. + """ + if pattern is None: + return False + return bool(re.search(r"(? list[str]: + """Patterns positively asserted somewhere, which prove a string can appear.""" + return [a.pattern for _, a in assertions if a.polarity == "needs-match" and a.pattern] + + +def _witnessed(pattern: str, witnesses: list[str]) -> bool: + """Whether any positive assertion establishes `pattern` can appear. + + Containment counts in one direction only. A positive assertion on a + *longer* string containing this pattern proves the pattern appears. A + positive assertion on a *shorter* one does not: `grep -qF 'checked'` + passing says nothing about `not checked:` ever being printed. + """ + return any(pattern in witness for witness in witnesses) + + +def _reads_as_sentence(pattern: str) -> bool: + """Whether a pattern reads as a whole line of output rather than a fragment. + + Narrow on purpose, and every clause of it is here because a wider version + fired on a substring match that was deliberate. Measured against the 160 + positive assertions in this organisation's own workflows: + + * **Ends with a full stop.** `'.dockerignore line 1 (.git)'` ends with a + bracket and is a column of a finding line, not a line. + * **Starts with a capital or a digit.** `'submodules. Read 3 gitlinks and + 3 .gitmodules entries.'` starts mid-word because it is the tail of + `Found 3 declarations ... across 3 submodules. Read ...`; a pattern + beginning lower-case has a line in front of it by construction. + * **Four words or more.** `'Nothing to check.'` is the end of a longer + sentence in the tool that prints it, and matching it as a substring is + the only thing that would work. + * **Unanchored.** A pattern with `^` or `$` in it is already being + matched as a line, by hand. + + This is the one judgement in `deadassert` that is about what somebody + meant rather than about what the shell does, which is why `--against` + exists: given real output, the same question is decided by comparison. + """ + text = pattern.strip() + if not text.endswith("."): + return False + if text.startswith("^") or text.endswith("$"): + return False + if not (text[0].isupper() or text[0].isdigit()): + return False + return len(text.split()) >= 4 + + +def _as_regex(assertion: Assertion) -> re.Pattern | None: + """The assertion's pattern as a Python regex, or None if not translatable.""" + if assertion.pattern is None: + return None + if assertion.fixed: + body = re.escape(assertion.pattern) + else: + # Basic and extended REs share enough with Python's for the operators + # that turn up in CI assertions; anything else is left undecided. + if re.search(r"\\[0-9({]|\[\[:", assertion.pattern): + return None + body = assertion.pattern + if assertion.anchored: + body = f"(?:{body})\\Z" + return re.compile(f"\\A{body}") + try: + return re.compile(body) + except re.error: + return None + + +def _prefix_detail( + assertion: Assertion, lines: list[str] | None, against: str | None +) -> str | None: + """Why this positive assertion is weaker than it reads, or None if it isn't. + + Without `--against`, the sentence heuristic, which is about intent. + + With `--against`, the pattern is compared to real output lines, and the + case worth finding is the one that first version of this function got + backwards. Three things a fixed, unanchored pattern `P` can be: + + * **A whole line.** `P` equals some line exactly. The assertion behaves + like a whole-line match today and will silently stop being one the + first time anything is appended. Reported whatever `P` looks like, + because here the output has settled the question that + `_reads_as_sentence` only guesses at. + * **The opening of a longer line.** Some line *starts with* `P` and has + more after it. The append has **already happened** and the assertion + did not notice -- it is passing over a line it does not name in full. + This is the stronger finding of the two, and the heuristic cannot see + it at all. Still gated on the pattern reading as a sentence, because a + short leading fragment like `'examples/a.txt:3:'` is a prefix match + somebody meant. + * **Inside a line.** `P` occurs, but never at the start. A deliberate + substring match, and no finding. This is what `--against` is for: + it takes findings away. + """ + pattern = assertion.pattern + if pattern is None or assertion.anchored or not assertion.fixed: + return None + + if lines is None: + if not _reads_as_sentence(pattern): + return None + return ( + "the pattern reads as a whole line of output, but -F without -x " + "matches it as a substring, so the assertion survives anything " + "appended to that line" + ) + + wanted = pattern.strip() + where = os.path.basename(against or "the output") + if any(line.strip() == wanted for line in lines): + return ( + f"the pattern is a whole line of {where}, but -F without -x matches it " + "as a substring, so the assertion survives anything appended to that line" + ) + grown = [ + line.strip() for line in lines if line.strip().startswith(wanted) and line.strip() != wanted + ] + if grown and _reads_as_sentence(pattern): + return ( + f"in {where} that line reads {grown[0]!r} -- the pattern is its opening, " + "so this assertion is already passing over a line it does not name in " + "full, which is the edit it was written to catch" + ) + return None + + +def analyse( + root: str, + paths: list[str] | None = None, + against: str | None = None, +) -> Result: + """Read the workflows under `root` and report the assertions that cannot fail.""" + if paths: + relative = [] + for path in paths: + if not os.path.isfile(path): + raise CorpusError(f"{path}: not a file") + relative.append(os.path.relpath(path, root) if root != "." else path) + else: + relative = workflows.find_workflows(root) + if not relative: + raise CorpusError( + f"{os.path.join(root, workflows.WORKFLOW_DIR)}: no workflow files here" + ) + + result = Result(root=root, against=against) + + output_lines: list[str] | None = None + if against is not None: + with open(against, encoding="utf-8") as handle: + output_lines = handle.read().split("\n") + + for rel in relative: + try: + workflow = workflows.load(root, rel) + except WorkflowError as exc: + result.unreadable.append((rel, str(exc))) + continue + result.files.append(rel) + for label, shell_name in workflow.foreign: + result.foreign.append((f"{os.path.basename(rel)} {label}", shell_name)) + for script in workflow.scripts: + result.scripts += 1 + for assertion in shell.scan(script.body): + where = f"{os.path.basename(rel)}:{script.line_of(assertion.line)}" + if assertion.polarity == "undecided": + result.undecided.append( + (where, assertion.undecided or "the shape was not recognised") + ) + continue + if _is_expansion(assertion.pattern): + # `grep -qF "$clean" out.txt` is a real assertion, but its + # pattern is whatever the variable held at the time. Judging + # it on the characters `$clean` would be judging a string + # that is never matched against anything. + result.undecided.append( + (where, "the pattern is a shell expansion, so its value is not known here") + ) + continue + result.assertions.append((script, assertion)) + + witnesses = _witnesses(result.assertions) + + for script, assertion in result.assertions: + if assertion.polarity == "needs-no-match": + if assertion.pattern is None: + continue + if _witnessed(assertion.pattern, witnesses): + continue + result.findings.append( + Finding( + kind="unwitnessed", + script=script, + assertion=assertion, + detail=( + "this passes when the pattern is absent, and nothing in " + "these workflows asserts that it is ever present -- so it " + "also passes if the pattern stops being produced at all" + ), + ) + ) + elif assertion.polarity == "needs-match": + detail = _prefix_detail(assertion, output_lines, against) + if detail: + result.findings.append( + Finding(kind="prefix", script=script, assertion=assertion, detail=detail) + ) + if output_lines is not None: + regex = _as_regex(assertion) + if regex and not any(regex.search(line) for line in output_lines): + result.absent.append( + ( + f"{os.path.basename(script.workflow)}:{script.line_of(assertion.line)}", + assertion.pattern or "", + ) + ) + elif assertion.polarity in ("swallowed", "masked", "decorative"): + detail = { + "swallowed": "its failure is discarded, so the step passes either way", + "masked": "its exit status is discarded by the rest of the pipeline, " + "so the step passes either way", + "decorative": "it chooses which branch runs, and neither branch can fail the step", + }[assertion.polarity] + result.findings.append( + Finding( + kind=assertion.polarity, + script=script, + assertion=assertion, + detail=detail, + ) + ) + + result.findings.sort(key=lambda f: (KIND_ORDER.index(f.kind), f.file, f.line)) + return result diff --git a/src/deadassert/cli.py b/src/deadassert/cli.py new file mode 100644 index 0000000..72751cf --- /dev/null +++ b/src/deadassert/cli.py @@ -0,0 +1,88 @@ +"""Command line. Exit 0 nothing to report, 1 findings, 2 no answer.""" + +from __future__ import annotations + +import argparse +import sys + +from . import report +from .analyse import CorpusError, analyse + + +def _os_error(error: OSError) -> str: + """An OSError as a sentence, rather than as its repr. + + `str(OSError)` is `[Errno 2] No such file or directory: '/tmp/x'`, which + is Python's, not ours. The rest of the family prints `: : + ` and so does this. + """ + reason = (error.strerror or "cannot be read").lower() + return f"{error.filename or '?'}: {reason}" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="deadassert", + description=( + "Say which of your CI assertions pass whether or not the thing they name is true." + ), + epilog=( + "Exit code is 0 when there is nothing to report, 1 when there is, and 2 " + "when deadassert could not get an answer at all." + ), + ) + parser.add_argument( + "root", + nargs="?", + default=".", + help="repository to read .github/workflows from (default: .)", + ) + parser.add_argument( + "-f", + "--file", + action="append", + default=[], + metavar="PATH", + help="read this workflow file instead of searching; repeatable", + ) + parser.add_argument( + "--against", + metavar="PATH", + help=( + "a file of real output the assertions run over. Decides the `prefix` " + "kind by comparison instead of by the shape of the pattern." + ), + ) + parser.add_argument("-q", "--quiet", action="store_true", help="one line per finding") + parser.add_argument("--json", action="store_true", help="machine-readable output") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + try: + result = analyse(args.root, paths=args.file or None, against=args.against) + except CorpusError as error: + print(f"deadassert: {error}", file=sys.stderr) + return 2 + except OSError as error: + print(f"deadassert: {_os_error(error)}", file=sys.stderr) + return 2 + + for line in report.error_lines(result): + print(line, file=sys.stderr) + + if not result.files: + # Nothing was read, so there is no measurement for a verdict to be + # about. Nothing on stdout either, which is what stops a failed run + # reading as a clean one. + print("deadassert: no workflow file could be read", file=sys.stderr) + return 2 + + if args.json: + print(report.as_json(result)) + else: + print(report.text(result, quiet=args.quiet)) + + return 1 if result.findings else 0 diff --git a/src/deadassert/report.py b/src/deadassert/report.py new file mode 100644 index 0000000..9309fdf --- /dev/null +++ b/src/deadassert/report.py @@ -0,0 +1,161 @@ +"""Output. One block per finding, in the order you would fix them.""" + +from __future__ import annotations + +import json +import textwrap + +from .analyse import Result + +HEADLINE = { + "unwitnessed": "passes whether or not the pattern is ever produced", + "prefix": "matches a whole sentence as a substring", + "swallowed": "cannot fail the step", + "masked": "cannot fail the step", + "decorative": "cannot fail the step", +} + + +def _plural(count: int, noun: str) -> str: + return f"{count} {noun}" if count == 1 else f"{count} {noun}s" + + +def _wrap(text: str, indent: str) -> str: + return textwrap.fill(text, width=92, initial_indent=indent, subsequent_indent=indent) + + +def _denominator(result: Result) -> str: + """What the verdict was measured over. + + logbook#59 and logbook#57: a verdict with no denominator cannot be told + apart from a verdict over nothing, and this tool is itself about + assertions that look like they are checking something and are not. + """ + return ( + f"{_plural(len(result.assertions), 'assertion')} in " + f"{_plural(result.scripts, 'script')} across " + f"{_plural(len(result.files), 'workflow file')}, " + f"{result.negated} of them negated." + ) + + +def nothing_to_check(result: Result) -> str: + """Said instead of an all-clear when there were no assertions at all.""" + return ( + f"No assertions here. Read {_plural(result.scripts, 'run: script')} in " + f"{_plural(len(result.files), 'workflow file')} and found no command whose " + "exit status decides a step. Nothing to check." + ) + + +def error_lines(result: Result) -> list[str]: + """Diagnostics, for stderr. Never part of any mode's stdout.""" + lines = [] + for path, why in result.unreadable: + lines.append(f"deadassert: {path}: {why}") + for where, shell_name in result.foreign: + lines.append(f"deadassert: {where}: skipped, shell is `{shell_name}`") + return lines + + +def text(result: Result, quiet: bool = False) -> str: + if result.nothing_to_check: + return nothing_to_check(result) + + out: list[str] = [] + if not result.findings: + out.append("Every assertion here fails when the thing it names stops being true.") + out.append(_denominator(result)) + if result.absent: + out.append("") + out.extend(_absent_block(result)) + if result.undecided: + out.append("") + out.extend(_undecided_block(result)) + return "\n".join(out) + + if quiet: + for finding in result.findings: + pattern = finding.assertion.pattern or "?" + out.append(f"{finding.file}:{finding.line}: {finding.kind}: {pattern!r}") + return "\n".join(out) + + for finding in result.findings: + pattern = finding.assertion.pattern or "?" + out.append(f"{finding.where} {finding.kind} {pattern!r}") + out.append(_wrap(f"{finding.assertion.program}: {HEADLINE[finding.kind]}", " ")) + out.append(_wrap(finding.detail, " ")) + out.append(f" {finding.assertion.text}") + out.append("") + + kinds = {} + for finding in result.findings: + kinds[finding.kind] = kinds.get(finding.kind, 0) + 1 + breakdown = ", ".join(f"{count} {kind}" for kind, count in kinds.items()) + out.append( + f"{_plural(len(result.findings), 'assertion')} here " + f"{'does' if len(result.findings) == 1 else 'do'} not check what " + f"{'it reads' if len(result.findings) == 1 else 'they read'} as checking " + f"({breakdown})." + ) + out.append(_denominator(result)) + + if result.absent: + out.append("") + out.extend(_absent_block(result)) + if result.undecided: + out.append("") + out.extend(_undecided_block(result)) + return "\n".join(out) + + +def _absent_block(result: Result) -> list[str]: + """Positive assertions whose pattern is nowhere in the `--against` file. + + Not a finding: a positive assertion that cannot match is already failing + CI, so it is either about a different step's output or the wrong file was + given. Worth printing either way, because it says the comparison is not + measuring what the reader thinks. + """ + lines = [f"not matched in {result.against}:"] + for where, pattern in result.absent: + lines.append(f" {where} {pattern!r}") + return lines + + +def _undecided_block(result: Result) -> list[str]: + lines = ["not decided:"] + for where, why in result.undecided: + lines.append(f" {where} {why}") + return lines + + +def as_json(result: Result) -> str: + payload = { + "files": result.files, + "scripts": result.scripts, + "assertions": len(result.assertions), + "negated": result.negated, + "findings": [ + { + "kind": finding.kind, + "file": finding.file, + "line": finding.line, + "job": finding.script.job, + "step": finding.script.label, + "program": finding.assertion.program, + "pattern": finding.assertion.pattern, + "fixed": finding.assertion.fixed, + "anchored": finding.assertion.anchored, + "polarity": finding.assertion.polarity, + "detail": finding.detail, + "command": finding.assertion.text, + } + for finding in result.findings + ], + "undecided": [{"where": w, "why": y} for w, y in result.undecided], + "foreign": [{"where": w, "shell": s} for w, s in result.foreign], + "unreadable": [{"path": p, "why": y} for p, y in result.unreadable], + "not_matched": [{"where": w, "pattern": p} for w, p in result.absent], + } + return json.dumps(payload, indent=2) diff --git a/tests/test_analyse.py b/tests/test_analyse.py new file mode 100644 index 0000000..3dd13b8 --- /dev/null +++ b/tests/test_analyse.py @@ -0,0 +1,260 @@ +"""Witnessing, the sentence rule, and what `--against` decides.""" + +from __future__ import annotations + +import pytest + +from deadassert import analyse + + +def repo(tmp_path, *steps: str) -> str: + """A repository whose single job runs each of `steps` as its own script.""" + body = ["on: push", "jobs:", " dogfood:", " runs-on: ubuntu-latest", " steps:"] + for step in steps: + body.append(" - run: |") + for line in step.strip("\n").split("\n"): + body.append(f" {line}") + directory = tmp_path / ".github" / "workflows" + directory.mkdir(parents=True) + (directory / "ci.yml").write_text("\n".join(body) + "\n") + return str(tmp_path) + + +def kinds(result) -> list[str]: + return [finding.kind for finding in result.findings] + + +def patterns(result, kind: str) -> list[str]: + return [f.assertion.pattern for f in result.findings if f.kind == kind] + + +class TestUnwitnessed: + def test_a_negated_assertion_with_no_witness_is_a_finding(self, tmp_path): + root = repo(tmp_path, "if grep -qF 'merge-upstream' out.txt; then\n exit 1\nfi") + result = analyse.analyse(root) + assert kinds(result) == ["unwitnessed"] + + def test_a_positive_assertion_on_the_same_pattern_witnesses_it(self, tmp_path): + root = repo( + tmp_path, + "grep -qF 'merge-upstream' expected.txt", + "if grep -qF 'merge-upstream' out.txt; then\n exit 1\nfi", + ) + assert analyse.analyse(root).findings == [] + + def test_a_longer_positive_pattern_containing_it_witnesses_it(self, tmp_path): + """Asserting a longer string proves the shorter one appears.""" + root = repo( + tmp_path, + "grep -qF 'reported merge-upstream twice' expected.txt", + "if grep -qF 'merge-upstream' out.txt; then\n exit 1\nfi", + ) + assert analyse.analyse(root).findings == [] + + def test_a_shorter_positive_pattern_does_not_witness_it(self, tmp_path): + """Containment counts one way only. + + `grep -qF 'checked'` passing says nothing about `not checked:` ever + being printed, so it cannot stand as the proof that it can be. + """ + root = repo( + tmp_path, + "grep -qF 'checked' expected.txt", + "if grep -qF 'not checked:' out.txt; then\n exit 1\nfi", + ) + assert kinds(analyse.analyse(root)) == ["unwitnessed"] + + def test_a_witness_in_another_job_still_counts(self, tmp_path): + directory = tmp_path / ".github" / "workflows" + directory.mkdir(parents=True) + (directory / "ci.yml").write_text( + "on: push\n" + "jobs:\n" + " a:\n" + " steps:\n" + " - run: grep -qF 'kind-x' expected.txt\n" + " b:\n" + " steps:\n" + " - run: |\n" + " if grep -qF 'kind-x' out.txt; then\n" + " exit 1\n" + " fi\n" + ) + assert analyse.analyse(str(tmp_path)).findings == [] + + def test_a_negated_assertion_is_counted_in_the_denominator(self, tmp_path): + root = repo(tmp_path, "if grep -qF 'x' out.txt; then\n exit 1\nfi") + assert analyse.analyse(root).negated == 1 + + +class TestSentenceRule: + """Every clause of this exists because a wider version fired on a + deliberate substring match in this organisation's real workflows.""" + + @pytest.mark.parametrize( + "pattern", + [ + "Every instruction has an effect on this build.", + "4 instructions with no effect on this build.", + "Found 3 declarations git never acts on, across 3 submodules.", + "10 lines here have no effect on what logrotate does. Read 5 rules in 6 files.", + ], + ) + def test_whole_summary_lines_are_findings(self, tmp_path, pattern): + root = repo(tmp_path, f"grep -qF '{pattern}' out.txt") + assert kinds(analyse.analyse(root)) == ["prefix"] + + @pytest.mark.parametrize( + "pattern,why", + [ + (".dockerignore line 1 (.git)", "ends with a bracket, so it is a column"), + ("submodules. Read 3 gitlinks and 3 entries.", "starts mid-sentence"), + ("Nothing to check.", "three words, and the tail of a longer sentence"), + ("merge-upstream", "a finding kind, not a line"), + ("unbuilt: ", "a prefix, matched as one on purpose"), + ("^not checked:", "already anchored by hand"), + ], + ) + def test_deliberate_substring_matches_are_not_findings(self, tmp_path, pattern, why): + root = repo(tmp_path, f"grep -qF {pattern!r} out.txt") + assert analyse.analyse(root).findings == [], why + + def test_an_anchored_assertion_is_never_a_prefix_finding(self, tmp_path): + root = repo(tmp_path, "grep -qxF 'Every instruction has an effect on this build.' o") + assert analyse.analyse(root).findings == [] + + def test_a_regex_assertion_is_not_a_prefix_finding(self, tmp_path): + # Without -F the pattern is a regex and `-x` is not the fix for it. + root = repo(tmp_path, "grep -q 'Every instruction has an effect on this build.' o") + assert analyse.analyse(root).findings == [] + + +class TestAgainst: + """Given real output, the question stops being about intent.""" + + def output(self, tmp_path, *lines: str) -> str: + path = tmp_path / "output.txt" + path.write_text("\n".join(lines) + "\n") + return str(path) + + def test_a_pattern_that_is_a_whole_line_is_a_finding(self, tmp_path): + root = repo(tmp_path, "grep -qF 'All clear.' out.txt") + against = self.output(tmp_path, "All clear.") + result = analyse.analyse(root, against=against) + assert kinds(result) == ["prefix"] + assert "whole line of output.txt" in result.findings[0].detail + + def test_a_short_pattern_that_is_a_whole_line_is_still_a_finding(self, tmp_path): + """The output has settled what the heuristic could only guess. + + `'All clear.'` is three words and would not pass the sentence rule, + but it provably *is* a whole line here, so matching it without -x is + latently wrong whatever it looks like. + """ + root = repo(tmp_path, "grep -qF 'ok' out.txt") + against = self.output(tmp_path, "ok") + assert kinds(analyse.analyse(root, against=against)) == ["prefix"] + + def test_a_pattern_that_is_the_opening_of_a_longer_line_is_a_finding(self, tmp_path): + """The append already happened and the assertion did not notice.""" + root = repo(tmp_path, "grep -qF 'Every rule here rotates something.' out.txt") + against = self.output(tmp_path, "Every rule here rotates something. Read 1 rule.") + result = analyse.analyse(root, against=against) + assert kinds(result) == ["prefix"] + assert "already passing over a line it does not name in full" in (result.findings[0].detail) + assert "Read 1 rule." in result.findings[0].detail + + def test_a_pattern_inside_a_line_is_not_a_finding(self, tmp_path): + """What `--against` is for: taking findings away.""" + root = repo(tmp_path, "grep -qF 'Read 3 rules in 2 files.' out.txt") + against = self.output(tmp_path, "Found 1 dead rule. Read 3 rules in 2 files.") + assert analyse.analyse(root, against=against).findings == [] + + def test_against_overrides_the_heuristic_both_ways(self, tmp_path): + """The same assertion, a finding without the output and not with it.""" + root = repo(tmp_path, "grep -qF 'Found 2 dead entries, across 3 files.' out.txt") + assert kinds(analyse.analyse(root)) == ["prefix"] + against = self.output(tmp_path, "note: Found 2 dead entries, across 3 files.") + assert analyse.analyse(root, against=against).findings == [] + + def test_a_positive_pattern_matching_nothing_is_listed_not_found(self, tmp_path): + root = repo(tmp_path, "grep -qF 'never printed' out.txt") + against = self.output(tmp_path, "something else") + result = analyse.analyse(root, against=against) + assert result.findings == [] + assert result.absent == [("ci.yml:7", "never printed")] + + def test_an_anchored_pattern_is_matched_as_a_whole_line(self, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt") + against = self.output(tmp_path, "note: All clear.") + # -x means the substring hit does not count, so this one is absent. + assert analyse.analyse(root, against=against).absent == [("ci.yml:7", "All clear.")] + + +class TestExpansions: + """A pattern the shell fills in is not a pattern this tool can judge.""" + + @pytest.mark.parametrize( + "pattern", ['"$clean"', '"${msg}"', '"$(cat expected.txt)"', '"`date`"'] + ) + def test_expansions_are_undecided_not_judged(self, tmp_path, pattern): + root = repo(tmp_path, f"if grep -qF {pattern} out.txt; then\n exit 1\nfi") + result = analyse.analyse(root) + assert result.findings == [] + assert len(result.undecided) == 1 + assert "shell expansion" in result.undecided[0][1] + + def test_an_escaped_dollar_is_a_literal_and_is_judged(self, tmp_path): + root = repo(tmp_path, "grep -qF 'costs \\$5 to run.' out.txt") + assert analyse.analyse(root).undecided == [] + + def test_an_expansion_is_not_counted_in_the_denominator(self, tmp_path): + root = repo(tmp_path, 'grep -qF "$msg" out.txt') + assert analyse.analyse(root).assertions == [] + + +class TestCannotFail: + @pytest.mark.parametrize( + "step,kind", + [ + ("grep -qF 'x' out.txt || true", "swallowed"), + ("grep -qF 'x' out.txt | tee -a log", "masked"), + ("if grep -qF 'x' out.txt; then\n echo yes\nelse\n echo no\nfi", "decorative"), + ], + ) + def test_each_shape(self, tmp_path, step, kind): + assert kinds(analyse.analyse(repo(tmp_path, step))) == [kind] + + +class TestNothingToCheck: + """logbook#62: an all-clear over nothing is not an all-clear.""" + + def test_a_workflow_with_no_assertions_says_so(self, tmp_path): + root = repo(tmp_path, "echo hello\nmake build") + result = analyse.analyse(root) + assert result.nothing_to_check + assert result.findings == [] + + def test_a_workflow_with_a_sound_assertion_is_not_nothing_to_check(self, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt") + result = analyse.analyse(root) + assert not result.nothing_to_check + assert result.findings == [] + + +class TestRefusals: + def test_a_tree_with_no_workflows_raises(self, tmp_path): + with pytest.raises(analyse.CorpusError, match="no workflow files here"): + analyse.analyse(str(tmp_path)) + + def test_an_unreadable_workflow_is_recorded_not_raised(self, tmp_path): + directory = tmp_path / ".github" / "workflows" + directory.mkdir(parents=True) + (directory / "ci.yml").write_text("name: [unclosed\n") + result = analyse.analyse(str(tmp_path)) + assert result.files == [] + assert len(result.unreadable) == 1 + + def test_a_named_file_that_is_not_there(self, tmp_path): + with pytest.raises(analyse.CorpusError, match="not a file"): + analyse.analyse(".", paths=[str(tmp_path / "nope.yml")]) From b6de4b9e80f1a825b4de19bd5278b1cb1edfad84 Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:44:47 +0000 Subject: [PATCH 4/5] feat: README, worked example, and CI that can go red The example holds one of every kind on purpose, and its expected output is committed and checked with diff rather than grep -- which is the README's own recommendation and the thing this repository is about. The dogfood job asserts the denominator as well as the verdict (logbook#57, logbook#59), and then mutates a copy of this file to prove the assertion can actually fail. deadassert finds only 2 assertions in its own CI because that CI uses diff and test where it can; the number is printed rather than hidden. CI runs ruff format --check alongside ruff check (logbook#60). --- .github/workflows/ci.yml | 165 +++++++++++++++++++++++++ README.md | 196 +++++++++++++++++++++++++++++- examples/.github/workflows/ci.yml | 79 ++++++++++++ examples/expected-against.txt | 25 ++++ examples/expected-quiet.txt | 4 + examples/output.txt | 14 +++ tests/__init__.py | 0 tests/test_cli.py | 187 ++++++++++++++++++++++++++++ tests/test_workflows.py | 165 +++++++++++++++++++++++++ 9 files changed, 833 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 examples/.github/workflows/ci.yml create mode 100644 examples/expected-against.txt create mode 100644 examples/expected-quiet.txt create mode 100644 examples/output.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_workflows.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2b4d7cb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,165 @@ +name: CI + +'on': + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ['3.10', '3.12'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - run: pip install -e ".[dev]" + - run: pytest --junit-xml=results.xml + + - name: the suite that pins polarity actually ran + run: | + set -euo pipefail + python - <<'PY' + import xml.etree.ElementTree as ET + root = ET.parse("results.xml").getroot() + suite = root if root.tag == "testsuite" else root.find("testsuite") + total = int(suite.get("tests", 0)) + skipped = int(suite.get("skipped", 0)) + print(f"{total} tests, {skipped} skipped") + assert total >= 110, f"only {total} tests collected" + assert skipped == 0, f"{skipped} tests skipped" + PY + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -e ".[dev]" + - run: ruff check . + # logbook#60: `ruff check` and `ruff format --check` are different + # questions, and a repo that only asks the first produces a diff the + # moment anyone runs the project's own formatter. + - run: ruff format --check . + + example: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -e . + + # `diff` rather than `grep`, here and below, on purpose. This whole + # repository is about assertions that match less than they look like, + # and a whole-output assertion written as `diff` has no substring + # weakness to have an opinion about. + - name: the shipped example prints exactly what the README quotes + run: | + set -uo pipefail + deadassert examples/ -q > got.txt + status=$? + set -e + test "$status" -eq 1 + diff -u examples/expected-quiet.txt got.txt + + - name: --against turns the prefix finding into the stronger one + run: | + set -uo pipefail + deadassert examples/ --against examples/output.txt > got.txt + status=$? + set -e + test "$status" -eq 1 + diff -u examples/expected-against.txt got.txt + + - name: every sound positive assertion in the example matches the shipped output + run: | + set -uo pipefail + deadassert examples/ --against examples/output.txt --json > out.json + status=$? + set -e + test "$status" -eq 1 + python - <<'PY' + import json + found = json.load(open("out.json"))["not_matched"] + assert found == [], f"examples/output.txt has drifted: {found}" + PY + + - name: exit codes + run: | + deadassert examples/ > /dev/null; test $? -eq 1 + mkdir -p clean/.github/workflows # a repo with one sound assertion + printf \ + "'on': push\njobs:\n j:\n steps:\n - run: grep -qxF 'ok' o\n" \ + > clean/.github/workflows/ci.yml + deadassert clean > /dev/null; test $? -eq 0 + deadassert /nonexistent > /dev/null 2>&1; test $? -eq 2 + + dogfood: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -e . + + # logbook#59: a dogfood step that runs the tool on a repository holding + # none of its subject asserts nothing, and passes while doing it. The + # subject here is assertions in workflow files, and this file is full of + # them, so the step below is one of the few in the org that cannot be + # vacuous. The denominator is asserted anyway, because that is the + # difference between a real all-clear and an empty one. + - name: every assertion in this repository can fail + run: | + set -euo pipefail + deadassert . | tee out.txt + grep -qxF \ + 'Every assertion here fails when the thing it names stops being true.' \ + out.txt + + # Two. Not many, and worth saying out loud rather than hiding behind a + # green tick: this file prefers `diff` and `test "$status"` to `grep` + # wherever a whole output is being checked, which is the right thing to + # do and also leaves this tool very little of its own subject to find. + # The richer corpus is `examples/`, which holds one of every kind and is + # checked by the `example` job above. What gives *this* step its teeth + # is the mutation below, not the size of the number. + - name: and it was measured over something + run: | + set -euo pipefail + deadassert . --json > out.json + python - <<'PY' + import json + d = json.load(open("out.json")) + print(f"{d['assertions']} assertions in {d['scripts']} scripts") + assert d["assertions"] >= 2, f"only {d['assertions']} assertions seen" + assert d["findings"] == [], d["findings"] + PY + + # The assertion above is only worth anything if it can go red. Break + # one assertion in a copy of this file and check that it does. + - name: a broken assertion in this file is caught + run: | + set -euo pipefail + mkdir -p mutant/.github/workflows + sed 's/grep -qxF/grep -qF/' .github/workflows/ci.yml \ + > mutant/.github/workflows/ci.yml + set +e + deadassert mutant > mutant.txt + status=$? + set -e + cat mutant.txt + test "$status" -eq 1 + grep -qF 'prefix' mutant.txt diff --git a/README.md b/README.md index f2da13e..8d8a53d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,197 @@ # deadassert -Say which of your CI assertions pass whether or not the thing they name is true. +Say which of your CI assertions pass whether or not the thing they name is +true. -Scaffold. The work is in the pull request. +If you have a workflow full of `grep -q 'all clear' out.txt` steps and you +want to know which of them would still be green after the output they check +stopped saying that, this tells you. It reads the `run:` scripts in +`.github/workflows`, works out which way round each assertion fails, and +reports the ones that cannot fail at all. + +## Install + +```sh +pip install -e . +``` + +Python 3.10+, PyYAML, nothing else. + +## Use + +```sh +deadassert # .github/workflows in the current repo +deadassert path/to/repo +deadassert -f .github/workflows/ci.yml +deadassert --against out.txt # decide `prefix` against real output +deadassert -q # one line per finding +deadassert --json +``` + +Exit code is 0 when there is nothing to report, 1 when there is, and 2 when +`deadassert` could not get an answer — no workflow files, or none that +parse. On exit 2 nothing is printed to stdout, so a run that failed can +never be read as a clean one. + +A workflow with no assertions in it exits 0, but says so in its own words +rather than reporting a clean one. + +## The thing this is about + +An assertion that rots does not rot symmetrically. + +```sh +grep -qF 'no dead entries' out.txt # fails when the string goes away +if grep -qF 'dead entry' out.txt; then # passes when the string goes away + exit 1 +fi +``` + +Both are reasonable lines to write. The first one looks after itself: change +the wording in the tool and CI goes red that afternoon and somebody fixes +it. The second is a promise that a particular string stays out of the +output, and the day the tool stops being able to produce that string at all, +the promise becomes free. It keeps passing. Nothing tells you it stopped +checking anything, because from the outside a negated assertion that is +working and a negated assertion that is dead look exactly the same: green. + +`deadassert` finds the second kind by asking what *else* is in your +workflows. If something, somewhere, positively asserts that the string can +appear, the negated assertion has a witness and is sound. If nothing does, +it is a check with no way of ever firing. + +## An example + +`examples/` is a repository whose dogfood job looks thorough. Four things +are wrong with it, in four different ways. + +```console +$ deadassert examples/ +ci.yml:51 unwitnessed 'countdead warning:' + grep: passes whether or not the pattern is ever produced + this passes when the pattern is absent, and nothing in these workflows asserts that it + is ever present -- so it also passes if the pattern stops being produced at all + if grep -qF 'countdead warning:' out.txt; then + +ci.yml:42 prefix 'Every entry here is read.' + grep: matches a whole sentence as a substring + the pattern reads as a whole line of output, but -F without -x matches it as a + substring, so the assertion survives anything appended to that line + grep -qF 'Every entry here is read.' clean.txt + +ci.yml:61 swallowed '"dead": 2' + grep: cannot fail the step + its failure is discarded, so the step passes either way + grep -qF '"dead": 2' out.json || true + +ci.yml:67 decorative 'skipped' + grep: cannot fail the step + it chooses which branch runs, and neither branch can fail the step + if grep -qF 'skipped' seen.txt; then + +4 assertions here do not check what they read as checking (1 unwitnessed, 1 prefix, 1 swallowed, 1 decorative). +7 assertions in 6 scripts across 1 workflow file, 1 of them negated. +``` + +## The kinds + +| kind | what it means | +| --- | --- | +| `unwitnessed` | fails only when its pattern appears, and nothing establishes that it can | +| `prefix` | names a whole output line but matches it as a substring | +| `swallowed` | its failure is discarded, by `\|\| true` or similar | +| `masked` | its exit status is thrown away by its position in a pipeline | +| `decorative` | it picks a branch, and neither branch can fail the step | + +## `--against`, and the one guess in here + +`prefix` is the only kind that rests on what somebody *meant*. `grep -qF 'x'` +matching a substring is not a bug; it is a bug only if the pattern was meant +to be the whole line. Without help, `deadassert` guesses from the shape of +the pattern — it has to end in a full stop, start with a capital or a digit, +be four words or more, and carry no `^` or `$`. Every one of those clauses is +there because a looser version fired on a substring match somebody meant. + +Give it real output and the guess goes away: + +```console +$ countdead examples/ > output.txt # whatever your tool prints +$ countdead clean >> output.txt # every variant, concatenated +$ deadassert examples/ --against output.txt +ci.yml:42 prefix 'Every entry here is read.' + grep: matches a whole sentence as a substring + in output.txt that line reads 'Every entry here is read. Read 0 entries in 1 file.' -- + the pattern is its opening, so this assertion is already passing over a line it does not + name in full, which is the edit it was written to catch +``` + +That is the finding worth having, and the shape-based guess cannot see it: +the line has **already** grown and the assertion did not notice. Pass +`--against` a file holding every variant of output your CI matches against — +it is compared line by line, so more is better. + +`--against` also takes findings away. A pattern that only ever appears +*inside* a line, never at its start, is a substring match on purpose and +stops being reported. + +## What it will not tell you + +- **It does not run anything.** It reads workflow files. A step that fails + for some other reason is not its business. +- **It reads POSIX shells only.** A `run:` under `pwsh` or `shell: python` + is skipped, and the skip is printed, so it never reads as a pass. +- **A pattern the shell fills in is never judged.** `grep -qF "$expected"` + is a real assertion, but the characters in the file are not the characters + grep gets, so it goes in `not decided:` and is left out of the counts. +- **It only knows about matchers.** `grep`, `egrep`, `fgrep`, `rg`. A + verdict reached by `diff`, `test` or a language's own assert is invisible + to it — which is a reason to prefer those, not a gap to fill. See below. +- **`unwitnessed` is a smell, not a proof.** It says nothing in *these + workflows* establishes the string can appear. The proof may well live in a + unit test, where it is arguably better placed. What the finding is really + telling you is that the CI step, read on its own, does not stand up. + +## `grep` is a poor way to assert a whole output + +Having written this, the conclusion it kept pushing me towards is that most +of these findings are a symptom of reaching for the wrong tool. If you are +checking an entire output, `diff` against a committed file says exactly what +you mean, fails loudly on any change, and has no substring semantics to get +wrong: + +```sh +mytool examples/ > got.txt +diff -u expected.txt got.txt +``` + +This repository's own CI does that wherever it can, which is why +`deadassert` run on itself finds only two assertions to examine. That number +is small on purpose, and it is printed rather than hidden. + +## Measured on something real + +Run across the 30 repositories of the organisation this was written in — 38 +workflow files, 337 `run:` scripts: + +| | count | +| --- | --- | +| assertions found | 157 | +| negated (`needs-no-match`) | 22 | +| `unwitnessed` | 12 | +| `prefix` | 21 | +| `swallowed` / `masked` / `decorative` | 0 | + +Twelve of the thirty repositories have at least one finding. The three +can't-fail kinds scoring zero is the result I did not expect: every `|| true` +in that organisation is on a *tool invocation*, deliberately allowing a +non-zero findings exit before the output is asserted, which is correct usage. +An early version of this tool reported all of them and was wrong to. + +`shellcheck` 0.9.0 reports none of these at default settings. With +`--enable=all` it finds the two pipeline cases as `SC2312` (info) and +suggests fixing them with `|| true`, which for an assertion is the opposite +of a fix. + +## Licence + +MIT. diff --git a/examples/.github/workflows/ci.yml b/examples/.github/workflows/ci.yml new file mode 100644 index 0000000..d0f5dfb --- /dev/null +++ b/examples/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: ci + +# A worked example for deadassert, in the shape these files really come in: +# a tool called `countdead` with a dogfood job that looks thorough. Seven of +# the lines below are assertions. Three of those are sound; four are not, in +# four different ways, and every one of the four is the sort of thing you +# write on purpose and then never look at again. +# +# The `test $? -eq N` and `countdead ... || test $? -le 1` lines are not +# assertions about output and deadassert says nothing about them. +# +# `examples/out.txt` is what `countdead examples/` prints, so that +# `deadassert --against examples/out.txt` can decide the `prefix` kind by +# comparison rather than by the shape of the pattern. + +on: + push: + branches: [main] + +jobs: + dogfood: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: the summary line is what the README quotes + run: | + countdead examples/ > out.txt || test $? -le 1 + cat out.txt + # Sound: -x, so appending a count to this line turns the step red. + grep -qxF 'Found 2 dead entries, across 3 files.' out.txt + # A finding kind, matched as a substring on purpose. Sound. + grep -qF 'never-read' out.txt + + - name: a clean tree reports a clean tree + run: | + mkdir -p clean && countdead clean > clean.txt + # NOT sound: `prefix`. This is the whole summary line, matched with + # -F and no -x, so it keeps passing when the line becomes + # `Every entry here is read. Read 0 entries in 1 file.` -- which is + # the exact edit logbook#59 asks every tool in the org to make. + grep -qF 'Every entry here is read.' clean.txt + + - name: internal diagnostics never reach stdout + run: | + countdead examples/ > out.txt || test $? -le 1 + # NOT sound: `unwitnessed`. This promises the string stays out of + # stdout, and nothing anywhere asserts it can appear at all. Rename + # the prefix to `countdead:` and this passes forever, checking + # nothing. Its positive twin would have gone red immediately. + if grep -qF 'countdead warning:' out.txt; then + echo 'a diagnostic reached stdout' >&2 + exit 1 + fi + + - name: the json mode carries the same counts + run: | + countdead examples/ --json > out.json || test $? -le 1 + # NOT sound: `swallowed`. The `|| true` was added to get past a + # flake and never taken out. The step cannot fail here. + grep -qF '"dead": 2' out.json || true + + - name: every file in the corpus was reached + run: | + countdead examples/ -v > seen.txt || test $? -le 1 + # NOT sound: `decorative`. Reads like a check, prints either way. + if grep -qF 'skipped' seen.txt; then + echo 'some files were skipped' + else + echo 'all files read' + fi + + - name: exit codes + run: | + # Sound, all three: the status is the assertion. + countdead clean > /dev/null; test $? -eq 0 + countdead examples/ > /dev/null; test $? -eq 1 + countdead /nonexistent > /dev/null 2>&1; test $? -eq 2 + countdead examples/ -q | grep -qxF 'examples/a.txt:3: never-read' diff --git a/examples/expected-against.txt b/examples/expected-against.txt new file mode 100644 index 0000000..38fd49a --- /dev/null +++ b/examples/expected-against.txt @@ -0,0 +1,25 @@ +ci.yml:51 unwitnessed 'countdead warning:' + grep: passes whether or not the pattern is ever produced + this passes when the pattern is absent, and nothing in these workflows asserts that it + is ever present -- so it also passes if the pattern stops being produced at all + if grep -qF 'countdead warning:' out.txt; then + +ci.yml:42 prefix 'Every entry here is read.' + grep: matches a whole sentence as a substring + in output.txt that line reads 'Every entry here is read. Read 0 entries in 1 file.' -- + the pattern is its opening, so this assertion is already passing over a line it does not + name in full, which is the edit it was written to catch + grep -qF 'Every entry here is read.' clean.txt + +ci.yml:61 swallowed '"dead": 2' + grep: cannot fail the step + its failure is discarded, so the step passes either way + grep -qF '"dead": 2' out.json || true + +ci.yml:67 decorative 'skipped' + grep: cannot fail the step + it chooses which branch runs, and neither branch can fail the step + if grep -qF 'skipped' seen.txt; then + +4 assertions here do not check what they read as checking (1 unwitnessed, 1 prefix, 1 swallowed, 1 decorative). +7 assertions in 6 scripts across 1 workflow file, 1 of them negated. diff --git a/examples/expected-quiet.txt b/examples/expected-quiet.txt new file mode 100644 index 0000000..7eb38e1 --- /dev/null +++ b/examples/expected-quiet.txt @@ -0,0 +1,4 @@ +.github/workflows/ci.yml:51: unwitnessed: 'countdead warning:' +.github/workflows/ci.yml:42: prefix: 'Every entry here is read.' +.github/workflows/ci.yml:61: swallowed: '"dead": 2' +.github/workflows/ci.yml:67: decorative: 'skipped' diff --git a/examples/output.txt b/examples/output.txt new file mode 100644 index 0000000..fac412c --- /dev/null +++ b/examples/output.txt @@ -0,0 +1,14 @@ +examples/a.txt:3: never-read +examples/b.txt:11: never-read +examples/a.txt:3: never-read: nothing in this tree opens it +examples/b.txt:11: never-read: nothing in this tree opens it + +Found 2 dead entries, across 3 files. +Every entry here is read. Read 0 entries in 1 file. +{ + "files": 3, + "dead": 2 +} +read examples/a.txt +read examples/b.txt +read examples/c.txt diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..ace9e9d --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,187 @@ +"""Exit codes, modes, and the example the README quotes.""" + +from __future__ import annotations + +import json +import os + +import pytest + +from deadassert.cli import main + +EXAMPLES = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +def run(capsys, *argv): + code = main(list(argv)) + captured = capsys.readouterr() + return code, captured.out, captured.err + + +def repo(tmp_path, *steps: str) -> str: + body = ["on: push", "jobs:", " j:", " runs-on: ubuntu-latest", " steps:"] + for step in steps: + body.append(" - run: |") + for line in step.strip("\n").split("\n"): + body.append(f" {line}") + directory = tmp_path / ".github" / "workflows" + directory.mkdir(parents=True) + (directory / "ci.yml").write_text("\n".join(body) + "\n") + return str(tmp_path) + + +class TestExitCodes: + def test_findings_exit_1(self, capsys, tmp_path): + root = repo(tmp_path, "if grep -qF 'kind-x' out.txt; then\n exit 1\nfi") + code, out, _ = run(capsys, root) + assert code == 1 + assert "unwitnessed" in out + + def test_nothing_to_report_exits_0(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt") + code, out, _ = run(capsys, root) + assert code == 0 + assert "Every assertion here fails when the thing it names stops being true." in out + + def test_no_workflows_exits_2_with_nothing_on_stdout(self, capsys, tmp_path): + code, out, err = run(capsys, str(tmp_path)) + assert code == 2 + assert out == "" + assert "no workflow files here" in err + + def test_an_unreadable_workflow_exits_2_with_nothing_on_stdout(self, capsys, tmp_path): + directory = tmp_path / ".github" / "workflows" + directory.mkdir(parents=True) + (directory / "ci.yml").write_text("name: [unclosed\n") + code, out, err = run(capsys, str(tmp_path)) + assert code == 2 + assert out == "" + assert "not valid YAML" in err + + def test_a_missing_against_file_exits_2(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qF 'x' out.txt") + code, out, err = run(capsys, root, "--against", str(tmp_path / "nope.txt")) + assert code == 2 + assert out == "" + assert "no such file or directory" in err + + def test_an_os_error_is_a_sentence_not_a_repr(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qF 'x' out.txt") + _, _, err = run(capsys, root, "--against", str(tmp_path / "nope.txt")) + assert "[Errno" not in err + assert err.startswith("deadassert: ") + + +class TestNothingToCheck: + """logbook#62: a clean verdict over no assertions is not a clean verdict.""" + + def test_a_workflow_with_no_assertions_does_not_claim_all_clear(self, capsys, tmp_path): + root = repo(tmp_path, "echo building\nmake all") + code, out, _ = run(capsys, root) + assert code == 0 + assert "No assertions here." in out + assert "Nothing to check." in out + assert "Every assertion here fails" not in out + + +class TestDenominator: + """logbook#59 and logbook#57: a verdict needs what it was measured over.""" + + def test_the_clean_verdict_carries_counts(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt") + _, out, _ = run(capsys, root) + assert "1 assertion in 1 script across 1 workflow file, 0 of them negated." in out + + def test_the_findings_verdict_carries_counts(self, capsys, tmp_path): + root = repo(tmp_path, "if grep -qF 'kind-x' out.txt; then\n exit 1\nfi") + _, out, _ = run(capsys, root) + assert "1 assertion in 1 script across 1 workflow file, 1 of them negated." in out + + +class TestModes: + def test_quiet_is_one_line_per_finding(self, capsys, tmp_path): + root = repo( + tmp_path, + "if grep -qF 'kind-x' out.txt; then\n exit 1\nfi", + "grep -qF 'Every entry here is read.' c.txt", + ) + code, out, _ = run(capsys, root, "-q") + assert code == 1 + lines = out.strip().split("\n") + assert len(lines) == 2 + assert all(line.startswith(".github/workflows/ci.yml:") for line in lines) + + def test_json_carries_the_counts_and_the_findings(self, capsys, tmp_path): + root = repo(tmp_path, "if grep -qF 'kind-x' out.txt; then\n exit 1\nfi") + code, out, _ = run(capsys, root, "--json") + assert code == 1 + payload = json.loads(out) + assert payload["assertions"] == 1 + assert payload["negated"] == 1 + assert payload["scripts"] == 1 + finding = payload["findings"][0] + assert finding["kind"] == "unwitnessed" + assert finding["pattern"] == "kind-x" + assert finding["polarity"] == "needs-no-match" + assert finding["job"] == "j" + + def test_json_is_valid_when_there_is_nothing_to_report(self, capsys, tmp_path): + root = repo(tmp_path, "grep -qxF 'All clear.' out.txt") + code, out, _ = run(capsys, root, "--json") + assert code == 0 + assert json.loads(out)["findings"] == [] + + def test_a_named_file_is_read_instead_of_searching(self, capsys, tmp_path): + root = repo(tmp_path, "if grep -qF 'kind-x' out.txt; then\n exit 1\nfi") + path = os.path.join(root, ".github", "workflows", "ci.yml") + code, out, _ = run(capsys, ".", "-f", path) + assert code == 1 + assert "unwitnessed" in out + + +class TestShippedExample: + """The example in `examples/` is what the README quotes, so it is pinned. + + `logbook#59`: a dogfood step that runs the tool on a tree with none of + its subject asserts nothing. This repository's own workflows do contain + assertions -- that is unavoidable, since the subject is assertions -- but + the example is the input that holds one of every kind on purpose. + """ + + def test_all_four_kinds_fire_with_their_lines(self, capsys): + code, out, _ = run(capsys, EXAMPLES, "-q") + assert code == 1 + assert out.strip().split("\n") == [ + ".github/workflows/ci.yml:51: unwitnessed: 'countdead warning:'", + ".github/workflows/ci.yml:42: prefix: 'Every entry here is read.'", + ".github/workflows/ci.yml:61: swallowed: '\"dead\": 2'", + ".github/workflows/ci.yml:67: decorative: 'skipped'", + ] + + def test_the_six_sound_assertions_are_not_findings(self, capsys): + _, out, _ = run(capsys, EXAMPLES, "--json") + payload = json.loads(out) + assert payload["assertions"] == 7 + assert len(payload["findings"]) == 4 + + def test_against_names_the_line_that_already_grew(self, capsys): + code, out, _ = run(capsys, EXAMPLES, "--against", os.path.join(EXAMPLES, "output.txt")) + assert code == 1 + # The detail is wrapped for the terminal, so compare on the words. + flat = " ".join(out.split()) + assert "Every entry here is read. Read 0 entries in 1 file." in flat + assert "already passing over a line it does not name in full" in flat + + def test_every_sound_positive_assertion_matches_the_shipped_output(self, capsys): + """If this fails, `examples/output.txt` has drifted from the workflow.""" + _, out, _ = run( + capsys, EXAMPLES, "--against", os.path.join(EXAMPLES, "output.txt"), "--json" + ) + assert json.loads(out)["not_matched"] == [] + + +class TestHelp: + def test_help_exits_0(self, capsys): + with pytest.raises(SystemExit) as exit_info: + main(["--help"]) + assert exit_info.value.code == 0 diff --git a/tests/test_workflows.py b/tests/test_workflows.py new file mode 100644 index 0000000..152970a --- /dev/null +++ b/tests/test_workflows.py @@ -0,0 +1,165 @@ +"""Reading workflow files, and the line numbers that make a finding findable.""" + +from __future__ import annotations + +import pytest + +from deadassert import workflows + + +def write(tmp_path, body: str, name: str = "ci.yml") -> str: + directory = tmp_path / ".github" / "workflows" + directory.mkdir(parents=True, exist_ok=True) + (directory / name).write_text(body) + return str(tmp_path) + + +class TestLineNumbers: + """A finding is one command inside a script, so `ci.yml` is not a location. + + PyYAML marks a block scalar on its `|`, with content starting the line + after, and a plain or quoted scalar on the content itself. Getting that + distinction wrong puts every finding in the file off by one. + """ + + def test_block_scalar_first_line_is_after_the_bar(self, tmp_path): + root = write( + tmp_path, + "name: ci\n" # 1 + "on: push\n" # 2 + "jobs:\n" # 3 + " j:\n" # 4 + " runs-on: ubuntu-latest\n" # 5 + " steps:\n" # 6 + " - run: |\n" # 7 + " echo one\n" # 8 + " echo two\n", # 9 + ) + script = workflows.load(root, ".github/workflows/ci.yml").scripts[0] + assert script.first_line == 8 + assert script.line_of(1) == 8 + assert script.line_of(2) == 9 + + def test_folded_scalar_is_marked_the_same_way(self, tmp_path): + root = write( + tmp_path, + "on: push\njobs:\n j:\n steps:\n - run: >\n echo one\n", + ) + script = workflows.load(root, ".github/workflows/ci.yml").scripts[0] + assert script.first_line == 6 + + def test_plain_scalar_is_on_its_own_line(self, tmp_path): + root = write( + tmp_path, + "on: push\n" # 1 + "jobs:\n" # 2 + " j:\n" # 3 + " steps:\n" # 4 + " - run: grep -qF 'x' out.txt\n", # 5 + ) + script = workflows.load(root, ".github/workflows/ci.yml").scripts[0] + assert script.first_line == 5 + assert script.line_of(1) == 5 + + def test_second_step_is_not_given_the_first_step_line(self, tmp_path): + root = write( + tmp_path, + "on: push\n" + "jobs:\n" + " j:\n" + " steps:\n" + " - run: |\n" + " echo first\n" + " - name: second\n" + " run: |\n" + " echo second\n", + ) + first, second = workflows.load(root, ".github/workflows/ci.yml").scripts + assert first.first_line == 6 + assert second.first_line == 9 + + +class TestShells: + def test_bash_is_the_default(self, tmp_path): + root = write(tmp_path, "on: push\njobs:\n j:\n steps:\n - run: 'true'\n") + assert workflows.load(root, ".github/workflows/ci.yml").scripts[0].shell == "bash" + + def test_windows_runner_defaults_to_pwsh_and_is_skipped(self, tmp_path): + root = write( + tmp_path, + "on: push\njobs:\n j:\n runs-on: windows-latest\n steps:\n - run: 'true'\n", + ) + workflow = workflows.load(root, ".github/workflows/ci.yml") + assert workflow.scripts == [] + assert workflow.foreign == [("steps[0]", "pwsh")] + + def test_python_shell_is_skipped_not_read_as_shell(self, tmp_path): + root = write( + tmp_path, + "on: push\njobs:\n j:\n steps:\n" + " - shell: python\n run: |\n print('grep -qF x')\n", + ) + workflow = workflows.load(root, ".github/workflows/ci.yml") + assert workflow.scripts == [] + assert workflow.foreign == [("steps[0]", "python")] + + def test_a_shell_given_as_a_command_line_is_still_bash(self, tmp_path): + root = write( + tmp_path, + "on: push\njobs:\n j:\n steps:\n" + " - shell: bash -eo pipefail {0}\n run: 'true'\n", + ) + assert workflows.load(root, ".github/workflows/ci.yml").scripts[0].posix + + def test_job_defaults_shell_applies(self, tmp_path): + root = write( + tmp_path, + "on: push\njobs:\n j:\n defaults:\n run:\n shell: pwsh\n" + " steps:\n - run: 'true'\n", + ) + assert workflows.load(root, ".github/workflows/ci.yml").scripts == [] + + +class TestRefusals: + def test_on_is_a_boolean_in_yaml_11_and_jobs_still_load(self, tmp_path): + """The thing to know in every tool in this family. + + `on:` comes back from PyYAML as `True`, not `"on"`. Nothing here reads + the trigger block, but a test pins the assumption so a future change + that starts reading it does not quietly find nothing. + """ + import yaml + + doc = yaml.safe_load("on: push\njobs: {}\n") + assert True in doc and "on" not in doc + + def test_empty_file(self, tmp_path): + root = write(tmp_path, "") + with pytest.raises(workflows.WorkflowError, match="empty file"): + workflows.load(root, ".github/workflows/ci.yml") + + def test_not_yaml(self, tmp_path): + root = write(tmp_path, "name: [unclosed\n") + with pytest.raises(workflows.WorkflowError, match="not valid YAML"): + workflows.load(root, ".github/workflows/ci.yml") + + def test_top_level_is_a_list(self, tmp_path): + root = write(tmp_path, "- one\n- two\n") + with pytest.raises(workflows.WorkflowError, match="not a mapping"): + workflows.load(root, ".github/workflows/ci.yml") + + def test_missing_file(self, tmp_path): + with pytest.raises(workflows.WorkflowError): + workflows.load(str(tmp_path), ".github/workflows/nope.yml") + + def test_find_workflows_sorted_and_filtered(self, tmp_path): + root = write(tmp_path, "on: push\njobs: {}\n", name="b.yml") + write(tmp_path, "on: push\njobs: {}\n", name="a.yaml") + write(tmp_path, "notes\n", name="README.md") + assert workflows.find_workflows(root) == [ + ".github/workflows/a.yaml", + ".github/workflows/b.yml", + ] + + def test_find_workflows_on_a_tree_with_no_workflow_dir(self, tmp_path): + assert workflows.find_workflows(str(tmp_path)) == [] From 9418c92329f364ea458032fd3a1f5d9d6d45ff8e Mon Sep 17 00:00:00 2001 From: "richmond-avenal[bot]" <323733923+richmond-avenal[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:47:30 +0000 Subject: [PATCH 5/5] fix: set +e before each exit-code check, since Actions runs bash -e The example job went red on its first run. GitHub invokes a step as `bash -e {0}`, so -e is on before the script's first line and `set -uo pipefail` does not turn it off: the script died on deadassert's exit 1 before reaching `status=$?`, printing nothing, so the log showed only "Process completed with exit code 1". `cmd; test $? -eq N` has the same problem with different punctuation, and the exit-codes step below would have failed next for that reason. Verified by extracting every run: block from this file and executing each under `bash -e`, which is what the first round of local testing should have done -- a subshell without -e is not the environment these run in. --- .github/workflows/ci.yml | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b4d7cb..2cc3414 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,13 +61,23 @@ jobs: python-version: '3.12' - run: pip install -e . - # `diff` rather than `grep`, here and below, on purpose. This whole - # repository is about assertions that match less than they look like, - # and a whole-output assertion written as `diff` has no substring - # weakness to have an opinion about. + # Two things about every step in this job. + # + # `diff` rather than `grep`, on purpose. This whole repository is about + # assertions that match less than they look like, and a whole-output + # assertion written as `diff` has no substring weakness to have an + # opinion about. + # + # `set +e` before each deadassert run, also on purpose. GitHub runs + # steps under `bash -e {0}`, so -e is on before the script starts and + # `set -uo pipefail` does not turn it off -- without `set +e` the + # script dies on deadassert's exit 1 and never reaches `status=$?`. + # That cost this PR a red run, which is this repository's own subject + # arriving by the front door. - name: the shipped example prints exactly what the README quotes run: | set -uo pipefail + set +e deadassert examples/ -q > got.txt status=$? set -e @@ -77,6 +87,7 @@ jobs: - name: --against turns the prefix finding into the stronger one run: | set -uo pipefail + set +e deadassert examples/ --against examples/output.txt > got.txt status=$? set -e @@ -86,6 +97,7 @@ jobs: - name: every sound positive assertion in the example matches the shipped output run: | set -uo pipefail + set +e deadassert examples/ --against examples/output.txt --json > out.json status=$? set -e @@ -96,15 +108,26 @@ jobs: assert found == [], f"examples/output.txt has drifted: {found}" PY + # `cmd; test $? -eq N` does not survive -e either: the script is gone + # on cmd's non-zero status before `test` is reached. Same trap as + # above, different punctuation. - name: exit codes run: | - deadassert examples/ > /dev/null; test $? -eq 1 + set +e mkdir -p clean/.github/workflows # a repo with one sound assertion printf \ "'on': push\njobs:\n j:\n steps:\n - run: grep -qxF 'ok' o\n" \ > clean/.github/workflows/ci.yml - deadassert clean > /dev/null; test $? -eq 0 - deadassert /nonexistent > /dev/null 2>&1; test $? -eq 2 + + deadassert examples/ > /dev/null; findings=$? + deadassert clean > /dev/null; sound=$? + deadassert /nonexistent > /dev/null 2>&1; nothing=$? + set -e + + echo "findings=$findings sound=$sound nothing=$nothing" + test "$findings" -eq 1 + test "$sound" -eq 0 + test "$nothing" -eq 2 dogfood: runs-on: ubuntu-latest