diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b43374..1099f15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -242,6 +242,48 @@ jobs: assert unread == [("in function", "grep -qF 'ok' out.txt")], unread PY + # logbook#71: a `$` with nothing after it is a regex anchor, and used to + # send the whole assertion to `not decided` as a "shell expansion". The + # three lines below are the three answers this has to keep apart, and + # the first one is 9 of the organisation's 28 undecided assertions. + # + # `printf` and `\$` for the same reason as the step above: these lines + # are arguments, not script, so the matchers in them stay out of this + # repository's own count -- which the `>= 2` assertion above would + # notice. + - name: the shapes logbook#71 was filed for + run: | + set -euo pipefail + mkdir -p dollars/.github/workflows + printf '%s\n' \ + "'on': push" \ + "jobs:" \ + " j:" \ + " steps:" \ + " - run: |" \ + " grep -q '^ mode *nothing\$' out.txt" \ + " grep -qF \"\$clean\" out.txt" \ + " grep -q 'job \`publish\` is gated on it' out.txt" \ + > dollars/.github/workflows/ci.yml + cat dollars/.github/workflows/ci.yml + deadassert dollars --json > dollars.json + python - <<'PY' + import json + d = json.load(open("dollars.json")) + + # The trailing anchor is read: one assertion, judged, and sound. + assert d["assertions"] == 1, d["assertions"] + assert d["findings"] == [], d["findings"] + + # The other two are not, and for the stated reason. `$clean` has a + # name after the `$`; the backticks are a substitution wherever they + # appear. Both stay undecided even though the quoting here makes + # them literal -- that is the conservatism, and it is the point. + where = sorted(x["where"] for x in d["undecided"]) + assert where == ["ci.yml:7", "ci.yml:8"], where + assert all("shell expansion" in x["why"] for x in d["undecided"]), d["undecided"] + PY + # The assertion above is only worth anything if it can go red. Break # one assertion in a copy of this file and check that it does. - name: a broken assertion in this file is caught diff --git a/README.md b/README.md index ab523e9..903c7ff 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,13 @@ stops being reported. is skipped, and the skip is printed, so it never reads as a pass. - **A pattern the shell fills in is never judged.** `grep -qF "$expected"` is a real assertion, but the characters in the file are not the characters - grep gets, so it goes in `not decided:` and is left out of the counts. + grep gets, so it goes in `not decided:` and is left out of the counts. A + `$` is only read that way when a name, a brace, a paren or a special + parameter follows it — a trailing `$`, as in `'^ command *nothing$'`, is + a regex anchor with no name to look up, and is judged like any other + pattern. The quoting is deliberately *not* consulted: `'$GITHUB_REPOSITORY'` + in single quotes is literal, but an author who meant it to expand writes + exactly those characters, and that is the mistake `not decided:` is for. - **It only knows about matchers.** `grep`, `egrep`, `fgrep`, `rg`. A verdict reached by `diff`, `test` or a language's own assert is not read — which is a reason to prefer those, not a gap to fill. See below. It is no diff --git a/src/deadassert/analyse.py b/src/deadassert/analyse.py index 7363f4e..e6f8a82 100644 --- a/src/deadassert/analyse.py +++ b/src/deadassert/analyse.py @@ -109,16 +109,36 @@ def nothing_to_check(self) -> bool: return not self.assertions +#: What has to follow a `$` for the shell to fill something in: a name, a +#: brace, a paren, or one of the special parameters. A `$` with none of these +#: after it -- at the end of a pattern, or before `|`, `)`, `/` -- is a regex +#: end-of-line anchor, and there is no quoting under which it expands. +_EXPANDS_AFTER_DOLLAR = re.compile(r"[A-Za-z_{(0-9@*#?!$-]") + + def _is_expansion(pattern: str | None) -> bool: """Whether a pattern's value depends on something the shell fills in. `$var`, `${var}`, `$(cmd)` and backticks all mean the characters in the workflow file are not the characters grep is given, so nothing here can say what the assertion matches. A literal `\\$` is not an expansion. + + A bare `$` is not one either. `'^ command *nothing$'` ends in an anchor, + and an anchor has no name after it to look up -- so unlike `$clean`, there + is no variable it could have been and no mistake it could be hiding. This + is deliberately narrower than asking about the quoting: `'$GITHUB_REPOSITORY'` + in single quotes is literal, but an author who meant it to expand writes + exactly those characters, and that is the mistake `not decided` is for. """ if pattern is None: return False - return bool(re.search(r"(? list[str]: diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 3dd13b8..c92b717 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -213,6 +213,71 @@ def test_an_expansion_is_not_counted_in_the_denominator(self, tmp_path): assert analyse.analyse(root).assertions == [] +class TestDollarWithNothingToExpand: + """A `$` the shell cannot fill in is an anchor, and the tool can read it. + + `logbook#71`: every unescaped `$` used to send the assertion to + `not decided`, which was 9 of the organisation's 28 undecided + assertions -- and all of them whole-line, anchored matches, so the + tool was blindest where the assertions were best written. + """ + + @pytest.mark.parametrize( + "pattern", + [ + "'^ command *nothing$'", # whichrc ci.yml:131, a trailing anchor + "'^[0-9]+ findings?$'", # blankout ci.yml:79 + "'^python-dotenv +ready$'", # envdrift ci.yml:61 + "'^a$|^b$'", # before an alternation + "'(^a$)'", # before a close paren + "'^a$' out.txt; grep -q 'b'", # not even last in the line + ], + ) + def test_a_dollar_with_no_name_after_it_is_judged(self, tmp_path, pattern): + root = repo(tmp_path, f"grep -q {pattern} out.txt") + result = analyse.analyse(root) + assert result.undecided == [] + assert len(result.assertions) >= 1 + + @pytest.mark.parametrize( + "pattern", + [ + '"$clean"', # a name + '"${msg}"', # a brace + '"$(cat expected.txt)"', # a substitution + '"`date`"', # a backtick: unlike `$`, what follows it is irrelevant + '"ready $1"', # a positional, usually empty in a run: script + '"ready $?"', # a special parameter + # blankout ci.yml:158 builds this pattern out of three quoted + # pieces so Actions cannot substitute it; what reaches grep is a + # literal `${{ ... }}`, and a `${` is still a brace expansion. + "'${{ steps.check.outputs.release }}'", + ], + ) + def test_a_dollar_with_something_to_expand_stays_undecided(self, tmp_path, pattern): + root = repo(tmp_path, f"grep -q {pattern} out.txt") + result = analyse.analyse(root) + assert len(result.undecided) == 1, result.undecided + assert "shell expansion" in result.undecided[0][1] + + def test_an_anchored_negated_assertion_is_now_witnessable(self, tmp_path): + """The point of deciding them: they can take part in the witness rule.""" + root = repo( + tmp_path, + "grep -q '^ mode *nothing$' expected.txt", + "if grep -q '^ mode *nothing$' out.txt; then\n exit 1\nfi", + ) + result = analyse.analyse(root) + assert result.undecided == [] + assert kinds(result) == [] + + def test_an_anchored_negated_assertion_with_no_witness_is_a_finding(self, tmp_path): + root = repo(tmp_path, "if grep -q '^ mode *nothing$' out.txt; then\n exit 1\nfi") + result = analyse.analyse(root) + assert result.undecided == [] + assert kinds(result) == ["unwitnessed"] + + class TestCannotFail: @pytest.mark.parametrize( "step,kind",