Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,48 @@ jobs:
assert unread == [("in function", "grep -qF 'ok' out.txt")], unread
PY

# logbook#71: a `$` with nothing after it is a regex anchor, and used to
# send the whole assertion to `not decided` as a "shell expansion". The
# three lines below are the three answers this has to keep apart, and
# the first one is 9 of the organisation's 28 undecided assertions.
#
# `printf` and `\$` for the same reason as the step above: these lines
# are arguments, not script, so the matchers in them stay out of this
# repository's own count -- which the `>= 2` assertion above would
# notice.
- name: the shapes logbook#71 was filed for
run: |
set -euo pipefail
mkdir -p dollars/.github/workflows
printf '%s\n' \
"'on': push" \
"jobs:" \
" j:" \
" steps:" \
" - run: |" \
" grep -q '^ mode *nothing\$' out.txt" \
" grep -qF \"\$clean\" out.txt" \
" grep -q 'job \`publish\` is gated on it' out.txt" \
> dollars/.github/workflows/ci.yml
cat dollars/.github/workflows/ci.yml
deadassert dollars --json > dollars.json
python - <<'PY'
import json
d = json.load(open("dollars.json"))

# The trailing anchor is read: one assertion, judged, and sound.
assert d["assertions"] == 1, d["assertions"]
assert d["findings"] == [], d["findings"]

# The other two are not, and for the stated reason. `$clean` has a
# name after the `$`; the backticks are a substitution wherever they
# appear. Both stay undecided even though the quoting here makes
# them literal -- that is the conservatism, and it is the point.
where = sorted(x["where"] for x in d["undecided"])
assert where == ["ci.yml:7", "ci.yml:8"], where
assert all("shell expansion" in x["why"] for x in d["undecided"]), d["undecided"]
PY

# The assertion above is only worth anything if it can go red. Break
# one assertion in a copy of this file and check that it does.
- name: a broken assertion in this file is caught
Expand Down
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,13 @@ stops being reported.
is skipped, and the skip is printed, so it never reads as a pass.
- **A pattern the shell fills in is never judged.** `grep -qF "$expected"`
is a real assertion, but the characters in the file are not the characters
grep gets, so it goes in `not decided:` and is left out of the counts.
grep gets, so it goes in `not decided:` and is left out of the counts. A
`$` is only read that way when a name, a brace, a paren or a special
parameter follows it — a trailing `$`, as in `'^ command *nothing$'`, is
a regex anchor with no name to look up, and is judged like any other
pattern. The quoting is deliberately *not* consulted: `'$GITHUB_REPOSITORY'`
in single quotes is literal, but an author who meant it to expand writes
exactly those characters, and that is the mistake `not decided:` is for.
- **It only knows about matchers.** `grep`, `egrep`, `fgrep`, `rg`. A
verdict reached by `diff`, `test` or a language's own assert is not read —
which is a reason to prefer those, not a gap to fill. See below. It is no
Expand Down
22 changes: 21 additions & 1 deletion src/deadassert/analyse.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,16 +109,36 @@ def nothing_to_check(self) -> bool:
return not self.assertions


#: What has to follow a `$` for the shell to fill something in: a name, a
#: brace, a paren, or one of the special parameters. A `$` with none of these
#: after it -- at the end of a pattern, or before `|`, `)`, `/` -- is a regex
#: end-of-line anchor, and there is no quoting under which it expands.
_EXPANDS_AFTER_DOLLAR = re.compile(r"[A-Za-z_{(0-9@*#?!$-]")


def _is_expansion(pattern: str | None) -> bool:
"""Whether a pattern's value depends on something the shell fills in.

`$var`, `${var}`, `$(cmd)` and backticks all mean the characters in the
workflow file are not the characters grep is given, so nothing here can
say what the assertion matches. A literal `\\$` is not an expansion.

A bare `$` is not one either. `'^ command *nothing$'` ends in an anchor,
and an anchor has no name after it to look up -- so unlike `$clean`, there
is no variable it could have been and no mistake it could be hiding. This
is deliberately narrower than asking about the quoting: `'$GITHUB_REPOSITORY'`
in single quotes is literal, but an author who meant it to expand writes
exactly those characters, and that is the mistake `not decided` is for.
"""
if pattern is None:
return False
return bool(re.search(r"(?<!\\)[$`]", pattern))
for match in re.finditer(r"(?<!\\)[$`]", pattern):
if match.group() == "`":
return True
after = pattern[match.end() : match.end() + 1]
if after and _EXPANDS_AFTER_DOLLAR.match(after):
return True
return False


def _witnesses(assertions: list[tuple[Script, Assertion]]) -> list[str]:
Expand Down
65 changes: 65 additions & 0 deletions tests/test_analyse.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,71 @@ def test_an_expansion_is_not_counted_in_the_denominator(self, tmp_path):
assert analyse.analyse(root).assertions == []


class TestDollarWithNothingToExpand:
"""A `$` the shell cannot fill in is an anchor, and the tool can read it.

`logbook#71`: every unescaped `$` used to send the assertion to
`not decided`, which was 9 of the organisation's 28 undecided
assertions -- and all of them whole-line, anchored matches, so the
tool was blindest where the assertions were best written.
"""

@pytest.mark.parametrize(
"pattern",
[
"'^ command *nothing$'", # whichrc ci.yml:131, a trailing anchor
"'^[0-9]+ findings?$'", # blankout ci.yml:79
"'^python-dotenv +ready$'", # envdrift ci.yml:61
"'^a$|^b$'", # before an alternation
"'(^a$)'", # before a close paren
"'^a$' out.txt; grep -q 'b'", # not even last in the line
],
)
def test_a_dollar_with_no_name_after_it_is_judged(self, tmp_path, pattern):
root = repo(tmp_path, f"grep -q {pattern} out.txt")
result = analyse.analyse(root)
assert result.undecided == []
assert len(result.assertions) >= 1

@pytest.mark.parametrize(
"pattern",
[
'"$clean"', # a name
'"${msg}"', # a brace
'"$(cat expected.txt)"', # a substitution
'"`date`"', # a backtick: unlike `$`, what follows it is irrelevant
'"ready $1"', # a positional, usually empty in a run: script
'"ready $?"', # a special parameter
# blankout ci.yml:158 builds this pattern out of three quoted
# pieces so Actions cannot substitute it; what reaches grep is a
# literal `${{ ... }}`, and a `${` is still a brace expansion.
"'${{ steps.check.outputs.release }}'",
],
)
def test_a_dollar_with_something_to_expand_stays_undecided(self, tmp_path, pattern):
root = repo(tmp_path, f"grep -q {pattern} out.txt")
result = analyse.analyse(root)
assert len(result.undecided) == 1, result.undecided
assert "shell expansion" in result.undecided[0][1]

def test_an_anchored_negated_assertion_is_now_witnessable(self, tmp_path):
"""The point of deciding them: they can take part in the witness rule."""
root = repo(
tmp_path,
"grep -q '^ mode *nothing$' expected.txt",
"if grep -q '^ mode *nothing$' out.txt; then\n exit 1\nfi",
)
result = analyse.analyse(root)
assert result.undecided == []
assert kinds(result) == []

def test_an_anchored_negated_assertion_with_no_witness_is_a_finding(self, tmp_path):
root = repo(tmp_path, "if grep -q '^ mode *nothing$' out.txt; then\n exit 1\nfi")
result = analyse.analyse(root)
assert result.undecided == []
assert kinds(result) == ["unwitnessed"]


class TestCannotFail:
@pytest.mark.parametrize(
"step,kind",
Expand Down
Loading