From b3a664903a9674c34dac97e066326a72fc322dd8 Mon Sep 17 00:00:00 2001 From: Nathan Storey Date: Sun, 4 Oct 2026 08:15:30 -0400 Subject: [PATCH 1/3] Guard digests against re-import routes, statically and at run time The static scanner now reports any route to a digest module outside pin.py: an imported alias (from .pin import hashlib), an attribute (pin.hashlib), a name, a string naming one, a sys.modules lookup that names one or is not a literal, and import_module or __import__ with a non-literal name. A run-time test records each hashlib constructor call by its calling frame while the five commands and every helper run, and fails on any typedstandards module but pin. The cold read's four routes join the offender tree. Red first: the old scanner returned [] over the four routes; temporary offending calls in _sidecar.py failed the guards, including one with names built at run time that only the run-time test caught. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ru4PYga7Zf8HANgotZKs4u Signed-off-by: Nathan Storey --- tests/guards.py | 57 ++++++++++++++++++++++++++++-------- tests/test_guards.py | 70 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 13 deletions(-) diff --git a/tests/guards.py b/tests/guards.py index 3df584b..3464aeb 100644 --- a/tests/guards.py +++ b/tests/guards.py @@ -4,7 +4,8 @@ - It passes the child process no environment but the inherited one: no ``env=`` on any call, and nothing that sets, unsets or replaces a variable for a child. - It computes none of the format's hashes: no module imports ``hashlib`` (or the modules behind - it), except the allowlisted P2 ``pin`` module, whose digest is a signed assertion. + it) or reaches one another way (a re-import from ``pin``, ``pin.hashlib``, ``sys.modules``, a + built import name), except the allowlisted ``pin`` module, whose digest is a signed assertion. Each scanner takes a directory and returns one line per offence, so a test can drive it over the package and over a fixture tree of offenders. @@ -119,8 +120,22 @@ def env_overrides(root: Path) -> list[str]: return found +def _is_sys_modules(node: ast.AST) -> bool: + return (isinstance(node, ast.Attribute) and node.attr == "modules") or ( + isinstance(node, ast.Name) and node.id == "modules" + ) + + +def _digest_name(value: object) -> bool: + return isinstance(value, str) and value.split(".")[0] in HASH_MODULES + + def hash_imports(root: Path, allow: frozenset[str] = HASH_ALLOWLIST) -> list[str]: - """Every import of a digest module outside the allowlist, static or dynamic.""" + """Every route to a digest module outside the allowlist: an import of one (static, or dynamic + with a literal name); any reference to one's name (a name, an attribute such as + ``pin.hashlib``, an imported alias such as ``from .pin import hashlib``, a string naming it); + a ``sys.modules`` lookup that names one or is not a literal; and ``import_module`` or + ``__import__`` with a name that is not a literal, since it can build any name.""" found = [] for path in python_files(root): where = path.relative_to(root) @@ -128,16 +143,34 @@ def hash_imports(root: Path, allow: frozenset[str] = HASH_ALLOWLIST) -> list[str continue tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) for node in ast.walk(tree): - modules: list[str] = [] + line = getattr(node, "lineno", 0) if isinstance(node, ast.Import): - modules = [alias.name for alias in node.names] - elif isinstance(node, ast.ImportFrom) and node.module: - modules = [node.module] + for alias in node.names: + if _digest_name(alias.name): + found.append(f"{where}:{line}: imports {alias.name}") + elif isinstance(node, ast.ImportFrom): + if _digest_name(node.module or ""): + found.append(f"{where}:{line}: imports {node.module}") + for alias in node.names: + if _digest_name(alias.name): + found.append(f"{where}:{line}: imports the name {alias.name} from {node.module or '.'}") elif isinstance(node, ast.Call) and _call_name(node) in {"import_module", "__import__"} and node.args: first = node.args[0] - if isinstance(first, ast.Constant) and isinstance(first.value, str): - modules = [first.value] - for module in modules: - if module.split(".")[0] in HASH_MODULES: - found.append(f"{where}:{node.lineno}: imports {module}") - return found + if not (isinstance(first, ast.Constant) and isinstance(first.value, str)): + found.append(f"{where}:{line}: {_call_name(node)} with a name that is not a literal") + elif _digest_name(first.value): + found.append(f"{where}:{line}: imports {first.value}") + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "get": + if _is_sys_modules(node.func.value): + found.append(f"{where}:{line}: looks up sys.modules") + elif isinstance(node, ast.Subscript) and _is_sys_modules(node.value): + key = node.slice + if not (isinstance(key, ast.Constant) and isinstance(key.value, str)) or _digest_name(key.value): + found.append(f"{where}:{line}: looks up sys.modules") + elif isinstance(node, ast.Name) and _digest_name(node.id): + found.append(f"{where}:{line}: refers to {node.id}") + elif isinstance(node, ast.Attribute) and _digest_name(node.attr): + found.append(f"{where}:{line}: refers to .{node.attr}") + elif isinstance(node, ast.Constant) and isinstance(node.value, str) and node.value in HASH_MODULES: + found.append(f"{where}:{line}: names {node.value}") + return sorted(set(found)) diff --git a/tests/test_guards.py b/tests/test_guards.py index 034cfd1..4afa27c 100644 --- a/tests/test_guards.py +++ b/tests/test_guards.py @@ -70,6 +70,11 @@ def test_package_runs_no_global_cli() -> None: "hash_dynamic.py": "import importlib\nh = importlib.import_module('hashlib')\n", "hash_hmac.py": "import hmac\n", "pin.py": "import hashlib\n", + # Re-import routes (the cold read's four), each outside the allowlist. + "reimport_from_pin.py": "from .pin import hashlib\n", + "pin_attribute.py": "from . import pin\ndigest = pin.hashlib.sha256(b'x')\n", + "sys_modules.py": "import sys\ndigest = sys.modules['hashlib'].sha256(b'x')\n", + "dynamic_name.py": "import importlib\nmodule = importlib.import_module('hash' + 'lib')\n", } @@ -100,7 +105,16 @@ def test_env_scanner_fails_on_offenders(offenders: Path) -> None: def test_hash_scanner_fails_on_offenders_and_allows_only_pin(offenders: Path) -> None: - assert _files(hash_imports(offenders)) == {"hash_import.py", "hash_from.py", "hash_dynamic.py", "hash_hmac.py"} + assert _files(hash_imports(offenders)) == { + "hash_import.py", + "hash_from.py", + "hash_dynamic.py", + "hash_hmac.py", + "reimport_from_pin.py", + "pin_attribute.py", + "sys_modules.py", + "dynamic_name.py", + } assert "pin.py" in _files(hash_imports(offenders, allow=frozenset())) @@ -125,6 +139,60 @@ def _drive_every_command() -> None: typedstandards.cli_version() +DIGEST_CONSTRUCTORS = ( + "new", "md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha3_224", "sha3_256", "sha3_384", + "sha3_512", "shake_128", "shake_256", "blake2b", "blake2s", +) # fmt: skip + + +def test_no_module_but_pin_computes_a_digest_at_run_time( + seed: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """While the five commands and every helper run, record each hashlib constructor call by the + module of the frame that made it. Only typedstandards.pin may appear; third-party libraries + (httpx, yaml) hashing internally are not typedstandards frames and are not counted.""" + import hashlib + import sys + + import httpx + from support import analysis_input, synthetic_notebook + + callers: list[str] = [] + + def recording(name: str, real: Any) -> Any: + def call(*args: Any, **kwargs: Any) -> Any: + caller = sys._getframe(1).f_globals.get("__name__", "") + if caller == "typedstandards" or caller.startswith("typedstandards."): + callers.append(f"{caller} called hashlib.{name}") + return real(*args, **kwargs) + + return call + + for name in DIGEST_CONSTRUCTORS: + monkeypatch.setattr(hashlib, name, recording(name, getattr(hashlib, name))) + + _drive_every_command() + notebook = tmp_path / "analysis.ipynb" + notebook.write_text(synthetic_notebook(), encoding="utf-8") + typedstandards.badge_cell( + "https://records.example.org/bundles/analysis.bundle.json", capture_method="script-run", notebook=notebook + ) + typedstandards.comparison_cell(notebook, {"rows": 1}, recompute="recompute()", captured_at="2026-10-04T00:00:00Z") + _, entry = typedstandards.pin( + "https://files.example.org/data.csv", transport=httpx.MockTransport(lambda r: httpx.Response(200, content=b"x")) + ) + signed = typedstandards.sign(analysis_input(queries=[entry]), output_file=notebook) + bundle = typedstandards.view(signed, visibility="public") + typedstandards.sidecar(bundle, notebook) + typedstandards.show(bundle) + typedstandards.show(bundle, typedstandards.verify(bundle), marimo=True) + + assert "typedstandards.pin called hashlib.sha256" in callers # the recorder sees pin's one digest + others = sorted({c for c in callers if not c.startswith("typedstandards.pin ")}) + if others: + pytest.fail(f"a module other than pin computed a digest: {others}") + + def test_every_child_inherits_the_environment(seed: str, spawned: list[dict[str, Any]]) -> None: before = dict(os.environ) _drive_every_command() From 4f0882532a568614ad6b63423c878cc5245f0d2d Mon Sep 17 00:00:00 2001 From: Nathan Storey Date: Sun, 4 Oct 2026 08:15:30 -0400 Subject: [PATCH 2/3] badge_cell: a host with a port is not read as a time The date and time check now leaves out the URL's authority (the host fact and the encoded host in the link), so 192.168.1.10:8080 no longer reads as 10:80. It also checks the URL past its authority as written and percent-decoded, so a time in the path, query or fragment is refused: the cell carries the URL encoded, which had hidden a time's colon from the check. Red first: the IP-with-port URL was refused, and a time in the query (raw or encoded) or fragment was accepted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ru4PYga7Zf8HANgotZKs4u Signed-off-by: Nathan Storey --- src/typedstandards/_badge.py | 22 +++++++++++++++++----- tests/test_badge.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/typedstandards/_badge.py b/src/typedstandards/_badge.py index 0c03802..ec52d53 100644 --- a/src/typedstandards/_badge.py +++ b/src/typedstandards/_badge.py @@ -18,7 +18,7 @@ import os import re -from urllib.parse import quote, urlsplit +from urllib.parse import quote, unquote, urlsplit from ._notebook import insert_cell, source_lines @@ -71,25 +71,33 @@ def _check_fact(name: str, value: str) -> str: return value.strip() -def refuse_hash_or_time(text: str) -> str: +def refuse_hash_or_time(text: str, *, authorities: tuple[str, ...] = ()) -> str: """Raise ``ValueError`` when ``text`` holds a 64-hex string, a date or a time of day. The badge cell is part of the signed bytes, written before signing: a hash in it cannot be the record's own, and a time in it cannot be the signing time, so either would mislead. + Each of ``authorities`` (a URL's host and port, as the text carries them) is left out of the + date and time check, so ``192.168.1.10:8080`` is not read as the time ``10:80``. """ if _HEX64.search(text): raise ValueError( "the badge cell would carry a 64-hex string; it is written before signing, so it names no hash" ) - if _DATE_OR_TIME.search(text): + scan = text + for authority in authorities: + if authority: + scan = scan.replace(authority, " ") + if _DATE_OR_TIME.search(scan): raise ValueError("the badge cell would carry a date or time; it is written before signing, so it names no time") return text def badge_text(bundle_url: str, *, capture_method: str, host: str | None = None) -> str: """The badge cell's Markdown: the badge, a two-row table (host, capture method), one sentence.""" + url = bundle_url.strip() + parts = urlsplit(url) if host is None: - host = urlsplit(bundle_url.strip()).netloc + host = parts.netloc host = _check_fact("host", host) capture_method = _check_fact("capture_method", capture_method) text = ( @@ -103,7 +111,11 @@ def badge_text(bundle_url: str, *, capture_method: str, host: str | None = None) "This cell is a reader affordance and is not authoritative: verification reads the signed record, " "not this cell, and the verifier shows the record's signer, hash and time.\n" ) - return refuse_hash_or_time(text) + # The cell carries the URL percent-encoded, which hides a time's ":"; so the URL past its + # authority (path, query, fragment) is checked as written and decoded too. + rest = url[len(parts.scheme) + 3 + len(parts.netloc) :] + refuse_hash_or_time(f"{rest}\n{unquote(rest)}") + return refuse_hash_or_time(text, authorities=(f"`{host}`", encode_uri_component(parts.netloc))) def _marimo_source(markdown: str) -> str: diff --git a/tests/test_badge.py b/tests/test_badge.py index fb746fa..3af8167 100644 --- a/tests/test_badge.py +++ b/tests/test_badge.py @@ -181,6 +181,37 @@ def test_a_url_carrying_a_hash_or_date_is_refused(tmp_path: Path, url: str) -> N assert _read(path) == original +@pytest.mark.parametrize( + "url", + [ + "https://192.168.1.10:8080/bundles/a.bundle.json", + "https://records.example.org:8443/a.bundle.json", + "http://127.0.0.1:12345/bundles/a.bundle.json", + ], +) +def test_a_host_with_a_port_is_not_read_as_a_time(tmp_path: Path, url: str) -> None: + path = _write(tmp_path, synthetic_notebook()) + text = badge_cell(url, capture_method="script-run", notebook=path) + assert text.splitlines()[0] == badge_markdown(url) + assert json.loads(_read(path))["cells"][0]["id"] == "typedstandards-badge" + + +@pytest.mark.parametrize( + ("url", "capture_method"), + [ + ("https://192.168.1.10:8080/2026-10-04/a.bundle.json", "script-run"), + ("https://records.example.org:8443/a.bundle.json?t=12:30", "script-run"), + ("https://records.example.org/a.bundle.json?t=12%3A30%3A45", "script-run"), + ("https://records.example.org/a.bundle.json#T12:30", "script-run"), + ("https://192.168.1.10:8080/a.bundle.json", "run-T12:30:45"), + ], + ids=["date-in-path", "time-in-query", "encoded-time-in-query", "time-in-fragment", "time-in-fact"], +) +def test_a_date_or_time_beside_a_port_is_still_refused(url: str, capture_method: str) -> None: + with pytest.raises(ValueError, match="date or time"): + badge_cell(url, capture_method=capture_method) + + def test_a_second_badge_is_refused(tmp_path: Path) -> None: path = _write(tmp_path, synthetic_notebook()) badge_cell(BUNDLE_URL, capture_method="script-run", notebook=path) From 83c486853536777516b19da9741b71f18cd570fb Mon Sep 17 00:00:00 2001 From: Nathan Storey Date: Sun, 4 Oct 2026 08:15:30 -0400 Subject: [PATCH 3/3] README: which calls need Node, and absolute links for the PyPI page Only the five commands, cli_version() and show without a result run the CLI; the README now says so, and that pin, badge_cell, comparison_cell, sidecar and show(record, result) run without Node. A test pins both halves with the Node override pointed at a missing file. The CLAUDE.md link is absolute, and a test fails on any relative link or image in the README. CHANGELOG entries under Unreleased. Red first: the link test caught [CLAUDE.md](CLAUDE.md). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ru4PYga7Zf8HANgotZKs4u Signed-off-by: Nathan Storey --- CHANGELOG.md | 5 ++++ README.md | 12 +++++--- tests/test_readme.py | 71 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 tests/test_readme.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 18da5ce..ba9dc38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,3 +21,8 @@ `trustRegistry`. - `show`: HTML for Jupyter (`_repr_html_`) or Marimo (`mo.Html`) from a record and its `verify --json` result. +- `badge_cell` no longer reads a URL's host and port as a time: `https://192.168.1.10:8080/…` is + accepted. A date or time in the URL's path, query or fragment (as written or percent-encoded) or + in a fact is refused. +- README: only the calls that run the CLI need Node (the five commands, `cli_version()`, and + `show` without a result); its links are absolute, so they resolve on the PyPI page. diff --git a/README.md b/README.md index 9375eac..ce2ef65 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,11 @@ The wrapper looks for Node in this order: 1. `TYPEDSTANDARDS_NODE`, when set: the path (or name on `PATH`) of a Node binary; 2. `node` on `PATH`. -With no Node, or one older than 20.19.0, every call raises `typedstandards.NodeLocatorError`, whose -message names the floor and `TYPEDSTANDARDS_NODE`. +Only the calls that run the CLI need Node: `sign`, `withdraw`, `attest`, `view`, `verify`, +`cli_version()`, and `show` without a precomputed result. With no Node, or one older than 20.19.0, +each of those raises `typedstandards.NodeLocatorError`, whose message names the floor and +`TYPEDSTANDARDS_NODE`. `pin`, `badge_cell`, `comparison_cell`, `sidecar` and +`show(record, result)` run without Node. Linux and macOS are tested. Windows is untested. @@ -162,7 +165,8 @@ ts.show(bundle) # in Jupyter; ts.show(bundle, marimo=True) in Marimo `typedstandards-badge` on nbformat 4.5). With `marimo=True`, it returns the source of a `mo.md(...)` cell to paste into the app. The cell is written before signing and is part of the signed bytes, so it names no hash and no time; a URL or value holding a 64-hex string, a date or - a time is refused. + a time is refused. The URL's host and port are not read as a time (`192.168.1.10:8080` is + accepted). - **`comparison_cell(notebook, values, *, recompute, captured_at)`** appends the comparison cell of spec §8.7.4 as the last cell (id `typedstandards-comparison`): the values as Python literals, `current = `, and a loop that prints each delta. Values are `None`, `bool`, `int`, @@ -195,7 +199,7 @@ that moves the pin. ## Development -See [CLAUDE.md](CLAUDE.md) for the development loop and the checks CI runs. +See [CLAUDE.md](https://github.com/npstorey/typedstandards-python/blob/main/CLAUDE.md) for the development loop and the checks CI runs. ## License diff --git a/tests/test_readme.py b/tests/test_readme.py new file mode 100644 index 0000000..fce9e53 --- /dev/null +++ b/tests/test_readme.py @@ -0,0 +1,71 @@ +"""The README is the wheel's long description, shown on PyPI, where a relative link does not +resolve. Every Markdown link and image in it is absolute (or an in-page ``#anchor``). + +It also pins the README's Node sentence: only the five commands, ``cli_version()`` and ``show`` +without a result run the CLI and so need Node; the other helpers run without one. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import httpx +import pytest +from support import FIXTURES, synthetic_notebook + +import typedstandards + +README = Path(__file__).parent.parent / "README.md" + +#: An inline link or image destination: ``](dest)`` or ``]()``; and a reference definition. +_INLINE = re.compile(r"\]\(\s*(<[^>]*>|[^)\s]+)") +_REFERENCE = re.compile(r"^\s{0,3}\[[^\]]+\]:\s*(\S+)", re.MULTILINE) +_ABSOLUTE = re.compile(r"(https?://|mailto:|#)") + + +def relative_links(markdown: str) -> list[str]: + """Every link or image destination outside fenced code that is neither absolute nor an anchor.""" + prose = re.sub(r"^```.*?^```", "", markdown, flags=re.MULTILINE | re.DOTALL) + found = [m.strip("<>") for m in _INLINE.findall(prose)] + _REFERENCE.findall(prose) + return [dest for dest in found if not _ABSOLUTE.match(dest)] + + +def test_the_checker_finds_relative_links() -> None: + text = "See [a](CLAUDE.md), ![i](docs/x.png), [b](), [c](#use), [d](https://x.org/y).\n\n[e]: e.md\n" + assert relative_links(text) == ["CLAUDE.md", "docs/x.png", "rel path.md", "e.md"] + + +def test_readme_has_no_relative_link() -> None: + assert relative_links(README.read_text(encoding="utf-8")) == [] + + +def test_the_helpers_need_no_node(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv(typedstandards.NODE_OVERRIDE, str(tmp_path / "no-node-here")) + with pytest.raises(typedstandards.NodeLocatorError): + typedstandards.locate_node() + + notebook = tmp_path / "analysis.ipynb" + notebook.write_text(synthetic_notebook(), encoding="utf-8") + typedstandards.badge_cell( + "https://records.example.org/bundles/analysis.bundle.json", capture_method="script-run", notebook=notebook + ) + typedstandards.comparison_cell(notebook, {"rows": 1}, recompute="recompute()", captured_at="2026-10-04T00:00:00Z") + typedstandards.pin( + "https://files.example.org/data.csv", transport=httpx.MockTransport(lambda r: httpx.Response(200, content=b"x")) + ) + bundle = json.loads((FIXTURES / "record-active.bundle.json").read_text(encoding="utf-8")) + result = json.loads((FIXTURES / "record-active.verify.json").read_text(encoding="utf-8")) + typedstandards.sidecar(bundle, notebook) + typedstandards.show(bundle, result) + + # The calls that run the CLI raise it, naming the floor and the override. + for call in ( + lambda: typedstandards.show(bundle), + lambda: typedstandards.verify(bundle), + lambda: typedstandards.view(FIXTURES / "record-active.signed.json", visibility="public"), + typedstandards.cli_version, + ): + with pytest.raises(typedstandards.NodeLocatorError, match="20.19"): + call()