diff --git a/scripts/check_corpus_headline_evidence.py b/scripts/check_corpus_headline_evidence.py new file mode 100644 index 000000000..c848b452c --- /dev/null +++ b/scripts/check_corpus_headline_evidence.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Require authenticated recorded native output when a PR changes its headline. + +Execute this file from the trusted PR base. Candidate Git objects and PR body +attachments are data. No shell text, candidate modules, or artifact code runs. +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import urllib.error +import urllib.parse +import urllib.request +import zipfile + +REPOSITORY = "pineforge-4pass/pineforge-engine" +CORPUS_REPOSITORY = "pineforge-4pass/pineforge-corpus" +PARITY = "scripts/check_corpus_parity.sh" +WORKFLOW = ".github/workflows/corpus-parity.yml" +SCHEMA = "pineforge-corpus-headline-v1" +SHA = re.compile(r"[0-9a-f]{40}") +DIGEST = re.compile(r"[0-9a-f]{64}") +PRODUCER_PATHS = ( + WORKFLOW, "scripts/record_corpus_headline_evidence.py", + "scripts/check_corpus_headline_evidence.py", PARITY, + "scripts/run_corpus.sh", "scripts/run_strategy.py", "scripts/verify_corpus.py", + "scripts/parity_paths.py", "scripts/derive_corpus_feeds.py", +) +# Unknown tracked inputs are INCLUDED. Only named guard/CI files and prose are +# excluded; the pin alone is normalized below. This includes build files, +# native/runtime code, grader dependencies, corpus gitlink and feed derivation. +NONSCIENTIFIC = frozenset(( + "AGENTS.md", "CLAUDE.md", "LICENSE", "NOTICE", + "scripts/check_corpus_headline_evidence.py", + "scripts/test_check_corpus_headline_evidence.py", + "scripts/record_corpus_headline_evidence.py", + "scripts/test_record_corpus_headline_evidence.py", + "scripts/ci_preflight.py", "scripts/test_ci_preflight.py", +)) +NATIVE_SUMMARY = re.compile(rb"ran ([1-9][0-9]*) strategies in [0-9]+s -- ok=([0-9]+) fail=([0-9]+)\n") +HEADLINE = re.compile( + rb"Verified ([1-9][0-9]*) strategies \xe2\x80\x94 excellent=([0-9]+), " + rb"strong=([0-9]+), moderate=([0-9]+), weak=([0-9]+), minimal=([0-9]+), " + rb"anomaly=([0-9]+), engine_only=([0-9]+), missing=([0-9]+)") + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha256(data): + return hashlib.sha256(data).hexdigest() + + +def json_bytes(value): + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def json_object(data): + def pairs(items): + result = {} + for key, value in items: + require(key not in result, "duplicate JSON key") + result[key] = value + return result + def constant(value): + raise ValueError("invalid JSON constant") + value = json.loads(data, object_pairs_hook=pairs, parse_constant=constant) + require(isinstance(value, dict), "expected JSON object") + return value + + +def git(repo, *args): + return subprocess.run(["git", "--no-optional-locks", "-C", str(repo), *args], + check=True, capture_output=True).stdout + + +def tree(repo, revision): + require(bool(SHA.fullmatch(revision)), "revision must be a full commit SHA") + result = {} + for row in git(repo, "ls-tree", "-rz", "--full-tree", revision).split(b"\0"): + if not row: + continue + info, name = row.split(b"\t", 1) + mode, kind, oid = info.decode("ascii").split() + path = name.decode("utf-8") + require(not path.startswith("/") and all(p not in ("", ".", "..") + for p in path.split("/")), "unsafe tree path") + result[path] = (mode, kind, oid) + return result + + +def blob(repo, revision, path): + entry = tree(repo, revision).get(path) + require(entry is not None and entry[0] in ("100644", "100755") + and entry[1] == "blob", f"not a regular tracked file: {path}") + return git(repo, "cat-file", "blob", entry[2]) + + +def assignment(script): + """One supported, literal, top-level assignment; never evaluate shell.""" + require(b"\r" not in script and b"\0" not in script, "unparseable shell text") + matches = [] + lines = script.splitlines(keepends=True) + for index, line in enumerate(lines): + if line.lstrip().startswith(b"#"): + continue + if line.startswith(b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-'): + match = re.fullmatch(rb'EXPECTED_VERIFY="\$\{EXPECTED_VERIFY:-([^"$`\\{}\r\n]*)\}"\n?', line) + else: + match = re.fullmatch(rb'EXPECTED_VERIFY=(?:"([^"$`\\{}\r\n]*)"|\'([^\'\r\n]*)\')\n?', line) + if match: + value = next(group for group in match.groups() if group is not None) + require(bool(value), "empty headline default") + matches.append((index, value)) + continue + references_removed = re.sub(rb"\$\{EXPECTED_VERIFY\}|\$EXPECTED_VERIFY\b", b"", line) + require(not re.search(rb"\bEXPECTED_VERIFY\b", references_removed), + "dynamic or ambiguous EXPECTED_VERIFY assignment/use") + require(len(matches) == 1, "expected exactly one literal headline assignment") + index, value = matches[0] + lines[index] = b"EXPECTED_VERIFY=\n" + return value, b"".join(lines) + + +def compared(repo, base, head): + require(bool(SHA.fullmatch(base)) and bool(SHA.fullmatch(head)), "full PR SHAs required") + ancestor = git(repo, "merge-base", "--all", base, head).decode().strip() + require(bool(SHA.fullmatch(ancestor)), "ambiguous or missing PR merge-base") + require(ancestor != head, "empty or reversed PR comparison") + return ancestor, assignment(blob(repo, ancestor, PARITY))[0], assignment(blob(repo, head, PARITY))[0] + + +def excluded(path): + return (path in NONSCIENTIFIC or path.startswith((".github/", "operator/")) + or (path.startswith("docs/") and Path(path).suffix in (".md", ".rst", ".svg", ".png")) + or ("/" not in path and path.endswith(".md"))) + + +def engine_identity(repo, revision): + rows = [] + for path, (mode, kind, oid) in sorted(tree(repo, revision).items()): + if excluded(path): + continue + if path == PARITY: + oid = "sha256:" + sha256(assignment(blob(repo, revision, path))[1]) + rows.append([path, mode, kind, oid]) + require(bool(rows), "empty scientific source") + return sha256(json_bytes(rows)) + + +def scientific(repo, revision): + entries = tree(repo, revision) + corpus = entries.get("corpus") + grader = entries.get("scripts/verify_corpus.py") + require(corpus is not None and corpus[:2] == ("160000", "commit"), "missing corpus gitlink") + require(grader is not None and grader[0] in ("100644", "100755") + and grader[1] == "blob", "missing grader") + return {"engine_sha256": engine_identity(repo, revision), + "grader_git_blob": grader[2], "corpus_commit": corpus[2]} + + +def population(entries): + """Mirror verifier discovery, not its grading or outcomes.""" + paths = set(entries) + probes = set() + for path in paths: + parts = path.split("/") + if len(parts) < 3 or parts[0] not in ("basic", "community", "validation"): + continue + probe = "/".join(parts[:2]) + if parts[1] == "symbol-specified" and probe + "/strategy.pine" not in paths: + continue + probes.add(probe) + require(bool(probes), "empty expected verifier population") + require(all(re.fullmatch(r"(basic|community|validation)/[A-Za-z0-9][A-Za-z0-9._-]*", p) + for p in probes), "unsupported probe name") + return sorted(probes) + + +def headline(raw): + rows = [row for row in raw.split(b"\n") if row.startswith(b"Verified ")] + require(len(rows) == 1, "missing or ambiguous native verifier headline") + match = HEADLINE.fullmatch(rows[0]) + require(match is not None, "unparseable native verifier headline") + counts = [int(value) for value in match.groups()] + require(sum(counts[1:]) == counts[0] and counts[-1] == 0, + "inconsistent or missing measured population") + return rows[0], counts[0] + + +PYTHON_TOOLCACHE = "/opt/hostedtoolcache/" +PYTHON_CACHE_ID = re.compile(r"Python/([0-9]+\.[0-9]+\.[0-9]+)/(x64|arm64)") + + +def loader_environment(env): + """Describe only the permitted loader binding, without absolute paths.""" + if "LD_LIBRARY_PATH" not in env: + return {"LD_LIBRARY_PATH": "absent"} + library = env["LD_LIBRARY_PATH"] + require(isinstance(library, str), "invalid loader environment value") + if library == "": + return {"LD_LIBRARY_PATH": "empty"} + location = env.get("pythonLocation") + require(isinstance(location, str) and location.startswith(PYTHON_TOOLCACHE), + "loader environment requires a standard pythonLocation") + relative = location[len(PYTHON_TOOLCACHE):] + require(PYTHON_CACHE_ID.fullmatch(relative) is not None + and library == location + "/lib", + "loader environment must be exactly pythonLocation/lib") + return { + "LD_LIBRARY_PATH": "pythonLocation/lib", "pythonLocation": relative, + "pythonLocation_sha256": sha256(location.encode()), + "LD_LIBRARY_PATH_sha256": sha256(library.encode()), + } + + +def validate_loader_environment(value, python_version): + require(isinstance(value, dict), "missing loader environment") + mode = value.get("LD_LIBRARY_PATH") + if mode in ("absent", "empty"): + require(value == {"LD_LIBRARY_PATH": mode}, "unbound loader environment fields") + return + require(mode == "pythonLocation/lib", "unsupported loader environment") + relative = value.get("pythonLocation") + match = PYTHON_CACHE_ID.fullmatch(relative) if isinstance(relative, str) else None + require(match is not None and match[1] == python_version, + "loader environment location/version mismatch") + location = PYTHON_TOOLCACHE + relative + require(value == loader_environment({ + "pythonLocation": location, "LD_LIBRARY_PATH": location + "/lib", + }), "loader environment binding/digest mismatch") + + +def validate_record(record, raw, summary, expected, inputs, probes, source, output_digest): + require(record.get("schema") == SCHEMA, "unrecorded evidence schema") + require(record.get("source_commit") == source, "stale measured source") + require(record.get("scientific") == inputs, "wrong scientific inputs") + require(record.get("population") == probes and bool(probes), "wrong expected population") + require(bool(DIGEST.fullmatch(record.get("inputs_sha256", ""))), "missing input digest") + verifier = record.get("verifier", {}) + require(type(verifier.get("exit")) is int and verifier["exit"] == 0, "failed/unexecuted verifier") + require(sha256(raw) == output_digest == verifier.get("sha256"), "raw output digest mismatch") + actual, total = headline(raw) + require(actual == expected, "measured headline bytes differ from candidate default") + require(total == len(probes) > 0, "partial measured verifier population") + require(record.get("native_summary_sha256") == sha256(summary), "native control digest mismatch") + match = NATIVE_SUMMARY.fullmatch(summary) + require(match is not None, "missing native completion control") + require(tuple(map(int, match.groups())) == (total, total, 0), "partial/failed native control") + native = record.get("native", {}) + require(all(type(native.get(key)) is int and native[key] == value + for key, value in (("exit", 0), ("total", total), ("ok", total), ("fail", 0))), + "failed or partial native run") + require(native.get("probes") == probes, "partial native probe set") + for key in ("outputs", "libraries"): + files = native.get(key) + require(isinstance(files, dict) and set(files) == set(probes) + and all(isinstance(d, str) and DIGEST.fullmatch(d) for d in files.values()), + f"incomplete native {key}") + require(bool(DIGEST.fullmatch(native.get("runtime_sha256", ""))), "missing native runtime") + derived = record.get("derived_inputs") + require(isinstance(derived, dict) and set(derived) == { + "ohlcv_ETH-USDT-USDT_15m.csv", "ohlcv_ETH-USDT-USDT_15m_window.csv"} + and all(isinstance(d, str) and DIGEST.fullmatch(d) for d in derived.values()), + "missing derived scientific input identity") + build = record.get("build", {}) + require(build.get("type") == "Release" and build.get("runner") == "ctypes" + and isinstance(build.get("cache_sha256"), str) and DIGEST.fullmatch(build["cache_sha256"]) + and all(isinstance(build.get(k), str) and re.fullmatch(r"[0-9][0-9.]+", build[k]) + for k in ("compiler", "cmake", "python")), "unsupported recording build") + validate_loader_environment(record.get("environment"), build["python"]) + return total + + +def attachment(value): + require(set(value) == {"run_id", "artifact_id", "artifact_sha256", "output_sha256"}, + "attachment must name run/artifact IDs and both SHA-256 digests") + require(all(type(value[k]) is int and value[k] > 0 for k in ("run_id", "artifact_id")), + "invalid run/artifact ID") + require(all(isinstance(value[k], str) and DIGEST.fullmatch(value[k]) + for k in ("artifact_sha256", "output_sha256")), "invalid attachment digest") + return value + + +def service_identity(run, artifact, attached, workflow_id): + require(run.get("id") == attached["run_id"] and run.get("workflow_id") == workflow_id + and run.get("path") == WORKFLOW and run.get("event") == "workflow_dispatch" + and run.get("status") == "completed" and run.get("conclusion") in ("success", "failure"), + "untrusted or incomplete producer run") + require(run.get("repository", {}).get("full_name") == REPOSITORY + and run.get("head_repository", {}).get("full_name") == REPOSITORY, + "wrong producer repository") + source, attempt = run.get("head_sha", ""), run.get("run_attempt") + require(bool(SHA.fullmatch(source)) and type(attempt) is int and attempt > 0, + "unreadable producer source/attempt") + association = artifact.get("workflow_run", {}) + require(artifact.get("id") == attached["artifact_id"] and artifact.get("expired") is False + and artifact.get("name") == f"corpus-headline-evidence-{attempt}" + and artifact.get("digest") == "sha256:" + attached["artifact_sha256"] + and association.get("id") == attached["run_id"] + and association.get("head_sha") == source + and type(run["repository"].get("id")) is int + and association.get("repository_id") == run["repository"]["id"] + and association.get("head_repository_id") == run["head_repository"].get("id"), + "wrong/unrecorded artifact association or service digest") + return source + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +class Service: + """Fixed GitHub authority; bearer token never follows an artifact redirect.""" + def request(self, path): + require(path.startswith("/repos/") and not path.startswith("//"), "invalid service path") + headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "pineforge-corpus-headline"} + if os.environ.get("GH_TOKEN"): + headers["Authorization"] = "Bearer " + os.environ["GH_TOKEN"] + return urllib.request.Request("https://api.github.com" + path, headers=headers) + + def read(self, path): + with urllib.request.build_opener(NoRedirect()).open(self.request(path), timeout=30) as response: + require(re.search(r'rel="?next"?', response.headers.get("Link", "")) is None, + "truncated service response") + data = response.read(8 * 1024 * 1024 + 1) + require(len(data) <= 8 * 1024 * 1024, "service response too large") + return json_object(data) + + def download(self, artifact_id): + path = f"/repos/{REPOSITORY}/actions/artifacts/{artifact_id}/zip" + try: + response = urllib.request.build_opener(NoRedirect()).open(self.request(path), timeout=30) + except urllib.error.HTTPError as error: + require(error.code == 302, "artifact download refused") + url = urllib.parse.urlsplit(error.headers.get("Location", "")) + require(url.scheme == "https" and url.hostname and url.port in (None, 443) + and url.username is None and url.password is None + and (url.hostname.endswith(".blob.core.windows.net") + or url.hostname.endswith(".actions.githubusercontent.com")), + "untrusted artifact redirect") + # New request, no API authorization header, and no further redirects. + response = urllib.request.build_opener(NoRedirect()).open(url.geturl(), timeout=30) + with response: + data = response.read(8 * 1024 * 1024 + 1) + require(0 < len(data) <= 8 * 1024 * 1024, "empty/oversized artifact") + return data + + def source(self, path, revision): + data = self.read(f"/repos/{REPOSITORY}/contents/{path}?ref={revision}") + require(data.get("type") == "file" and data.get("encoding") == "base64", "missing producer source") + raw = base64.b64decode("".join(data["content"].split()), validate=True) + require(hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest() == data.get("sha"), + "producer source blob mismatch") + return raw + + +def unpack(data, digest): + require(sha256(data) == digest, "service artifact digest mismatch") + with zipfile.ZipFile(io.BytesIO(data)) as archive: + names = ["record.json", "verify_corpus.log", "native-summary.log"] + require(sorted(archive.namelist()) == sorted(names), "unexpected/duplicate artifact members") + for entry in archive.infolist(): + mode = entry.external_attr >> 16 + require(not stat.S_ISLNK(mode) and entry.file_size <= 4 * 1024 * 1024 + and not entry.flag_bits & 1, "unsafe/oversized artifact member") + return (json_object(archive.read(names[0])), archive.read(names[1]), archive.read(names[2])) + + +def check_service(service, attached, repo, base, head, expected): + attached = attachment(attached) + prefix = f"/repos/{REPOSITORY}" + workflow = service.read(prefix + "/actions/workflows/corpus-parity.yml") + require(workflow.get("path") == WORKFLOW and type(workflow.get("id")) is int, "wrong producer workflow") + run = service.read(prefix + f"/actions/runs/{attached['run_id']}") + artifact = service.read(prefix + f"/actions/artifacts/{attached['artifact_id']}") + source = service_identity(run, artifact, attached, workflow["id"]) + for path in PRODUCER_PATHS: + trusted, actual = blob(repo, base, path), service.source(path, source) + if path == PARITY: + trusted, actual = assignment(trusted)[1], assignment(actual)[1] + require(actual == trusted, f"producer differs from accepted base: {path}") + jobs = service.read(prefix + f"/actions/runs/{attached['run_id']}/attempts/{run['run_attempt']}/jobs?per_page=100") + rows = jobs.get("jobs") + require(isinstance(rows, list) and type(jobs.get("total_count")) is int + and len(rows) == jobs["total_count"] <= 100, "incomplete producer jobs") + selected = [j for j in rows if j.get("name") == "corpus-parity"] + require(len(selected) == 1 and selected[0].get("status") == "completed" + and selected[0].get("conclusion") in ("success", "failure"), "native producer did not run") + steps = selected[0].get("steps", []) + for name, verdicts in ( + ("Check TradingView parity against the committed oracle", ("success", "failure")), + ("Retain recorded headline evidence", ("success",)), + ): + found = [s for s in steps if s.get("name") == name] + require(len(found) == 1 and found[0].get("status") == "completed" + and found[0].get("conclusion") in verdicts, "missing/failed producer step: " + name) + record, raw, summary = unpack(service.download(attached["artifact_id"]), attached["artifact_sha256"]) + inputs = scientific(repo, head) + corpus_tree = service.read(f"/repos/{CORPUS_REPOSITORY}/git/trees/{inputs['corpus_commit']}?recursive=1") + require(corpus_tree.get("sha") == inputs["corpus_commit"] + and corpus_tree.get("truncated") is False + and isinstance(corpus_tree.get("tree"), list), "incomplete corpus input tree") + probes = population({entry["path"]: entry for entry in corpus_tree["tree"] + if entry.get("type") == "blob"}) + total = validate_record(record, raw, summary, expected, inputs, probes, source, attached["output_sha256"]) + latest = service.read(prefix + f"/actions/runs/{attached['run_id']}") + latest_artifact = service.read(prefix + f"/actions/artifacts/{attached['artifact_id']}") + require(latest.get("run_attempt") == run["run_attempt"] + and service_identity(latest, latest_artifact, attached, workflow["id"]) == source, + "recording changed while evidence was read") + return total + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path(".")) + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + source = parser.add_mutually_exclusive_group() + source.add_argument("--pr", type=int) + source.add_argument("--attachment", type=Path) + args = parser.parse_args(argv) + try: + ancestor, old, new = compared(args.repo, args.base, args.head) + if old == new: + print(f"corpus headline unchanged: merge-base {ancestor}; no new evidence required") + return 0 + require(args.pr is not None or args.attachment is not None, + "changed headline requires a recorded run/artifact attachment") + service = Service() + if args.pr is not None: + require(args.pr > 0, "invalid PR number") + pr = service.read(f"/repos/{REPOSITORY}/pulls/{args.pr}") + require(pr.get("head", {}).get("sha") == args.head + and pr.get("base", {}).get("sha") == args.base + and pr.get("head", {}).get("repo", {}).get("full_name") == REPOSITORY + and pr.get("head", {}).get("repo", {}).get("fork") is False, + "live PR no longer matches admitted base/head") + body = pr.get("body") or "" + require(body.count("```corpus-headline-evidence") == 1, "ambiguous PR attachment") + blocks = re.findall(r"(?m)^```corpus-headline-evidence\n([^\n]+)\n```[ \t]*$", body) + require(len(blocks) == 1, "expected one corpus-headline-evidence PR attachment") + attached = json_object(blocks[0]) + else: + attached = json_object(args.attachment.read_bytes()) + total = check_service(service, attached, args.repo, args.base, args.head, new) + print(f"corpus headline evidence: {total}/{total} native probes; exact measured bytes and scientific inputs") + print("Coverage: this recorded full native verifier run; no baseline/parity or review approval implied") + return 0 + except Exception as error: + print("corpus headline evidence refused: " + " ".join(str(error).splitlines())) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_preflight.py b/scripts/ci_preflight.py index eac23b118..db0cc7d73 100644 --- a/scripts/ci_preflight.py +++ b/scripts/ci_preflight.py @@ -599,6 +599,142 @@ def _release_split_requested(jobs: dict[str, str]) -> bool: for _anchor, _addition in _GKE_CORPUS_RESULT_INSERTIONS: GKE_CORPUS_RESULT_STEP = GKE_CORPUS_RESULT_STEP.replace(_anchor, _anchor + _addition, 1) GKE_ARM_CORPUS_RESULT_STEP = GKE_ARM_CORPUS_RESULT_STEP.replace(_anchor, _anchor + _addition, 1) +# B4 pins the complete result job, not a marker or just its terminal gate. +# The four existing arm/corpus families each have exactly two public forms. +GKE_RESULT_CLASSIFIER_STEPS = r''' - name: Checkout the base classifier + if: ${{ needs.admit.result == 'success' }} + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: ${{ github.repository }} + ref: ${{ needs.admit.outputs.base_sha }} + fetch-depth: 1 + persist-credentials: false + + - name: Re-read the payload-bound compare list + id: second-classify + if: ${{ needs.admit.result == 'success' }} + env: + REPO: ${{ github.repository }} + BASE_SHA: ${{ needs.admit.outputs.base_sha }} + HEAD_SHA: ${{ needs.admit.outputs.head_sha }} + GH_TOKEN: ${{ github.token }} + PATH_FILE: ${{ runner.temp }}/gke-pr-result-paths + shell: python3 {0} + run: | + import json + import os + from pathlib import Path + import subprocess + + paths = [] + endpoint = (f"repos/{os.environ['REPO']}/compare/" + f"{os.environ['BASE_SHA']}...{os.environ['HEAD_SHA']}") + response = json.loads(subprocess.run( + ["gh", "api", endpoint], check=True, + capture_output=True, text=True).stdout) + rows = response.get("files") + if not isinstance(rows, list): + raise SystemExit("second compare response is unreadable") + if len(rows) >= 300: + raise SystemExit("second compare response is truncated at 300 files") + for row in rows: + if not isinstance(row, dict): + raise SystemExit("second compare response has an unreadable file row") + filename = row.get("filename") + if (not isinstance(filename, str) or "\x00" in filename + or any(part in ("", ".", "..") for part in filename.split("/"))): + raise SystemExit("second compare filename is unreadable") + paths.append(filename) + if row.get("status") == "renamed" and "previous_filename" not in row: + raise SystemExit("second compare response is missing a rename origin") + if "previous_filename" in row: + previous = row["previous_filename"] + if (not isinstance(previous, str) or "\x00" in previous + or any(part in ("", ".", "..") for part in previous.split("/"))): + raise SystemExit("second previous filename is unreadable") + paths.append(previous) + if not paths: + raise SystemExit("second compare response has no changed paths") + Path(os.environ["PATH_FILE"]).write_bytes( + b"\0".join(path.encode("utf-8") for path in paths) + + (b"\0" if paths else b"")) + subprocess.run(["python3", "scripts/ci_docs_only.py", + "--paths", os.environ["PATH_FILE"], + "--github-output", os.environ["GITHUB_OUTPUT"]], + check=True) + + - name: Apply the one-way code-extension veto + id: veto + if: ${{ needs.admit.result == 'success' }} + env: + PATH_FILE: ${{ runner.temp }}/gke-pr-result-paths + shell: python3 {0} + run: | + import os + from pathlib import Path + + data = Path(os.environ["PATH_FILE"]).read_bytes() + paths = [] if not data else [item.decode("utf-8") for item in data.rstrip(b"\0").split(b"\0")] + extensions = {".cpp", ".hpp", ".h", ".c", ".cc", ".cmake", ".py", ".sh", ".txt"} + code = any(Path(path).suffix in extensions and not path.startswith("docs/") + for path in paths) + Path(os.environ["GITHUB_OUTPUT"]).open("a").write( + f"code_extension={'true' if code else 'false'}\n") + +''' +GKE_HEADLINE_EVIDENCE_STEP = ''' - name: Require recorded native corpus headline evidence + id: headline + if: ${{ needs.admit.result == 'success' }} + env: + GH_TOKEN: ${{ github.token }} + BASE_SHA: ${{ needs.admit.outputs.base_sha }} + HEAD_SHA: ${{ needs.admit.outputs.head_sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + git fetch --no-tags origin "$BASE_SHA" "$HEAD_SHA" + python3 scripts/check_corpus_headline_evidence.py --repo . --base "$BASE_SHA" --head "$HEAD_SHA" --pr "$PR_NUMBER" + +''' + + +def _gke_result_contract(arm: bool, corpus: bool, *, headline: bool) -> str: + needs = 'admit, preflight, build-x86-gcc133' + gate = GKE_ARM_RESULT_STEP if arm else GKE_RESULT_STEP + if corpus: + needs += ', corpus-gitlink-result' + gate = GKE_ARM_CORPUS_RESULT_STEP if arm else GKE_CORPUS_RESULT_STEP + permissions = '{contents: read}' + classifiers = GKE_RESULT_CLASSIFIER_STEPS + if headline: + permissions = '{contents: read, actions: read, pull-requests: read}' + classifiers = classifiers.replace(' fetch-depth: 1\n', + ' fetch-depth: 0\n', 1) + classifiers += GKE_HEADLINE_EVIDENCE_STEP + for anchor, addition in ( + (' PREFLIGHT_RESULT: ${{ needs.preflight.result }}\n', + ' HEADLINE_RESULT: ${{ steps.headline.outcome }}\n'), + (' test "$PREFLIGHT_RESULT" = success || fail "preflight did not pass"\n', + ' test "$HEADLINE_RESULT" = success || fail "corpus headline evidence did not pass"\n')): + assert gate.count(anchor) == 1, 'headline result insertion anchor drifted' + gate = gate.replace(anchor, anchor + addition, 1) + return (f'\n needs: [{needs}, legs]\n' + ' if: always()\n' + ' runs-on: ubuntu-24.04\n' + f' permissions: {permissions}\n' + ' steps:\n' + classifiers + gate) + + +GKE_RESULT_CONTRACTS = { + (arm, corpus): { + 'OLD': _gke_result_contract(arm, corpus, headline=False), + 'NEW': _gke_result_contract(arm, corpus, headline=True), + } + for arm in (False, True) for corpus in (False, True) +} +GKE_RESULT_CONTRACT_FINDING = ( + 'gke-pr.yml result must match one complete OLD or NEW headline contract') + BUILD_CACHE_KEY = 'ccache-main-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-' PREFLIGHT_LINTER_RUN = ' '.join(r'''| set -euo pipefail @@ -1734,6 +1870,19 @@ def _gke_pr_corpus_variant(name: str, jobs: dict[str, str]) -> bool: return name == 'gke-pr.yml' and 'corpus-gitlink-result' in jobs +def gke_result_contract_version(workflow: str) -> str | None: + """Match a whole executable job before selecting either bridge form.""" + headers = _JOB_HEADER.finditer(_jobs_text(workflow) or '') + if sum(header.group(1) == 'result' for header in headers) != 1: + return None + jobs = _jobs(workflow) + family = (_gke_pr_arm_variant('gke-pr.yml', jobs), + _gke_pr_corpus_variant('gke-pr.yml', jobs)) + actual = _code(jobs.get('result', '')) + return next((version for version, body in GKE_RESULT_CONTRACTS[family].items() + if actual == _code(body)), None) + + GKE_PR_SMOKE_STEPS = r''' - name: Checkout the admitted head uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -1780,6 +1929,16 @@ def gke_admit_workflow_findings(workflow: str, name: str = 'gke-pr.yml') -> list findings = [] if not workflow: return [f'{name}: admission workflow is missing'] + if name == 'gke-pr.yml': + version = gke_result_contract_version(workflow) + if version is None: + findings.append(GKE_RESULT_CONTRACT_FINDING) + elif version == 'NEW': + # Only a complete NEW match reaches this fixed existing-policy + # adapter. Every other job and all workflow-level policy stay live. + jobs = _jobs(workflow) + family = (_gke_pr_arm_variant(name, jobs), _gke_pr_corpus_variant(name, jobs)) + workflow = workflow.replace(jobs['result'], GKE_RESULT_CONTRACTS[family]['OLD'], 1) if _events(workflow) != {'pull_request_target'}: findings.append(f'{name}: trigger must be exactly pull_request_target') if ' branches: [main]' not in workflow: @@ -2190,6 +2349,90 @@ def admit_rule_findings(workflows_dir: Path, *, run_actionlint: bool = True) -> return list(dict.fromkeys(action_findings)) +def _headline_window_git(source: Path, *args: str) -> bytes: + return subprocess.run( + ['git', '--no-replace-objects', '--no-optional-locks', '-C', str(source), *args], + check=True, capture_output=True).stdout + + +def headline_window_findings(source: Path, workflows_dir: Path) -> list[str]: + """Native base/head no-repin window, using the event and actual Git objects. + + There is no CLI context override or marker-based opt-out. The trusted main + entry calls this independently of the fetched workflow's shape verdict. + """ + import check_corpus_headline_evidence as evidence + + try: + require = evidence.require + repository = 'pineforge-4pass/pineforge-engine' + workflow_path = '.github/workflows/gke-pr.yml' + require(os.environ.get('GITHUB_ACTIONS') == 'true' + and os.environ.get('GITHUB_EVENT_NAME') == 'pull_request_target' + and os.environ.get('GITHUB_REPOSITORY') == repository + and os.environ.get('GITHUB_REF') == 'refs/heads/main' + and os.environ.get('GITHUB_WORKFLOW_REF') + == repository + '/' + workflow_path + '@refs/heads/main', + 'untrusted native admission context') + event = evidence.json_object(Path(os.environ['GITHUB_EVENT_PATH']).read_bytes()) + pr = event.get('pull_request', {}) + base, head = pr.get('base', {}), pr.get('head', {}) + payload_base_sha, head_sha = base.get('sha'), head.get('sha') + executing_sha = os.environ.get('GITHUB_SHA') + require(event.get('repository', {}).get('full_name') == repository + and type(event.get('number')) is int and event['number'] > 0 + and type(pr.get('number')) is int and pr['number'] == event['number'] + and pr.get('draft') is False and base.get('ref') == 'main' + and base.get('repo', {}).get('full_name') == repository + and head.get('repo', {}).get('full_name') == repository + and head.get('repo', {}).get('fork') is False + and isinstance(payload_base_sha, str) + and evidence.SHA.fullmatch(payload_base_sha) is not None + and isinstance(head_sha, str) and evidence.SHA.fullmatch(head_sha) is not None, + 'invalid immutable PR identity') + require(isinstance(executing_sha, str) + and evidence.SHA.fullmatch(executing_sha) is not None, + 'untrusted native admission context') + require(_headline_window_git(source, 'rev-parse', '--verify', 'HEAD').strip() + == executing_sha.encode(), 'checker checkout differs from executing SHA') + print(f'native admission base: executing_sha={executing_sha} ' + f'payload_base_sha={payload_base_sha} (informational)', file=sys.stderr) + shallow = _headline_window_git(source, 'rev-parse', '--is-shallow-repository').strip() + require(shallow in (b'true', b'false'), 'unreadable Git history depth') + fetch = ['fetch', '--no-tags', '--no-write-fetch-head'] + if shallow == b'true': + fetch.append('--unshallow') + # The existing trusted checkout is depth 1 with credentials disabled. + # Fetch the public canonical repository, never a head-controlled URL/ref. + _headline_window_git(source, *fetch, + 'https://github.com/' + repository + '.git', executing_sha, head_sha) + require(_headline_window_git(source, 'rev-parse', '--is-shallow-repository').strip() + == b'false', 'incomplete Git history') + for revision in (executing_sha, head_sha): + require(_headline_window_git(source, 'rev-parse', '--verify', + revision + '^{commit}').strip() == revision.encode(), + 'event revision is not an available commit') + fetched = (workflows_dir / workflow_path).read_bytes() + require(fetched == evidence.blob(source, head_sha, workflow_path), + 'fetched admission workflow differs from event head') + head_version = gke_result_contract_version(fetched.decode('utf-8')) + require(head_version is not None, GKE_RESULT_CONTRACT_FINDING) + # GITHUB_SHA and checkout HEAD bind the executing base workflow. The + # payload base is informational; the pin diff still uses the merge-base. + base_workflow = evidence.blob(source, executing_sha, workflow_path).decode('utf-8') + base_version = gke_result_contract_version(base_workflow) + require(base_version is not None, GKE_RESULT_CONTRACT_FINDING) + _, old, new = evidence.compared(source, executing_sha, head_sha) + require((base_version == 'NEW' and head_version == 'NEW') or old == new, + 'OLD result contract cannot change EXPECTED_VERIFY') + return [] + except Exception as error: + # Do not echo subprocess commands, event contents or credentials. + detail = ('Git history unavailable' if isinstance(error, subprocess.CalledProcessError) + else ' '.join(str(error).splitlines())) + return ['corpus headline bridge refused: ' + detail] + + def _job_values(body: str, key: str, *, unquote: bool = True) -> list[str]: """Every value of a job-level key, a block value's lines folded onto one.""" lines = body.split('\n') @@ -6061,6 +6304,8 @@ def check_commands(source: Path, output: Path | None = None, [sys.executable, str(source / 'scripts/test_validate_corpus_sha.py')]), ('corpus-gitlink-result-tests', [sys.executable, str(source / 'scripts/test_check_corpus_gitlink_result.py')]), + ('corpus-headline-evidence-tests', + [sys.executable, str(source / 'scripts/test_check_corpus_headline_evidence.py')]), ('corpus-file-digest-tests', [sys.executable, str(source / 'scripts/test_write_corpus_file_digests.py')]), ('corpus-admission-variant-tests', @@ -6304,7 +6549,8 @@ def main() -> int: if args.workflows_dir is None: print('--rules-only requires --workflows-dir') return 1 - findings = admit_rule_findings(args.workflows_dir) + findings = headline_window_findings(ROOT, args.workflows_dir) + findings += admit_rule_findings(args.workflows_dir) for finding in findings: print(finding) if not findings: diff --git a/scripts/test_check_corpus_headline_evidence.py b/scripts/test_check_corpus_headline_evidence.py new file mode 100644 index 000000000..5dea70836 --- /dev/null +++ b/scripts/test_check_corpus_headline_evidence.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Logic fixtures only: these tests do not certify a recorded native run.""" +import copy +import hashlib +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock +import zipfile + +import check_corpus_headline_evidence as guard + + +HEADLINE = ("Verified 2 strategies — excellent=2, strong=0, moderate=0, weak=0, " + "minimal=0, anomaly=0, engine_only=0, missing=0").encode() + + +class HeadlineTextTests(unittest.TestCase): + def test_literal_default_is_read_without_environment_or_shell(self): + script = b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-' + HEADLINE + b'}"\n' + with mock.patch.dict("os.environ", {"EXPECTED_VERIFY": "unrelated"}): + self.assertEqual(guard.assignment(script)[0], HEADLINE) + + def test_ambiguous_and_dynamic_assignments_refuse(self): + for script in ( + b'EXPECTED_VERIFY="$(touch forbidden)"\n', + b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-$OTHER}"\n', + b'EXPECTED_VERIFY=x\nEXPECTED_VERIFY=y\n', + b'export EXPECTED_VERIFY="x"\n', + b'read EXPECTED_VERIFY\n', + b'# EXPECTED_VERIFY="x"\n', + ): + with self.subTest(script=script), self.assertRaises(ValueError): + guard.assignment(script) + + def test_native_output_has_exactly_one_nonzero_consistent_headline(self): + self.assertEqual(guard.headline(b"\n" + HEADLINE + b"\n"), (HEADLINE, 2)) + for output in (b"", HEADLINE + b"\r\n", HEADLINE + b"\n" + HEADLINE, + HEADLINE.replace(b"Verified 2", b"Verified 0"), + HEADLINE.replace(b"excellent=2", b"excellent=1")): + with self.subTest(output=output), self.assertRaises(ValueError): + guard.headline(output) + + def test_duplicate_json_keys_cannot_override_identity(self): + with self.assertRaises(ValueError): + guard.json_object(b'{"run_id":1,"run_id":2}') + + +class GitComparisonTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.repo = Path(self.temp.name) + self.git("init", "-q") + self.git("config", "user.name", "Fixture") + self.git("config", "user.email", "fixture@example.invalid") + (self.repo / "scripts").mkdir() + self.script = self.repo / guard.PARITY + self.script.write_bytes(b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-old}"\n') + self.base = self.commit("base") + + def git(self, *args, data=None): + return subprocess.run(["git", "-C", str(self.repo), *args], check=True, + input=data, capture_output=True).stdout.decode().strip() + + def commit(self, message): + # Private Git objects, without staging or invoking git commit. + def write_tree(directory): + rows = [] + for path in sorted(directory.iterdir()): + if path.name == ".git": + continue + if path.is_dir(): + oid = write_tree(path) + rows.append(f"040000 tree {oid}\t{path.name}\n") + else: + oid = self.git("hash-object", "-w", "--stdin", data=path.read_bytes()) + rows.append(f"100644 blob {oid}\t{path.name}\n") + return self.git("mktree", data="".join(rows).encode()) + parents = ["-p", self.parent] if hasattr(self, "parent") else [] + self.parent = self.git("commit-tree", write_tree(self.repo), *parents, "-m", message) + self.git("update-ref", "HEAD", self.parent) + return self.parent + + def test_change_in_earlier_pr_commit_is_not_hidden_by_last_commit(self): + self.script.write_bytes(b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n') + self.commit("change") + (self.repo / "README.md").write_text("later unrelated commit\n") + head = self.commit("docs") + self.assertEqual(guard.compared(self.repo, self.base, head)[1:], (b"old", b"new")) + + def test_unchanged_control_needs_no_service_or_attachment(self): + (self.repo / "README.md").write_text("unrelated\n") + head = self.commit("docs") + with mock.patch.object(guard, "Service", side_effect=AssertionError("network")): + self.assertEqual(guard.main(["--repo", str(self.repo), + "--base", self.base, "--head", head]), 0) + + def test_changed_without_attachment_fails_closed(self): + self.script.write_bytes(b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n') + head = self.commit("change") + self.assertEqual(guard.main(["--repo", str(self.repo), + "--base", self.base, "--head", head]), 1) + + def test_empty_or_wrong_ref_comparison_refuses(self): + for base, head in ((self.base, self.base), ("HEAD", self.base), ("0" * 40, self.base)): + with self.subTest(base=base), self.assertRaises((ValueError, subprocess.CalledProcessError)): + guard.compared(self.repo, base, head) + + def test_scientific_identity_ignores_docs_and_pin_but_binds_grader(self): + (self.repo / "scripts/verify_corpus.py").write_text("grader = 1\n") + first = self.commit("grader") + self.script.write_bytes(b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n') + (self.repo / "README.md").write_text("docs\n") + second = self.commit("pin and docs") + self.assertEqual(guard.engine_identity(self.repo, first), + guard.engine_identity(self.repo, second)) + (self.repo / "scripts/verify_corpus.py").write_text("grader = 2\n") + third = self.commit("different grader") + self.assertNotEqual(guard.engine_identity(self.repo, second), + guard.engine_identity(self.repo, third)) + + +class RecordLogicTests(unittest.TestCase): + def setUp(self): + self.population = ["validation/a", "validation/b"] + self.scientific = {"engine_sha256": "a" * 64, "grader_git_blob": "b" * 40, + "corpus_commit": "c" * 40} + self.raw = b"\n" + HEADLINE + b"\n" + self.summary = b"ran 2 strategies in 1s -- ok=2 fail=0\n" + self.record = { + "schema": guard.SCHEMA, "source_commit": "d" * 40, + "scientific": self.scientific, "population": self.population, + "inputs_sha256": "e" * 64, + "environment": {"LD_LIBRARY_PATH": "absent"}, + "derived_inputs": {"ohlcv_ETH-USDT-USDT_15m.csv": "3" * 64, + "ohlcv_ETH-USDT-USDT_15m_window.csv": "4" * 64}, + "native": {"exit": 0, "total": 2, "ok": 2, "fail": 0, + "probes": self.population, + "outputs": {p: "f" * 64 for p in self.population}, + "libraries": {p: "1" * 64 for p in self.population}, + "runtime_sha256": "2" * 64}, + "verifier": {"exit": 0, "sha256": guard.sha256(self.raw)}, + "native_summary_sha256": guard.sha256(self.summary), + "build": {"type": "Release", "runner": "ctypes", + "compiler": "13.3.0", "cmake": "3.28.3", "python": "3.12.3", + "cache_sha256": "5" * 64}, + } + + def validate(self, record=None, raw=None, expected=HEADLINE): + return guard.validate_record(self.record if record is None else record, + self.raw if raw is None else raw, + self.summary, expected, self.scientific, self.population, + "d" * 40, guard.sha256(self.raw)) + + def test_structural_fixture_acceptance_is_only_parser_evidence(self): + self.assertEqual(self.validate(), 2) + + def test_recorded_loader_environment_requires_the_exact_binding(self): + location = "/opt/hostedtoolcache/Python/3.12.3/x64" + allowed = { + "LD_LIBRARY_PATH": "pythonLocation/lib", + "pythonLocation": "Python/3.12.3/x64", + "pythonLocation_sha256": hashlib.sha256(location.encode()).hexdigest(), + "LD_LIBRARY_PATH_sha256": hashlib.sha256((location + "/lib").encode()).hexdigest(), + } + for environment in ({"LD_LIBRARY_PATH": "absent"}, + {"LD_LIBRARY_PATH": "empty"}, allowed): + record = copy.deepcopy(self.record) + record["environment"] = environment + with self.subTest(environment=environment): + self.assertEqual(self.validate(record), 2) + invalid = [ + None, {}, {"LD_LIBRARY_PATH": "unknown"}, + {"LD_LIBRARY_PATH": "/other"}, + {"LD_LIBRARY_PATH": "absent", "pythonLocation": "unbound"}, + {**allowed, "pythonLocation": "Python/3.12.3/x64/.."}, + {**allowed, "pythonLocation": location}, + {**allowed, "pythonLocation_sha256": "0" * 64}, + {**allowed, "LD_LIBRARY_PATH_sha256": "0" * 64}, + {key: value for key, value in allowed.items() if key != "pythonLocation"}, + ] + for environment in invalid: + record = copy.deepcopy(self.record) + record["environment"] = environment + with self.subTest(environment=environment), self.assertRaisesRegex( + ValueError, "loader environment"): + self.validate(record) + record = copy.deepcopy(self.record) + del record["environment"] + with self.assertRaisesRegex(ValueError, "loader environment"): + self.validate(record) + record = copy.deepcopy(self.record) + record["environment"] = allowed + record["build"]["python"] = "3.12.4" + with self.assertRaisesRegex(ValueError, "loader environment"): + self.validate(record) + + def test_wrong_output_digest_and_wrong_headline_are_independent(self): + bad = copy.deepcopy(self.record) + bad["verifier"]["sha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "digest"): + self.validate(bad) + with self.assertRaisesRegex(ValueError, "headline"): + self.validate(expected=HEADLINE.replace(b"excellent=2", b"excellent=1")) + + def test_wrong_scientific_inputs_and_stale_source_refuse(self): + for field in self.scientific: + bad = copy.deepcopy(self.record) + bad["scientific"][field] = "0" * len(bad["scientific"][field]) + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "scientific"): + self.validate(bad) + bad = copy.deepcopy(self.record) + bad["source_commit"] = "0" * 40 + with self.assertRaisesRegex(ValueError, "source"): + self.validate(bad) + + def test_empty_partial_and_failed_measurements_refuse(self): + with self.assertRaises(ValueError): + self.validate(raw=b"") + for key, value in (("probes", self.population[:1]), ("total", 1), + ("ok", 0), ("fail", 1), ("exit", 1), ("outputs", {})): + bad = copy.deepcopy(self.record) + bad["native"][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + self.validate(bad) + bad = copy.deepcopy(self.record) + bad["verifier"]["exit"] = 1 + with self.assertRaises(ValueError): + self.validate(bad) + + def test_archive_digest_checked_before_inner_bytes(self): + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("record.json", json.dumps(self.record)) + archive.writestr("verify_corpus.log", self.raw) + archive.writestr("native-summary.log", self.summary) + actual = data.getvalue() + with self.assertRaisesRegex(ValueError, "artifact digest"): + guard.unpack(actual, "0" * 64) + record, raw, summary = guard.unpack(actual, guard.sha256(actual)) + self.assertEqual((record, raw, summary), (self.record, self.raw, self.summary)) + + def test_service_path_requires_actual_download_and_accepted_producer_bytes(self): + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("record.json", json.dumps(self.record)) + archive.writestr("verify_corpus.log", self.raw) + archive.writestr("native-summary.log", self.summary) + archive_bytes = data.getvalue() + attached = {"run_id": 7, "artifact_id": 8, "artifact_sha256": guard.sha256(archive_bytes), + "output_sha256": guard.sha256(self.raw)} + run = {"id": 7, "run_attempt": 1, "workflow_id": 9, "path": guard.WORKFLOW, + "event": "workflow_dispatch", "status": "completed", "conclusion": "failure", + "head_sha": "d" * 40, "repository": {"id": 11, "full_name": guard.REPOSITORY}, + "head_repository": {"id": 11, "full_name": guard.REPOSITORY}} + artifact = {"id": 8, "name": "corpus-headline-evidence-1", "expired": False, + "digest": "sha256:" + attached["artifact_sha256"], + "workflow_run": {"id": 7, "repository_id": 11, "head_repository_id": 11, + "head_sha": "d" * 40}} + jobs = {"total_count": 1, "jobs": [{"name": "corpus-parity", "status": "completed", + "conclusion": "failure", "steps": [ + {"name": "Check TradingView parity against the committed oracle", + "status": "completed", "conclusion": "failure"}, + {"name": "Retain recorded headline evidence", "status": "completed", + "conclusion": "success"}]}]} + corpus = {"sha": "c" * 40, "truncated": False, + "tree": [{"path": p + "/generated.cpp", "type": "blob"} for p in self.population]} + def read(path): + if "/actions/workflows/" in path: + return {"path": guard.WORKFLOW, "id": 9} + if "/jobs?" in path: + return jobs + if "/actions/runs/" in path: + return run + if "/actions/artifacts/" in path: + return artifact + return corpus + service = mock.Mock() + service.read.side_effect = read + service.download.return_value = archive_bytes + accepted = lambda path: (b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-old}"\n' + if path == guard.PARITY else b"accepted producer\n") + service.source.side_effect = lambda path, revision: accepted(path) + with mock.patch.object(guard, "blob", side_effect=lambda repo, rev, path: accepted(path)), \ + mock.patch.object(guard, "scientific", return_value=self.scientific): + self.assertEqual(guard.check_service(service, attached, ".", "a" * 40, + "b" * 40, HEADLINE), 2) + service.download.assert_called_with(8) + service.source.side_effect = lambda path, revision: b"hand-written replacement\n" + with self.assertRaisesRegex(ValueError, "producer"): + guard.check_service(service, attached, ".", "a" * 40, "b" * 40, HEADLINE) + service.source.side_effect = lambda path, revision: accepted(path) + service.download.return_value = b"unrecorded bytes" + with self.assertRaisesRegex(ValueError, "artifact digest"): + guard.check_service(service, attached, ".", "a" * 40, "b" * 40, HEADLINE) + + +class ServiceTrustTests(unittest.TestCase): + def test_service_identity_rejects_wrong_repository_ref_attempt_and_producer(self): + run = {"id": 7, "run_attempt": 1, "workflow_id": 9, + "path": guard.WORKFLOW, "event": "workflow_dispatch", + "status": "completed", "conclusion": "failure", "head_sha": "a" * 40, + "repository": {"id": 11, "full_name": guard.REPOSITORY}, + "head_repository": {"id": 11, "full_name": guard.REPOSITORY}} + artifact = {"id": 8, "name": "corpus-headline-evidence-1", + "expired": False, "digest": "sha256:" + "b" * 64, + "workflow_run": {"id": 7, "repository_id": 11, + "head_repository_id": 11, "head_sha": "a" * 40}} + attachment = {"run_id": 7, "artifact_id": 8, + "artifact_sha256": "b" * 64, "output_sha256": "c" * 64} + self.assertEqual(guard.service_identity(run, artifact, attachment, 9), "a" * 40) + for field, value in (("head_sha", "wrong"), ("event", "pull_request"), + ("workflow_id", 10), ("run_attempt", 2), + ("conclusion", "cancelled"), ("head_repository", {})): + bad = copy.deepcopy(run) + bad[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + guard.service_identity(bad, artifact, attachment, 9) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_ci_preflight.py b/scripts/test_ci_preflight.py index 4748ac96e..4ee0e9a6e 100644 --- a/scripts/test_ci_preflight.py +++ b/scripts/test_ci_preflight.py @@ -5188,7 +5188,7 @@ def test_complete_stage_inventory_and_tier_counts(self): 'native-c-surface-tests', 'native-x86-release-tests', 'kernel-residuals-tests', 'corpus-parity-identity-tests', 'corpus-parity-subset-cover-tests', 'corpus-sha-validation-tests', - 'corpus-gitlink-result-tests', 'corpus-file-digest-tests', + 'corpus-gitlink-result-tests', 'corpus-headline-evidence-tests', 'corpus-file-digest-tests', 'corpus-admission-variant-tests', 'report-schema-key-tests', 'selected-window-ownership-tests', 'report-outputs-key-tests', 'release-version-tests', 'release-gate-tests', 'rerun-preempted-tests', 'prebuilt-bookworm-tests', 'prebuilt-glibc-tests', @@ -5208,8 +5208,10 @@ def test_complete_stage_inventory_and_tier_counts(self): with mock.patch.object(ci_preflight.sys, 'platform', 'linux'): self.assertEqual([entry[0] for entry in check_commands(ROOT, env={})], expected) # main() prepends actionlint-version to all/fast/github-only. - self.assertEqual(len(expected) + 1, 71) - self.assertEqual(len(check_commands(ROOT, tier='fast')), 66) + # expectation corrected: all 71 -> 72, fast 66 -> 67, because B4 + # registers the ordinary standalone headline checker tests once. + self.assertEqual(len(expected) + 1, 72) + self.assertEqual(len(check_commands(ROOT, tier='fast')), 67) self.assertEqual(len(check_commands(ROOT, tier='slow')), 4) self.assertEqual(len(check_commands(ROOT, tier='github-only')), 7) for platform in ('darwin', 'win32'): @@ -5218,7 +5220,8 @@ def test_complete_stage_inventory_and_tier_counts(self): if name not in ('prebuilt-bookworm-tests', 'prebuilt-glibc-tests')] self.assertEqual([entry[0] for entry in check_commands(ROOT, env={})], expected_platform) - self.assertEqual(len(check_commands(ROOT, tier='fast')), 64) + # expectation corrected: 64 -> 65, for the same added stage. + self.assertEqual(len(check_commands(ROOT, tier='fast')), 65) class PrebuiltGlibcRegistration(unittest.TestCase): @@ -12271,6 +12274,8 @@ def test_arm_job_without_new_outputs_and_gates_is_refused(self) -> None: marker = ' corpus-parity-subset:\n' changed = self.original.replace(marker, self.ARM_JOB + marker, 1) self.assert_arm_refused(changed, [], baseline=self.original, shape=( + # B4 additionally binds the result body to this job family. + 'gke-pr.yml result must match one complete OLD or NEW headline contract', 'gke-pr.yml admit must publish only the base classifier outputs', 'gke-pr.yml legs must require every selected proof', 'gke-pr.yml result must require admission, preflight, native x86 and consistent proof results')) @@ -12278,6 +12283,7 @@ def test_arm_job_without_new_outputs_and_gates_is_refused(self) -> None: def test_arm_job_removed_without_reverting_new_gates_is_refused(self) -> None: changed = self.workflow.replace(self.ARM_JOB, '', 1) self.assert_arm_refused(changed, [], shape=( + 'gke-pr.yml result must match one complete OLD or NEW headline contract', 'gke-pr.yml admit must publish only the base classifier outputs', 'gke-pr.yml legs must require every selected proof', 'gke-pr.yml result must require admission, preflight, native x86 and consistent proof results')) @@ -12317,6 +12323,7 @@ def test_old_result_with_new_legs_is_refused(self) -> None: self.assertTrue(new_step) changed = self.workflow.replace(new_step, old_step, 1) self.assert_arm_refused(changed, [], shape=( + 'gke-pr.yml result must match one complete OLD or NEW headline contract', 'gke-pr.yml result must require admission, preflight, native x86 and consistent proof results',)) def test_old_legs_with_new_result_is_refused(self) -> None: @@ -12375,6 +12382,7 @@ def test_quoted_arm_job_key_keeps_exact_existing_refusals(self) -> None: self.assertNotIn('build-arm', _jobs(changed)) # The plain-key job reader folds this job into build-debug and selects OLD. shape = [ + 'gke-pr.yml result must match one complete OLD or NEW headline contract', 'gke-pr.yml job build-debug must keep its read-only permissions', f'gke-pr.yml job build-debug must use {HOSTED_X86}', 'gke-pr.yml admit must publish only the base classifier outputs', @@ -12436,7 +12444,11 @@ def test_arm_setup_and_gate_bypasses_are_refused(self) -> None: 'gke-pr.yml result must require admission, preflight, native x86 and consistent proof results')): with self.subTest(job=job, change=after): changed = in_job(self.workflow, job, before, after) - self.assert_arm_refused(changed, [], shape=(finding,)) + # expectation corrected: one -> two findings for the result + # mutation, because B4 also pins the complete result contract. + shape = (('gke-pr.yml result must match one complete OLD or NEW headline contract', + finding) if job == 'result' else (finding,)) + self.assert_arm_refused(changed, [], shape=shape) def test_existing_required_jobs_stay_required(self) -> None: for job, body in _jobs(self.original).items(): @@ -13530,6 +13542,515 @@ def test_old_and_new_forms_keep_producer_head_and_failure_semantics(self): self.assert_refused(in_job(workflow, job, before, after), finding) +class HeadlineResultBridgeTests(unittest.TestCase): + """Independent complete result fixtures, separate from the policy catalog.""" + + CONTRACT = 'gke-pr.yml result must match one complete OLD or NEW headline contract' + STEP = ''' - name: Require recorded native corpus headline evidence + id: headline + if: ${{ needs.admit.result == 'success' }} + env: + GH_TOKEN: ${{ github.token }} + BASE_SHA: ${{ needs.admit.outputs.base_sha }} + HEAD_SHA: ${{ needs.admit.outputs.head_sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + git fetch --no-tags origin "$BASE_SHA" "$HEAD_SHA" + python3 scripts/check_corpus_headline_evidence.py --repo . --base "$BASE_SHA" --head "$HEAD_SHA" --pr "$PR_NUMBER" + +''' + OUTCOME = ' HEADLINE_RESULT: ${{ steps.headline.outcome }}\n' + REQUIRE = (' test "$HEADLINE_RESULT" = success || ' + 'fail "corpus headline evidence did not pass"\n') + + def setUp(self): + path = ROOT / 'scripts/fixtures/ci_preflight/gke_pr_legacy.yml' + announce_compare_fixture_paths(path) + data = path.read_bytes() + self.assertEqual(hashlib.sha256(data).hexdigest(), + '21025075090a0a40bfd7360e54a36a0c9bbac9bb8109fbbaa86edce347c27b87') + self.arm = legacy_gke_arm_smoke(data.decode('utf-8')) + + def fixture(self, *, arm=True, corpus=False, new=False): + workflow = self.arm + if not arm: + self.assertEqual(workflow.count(GkeArmAdmissionVariantTests.ARM_JOB), 1) + workflow = workflow.replace(GkeArmAdmissionVariantTests.ARM_JOB, '', 1) + workflow = in_job( + workflow, 'legs', + ' needs: [admit, build, build-debug, build-arm, build-x86-gcc133, corpus-parity-subset]\n', + ' needs: [admit, build, build-debug, build-x86-gcc133, corpus-parity-subset]\n') + for job, addition in ( + ('admit', ' arm_relevant: ${{ steps.classify.outputs.arm_relevant }}\n'), + ('legs', ' ARM_RESULT: ${{ needs.build-arm.result }}\n'), + ('legs', ' ARM_RELEVANT: ${{ needs.admit.outputs.arm_relevant }}\n'), + ('legs', ' arm_passed() {\n' + ' case "$1:$2" in\n' + ' true:success|false:success|false:skipped) return 0 ;;\n' + ' *) return 1 ;;\n' + ' esac\n' + ' }\n'), + ('legs', ' arm_passed "$ARM_RELEVANT" "$ARM_RESULT"\n'), + ('result', ' ADMIT_ARM_RELEVANT: ${{ needs.admit.outputs.arm_relevant }}\n'), + ('result', ' SECOND_ARM_RELEVANT: ${{ steps.second-classify.outputs.arm_relevant }}\n'), + ('result', ' test "$SECOND_ARM_RELEVANT" = "$ADMIT_ARM_RELEVANT" || fail "arm relevance classification changed"\n' + ' case "$ADMIT_ARM_RELEVANT" in\n' + ' true|false) ;;\n' + ' *) fail "arm relevance classification is unanswered" ;;\n' + ' esac\n')): + workflow = in_job(workflow, job, addition, '') + if corpus: + # This is an existing independent literal fixture, not policy data. + from test_ci_corpus_variant import CORPUS_JOB + workflow = workflow.replace(' legs:\n', CORPUS_JOB + ' legs:\n', 1) + workflow = in_job( + workflow, 'result', + ' needs: [admit, preflight, build-x86-gcc133, legs]\n', + ' needs: [admit, preflight, build-x86-gcc133, corpus-gitlink-result, legs]\n') + for anchor, addition in ( + (' X86_RESULT: ${{ needs.build-x86-gcc133.result }}\n', + ' CORPUS_RESULT: ${{ needs.corpus-gitlink-result.result }}\n'), + (' test "$X86_RESULT" = success || fail "native x86 GNU 13.3.0 build did not pass"\n', + ' test "$CORPUS_RESULT" = success || fail "corpus validation did not pass"\n')): + workflow = in_job(workflow, 'result', anchor, anchor + addition) + if new: + workflow = in_job( + workflow, 'result', ' permissions: {contents: read}\n', + ' permissions: {contents: read, actions: read, pull-requests: read}\n') + workflow = in_job(workflow, 'result', + ' fetch-depth: 1\n', ' fetch-depth: 0\n') + marker = ' - name: Require an independently consistent successful result\n' + workflow = in_job(workflow, 'result', marker, self.STEP + marker) + for anchor, addition in ( + (' PREFLIGHT_RESULT: ${{ needs.preflight.result }}\n', self.OUTCOME), + (' test "$PREFLIGHT_RESULT" = success || fail "preflight did not pass"\n', + self.REQUIRE)): + workflow = in_job(workflow, 'result', anchor, anchor + addition) + return workflow + + def test_complete_old_and_new_contracts_cover_each_existing_family(self): + for arm in (False, True): + for corpus in (False, True): + for new in (False, True): + with self.subTest(arm=arm, corpus=corpus, new=new): + workflow = self.fixture(arm=arm, corpus=corpus, new=new) + self.assertEqual(ci_preflight.gke_result_contract_version(workflow), + 'NEW' if new else 'OLD') + self.assertEqual(gke_admit_workflow_findings(workflow), []) + self.assertEqual(main_admission_data_findings({'gke-pr.yml': workflow}), []) + + def test_partial_and_modified_result_contracts_have_a_named_refusal(self): + for arm in (False, True): + for corpus in (False, True): + old = self.fixture(arm=arm, corpus=corpus) + new = self.fixture(arm=arm, corpus=corpus, new=True) + marker = ' - name: Require an independently consistent successful result\n' + cases = [ + in_job(old, 'result', marker, self.STEP + marker), + in_job(old, 'result', 'fetch-depth: 1', 'fetch-depth: 0'), + in_job(new, 'result', self.STEP, ''), + in_job(new, 'result', self.OUTCOME, ''), + in_job(new, 'result', self.REQUIRE, ''), + in_job(new, 'result', self.OUTCOME, + self.OUTCOME.replace('steps.headline.outcome', "'success'")), + in_job(new, 'result', self.OUTCOME, + self.OUTCOME.replace('steps.headline.outcome', + "steps.headline.outcome || 'success'")), + in_job(new, 'result', '--head "$HEAD_SHA"', '--head "$BASE_SHA"'), + in_job(new, 'result', + ' python3 scripts/check_corpus_headline_evidence.py ' + '--repo . --base "$BASE_SHA" --head "$HEAD_SHA" --pr "$PR_NUMBER"\n', + ' true\n'), + in_job(new, 'result', self.STEP, self.STEP + self.STEP), + in_job(new, 'result', ' id: headline\n', + ' id: headline\n continue-on-error: true\n'), + in_job(new, 'result', ' steps:\n', + ' env:\n BASH_ENV: /outside\n steps:\n'), + in_job(old, 'result', ' steps:\n', + ' defaults:\n run:\n shell: sh\n steps:\n'), + new + '\n result:' + _jobs(new)['result'], + ] + for index, workflow in enumerate(cases): + with self.subTest(arm=arm, corpus=corpus, case=index): + self.assertNotIn(workflow, (old, new)) + self.assertIsNone(ci_preflight.gke_result_contract_version(workflow)) + self.assertIn(self.CONTRACT, gke_admit_workflow_findings(workflow)) + self.assertIn(self.CONTRACT, main_admission_data_findings( + {'gke-pr.yml': workflow})) + + def test_new_adapter_retains_the_existing_policy_outside_result(self): + for arm in (False, True): + for corpus in (False, True): + original = self.fixture(arm=arm, corpus=corpus, new=True) + changed = in_job(original, 'preflight', '--tier "$TIER"', '--tier all') + finding = 'gke-pr.yml preflight must run the selected fast or github-only tier' + with self.subTest(arm=arm, corpus=corpus): + self.assertEqual(ci_preflight.gke_result_contract_version(changed), 'NEW') + self.assertIn(finding, gke_admit_workflow_findings(changed)) + self.assertIn(finding, main_admission_data_findings({'gke-pr.yml': changed})) + + def test_new_gate_requires_explicit_headline_success_and_native_x86(self): + for arm in (False, True): + for corpus in (False, True): + workflow = self.fixture(arm=arm, corpus=corpus, new=True) + script = job_script(workflow, 'result', + 'Require an independently consistent successful result') + for docs, github, relevant in ( + ('false', 'false', 'true'), ('false', 'false', 'false'), + ('true', 'false', 'false'), ('false', 'true', 'false')): + values = { + 'ADMIT_RESULT': 'success', 'ALLOWED': 'true', + 'PREFLIGHT_RESULT': 'success', 'X86_RESULT': 'success', + 'HEADLINE_RESULT': 'success', 'CORPUS_RESULT': 'success', + 'LEGS_RESULT': 'skipped' if 'true' in (docs, github) else 'success', + 'ADMIT_DOCS_ONLY': docs, 'SECOND_DOCS_ONLY': docs, + 'ADMIT_GITHUB_ONLY': github, 'SECOND_GITHUB_ONLY': github, + 'ADMIT_ARM_RELEVANT': relevant, 'SECOND_ARM_RELEVANT': relevant, + 'CODE_EXTENSION': 'false', + } + def run(environment): + return subprocess.run( + ['bash', '--noprofile', '--norc', '-c', script], + env={'PATH': '/usr/bin:/bin', **environment}, + capture_output=True, text=True) + with self.subTest(arm=arm, corpus=corpus, docs=docs, github=github, + relevant=relevant): + self.assertEqual(run(values).returncode, 0) + for outcome in ('', 'failure', 'cancelled', 'skipped'): + refused = run({**values, 'HEADLINE_RESULT': outcome}) + self.assertNotEqual(refused.returncode, 0) + self.assertIn('corpus headline evidence did not pass', refused.stdout) + missing = dict(values) + del missing['HEADLINE_RESULT'] + self.assertNotEqual(run(missing).returncode, 0) + refused = run({**values, 'X86_RESULT': 'failure'}) + self.assertNotEqual(refused.returncode, 0) + self.assertIn('native x86 GNU 13.3.0 build did not pass', refused.stdout) + + +class HeadlineWindowNativeTests(unittest.TestCase): + """Synthetic event/transport, real private Git history; no hosted evidence.""" + + PREFIX = 'corpus headline bridge refused: ' + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.repo = self.directory / 'repo' + self.repo.mkdir() + self.git('init', '-q') + self.git('config', 'user.name', 'Fixture') + self.git('config', 'user.email', 'fixture@example.invalid') + fixtures = HeadlineResultBridgeTests() + fixtures.setUp() + self.workflow = fixtures.fixture() + self.files = { + '.github/workflows/gke-pr.yml': self.workflow.encode(), + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-old}"\n', + } + self.base = self.snapshot() + self.head = self.snapshot(self.base, extra={'README.md': b'ordinary change\n'}) + self.git('update-ref', 'HEAD', self.base) + self.fetched = self.directory / 'fetched' + self.target = self.fetched / '.github/workflows/gke-pr.yml' + self.target.parent.mkdir(parents=True) + self.target.write_text(self.workflow) + self.event_path = self.directory / 'event.json' + repository = {'full_name': 'pineforge-4pass/pineforge-engine', 'fork': False} + self.event = { + 'number': 17, 'repository': repository, + 'pull_request': { + 'number': 17, 'draft': False, + 'base': {'ref': 'main', 'sha': self.base, 'repo': repository}, + 'head': {'sha': self.head, 'repo': repository}, + }, + } + self.environment = { + 'GITHUB_ACTIONS': 'true', 'GITHUB_EVENT_NAME': 'pull_request_target', + 'GITHUB_EVENT_PATH': str(self.event_path), + 'GITHUB_REPOSITORY': 'pineforge-4pass/pineforge-engine', + 'GITHUB_SHA': self.base, 'GITHUB_REF': 'refs/heads/main', + 'GITHUB_WORKFLOW_REF': + 'pineforge-4pass/pineforge-engine/.github/workflows/gke-pr.yml@refs/heads/main', + } + + def git(self, *args, data=None): + return subprocess.run(['git', '-C', str(self.repo), *args], input=data, + check=True, capture_output=True).stdout.decode().strip() + + def snapshot(self, *parents, extra=None, omit=()): + nested = {} + for path, data in {**self.files, **(extra or {})}.items(): + if path in omit: + continue + current = nested + parts = path.split('/') + for part in parts[:-1]: + current = current.setdefault(part, {}) + current[parts[-1]] = data + def tree(entries): + rows = [] + for name, value in sorted(entries.items()): + if isinstance(value, dict): + rows.append(f'040000 tree {tree(value)}\t{name}\n') + else: + oid = self.git('hash-object', '-w', '--stdin', data=value) + rows.append(f'100644 blob {oid}\t{name}\n') + return self.git('mktree', data=''.join(rows).encode()) + argv = ['commit-tree', tree(nested)] + for parent in parents: + argv += ['-p', parent] + return self.git(*argv, '-m', 'private fixture') + + def bind_pr(self, base, head, workflow): + self.git('update-ref', 'HEAD', base) + self.environment['GITHUB_SHA'] = base + self.event['pull_request']['base']['sha'] = base + self.event['pull_request']['head']['sha'] = head + self.target.write_text(workflow) + + def verdict(self, *, fetch_source=None): + self.event_path.write_text(json.dumps(self.event)) + self.diagnostics = io.StringIO() + actual = ci_preflight._headline_window_git + def transport(repo, *args): + if args[0] == 'fetch': + self.assertEqual(args[:3], ('fetch', '--no-tags', '--no-write-fetch-head')) + self.assertEqual(args[-3], 'https://github.com/pineforge-4pass/pineforge-engine.git') + self.assertEqual(args[-2:], (self.environment['GITHUB_SHA'], + self.event['pull_request']['head']['sha'])) + source = self.repo if fetch_source is None else fetch_source + local = [source.as_uri() if arg == + 'https://github.com/pineforge-4pass/pineforge-engine.git' + else arg for arg in args] + return actual(repo, *local) + return actual(repo, *args) + with mock.patch.dict(os.environ, self.environment, clear=True), \ + mock.patch.object(ci_preflight, '_headline_window_git', side_effect=transport), \ + contextlib.redirect_stderr(self.diagnostics): + return ci_preflight.headline_window_findings(self.repo, self.fetched) + + def test_equal_payload_and_executing_checkout_has_an_actual_git_control(self): + self.assertEqual(self.verdict(), []) + + def test_stale_payload_base_allows_later_executing_checkout(self): + executing = self.snapshot(self.base, extra={'main-only.md': b'base advancement\n'}) + self.git('update-ref', 'HEAD', executing) + self.environment['GITHUB_SHA'] = executing + # The valid PR payload predates the workflow's actual main checkout. + self.assertNotEqual(self.event['pull_request']['base']['sha'], executing) + self.assertEqual(self.verdict(), []) + self.assertEqual(self.diagnostics.getvalue(), + f'native admission base: executing_sha={executing} ' + f'payload_base_sha={self.base} (informational)\n') + + def test_old_window_compares_merge_base_and_complete_head(self): + changed = self.snapshot(self.base, extra={ + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n'}) + later = self.snapshot(changed, extra={ + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n', + 'README.md': b'later unrelated change\n'}) + # The current base has independently acquired the same headline. It + # cannot replace the PR merge-base as the no-repin comparison source. + advanced = self.snapshot(self.base, extra={ + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n', + 'main-only.md': b'base advancement\n'}) + self.git('update-ref', 'HEAD', advanced) + self.environment['GITHUB_SHA'] = advanced + self.event['pull_request']['base']['sha'] = advanced + self.event['pull_request']['head']['sha'] = later + self.assertEqual(self.verdict(), [ + self.PREFIX + 'OLD result contract cannot change EXPECTED_VERIFY']) + + def test_actual_old_base_refuses_new_head_changed_pin(self): + fixtures = HeadlineResultBridgeTests() + fixtures.setUp() + for arm in (False, True): + for corpus in (False, True): + old = fixtures.fixture(arm=arm, corpus=corpus) + new = fixtures.fixture(arm=arm, corpus=corpus, new=True) + base = self.snapshot(self.base, extra={ + '.github/workflows/gke-pr.yml': old.encode()}) + head = self.snapshot(base, extra={ + '.github/workflows/gke-pr.yml': new.encode(), + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n'}) + self.bind_pr(base, head, new) + # pull_request_target still executes the base's OLD result job. + with self.subTest(arm=arm, corpus=corpus): + self.assertEqual(self.verdict(), [ + self.PREFIX + 'OLD result contract cannot change EXPECTED_VERIFY']) + + def test_stale_new_payload_cannot_defer_actual_old_base_changed_pin(self): + fixtures = HeadlineResultBridgeTests() + fixtures.setUp() + for arm in (False, True): + for corpus in (False, True): + old = fixtures.fixture(arm=arm, corpus=corpus) + new = fixtures.fixture(arm=arm, corpus=corpus, new=True) + payload = self.snapshot(self.base, extra={ + '.github/workflows/gke-pr.yml': new.encode()}) + # Main has reverted to OLD since the payload was recorded. + executing = self.snapshot(payload, extra={ + '.github/workflows/gke-pr.yml': old.encode()}) + head = self.snapshot(payload, extra={ + '.github/workflows/gke-pr.yml': new.encode(), + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n'}) + self.bind_pr(executing, head, new) + self.event['pull_request']['base']['sha'] = payload + with self.subTest(arm=arm, corpus=corpus): + self.assertEqual(self.verdict(), [ + self.PREFIX + 'OLD result contract cannot change EXPECTED_VERIFY']) + + def test_actual_new_base_and_new_head_defer_changed_pin_to_their_checker(self): + fixtures = HeadlineResultBridgeTests() + fixtures.setUp() + for arm in (False, True): + for corpus in (False, True): + new = fixtures.fixture(arm=arm, corpus=corpus, new=True) + base = self.snapshot(self.base, extra={ + '.github/workflows/gke-pr.yml': new.encode()}) + # Both branches adopt NEW after an OLD merge-base. Only the + # actual executing base determines which workflow executes. + head = self.snapshot(self.base, extra={ + '.github/workflows/gke-pr.yml': new.encode(), + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n'}) + for payload in (base, self.base): + self.bind_pr(base, head, new) + self.event['pull_request']['base']['sha'] = payload + with self.subTest(arm=arm, corpus=corpus, payload=payload): + self.assertEqual(self.verdict(), []) + + def test_missing_actual_base_workflow_fails_closed(self): + fixtures = HeadlineResultBridgeTests() + fixtures.setUp() + new = fixtures.fixture(new=True) + base = self.snapshot(self.base, omit=('.github/workflows/gke-pr.yml',)) + head = self.snapshot(base, extra={ + '.github/workflows/gke-pr.yml': new.encode(), + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n'}) + self.bind_pr(base, head, new) + self.assertEqual(self.verdict(), [ + self.PREFIX + 'not a regular tracked file: .github/workflows/gke-pr.yml']) + + def test_unclassifiable_actual_base_workflow_fails_closed(self): + fixtures = HeadlineResultBridgeTests() + fixtures.setUp() + new = fixtures.fixture(new=True) + partial = in_job(new, 'result', fixtures.REQUIRE, '') + base = self.snapshot(self.base, extra={ + '.github/workflows/gke-pr.yml': partial.encode()}) + head = self.snapshot(base, extra={ + '.github/workflows/gke-pr.yml': new.encode(), + 'scripts/check_corpus_parity.sh': b'EXPECTED_VERIFY="${EXPECTED_VERIFY:-new}"\n'}) + self.bind_pr(base, head, new) + self.assertEqual(self.verdict(), [self.PREFIX + fixtures.CONTRACT]) + + def test_event_identity_and_native_entry_cannot_be_omitted(self): + for key, value in ( + ('GITHUB_ACTIONS', ''), ('GITHUB_EVENT_NAME', 'pull_request'), + ('GITHUB_REPOSITORY', 'elsewhere/repo'), + ('GITHUB_REF', 'refs/heads/feature'), ('GITHUB_WORKFLOW_REF', 'elsewhere')): + with self.subTest(key=key), mock.patch.dict(self.environment, {key: value}): + self.assertEqual(self.verdict(), [self.PREFIX + 'untrusted native admission context']) + for kind in ('draft', 'fork', 'base-ref', 'number', 'base-sha', 'sha'): + original = json.loads(json.dumps(self.event)) + if kind == 'draft': + self.event['pull_request']['draft'] = True + elif kind == 'fork': + self.event['pull_request']['head']['repo'] = { + 'full_name': 'pineforge-4pass/pineforge-engine', 'fork': True} + elif kind == 'base-ref': + self.event['pull_request']['base']['ref'] = 'feature' + elif kind == 'number': + self.event['number'] = True + elif kind == 'base-sha': + self.event['pull_request']['base']['sha'] = 'HEAD' + else: + self.event['pull_request']['head']['sha'] = 'HEAD' + with self.subTest(kind=kind): + self.assertEqual(self.verdict(), [self.PREFIX + 'invalid immutable PR identity']) + self.event = original + + def test_missing_or_malformed_executing_sha_fails_closed(self): + for value in (None, '', 'HEAD', self.base[:-1], 'A' * 40): + with self.subTest(value=value), mock.patch.dict(self.environment): + if value is None: + self.environment.pop('GITHUB_SHA') + else: + self.environment['GITHUB_SHA'] = value + self.assertEqual(self.verdict(), [ + self.PREFIX + 'untrusted native admission context']) + + def test_fetched_marker_or_new_job_cannot_replace_head_bytes(self): + self.target.write_text(self.workflow + '\n# headline marker\n') + self.assertEqual(self.verdict(), [self.PREFIX + 'fetched admission workflow differs from event head']) + + def test_checker_checkout_must_match_executing_sha(self): + for checkout, executing in ((self.head, self.base), (self.base, self.head)): + self.git('update-ref', 'HEAD', checkout) + self.environment['GITHUB_SHA'] = executing + with self.subTest(checkout=checkout, executing=executing): + self.assertEqual(self.verdict(), [ + self.PREFIX + 'checker checkout differs from executing SHA']) + + def test_event_head_must_be_an_available_exact_commit(self): + self.git('tag', '-a', '-m', 'not a commit identity', 'event-head', self.head) + tag = self.git('rev-parse', 'refs/tags/event-head') + for revision, reason in ( + ('0' * 40, 'Git history unavailable'), + (tag, 'event revision is not an available commit')): + self.event['pull_request']['head']['sha'] = revision + with self.subTest(revision=revision): + self.assertEqual(self.verdict(), [self.PREFIX + reason]) + + def test_fetch_failure_cannot_claim_complete_history(self): + actual = ci_preflight._headline_window_git + def failed_fetch(repo, *args): + if args[0] == 'fetch': + raise subprocess.CalledProcessError(128, ['git', 'fetch'], stderr=b'unavailable') + return actual(repo, *args) + self.event_path.write_text(json.dumps(self.event)) + with mock.patch.dict(os.environ, self.environment, clear=True), \ + mock.patch.object(ci_preflight, '_headline_window_git', side_effect=failed_fetch): + findings = ci_preflight.headline_window_findings(self.repo, self.fetched) + self.assertEqual(findings, [self.PREFIX + 'Git history unavailable']) + + def test_shallow_history_must_actually_be_completed_before_comparison(self): + mirror = self.repo + self.git('update-ref', 'refs/heads/candidate', self.head) + shallow_mirror = self.directory / 'shallow-mirror' + subprocess.run(['git', 'clone', '-q', '--bare', '--depth', '1', '--no-single-branch', + mirror.as_uri(), str(shallow_mirror)], check=True, capture_output=True) + shallow = self.directory / 'shallow' + subprocess.run(['git', 'clone', '-q', '--depth', '1', '--no-checkout', + shallow_mirror.as_uri(), str(shallow)], check=True, capture_output=True) + self.repo = shallow + self.assertEqual(self.git('rev-parse', '--is-shallow-repository'), 'true') + self.assertEqual(self.verdict(fetch_source=shallow_mirror), [ + self.PREFIX + 'incomplete Git history']) + # Only the transport is redirected to a private fixture. The real + # unshallow, merge-base and blob reads must now succeed. + self.assertEqual(self.verdict(fetch_source=mirror), []) + self.assertEqual(self.git('rev-parse', '--is-shallow-repository'), 'false') + + def test_rules_only_calls_window_even_when_data_rules_refuse(self): + output = io.StringIO() + with mock.patch.object(sys, 'argv', ['ci_preflight.py', '--rules-only', + '--workflows-dir', str(self.fetched)]), \ + mock.patch.object(ci_preflight, 'headline_window_findings', + return_value=[self.PREFIX + 'window control']) as window, \ + mock.patch.object(ci_preflight, 'admit_rule_findings', + return_value=['data rule control']) as data, \ + contextlib.redirect_stdout(output): + self.assertEqual(ci_preflight.main(), 1) + window.assert_called_once_with(ci_preflight.ROOT, self.fetched) + data.assert_called_once_with(self.fetched) + self.assertEqual(output.getvalue().splitlines(), + [self.PREFIX + 'window control', 'data rule control']) + + if __name__ == '__main__': if any(arg == '--slice' or arg.startswith('--slice=') for arg in sys.argv[1:]): parser = argparse.ArgumentParser(description=__doc__)