From 3b93d1a9c052f174b2ef6e6c0639fcc580e531d6 Mon Sep 17 00:00:00 2001 From: Ali <268342250+aliengineering-byte@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:33:16 -0400 Subject: [PATCH 1/4] Bind generated regressions to portable evidence --- CHANGELOG.md | 7 ++++ README.md | 5 +++ src/phaseprobe/cli.py | 2 ++ src/phaseprobe/generate.py | 66 ++++++++++++++++++++++++++++++++-- tests/test_artifacts_replay.py | 31 ++++++++++++++++ tests/test_cli.py | 1 + 6 files changed, 110 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0d6437..63a04b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes are documented here. PhaseProbe follows semantic versioning. +## Unreleased + +- `generate-test` now writes a path-portable claim/decision evidence record that binds the + validated replay verdict to SHA-256 hashes of the copied fixture and generated pytest, the + exact reproduction command, PhaseProbe attribution, and explicit scientific limitations. +- Identical generated evidence is idempotent; conflicting evidence is rejected without overwrite. + ## 0.2.1 — 2026-08-31 - Fixed Issue #4: all four SciPy quick-start configurations now ship inside the importable diff --git a/README.md b/README.md index d92fd8d..e03367f 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,11 @@ manifest.json sizes and SHA-256 hashes Model names used for generated test paths are sanitized. The generated source comes from a fixed template; configuration strings never become executable Python. +`generate-test` validates the replay before writing anything. Alongside the fixed pytest template +and copied fixture, it writes `-pytest-evidence.json`: a path-portable claim/decision record +containing repository/version attribution, the declared replay comparisons, SHA-256 hashes for +both executable artifacts, the exact pytest command, and explicit scientific limitations. + ## What PhaseProbe adds—and what it does not | existing category | established strength | PhaseProbe’s narrower job | diff --git a/src/phaseprobe/cli.py b/src/phaseprobe/cli.py index 983639c..ad08747 100644 --- a/src/phaseprobe/cli.py +++ b/src/phaseprobe/cli.py @@ -172,6 +172,7 @@ def _generate_command(args: argparse.Namespace) -> int: "status": "PYTEST REGRESSION GENERATED", "test": str(generated.test_path), "fixture": str(generated.fixture_path), + "evidence": str(generated.evidence_path), } if bool(getattr(args, "json", False)): sys.stdout.write(json_report(data)) @@ -180,6 +181,7 @@ def _generate_command(args: argparse.Namespace) -> int: print() print(f"Test: {generated.test_path}") print(f"Replay fixture: {generated.fixture_path}") + print(f"Execution evidence: {generated.evidence_path}") return int(ExitCode.OK) diff --git a/src/phaseprobe/generate.py b/src/phaseprobe/generate.py index b91832b..8ca5353 100644 --- a/src/phaseprobe/generate.py +++ b/src/phaseprobe/generate.py @@ -2,19 +2,23 @@ from __future__ import annotations +import hashlib +import json import re from dataclasses import dataclass from pathlib import Path +from phaseprobe import __version__ from phaseprobe.replay import validate_fixture, verify_replay @dataclass(frozen=True, slots=True) class GeneratedTest: - """Generated fixed-template test and copied integrity-protected fixture.""" + """Generated test, copied fixture, and machine-readable execution evidence.""" test_path: Path fixture_path: Path + evidence_path: Path def _safe_name(value: object) -> str: @@ -33,6 +37,14 @@ def _reject_conflict(path: Path, expected: bytes, description: str) -> bool: return False +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _classification(payload: object) -> object: + return payload.get("classification") if isinstance(payload, dict) else None + + def generate_regression_test(fixture: Path, output_directory: Path) -> GeneratedTest: """Validate evidence, copy its fixture, and emit a non-extensible pytest template.""" @@ -46,6 +58,7 @@ def generate_regression_test(fixture: Path, output_directory: Path) -> Generated fixture_directory = output_directory / "fixtures" copied_fixture = fixture_directory / f"{model_name}-replay.json" test_path = output_directory / f"test_{model_name}_transition.py" + evidence_path = output_directory / f"{model_name}-pytest-evidence.json" baseline = payload.get("baseline") metadata = baseline.get("execution_metadata") if isinstance(baseline, dict) else None adapter_configuration = ( @@ -77,12 +90,61 @@ def test_{model_name}_transition_replays() -> None: ''' fixture_bytes = fixture.read_bytes() source_bytes = source.encode("utf-8") + evidence = { + "schema_version": "1.0", + "producer": { + "repository": "aliengineering-byte/phaseprobe", + "version": __version__, + "capability": "validated-replay-to-pytest", + "documentation": "https://github.com/aliengineering-byte/phaseprobe#five-minute-quick-start", + }, + "claim": { + "kind": "qualitative-simulation-regression", + "model": payload.get("model"), + "baseline_classification": _classification(payload.get("baseline")), + "changed_classification": _classification(payload.get("changed")), + "reproducible_at_generation": payload.get("reproducible"), + }, + "decision": { + "status": "REPLAY_VERIFIED", + "comparison_mode": verification.mode, + "comparisons": [dict(item) for item in verification.comparisons], + }, + "artifacts": { + "replay_fixture": { + "path": copied_fixture.relative_to(output_directory).as_posix(), + "sha256": _sha256(fixture_bytes), + }, + "pytest_regression": { + "path": test_path.relative_to(output_directory).as_posix(), + "sha256": _sha256(source_bytes), + }, + }, + "reproduction": { + "working_directory": ".", + "command": f"python -m pytest -q {test_path.name}", + }, + "limitations": [ + "Generation-time replay verifies only the fixture's declared exact or tolerance policy.", + "The generated pytest detects future mismatch; it does not prove an exact bifurcation point or global minimality.", + ], + } + evidence_bytes = ( + json.dumps(evidence, allow_nan=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") write_fixture = _reject_conflict(copied_fixture, fixture_bytes, "replay fixture") write_test = _reject_conflict(test_path, source_bytes, "generated pytest") + write_evidence = _reject_conflict(evidence_path, evidence_bytes, "generated pytest evidence") output_directory.mkdir(parents=True, exist_ok=True) fixture_directory.mkdir(exist_ok=True) if write_fixture: copied_fixture.write_bytes(fixture_bytes) if write_test: test_path.write_bytes(source_bytes) - return GeneratedTest(test_path=test_path, fixture_path=copied_fixture) + if write_evidence: + evidence_path.write_bytes(evidence_bytes) + return GeneratedTest( + test_path=test_path, + fixture_path=copied_fixture, + evidence_path=evidence_path, + ) diff --git a/tests/test_artifacts_replay.py b/tests/test_artifacts_replay.py index 2ddb32e..4e418ad 100644 --- a/tests/test_artifacts_replay.py +++ b/tests/test_artifacts_replay.py @@ -54,6 +54,28 @@ def test_replay_rejects_tampering(artifact_run: Path) -> None: def test_generated_pytest_genuinely_executes(artifact_run: Path, tmp_path: Path) -> None: generated = generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") assert str(tmp_path) not in generated.test_path.read_text(encoding="utf-8") + evidence = json.loads(generated.evidence_path.read_text(encoding="utf-8")) + assert str(tmp_path) not in generated.evidence_path.read_text(encoding="utf-8") + assert evidence["producer"] == { + "capability": "validated-replay-to-pytest", + "documentation": "https://github.com/aliengineering-byte/phaseprobe#five-minute-quick-start", + "repository": "aliengineering-byte/phaseprobe", + "version": "0.2.1", + } + assert evidence["claim"]["baseline_classification"] == "bounded-positive-oscillation" + assert evidence["claim"]["changed_classification"] is None + assert evidence["decision"]["status"] == "REPLAY_VERIFIED" + assert evidence["artifacts"]["replay_fixture"]["path"] == ("fixtures/predator_prey-replay.json") + assert evidence["artifacts"]["pytest_regression"]["path"] == ( + "test_predator_prey_transition.py" + ) + for artifact in evidence["artifacts"].values(): + digest = hashlib.sha256((generated.test_path.parent / artifact["path"]).read_bytes()) + assert digest.hexdigest() == artifact["sha256"] + assert evidence["reproduction"] == { + "command": "python -m pytest -q test_predator_prey_transition.py", + "working_directory": ".", + } assert ( generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") == generated ) @@ -84,6 +106,15 @@ def test_generated_pytest_refuses_conflicting_overwrite(artifact_run: Path, tmp_ assert generated.test_path.read_text(encoding="utf-8") == "# user-owned test\n" +def test_generated_pytest_refuses_conflicting_evidence(artifact_run: Path, tmp_path: Path) -> None: + output_directory = tmp_path / "generated" + generated = generate_regression_test(artifact_run / "replay.json", output_directory) + generated.evidence_path.write_text("{}\n", encoding="utf-8") + with pytest.raises(FileExistsError, match="generated pytest evidence"): + generate_regression_test(artifact_run / "replay.json", output_directory) + assert generated.evidence_path.read_text(encoding="utf-8") == "{}\n" + + def test_html_report_is_self_contained_and_offline(artifact_run: Path) -> None: report = (artifact_run / "report.html").read_text(encoding="utf-8") assert "" in report diff --git a/tests/test_cli.py b/tests/test_cli.py index 226fd7d..3747d0d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -118,6 +118,7 @@ def test_replay_generate_and_report_commands( assert generate_code == ExitCode.OK generated_payload = json.loads(capsys.readouterr().out) assert Path(generated_payload["test"]).exists() + assert Path(generated_payload["evidence"]).exists() report_code = main(["report", str(cli_artifact), "--format", "all"]) assert report_code == ExitCode.OK From fb9969ac662e179af2e398126a2d762b8dfac922 Mon Sep 17 00:00:00 2001 From: Ali <268342250+aliengineering-byte@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:52:35 -0400 Subject: [PATCH 2/4] docs: narrow generated evidence claims --- CHANGELOG.md | 2 ++ README.md | 3 +++ src/phaseprobe/generate.py | 16 +++++++++++++--- tests/test_artifacts_replay.py | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63a04b7..6030afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ All notable changes are documented here. PhaseProbe follows semantic versioning. validated replay verdict to SHA-256 hashes of the copied fixture and generated pytest, the exact reproduction command, PhaseProbe attribution, and explicit scientific limitations. - Identical generated evidence is idempotent; conflicting evidence is rejected without overwrite. +- Invariant-only fixtures now use the narrower `simulation-replay-regression` claim kind, and the + unsigned evidence-integrity boundary is explicit. ## 0.2.1 — 2026-08-31 diff --git a/README.md b/README.md index e03367f..1f0eb15 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,8 @@ Model names used for generated test paths are sanitized. The generated source co and copied fixture, it writes `-pytest-evidence.json`: a path-portable claim/decision record containing repository/version attribution, the declared replay comparisons, SHA-256 hashes for both executable artifacts, the exact pytest command, and explicit scientific limitations. +The record is unsigned: consumers must recompute the two artifact hashes to detect a recomputed or +substituted record. It is portable integrity metadata, not an authentication signature. ## What PhaseProbe adds—and what it does not @@ -266,6 +268,7 @@ python scripts/hygiene.py ``` See [CONTRIBUTING.md](CONTRIBUTING.md) and the concrete good-first-issue templates in `.github/ISSUE_TEMPLATE/`. +Report a reproducible defect with the [bug form](https://github.com/aliengineering-byte/phaseprobe/issues/new?template=bug_report.yml), or propose a bounded adapter or workflow with the [feature form](https://github.com/aliengineering-byte/phaseprobe/issues/new?template=feature_request.yml). Please do not include secrets or private model data. ## License and citation diff --git a/src/phaseprobe/generate.py b/src/phaseprobe/generate.py index 8ca5353..70dda38 100644 --- a/src/phaseprobe/generate.py +++ b/src/phaseprobe/generate.py @@ -90,6 +90,15 @@ def test_{model_name}_transition_replays() -> None: ''' fixture_bytes = fixture.read_bytes() source_bytes = source.encode("utf-8") + baseline_classification = _classification(payload.get("baseline")) + changed_classification = _classification(payload.get("changed")) + claim_kind = ( + "qualitative-simulation-regression" + if baseline_classification is not None + and changed_classification is not None + and baseline_classification != changed_classification + else "simulation-replay-regression" + ) evidence = { "schema_version": "1.0", "producer": { @@ -99,10 +108,10 @@ def test_{model_name}_transition_replays() -> None: "documentation": "https://github.com/aliengineering-byte/phaseprobe#five-minute-quick-start", }, "claim": { - "kind": "qualitative-simulation-regression", + "kind": claim_kind, "model": payload.get("model"), - "baseline_classification": _classification(payload.get("baseline")), - "changed_classification": _classification(payload.get("changed")), + "baseline_classification": baseline_classification, + "changed_classification": changed_classification, "reproducible_at_generation": payload.get("reproducible"), }, "decision": { @@ -127,6 +136,7 @@ def test_{model_name}_transition_replays() -> None: "limitations": [ "Generation-time replay verifies only the fixture's declared exact or tolerance policy.", "The generated pytest detects future mismatch; it does not prove an exact bifurcation point or global minimality.", + "Artifact hashes are unsigned and must be recomputed by a consumer; the evidence record does not authenticate itself.", ], } evidence_bytes = ( diff --git a/tests/test_artifacts_replay.py b/tests/test_artifacts_replay.py index 4e418ad..cc3eb89 100644 --- a/tests/test_artifacts_replay.py +++ b/tests/test_artifacts_replay.py @@ -64,6 +64,7 @@ def test_generated_pytest_genuinely_executes(artifact_run: Path, tmp_path: Path) } assert evidence["claim"]["baseline_classification"] == "bounded-positive-oscillation" assert evidence["claim"]["changed_classification"] is None + assert evidence["claim"]["kind"] == "simulation-replay-regression" assert evidence["decision"]["status"] == "REPLAY_VERIFIED" assert evidence["artifacts"]["replay_fixture"]["path"] == ("fixtures/predator_prey-replay.json") assert evidence["artifacts"]["pytest_regression"]["path"] == ( From aa85e90fa1d8bb0d9a0a881832548f914def158d Mon Sep 17 00:00:00 2001 From: Ali <268342250+aliengineering-byte@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:11:02 -0400 Subject: [PATCH 3/4] chore: identify unreleased PhaseProbe 0.3.0 --- CHANGELOG.md | 1 + CITATION.cff | 3 +-- README.md | 4 ++++ pyproject.toml | 2 +- src/phaseprobe/__init__.py | 2 +- tests/test_artifacts_replay.py | 2 +- uv.lock | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6030afb..25f19c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes are documented here. PhaseProbe follows semantic versioning. ## Unreleased +- The unreleased package identity is now `0.3.0` for the new generated-evidence contract. - `generate-test` now writes a path-portable claim/decision evidence record that binds the validated replay verdict to SHA-256 hashes of the copied fixture and generated pytest, the exact reproduction command, PhaseProbe attribution, and explicit scientific limitations. diff --git a/CITATION.cff b/CITATION.cff index d221e29..a594a6b 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,8 +4,7 @@ title: "PhaseProbe" type: software authors: - name: "Ali" -version: 0.2.1 -date-released: 2026-08-31 +version: 0.3.0 url: "https://github.com/aliengineering-byte/phaseprobe" repository-code: "https://github.com/aliengineering-byte/phaseprobe" license: Apache-2.0 diff --git a/README.md b/README.md index 1f0eb15..af69044 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,10 @@ both executable artifacts, the exact pytest command, and explicit scientific lim The record is unsigned: consumers must recompute the two artifact hashes to detect a recomputed or substituted record. It is portable integrity metadata, not an authentication signature. +If `generate-test` reports a conflict, it has left the existing output untouched. An identical +artifact can be generated again safely; for different content, inspect the existing files and +choose a fresh output directory (or remove them only after deciding they are no longer needed). + ## What PhaseProbe adds—and what it does not | existing category | established strength | PhaseProbe’s narrower job | diff --git a/pyproject.toml b/pyproject.toml index 1ab2d50..6a576dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "phaseprobe" -version = "0.2.1" +version = "0.3.0" description = "Find reproducible qualitative simulation transitions and turn them into regression tests." readme = "README.md" requires-python = ">=3.10" diff --git a/src/phaseprobe/__init__.py b/src/phaseprobe/__init__.py index aba796e..32f48e3 100644 --- a/src/phaseprobe/__init__.py +++ b/src/phaseprobe/__init__.py @@ -27,4 +27,4 @@ "run_simulation", ] -__version__ = "0.2.1" +__version__ = "0.3.0" diff --git a/tests/test_artifacts_replay.py b/tests/test_artifacts_replay.py index cc3eb89..01394bf 100644 --- a/tests/test_artifacts_replay.py +++ b/tests/test_artifacts_replay.py @@ -60,7 +60,7 @@ def test_generated_pytest_genuinely_executes(artifact_run: Path, tmp_path: Path) "capability": "validated-replay-to-pytest", "documentation": "https://github.com/aliengineering-byte/phaseprobe#five-minute-quick-start", "repository": "aliengineering-byte/phaseprobe", - "version": "0.2.1", + "version": "0.3.0", } assert evidence["claim"]["baseline_classification"] == "bounded-positive-oscillation" assert evidence["claim"]["changed_classification"] is None diff --git a/uv.lock b/uv.lock index 5464445..7fa1e69 100644 --- a/uv.lock +++ b/uv.lock @@ -390,7 +390,7 @@ wheels = [ [[package]] name = "phaseprobe" -version = "0.2.1" +version = "0.3.0" source = { editable = "." } [package.optional-dependencies] From d93fcafbfff8c24ae129ee974681b4e6de31946b Mon Sep 17 00:00:00 2001 From: Ali <268342250+aliengineering-byte@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:30:59 -0400 Subject: [PATCH 4/4] feat: verify generated evidence offline --- CHANGELOG.md | 2 + README.md | 7 + src/phaseprobe/cli.py | 27 +++- src/phaseprobe/generate.py | 281 +++++++++++++++++++++++++++++---- tests/test_artifacts_replay.py | 100 +++++++++++- tests/test_cli.py | 26 +++ 6 files changed, 409 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25f19c5..5db0e4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes are documented here. PhaseProbe follows semantic versioning. ## Unreleased +- Added bounded offline `verify-evidence` validation for generated pytest evidence, replay + decisions, artifact hashes, safe relative paths, and the fixed regression template. - The unreleased package identity is now `0.3.0` for the new generated-evidence contract. - `generate-test` now writes a path-portable claim/decision evidence record that binds the validated replay verdict to SHA-256 hashes of the copied fixture and generated pytest, the diff --git a/README.md b/README.md index af69044..880c499 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ pip install phaseprobe pytest phaseprobe scan --example logistic phaseprobe replay .phaseprobe/runs//replay.json phaseprobe generate-test .phaseprobe/runs//replay.json +phaseprobe verify-evidence tests/generated/logistic_map-pytest-evidence.json python -m pytest -q tests/generated phaseprobe report .phaseprobe/runs/ ``` @@ -134,6 +135,7 @@ does not import it. See [the audited contract](docs/SCIPY_SOLVE_IVP_AUDIT.md), | `check` | Execute a declared configuration policy for CI | Exit `1` only when policy fails | | `replay` | Validate fixture integrity and re-execute model, parameters, seed, initial state, tolerances, and retention | Declared `exact` or `tolerance` comparison passes | | `generate-test` | Validate and copy a fixture into a non-extensible pytest template without conflicting overwrites | Executable test under `tests/generated/` | +| `verify-evidence` | Re-derive claims/replay decisions and validate bounded strict JSON, artifact hashes, paths, and the fixed template offline | Stable verified or invalid-input exit | | `report` | Regenerate terminal, versioned JSON, and self-contained offline HTML evidence | Local report files | Common options: @@ -184,6 +186,11 @@ Model names used for generated test paths are sanitized. The generated source co and copied fixture, it writes `-pytest-evidence.json`: a path-portable claim/decision record containing repository/version attribution, the declared replay comparisons, SHA-256 hashes for both executable artifacts, the exact pytest command, and explicit scientific limitations. +Run `phaseprobe verify-evidence ` before pytest to validate the strict schema, +recompute the hashes, re-execute the replay policy, derive the claim/decision fields, and confirm +that the regression still matches PhaseProbe's fixed template. Validation is offline, bounded to a +1 MiB evidence document and 32 MiB per generated artifact, rejects duplicate keys and unsafe paths, +and exits `2` for invalid input. The record is unsigned: consumers must recompute the two artifact hashes to detect a recomputed or substituted record. It is portable integrity metadata, not an authentication signature. diff --git a/src/phaseprobe/cli.py b/src/phaseprobe/cli.py index ad08747..9d5967b 100644 --- a/src/phaseprobe/cli.py +++ b/src/phaseprobe/cli.py @@ -19,7 +19,7 @@ NumericalFailure, PhaseProbeError, ) -from phaseprobe.generate import generate_regression_test +from phaseprobe.generate import generate_regression_test, verify_generated_evidence from phaseprobe.replay import verify_replay from phaseprobe.reporting import json_report, regenerate_reports, terminal_report @@ -81,6 +81,12 @@ def build_parser() -> argparse.ArgumentParser: generate.add_argument("--output-directory", type=Path, default=Path("tests") / "generated") generate.add_argument("--json", action="store_true") + verify_evidence = commands.add_parser( + "verify-evidence", help="verify generated pytest evidence and artifacts offline" + ) + verify_evidence.add_argument("evidence", type=Path) + verify_evidence.add_argument("--json", action="store_true") + report = commands.add_parser( "report", help="regenerate terminal, JSON, and offline HTML evidence" ) @@ -185,6 +191,23 @@ def _generate_command(args: argparse.Namespace) -> int: return int(ExitCode.OK) +def _verify_evidence_command(args: argparse.Namespace) -> int: + evidence = args.evidence + if not isinstance(evidence, Path): + raise ConfigurationError("evidence must be a path") + result = verify_generated_evidence(evidence) + data = result.as_dict() + if bool(getattr(args, "json", False)): + sys.stdout.write(json_report(data)) + else: + print(data["status"]) + print() + print(f"Evidence SHA-256: {result.evidence_sha256}") + print(f"Replay fixture: {result.fixture_path}") + print(f"Pytest regression: {result.test_path}") + return int(ExitCode.OK) + + def _report_command(args: argparse.Namespace) -> int: run_directory = args.run_directory report_format = args.format @@ -223,6 +246,8 @@ def main(argv: Sequence[str] | None = None) -> int: return _replay_command(args) if args.command == "generate-test": return _generate_command(args) + if args.command == "verify-evidence": + return _verify_evidence_command(args) if args.command == "report": return _report_command(args) raise ConfigurationError(f"unknown command {args.command!r}") diff --git a/src/phaseprobe/generate.py b/src/phaseprobe/generate.py index 70dda38..3527ff8 100644 --- a/src/phaseprobe/generate.py +++ b/src/phaseprobe/generate.py @@ -7,10 +7,30 @@ import re from dataclasses import dataclass from pathlib import Path +from typing import Any from phaseprobe import __version__ +from phaseprobe.errors import IntegrityError from phaseprobe.replay import validate_fixture, verify_replay +MAX_EVIDENCE_BYTES = 1_048_576 +MAX_GENERATED_ARTIFACT_BYTES = 33_554_432 +MAX_JSON_DEPTH = 32 +MAX_JSON_NODES = 10_000 + +_PRODUCER = { + "repository": "aliengineering-byte/phaseprobe", + "version": __version__, + "capability": "validated-replay-to-pytest", + "documentation": "https://github.com/aliengineering-byte/phaseprobe#five-minute-quick-start", +} + +_LIMITATIONS = [ + "Generation-time replay verifies only the fixture's declared exact or tolerance policy.", + "The generated pytest detects future mismatch; it does not prove an exact bifurcation point or global minimality.", + "Artifact hashes are unsigned and must be recomputed by a consumer; the evidence record does not authenticate itself.", +] + @dataclass(frozen=True, slots=True) class GeneratedTest: @@ -21,6 +41,29 @@ class GeneratedTest: evidence_path: Path +@dataclass(frozen=True, slots=True) +class GeneratedEvidenceVerification: + """Bounded offline validation result for generated regression evidence.""" + + evidence_path: Path + fixture_path: Path + test_path: Path + evidence_sha256: str + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": "1.0", + "status": "GENERATED EVIDENCE VERIFIED", + "integrity": "unsigned-recomputable", + "authentication": "none", + "evidence_sha256": self.evidence_sha256, + "artifacts": { + "replay_fixture": str(self.fixture_path), + "pytest_regression": str(self.test_path), + }, + } + + def _safe_name(value: object) -> str: candidate = re.sub(r"[^a-z0-9]+", "_", str(value).lower()).strip("_") return candidate[:64] or "model" @@ -45,6 +88,209 @@ def _classification(payload: object) -> object: return payload.get("classification") if isinstance(payload, dict) else None +def _generated_source(model_name: str, scipy_backed: bool) -> bytes: + scipy_import = "import pytest\n" if scipy_backed else "" + scipy_guard = ( + '\npytest.importorskip("scipy")\npytestmark = pytest.mark.scipy\n' if scipy_backed else "" + ) + source = f'''"""Generated by PhaseProbe from a validated replay fixture.""" + +from pathlib import Path + +{scipy_import} +from phaseprobe.replay import verify_replay +{scipy_guard} + +FIXTURE = Path(__file__).parent / "fixtures" / "{model_name}-replay.json" + + +def test_{model_name}_transition_replays() -> None: + """Re-execute model/config/seed and verify the fixture's declared replay policy.""" + result = verify_replay(FIXTURE) + assert result.ok, result.as_dict() +''' + return source.encode("utf-8") + + +def _strict_object(value: object, required: set[str], description: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != required: + raise IntegrityError(f"{description} must contain exactly: {', '.join(sorted(required))}") + return value + + +def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise IntegrityError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + +def _bounded_json(path: Path, maximum: int) -> tuple[bytes, dict[str, Any]]: + if not path.is_file(): + raise IntegrityError(f"evidence must be a regular file: {path}") + if path.stat().st_size > maximum: + raise IntegrityError(f"evidence exceeds {maximum} bytes") + raw = path.read_bytes() + try: + text = raw.decode("utf-8") + parsed = json.loads(text, object_pairs_hook=_reject_duplicate_keys) + except (UnicodeError, json.JSONDecodeError, RecursionError) as exc: + raise IntegrityError(f"evidence is not strict UTF-8 JSON: {exc}") from exc + if not isinstance(parsed, dict): + raise IntegrityError("evidence root must be an object") + stack: list[tuple[object, int]] = [(parsed, 1)] + nodes = 0 + while stack: + current, depth = stack.pop() + nodes += 1 + if nodes > MAX_JSON_NODES: + raise IntegrityError(f"evidence exceeds {MAX_JSON_NODES} JSON nodes") + if depth > MAX_JSON_DEPTH: + raise IntegrityError(f"evidence exceeds JSON depth {MAX_JSON_DEPTH}") + if isinstance(current, dict): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) + return raw, parsed + + +def _artifact_path(base: Path, value: object, description: str) -> Path: + if not isinstance(value, str) or not value or "\\" in value: + raise IntegrityError(f"{description} path must be a non-empty POSIX relative path") + path = Path(value) + if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts): + raise IntegrityError(f"unsafe {description} path {value!r}") + try: + resolved = (base / path).resolve(strict=True) + resolved.relative_to(base.resolve(strict=True)) + except (OSError, ValueError) as exc: + raise IntegrityError(f"{description} escapes or is missing: {value!r}") from exc + if not resolved.is_file(): + raise IntegrityError(f"{description} must resolve to a regular file") + if resolved.stat().st_size > MAX_GENERATED_ARTIFACT_BYTES: + raise IntegrityError(f"{description} exceeds {MAX_GENERATED_ARTIFACT_BYTES} bytes") + return resolved + + +def _artifact_record(artifacts: dict[str, Any], name: str, base: Path) -> tuple[Path, bytes]: + record = _strict_object(artifacts.get(name), {"path", "sha256"}, name) + path = _artifact_path(base, record["path"], name) + raw = path.read_bytes() + expected_hash = record["sha256"] + if not isinstance(expected_hash, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_hash): + raise IntegrityError(f"{name} sha256 must be 64 lowercase hex characters") + if _sha256(raw) != expected_hash: + raise IntegrityError(f"{name} sha256 mismatch") + return path, raw + + +def verify_generated_evidence(evidence_path: Path) -> GeneratedEvidenceVerification: + """Verify bounded generated-test evidence and derive its important claims offline.""" + + evidence_raw, evidence = _bounded_json(evidence_path, MAX_EVIDENCE_BYTES) + evidence = _strict_object( + evidence, + { + "schema_version", + "producer", + "claim", + "decision", + "artifacts", + "reproduction", + "limitations", + }, + "evidence", + ) + if evidence["schema_version"] != "1.0": + raise IntegrityError("unsupported generated-evidence schema_version") + if _strict_object(evidence["producer"], set(_PRODUCER), "producer") != _PRODUCER: + raise IntegrityError("producer identity does not match this PhaseProbe release") + claim = _strict_object( + evidence["claim"], + { + "kind", + "model", + "baseline_classification", + "changed_classification", + "reproducible_at_generation", + }, + "claim", + ) + decision = _strict_object( + evidence["decision"], {"status", "comparison_mode", "comparisons"}, "decision" + ) + artifacts = _strict_object( + evidence["artifacts"], {"replay_fixture", "pytest_regression"}, "artifacts" + ) + reproduction = _strict_object( + evidence["reproduction"], {"working_directory", "command"}, "reproduction" + ) + if evidence["limitations"] != _LIMITATIONS: + raise IntegrityError("limitations do not match the generated-evidence contract") + + base = evidence_path.resolve(strict=True).parent + fixture_path, fixture_raw = _artifact_record(artifacts, "replay_fixture", base) + test_path, test_raw = _artifact_record(artifacts, "pytest_regression", base) + fixture = validate_fixture(fixture_path) + replay = verify_replay(fixture_path) + if not replay.ok: + raise IntegrityError("replay fixture no longer satisfies its declared policy") + model_name = _safe_name(fixture.get("model")) + if artifacts["replay_fixture"]["path"] != f"fixtures/{model_name}-replay.json": + raise IntegrityError("replay fixture path conflicts with the derived model name") + if artifacts["pytest_regression"]["path"] != f"test_{model_name}_transition.py": + raise IntegrityError("pytest path conflicts with the derived model name") + baseline = fixture.get("baseline") + metadata = baseline.get("execution_metadata") if isinstance(baseline, dict) else None + adapter_configuration = ( + metadata.get("adapter_configuration") if isinstance(metadata, dict) else None + ) + scipy_backed = ( + isinstance(adapter_configuration, dict) + and adapter_configuration.get("adapter") == "scipy.solve_ivp" + ) + if test_raw != _generated_source(model_name, scipy_backed): + raise IntegrityError("pytest regression is not the fixed template derived from the fixture") + baseline_classification = _classification(fixture.get("baseline")) + changed_classification = _classification(fixture.get("changed")) + expected_kind = ( + "qualitative-simulation-regression" + if baseline_classification is not None + and changed_classification is not None + and baseline_classification != changed_classification + else "simulation-replay-regression" + ) + if claim != { + "kind": expected_kind, + "model": fixture.get("model"), + "baseline_classification": baseline_classification, + "changed_classification": changed_classification, + "reproducible_at_generation": fixture.get("reproducible"), + }: + raise IntegrityError("claim conflicts with the replay fixture") + if decision != { + "status": "REPLAY_VERIFIED", + "comparison_mode": replay.mode, + "comparisons": [dict(item) for item in replay.comparisons], + }: + raise IntegrityError("decision conflicts with replay verification") + if reproduction != { + "working_directory": ".", + "command": f"python -m pytest -q {test_path.name}", + }: + raise IntegrityError("reproduction command conflicts with the generated artifact") + if _sha256(fixture_raw) != artifacts["replay_fixture"]["sha256"]: + raise IntegrityError("replay fixture changed during verification") + return GeneratedEvidenceVerification( + evidence_path=evidence_path.resolve(strict=True), + fixture_path=fixture_path, + test_path=test_path, + evidence_sha256=_sha256(evidence_raw), + ) + + def generate_regression_test(fixture: Path, output_directory: Path) -> GeneratedTest: """Validate evidence, copy its fixture, and emit a non-extensible pytest template.""" @@ -68,28 +314,8 @@ def generate_regression_test(fixture: Path, output_directory: Path) -> Generated isinstance(adapter_configuration, dict) and adapter_configuration.get("adapter") == "scipy.solve_ivp" ) - scipy_import = "import pytest\n" if scipy_backed else "" - scipy_guard = ( - '\npytest.importorskip("scipy")\npytestmark = pytest.mark.scipy\n' if scipy_backed else "" - ) - source = f'''"""Generated by PhaseProbe from a validated replay fixture.""" - -from pathlib import Path - -{scipy_import} -from phaseprobe.replay import verify_replay -{scipy_guard} - -FIXTURE = Path(__file__).parent / "fixtures" / "{model_name}-replay.json" - - -def test_{model_name}_transition_replays() -> None: - """Re-execute model/config/seed and verify the fixture's declared replay policy.""" - result = verify_replay(FIXTURE) - assert result.ok, result.as_dict() -''' fixture_bytes = fixture.read_bytes() - source_bytes = source.encode("utf-8") + source_bytes = _generated_source(model_name, scipy_backed) baseline_classification = _classification(payload.get("baseline")) changed_classification = _classification(payload.get("changed")) claim_kind = ( @@ -101,12 +327,7 @@ def test_{model_name}_transition_replays() -> None: ) evidence = { "schema_version": "1.0", - "producer": { - "repository": "aliengineering-byte/phaseprobe", - "version": __version__, - "capability": "validated-replay-to-pytest", - "documentation": "https://github.com/aliengineering-byte/phaseprobe#five-minute-quick-start", - }, + "producer": _PRODUCER, "claim": { "kind": claim_kind, "model": payload.get("model"), @@ -133,11 +354,7 @@ def test_{model_name}_transition_replays() -> None: "working_directory": ".", "command": f"python -m pytest -q {test_path.name}", }, - "limitations": [ - "Generation-time replay verifies only the fixture's declared exact or tolerance policy.", - "The generated pytest detects future mismatch; it does not prove an exact bifurcation point or global minimality.", - "Artifact hashes are unsigned and must be recomputed by a consumer; the evidence record does not authenticate itself.", - ], + "limitations": _LIMITATIONS, } evidence_bytes = ( json.dumps(evidence, allow_nan=False, indent=2, sort_keys=True) + "\n" diff --git a/tests/test_artifacts_replay.py b/tests/test_artifacts_replay.py index 01394bf..1fcf163 100644 --- a/tests/test_artifacts_replay.py +++ b/tests/test_artifacts_replay.py @@ -15,7 +15,11 @@ from phaseprobe.config import load_example from phaseprobe.engine import run_check from phaseprobe.errors import IntegrityError -from phaseprobe.generate import generate_regression_test +from phaseprobe.generate import ( + MAX_EVIDENCE_BYTES, + generate_regression_test, + verify_generated_evidence, +) from phaseprobe.replay import validate_fixture, verify_replay from phaseprobe.reporting import regenerate_reports @@ -98,6 +102,100 @@ def test_generated_pytest_genuinely_executes(artifact_run: Path, tmp_path: Path) assert "1 passed" in completed.stdout +def test_generated_evidence_verifies_offline(artifact_run: Path, tmp_path: Path) -> None: + generated = generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") + result = verify_generated_evidence(generated.evidence_path) + assert result.fixture_path == generated.fixture_path.resolve() + assert result.test_path == generated.test_path.resolve() + assert len(result.evidence_sha256) == 64 + + +@pytest.mark.parametrize( + ("target", "replacement", "message"), + [ + (("claim", "kind"), "fabricated", "claim conflicts"), + (("decision", "status"), "TRUST_RECORDED_BOOLEAN", "decision conflicts"), + (("schema_version",), "99.0", "unsupported"), + (("artifacts", "replay_fixture", "path"), "../replay.json", "unsafe"), + ], +) +def test_generated_evidence_rejects_conflicts( + artifact_run: Path, + tmp_path: Path, + target: tuple[str, ...], + replacement: object, + message: str, +) -> None: + generated = generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") + payload = json.loads(generated.evidence_path.read_text(encoding="utf-8")) + current = payload + for key in target[:-1]: + current = current[key] + current[target[-1]] = replacement + generated.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(IntegrityError, match=message): + verify_generated_evidence(generated.evidence_path) + + +def test_generated_evidence_rejects_artifact_tampering_and_recomputed_hash( + artifact_run: Path, tmp_path: Path +) -> None: + generated = generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") + generated.test_path.write_text("# substituted test\n", encoding="utf-8") + with pytest.raises(IntegrityError, match="sha256 mismatch"): + verify_generated_evidence(generated.evidence_path) + payload = json.loads(generated.evidence_path.read_text(encoding="utf-8")) + payload["artifacts"]["pytest_regression"]["sha256"] = hashlib.sha256( + generated.test_path.read_bytes() + ).hexdigest() + generated.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(IntegrityError, match="not the fixed template"): + verify_generated_evidence(generated.evidence_path) + + +def test_generated_evidence_rejects_duplicate_keys_and_resource_exhaustion( + artifact_run: Path, tmp_path: Path +) -> None: + generated = generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") + generated.evidence_path.write_text( + '{"schema_version":"1.0","schema_version":"1.0"}', encoding="utf-8" + ) + with pytest.raises(IntegrityError, match="duplicate JSON key"): + verify_generated_evidence(generated.evidence_path) + generated.evidence_path.write_bytes(b" " * (MAX_EVIDENCE_BYTES + 1)) + with pytest.raises(IntegrityError, match="exceeds"): + verify_generated_evidence(generated.evidence_path) + + +def test_generated_evidence_rejects_excessive_depth_and_node_count( + artifact_run: Path, tmp_path: Path +) -> None: + generated = generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") + nested: object = [] + for _ in range(33): + nested = [nested] + generated.evidence_path.write_text(json.dumps({"nested": nested}), encoding="utf-8") + with pytest.raises(IntegrityError, match="JSON depth"): + verify_generated_evidence(generated.evidence_path) + generated.evidence_path.write_text(json.dumps({"nodes": [None] * 10_001}), encoding="utf-8") + with pytest.raises(IntegrityError, match="JSON nodes"): + verify_generated_evidence(generated.evidence_path) + + +def test_generated_evidence_rejects_missing_and_malformed_unicode( + artifact_run: Path, tmp_path: Path +) -> None: + generated = generate_regression_test(artifact_run / "replay.json", tmp_path / "generated") + payload = json.loads(generated.evidence_path.read_text(encoding="utf-8")) + del payload["decision"] + generated.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(IntegrityError, match="must contain exactly"): + verify_generated_evidence(generated.evidence_path) + generated.evidence_path.write_bytes(b"\xff") + with pytest.raises(IntegrityError, match="strict UTF-8 JSON"): + verify_generated_evidence(generated.evidence_path) + + def test_generated_pytest_refuses_conflicting_overwrite(artifact_run: Path, tmp_path: Path) -> None: output_directory = tmp_path / "generated" generated = generate_regression_test(artifact_run / "replay.json", output_directory) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3747d0d..3a46445 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -120,6 +120,12 @@ def test_replay_generate_and_report_commands( assert Path(generated_payload["test"]).exists() assert Path(generated_payload["evidence"]).exists() + verify_code = main(["verify-evidence", generated_payload["evidence"], "--json"]) + assert verify_code == ExitCode.OK + verify_payload = json.loads(capsys.readouterr().out) + assert verify_payload["status"] == "GENERATED EVIDENCE VERIFIED" + assert verify_payload["integrity"] == "unsigned-recomputable" + report_code = main(["report", str(cli_artifact), "--format", "all"]) assert report_code == ExitCode.OK report_output = capsys.readouterr().out @@ -149,6 +155,26 @@ def test_tampered_replay_is_invalid_input(cli_artifact: Path) -> None: assert main(["replay", str(fixture)]) == ExitCode.INVALID_INPUT +def test_tampered_generated_evidence_is_invalid_input(cli_artifact: Path, tmp_path: Path) -> None: + generated_directory = tmp_path / "generated" + assert ( + main( + [ + "generate-test", + str(cli_artifact / "replay.json"), + "--output-directory", + str(generated_directory), + ] + ) + == ExitCode.OK + ) + evidence = next(generated_directory.glob("*-pytest-evidence.json")) + payload = json.loads(evidence.read_text(encoding="utf-8")) + payload["decision"]["status"] = "FABRICATED" + evidence.write_text(json.dumps(payload), encoding="utf-8") + assert main(["verify-evidence", str(evidence)]) == ExitCode.INVALID_INPUT + + def test_internal_defect_is_four(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def broken_execute(_config: object) -> object: raise RuntimeError("synthetic containment test")