diff --git a/CHANGELOG.md b/CHANGELOG.md index 2024630..ee68ae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ AgentDiff is pre-release software. APIs and artifact schemas may change before the first stable release. +## 0.2.1 + +### Security + +- **Promotion crash window closed**: every filesystem mutation is now preceded by a persisted `APPLY_INTENT` journal entry and followed by post-state verification and a persisted `APPLIED` entry. Recovery distinguishes "mutation occurred", "mutation did not occur", and "ambiguous" by comparing current host state to the recorded base/result, and fails closed on ambiguity. +- **Workspace lease inode race fixed**: `promotion.lock` is a persistent lock file that is never unlinked; only the OS-level advisory lock (flock / msvcrt) is released, so concurrent AgentDiff promotions cannot both hold the lease. +- **Corrupt journal fails closed**: journal loading returns explicit `NO_JOURNAL` / `VALID_JOURNAL` / `CORRUPT_JOURNAL` outcomes; a corrupt journal blocks promotion instead of reading as "no journal". Legacy schema-2 journals remain recoverable under their old semantics. +- **Recovery path/digest hardening**: journal paths are validated (normalized, below root and the approved backup directory, no symlink parents, no AgentDiff-internal targets); backups are re-verified by SHA-256/size and opened identity before restore; restore writes through an fsynced temp and restores content plus mode. Legacy "applied: bool" journals map onto the new state machine. +- **Verifier independence (proof)**: proof now runs a baseline verification that restores the sealed pre-run verifier files over the patched product code, so agent-modified tests cannot silently weaken the verifier; PROVEN fails closed when verifier-related files were modified and the baseline cannot confirm. +- **Proof-strength metadata**: deterministic L0-L4 levels with WEAK/REVIEW/STRONG labels and verifier-independence ratings; the PROVEN/NOT_PROVEN verdict remains deterministic. +- **Terminology corrections**: `copy_file_range` is no longer described as a guaranteed reflink (CLONE = FICLONE only when supported, FAST_COPY = accelerated copy, STREAM_COPY = fallback); the capsule aggregate digest is renamed Capsule Root Digest, not "Merkle root". +- **Materializer mode preservation**: the executable bit and file modes are preserved on every copy strategy; symlinks, hardlinks, and special files are rejected instead of silently dropped; O_NOFOLLOW identity checks prevent source substitution. +- **Runtime capabilities**: explicit static `RuntimeCapabilities` (filesystem/network/process/privilege control levels, private workspace, live safety, source snapshot support) replace `hasattr`/`getattr` string sniffing before execution. +- **Hybrid safety watcher integrated**: OS event hints feed dirty-path targeted checks with periodic authoritative full reconciliation; backend failures degrade to polling with recorded status; final after-state always comes from an authoritative capture. +- **Capsule v1/v2 separation**: `verify_integrity` routes by capsule version; legacy v1 capsules verify under their original guarantees, and a schema-2 mirror without the structured manifest fails closed. +- **Content-addressed object store**: immutable `ObjectStore` (`.agentdiff/objects`) with write-once semantics, digest-validated paths, and fail-closed reads; capsule layout (spec v2) is unchanged, with the object store as the incremental spec-v3/export foundation. + +### Changed + +- `RuntimeBackend` protocol gains `capabilities`, `configure_source`, `configure_safety`, and `close`; all runtimes implement them. +- `WorkspaceMaterializer` strategies renamed (`CLONE`/`FAST_COPY`/`STREAM_COPY`; `REFLINK`/`COPY` retained as aliases) and reports the strategy actually used. +- Docker runtime materializes the private workspace through `WorkspaceMaterializer` and records materialization evidence in `runtime.json`. +- `agentdiff inspect` and proof JSON now include proof-strength, baseline-verifier, and watcher evidence. + +### Tests + +- Added adversarial promotion fault-injection coverage: corrupt/malformed journals, traversal, backup symlink/hardlink, crash at every write-ahead transition, mode restoration, ambiguous recovery, legacy journal recovery, cross-process lease exclusion. +- Added proof-strength matrix, verifier-mutation classifier, baseline overlay end-to-end, and tamper-blocking tests. +- Added materializer security tests (mode preservation per strategy, symlink/hardlink/special-file rejection, target symlink), capsule v1/v2 verification tests, CAS object-store tests, and hybrid watcher degradation/overflow tests. + ## 0.2.0 ### Added @@ -9,9 +39,9 @@ AgentDiff is pre-release software. APIs and artifact schemas may change before t - **Proof Trust Provenance (P0 Security)**: Base-snapshot verification plan auto-discovery with deterministic tamper rejection when patches modify build or test configuration files (`package.json`, `pyproject.toml`, `conftest.py`, `Makefile`, etc.) without an explicit policy override. - **Crash-Consistent Promotion Gate**: Multi-file promotion with advisory workspace lease locking (`WorkspaceLease`), write-ahead transaction logging (`PromotionJournal`), two-phase staging with `fsync` validation (`PromotionStager`), and automatic crash recovery (`PromotionRecovery`). - **Policy Schema v2**: Added first-class `proof:` section supporting container image, network mode, setup, build, and test command sequences. -- **Capsule Spec v2 & Merkle Validation**: Structured integrity manifests, content-addressed blob references, deterministic Merkle root hashing, and backward compatibility with v1 flat capsules. +- **Capsule Spec v2**: Structured integrity manifests, blob references, a deterministic Capsule Root Digest (flat aggregate — not a Merkle tree), and backward compatibility with v1 flat capsules. - **Hybrid Safety Watcher**: Blends filesystem notification hints with deterministic snapshot validation and budget enforcement. -- **High-Speed Workspace Materializer**: Fast copy-on-write / reflink / copy directory materializer for isolated container workspaces. +- **High-Speed Workspace Materializer**: Fast clone (FICLONE where supported) / accelerated copy / streaming directory materializer for isolated container workspaces. ### Changed diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index f1b7fda..3f317db 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -18,16 +18,20 @@ The primary path is `agentdiff run --task "…" -- `. Experimental memo 6. Evidence is redacted, bounded, and private by default. 7. Isolation, tracing, and agent protocols are integration seams, not features to rebuild. -## Current `0.1.0` surface +## Current `0.2.x` surface -### Beta +### Beta — trust pipeline -- No-follow filesystem manifests and private run capsules. -- Deterministic `allow`, `review`, and `deny` policy with provenance. -- Local shell-free command execution with timeout and best-effort process evidence. -- Explainable, capped blast-radius scoring. +- No-follow filesystem manifests and private run capsules (spec v2; legacy v1 capsules remain verifiable under their original guarantees). +- Deterministic `allow`, `review`, and `deny` policy with provenance and live budget enforcement. +- Local shell-free execution with timeout, process, and port evidence; Docker runtime with a private writable workspace (never a writable host repo), cap-drop, no-new-privileges, and explicit network modes. +- Explainable, capped immediate blast-radius scoring; separate future-execution-risk analysis. +- Hybrid safety watcher: OS event hints feed dirty-path targeted checks, authoritative full reconciliation runs on schedule/overflow/force, and backend failures degrade to polling with recorded status. +- Clean-room proof: trusted verification plan from pre-run evidence, patched tests, and an independent baseline verifier (pre-run verifier files over patched product code) with deterministic proof-strength metadata (L0-L4). +- Crash-consistent promotion gate: write-ahead journal with per-entry state machine, persistent workspace lease (never-unlinked lock file), validated backup restore with digest/mode checks, and fail-closed recovery on corrupt or ambiguous state. - Run listing, inspection, checksum verification, and exact-identity cleanup. - Conflict-safe recovery for eligible regular files. +- Content-addressed immutable object store as the migration foundation for spec-v3 artifact references and future export/import. - Linux, macOS, and native Windows CI on Python 3.12–3.14. ### Experimental @@ -46,7 +50,7 @@ The primary path is `agentdiff run --task "…" -- `. Experimental memo - Artifact migration and compatibility tooling. - Larger external-state benchmark coverage. -An HTTP API, hosted dashboard, Docker backend, bundled sandbox, universal network blocking, and arbitrary external-state rollback are not implemented. +An HTTP API, hosted dashboard, bundled sandbox, universal network blocking, and arbitrary external-state rollback are not implemented. The Docker backend implements the isolation boundary this plan targets; it is a capability-bearing container boundary, not a virtual machine. ## Release gates @@ -79,12 +83,14 @@ An HTTP API, hosted dashboard, Docker backend, bundled sandbox, universal networ ### Differentiated safety core -1. Add clean-room proof by replaying a captured patch in a fresh worktree before promotion. -2. Detect future execution risk in package scripts and GitHub Actions changes, then extend to Dockerfiles, Makefiles, hooks, and editor tasks. -3. Add an experimental copy-on-write Docker runtime where the real repository is changed only by an explicit, policy-filtered promotion step. +1. [x] Clean-room proof replays the captured patch in a fresh environment before promotion. +2. [x] Future execution risk analysis covers package scripts, GitHub Actions, Dockerfiles, Makefiles, hooks, and editor tasks. +3. [x] Docker runtime materializes a private workspace and the real repository is changed only by an explicit, policy-filtered promotion step. +4. Harden verifier independence further: external signed CI verification (proof strength L4) and verifier-file policy controls. ### Evidence moat -1. Add signed, shareable capsule export and standardized telemetry. -2. Add run attribution for changed lines and evidence-based comparison of parallel agent attempts. -3. Keep adversarial race, path, hardlink, redaction, and rollback tests ahead of new claims. +1. Add signed (authenticated) capsule support; current checksums are tamper-evident, not authenticated. +2. Add shareable capsule export/import (the CAS object store is the hydration foundation). +3. Add run attribution for changed lines and evidence-based comparison of parallel agent attempts. +4. Keep adversarial race, path, hardlink, redaction, promotion-crash, and rollback tests ahead of new claims. diff --git a/README.md b/README.md index 75ea430..592d009 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@

AgentDiff

- See what the agent changed. Undo only the collateral.
- Independent state observation, deterministic intent policy, explainable blast radius, and conflict-safe selective recovery for AI-agent commands. + DON'T TRUST AN AI PATCH. PROVE IT.
+ Isolate the agent, observe real state, enforce deterministic policy, reproduce the patch in a clean room, and promote only proven work through a crash-consistent gate.

@@ -42,7 +42,7 @@ A command can exit successfully while leaving one intended edit, one dependency ## Start in under a minute -AgentDiff `0.1.0` requires Python 3.12+ and is currently installed from source: +AgentDiff `0.2.x` requires Python 3.12+ and is currently installed from source: ```bash git clone https://github.com/kam6l/agentdiff.git @@ -50,62 +50,90 @@ cd agentdiff uv tool install . ``` -From the project you want to observe: +From the project you want to guard, run the agent inside an isolated private +workspace, prove the result in a clean room, then promote only proven work +through the crash-consistent gate: ```bash agentdiff policy init -agentdiff run --task "Fix authentication" -- codex -``` -The summary leads with the decision you need: +agentdiff run \ + --runtime docker \ + --task "Fix authentication" \ + -- codex -```text -Task completed +agentdiff inspect -Expected changes: 4 -Unexpected changes: 3 -Protected changes: 1 +agentdiff prove -Blast Radius: HIGH (72/100) -Recovery available: YES -Policy outcome: DENY +agentdiff promote --dry-run --safe-only +agentdiff promote --safe-only + +agentdiff verify ``` -Then inspect the durable capsule or recover unchanged collateral: +`agentdiff prove` reproduces the base-plus-patch workspace in a clean Docker +container, runs the **patched** tests, then re-runs the trusted **baseline** +tests (the pre-run verifier files restored over the patched product code) so +an agent cannot hide behind weakened tests. `agentdiff promote` applies only +proven, policy-selected changes with a write-ahead journal and automatic +crash recovery; `--safe-only` selects only `ALLOW` changes. -```bash -agentdiff inspect -agentdiff verify -agentdiff rollback --safe-only +The trust report shows immediate vs future blast radius, verifier changes, +proof strength, and a single deterministic verdict: + +```text +Runtime Docker / private isolated workspace +Policy ALLOW +Immediate Blast 12 / LOW +Future Blast 6 / LOW +Trusted Plan YES +Baseline Tests 184 / 184 +Patched Tests 191 / 191 +Verifier Changes 2 +Proof Strength L3 / STRONG +Promotion CRASH-CONSISTENT / SAFE + ✓ PROVEN ``` -[Run the reproducible five-minute example](https://kam6l.github.io/agentdiff/docs/quickstart/) +For lower isolation, `--runtime local` runs the agent directly on the host +with observation and recovery, not a sandbox. + +[Run the reproducible example](https://kam6l.github.io/agentdiff/docs/quickstart/) ## How it works | Stage | Result | |---|---| -| **Capture** | No-follow before-state manifest and bounded recovery backups | -| **Execute** | Exact argv, exit status, owned-process evidence, and machine-wide port observations | -| **Evaluate** | `allow` / `review` / `deny` decisions, rule provenance, warnings, and a 0-100 score | -| **Recover** | Exact post-state conflict checks before eligible collateral is changed | +| **Isolate** | Docker private workspace (never a writable host repo) or observed local run | +| **Observe** | No-follow before/after manifests, live hybrid safety watcher, owned-process and port evidence | +| **Control** | Deterministic `allow` / `review` / `deny` policy plus budget enforcement | +| **Analyze** | Immediate and future blast radius stay separate | +| **Prove** | Clean-room reproduction with trusted baseline + patched verification | +| **Promote** | Write-ahead journal, crash-consistent recovery, workspace lease | +| **Evidence** | Tamper-evident sealed capsules (spec v2, v1 still verifiable) | ## Feature status | Status | Surface | |---|---| -| **Beta** | Local transactions, policy, capsules, verification, scoring, and regular-file recovery (tested on Python 3.12-3.14) | -| **Experimental** | Cortex evidence memory and provider routing, Anthropic `srt` adapter, transport-neutral MCP policy hook, and LangChain callback | -| **Planned** | PyPI/binary releases, authenticated evidence, telemetry export, and a maintained hosted sandbox integration | +| **Beta** | Transactions, policy, blast radius, capsules, clean-room proof, baseline verifier, promotion gate with crash recovery, and Docker/local runtimes (tested on Python 3.12-3.14, Linux/macOS/Windows) | +| **Experimental** | Cortex evidence memory and provider routing, Anthropic `srt` adapter, transport-neutral MCP policy hook, LangChain callback, content-addressed object store (spec-v3 migration foundation) | +| **Planned** | PyPI/binary releases, authenticated (signed) capsules, capsule export/import CLI, hosted dashboard, maintained hosted sandbox integration | -There is no HTTP server, hosted dashboard, Docker backend, bundled sandbox, or claimed PyPI release today. +Capsule checksums are **tamper-evident, not authenticated**: they detect +accidental or modest modification but an attacker who can rewrite the whole +capsule can produce a new self-consistent one. Signing remains future work. +There is no HTTP server, hosted dashboard, or claimed PyPI release today. ## CLI | Command | Purpose | |---|---| -| `agentdiff run -- ` | Wrap an explicit argv in a transaction | -| `agentdiff runs` / `inspect` / `verify` | Find and validate local evidence capsules | +| `agentdiff run -- ` | Wrap an explicit argv in a transaction (`--runtime docker` for isolation) | +| `agentdiff runs` / `inspect` / `verify` | Find, inspect, and validate evidence capsules | +| `agentdiff prove ` | Clean-room reproduction + trusted baseline/patched verification | +| `agentdiff promote [--dry-run] [--safe-only]` | Crash-consistent, proof-gated promotion | | `agentdiff rollback --safe-only` | Recover eligible `review` and `deny` changes | | `agentdiff cleanup ` | Signal exact PID/create-time identities recorded for a run | | `agentdiff doctor` | Report implemented capabilities and limits | @@ -129,7 +157,20 @@ agentdiff cortex advise ## Trust boundary -AgentDiff records symlinks without traversing them, redacts common secret-bearing values, verifies backups and capsule checksums, and identifies processes by PID plus creation time. It does **not** authenticate a capsule against an attacker who can replace the whole directory, attribute machine-wide port changes to one process, or undo APIs, databases, network effects, hardlinks, symlinks, and unbacked files. +AgentDiff never trusts the agent's explanation, environment, verifier, or +generated state. It observes independent filesystem state, reproduces patches +with trusted pre-run verification commands, and promotes only when evidence +supports it. Promotion recovery fails closed when host state is ambiguous or +the journal is corrupt, and the workspace lease never deletes its lock file +(the OS lock itself is released instead), so concurrent promotions cannot +both hold the lock. + +It does **not** authenticate a capsule against an attacker who can replace +the whole directory, attribute machine-wide port changes to one process, or +undo APIs, databases, network effects, hardlinks, symlinks, and unbacked +files. Cortex (the experimental LLM surface) can read verified evidence and +generate advice but never decides policy, proof, blast radius, or promotion +outcomes. Read the [runtime model](https://kam6l.github.io/agentdiff/docs/concepts/runtime/), [recovery guarantees](https://kam6l.github.io/agentdiff/docs/concepts/recovery/), and [security limits](https://kam6l.github.io/agentdiff/docs/trust/). diff --git a/SECURITY.md b/SECURITY.md index eda9a6d..b92d6b6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -101,6 +101,28 @@ Rollback is not guaranteed for: Use version control, disposable workspaces, and real sandboxing alongside AgentDiff for high-risk runs. +### Promotion gate + +Promotion applies only proof-PROVEN, policy-selected regular-file changes to the real repository. Its core invariants: + +- the workspace lease is an OS-level advisory lock on a **persistent** lock file that is never unlinked, so concurrent AgentDiff promotions cannot both hold the lease (the lock coordinates AgentDiff-aware processes only, not arbitrary external writers); +- every filesystem mutation is preceded by a persisted write-ahead journal entry (`APPLY_INTENT`), followed by post-state verification and a persisted `APPLIED` entry; +- recovery treats the journal as untrusted input: every path is validated (normalized, below the root and the approved backup directory, no symlink parents, no AgentDiff-internal targets), backups are re-verified by SHA-256/size before restore, and restore writes through an fsynced temp so a swapped backup can never be moved into the tree; +- a corrupt journal is **not** the same as no journal — promotion fails closed (`PromotionRecoveryError`) whenever recovery state cannot be established; +- recovery fails closed on ambiguous host state and never overwrites legitimate concurrent host edits; and +- proof must be PROVEN and bound to the same patch digest and immutable manifest before promotion is allowed. + +Promotion is not atomic across files: it is crash-consistent (recoverable to the recorded base after any crash point), not an all-or-nothing transaction. Recovery restores content and mode where the platform permits; it cannot restore unbacked external effects. + +### Clean-room proof + +Proof replays the sealed base-plus-patch workspace in a fresh container using a verification plan whose commands come only from trusted pre-run evidence (explicit policy, sealed pre-run repository state, or conservative defaults). It runs two verifications: + +- **patched verification** — the complete patched project tests; and +- **baseline verification** — the pre-run verifier files (tests, fixtures, runner config, manifests, lockfiles, CI workflows) restored over the patched product code, so an agent that modified the tests cannot silently weaken the verifier. + +When verifier-related files were modified and the baseline cannot confirm the patched run, the verdict is NOT_PROVEN. Proof-strength metadata (L0-L4) is explanatory; the PROVEN/NOT_PROVEN verdict is deterministic and never an LLM decision. Capsule checksums are tamper-evident, not authenticated: an attacker able to rewrite the entire capsule can produce a new self-consistent one, and signing remains future work. + ## Safer operating guidance 1. Run agents in an unprivileged disposable workspace. @@ -115,4 +137,4 @@ Use version control, disposable workspaces, and real sandboxing alongside AgentD ## Security testing -Security-sensitive changes should include regression tests for path confinement, symlink behavior, identity checks, redaction, backup integrity, and post-run divergence. Pull requests run CodeQL, Bandit, `pip-audit`, Dependency Review, cross-platform tests, and package validation. +Security-sensitive changes should include regression tests for path confinement, symlink behavior, identity checks, redaction, backup integrity, and post-run divergence. Promotion changes should include adversarial crash/fault-injection coverage (corrupt journals, path traversal, backup symlink/hardlink, crash at every write-ahead transition, ambiguous recovery). Pull requests run CodeQL, Bandit, `pip-audit`, Dependency Review, cross-platform tests, and package validation. diff --git a/benchmarks/materialize_bench.py b/benchmarks/materialize_bench.py new file mode 100644 index 0000000..580e8ae --- /dev/null +++ b/benchmarks/materialize_bench.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Measured workspace-materializer throughput benchmark (not a unit test). + +Run with: uv run python3 benchmarks/materialize_bench.py --output materialize-bench.json + +Measures streaming copy throughput and per-file cost across file counts so +materializer performance claims are measured, not assumed. +""" + +from __future__ import annotations + +import argparse +import json +import tempfile +from dataclasses import asdict, dataclass +from pathlib import Path +from time import perf_counter + +from agentdiff.runtime import MaterializationStrategy, WorkspaceMaterializer + + +@dataclass(frozen=True, slots=True) +class BenchCase: + files: int + bytes_per_file: int + strategy: str + duration_seconds: float + files_per_second: float + mib_per_second: float + + +def build_tree(root: Path, files: int, bytes_per_file: int) -> None: + payload = bytes((index % 251 for index in range(bytes_per_file))) + for index in range(files): + subdir = root / f"d{index % 50}" + subdir.mkdir(exist_ok=True) + (subdir / f"f{index}.bin").write_bytes(payload) + + +def run_case(root: Path, files: int, bytes_per_file: int, strategy: object) -> BenchCase: + src = root / f"src-{files}-{bytes_per_file}" + dst = root / f"dst-{files}-{bytes_per_file}-{strategy.value}" + src.mkdir(exist_ok=True) + build_tree(src, files, bytes_per_file) + started = perf_counter() + report = WorkspaceMaterializer(strategy=strategy).materialize(src, dst) + duration = perf_counter() - started + total_bytes = files * bytes_per_file + return BenchCase( + files=files, + bytes_per_file=bytes_per_file, + strategy=report.strategy_used, + duration_seconds=duration, + files_per_second=files / duration if duration else 0.0, + mib_per_second=(total_bytes / (1024 * 1024)) / duration if duration else 0.0, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", default="materialize-bench.json") + args = parser.parse_args() + + cases: list[BenchCase] = [] + with tempfile.TemporaryDirectory(prefix="agentdiff-bench-") as temporary: + root = Path(temporary) + for files, size in [(1_000, 4_096), (10_000, 4_096)]: + for strategy in ( + MaterializationStrategy.STREAM_COPY, + MaterializationStrategy.FAST_COPY, + MaterializationStrategy.AUTO, + ): + cases.append(run_case(root, files, size, strategy)) + print( + f"{files:>7} files x {size} B {strategy.value:12} -> " + f"{cases[-1].files_per_second:9.0f} files/s " + f"{cases[-1].mib_per_second:8.1f} MiB/s" + ) + payload = { + "benchmark": "agentdiff-materializer", + "schema_version": 1, + "cases": [asdict(case) for case in cases], + } + Path(args.output).write_text(json.dumps(payload, indent=2), encoding="utf-8") + print(f"Results written to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs_src/docs/api-reference.md b/docs_src/docs/api-reference.md index a28117a..6a3a993 100644 --- a/docs_src/docs/api-reference.md +++ b/docs_src/docs/api-reference.md @@ -1,6 +1,6 @@ --- title: Capsule reference -description: Files and trust properties of an AgentDiff 0.1.0 run capsule. +description: Files and trust properties of an AgentDiff 0.2.x run capsule. --- # Capsule reference diff --git a/docs_src/docs/cli/prove-promote.md b/docs_src/docs/cli/prove-promote.md index 132f719..3e5f737 100644 --- a/docs_src/docs/cli/prove-promote.md +++ b/docs_src/docs/cli/prove-promote.md @@ -24,7 +24,7 @@ agentdiff promote RUN_ID \ [--format summary|json] ``` -Exit `0` means the dry-run plan is safe or selected changes were promoted. Exit `8` means conflict/refusal. Promotion is unavailable without valid `PROVEN` evidence. +Exit `0` means the dry-run plan is safe or selected changes were promoted; exit `4` means plan conflicts; exit `8` means another refusal; exit `9` means promotion was blocked (missing/corrupt proof evidence or recovery could not establish a safe state). Promotion is unavailable without valid `PROVEN` evidence, and a corrupt or ambiguous recovery journal blocks promotion instead of being treated as absent. `--safe-only` selects `ALLOW` entries. Without it, `REVIEW` entries still require explicit `--path`; `DENY` entries are never promotable. Use `--dry-run --safe-only` before any real promotion. diff --git a/docs_src/docs/index.md b/docs_src/docs/index.md index b5fd883..8ca6c86 100644 --- a/docs_src/docs/index.md +++ b/docs_src/docs/index.md @@ -132,7 +132,7 @@ The following shape is taken from a real repository run. AgentDiff reported a su

- **Install AgentDiff** — supported source installation and requirements. [Installation →](installation.md) -- **Use the CLI** — exact commands and flags implemented in `v0.1.0`. [CLI reference →](cli/index.md) +- **Use the CLI** — exact commands and flags implemented in `v0.2.x`. [CLI reference →](cli/index.md) - **Integrate a command or framework** — use the transaction API, or the separately labeled experimental compatibility hooks. [Integrations →](integrations/custom.md) - **Connect an AI provider** — use verified repository memory with Claude, Codex/OpenAI, or Ollama. [Cortex →](concepts/cortex.md) - **Review the security boundary** — explicit guarantees, observations, and non-goals. [Security & limits →](trust.md) diff --git a/docs_src/docs/installation.md b/docs_src/docs/installation.md index 1a2dec7..238f55e 100644 --- a/docs_src/docs/installation.md +++ b/docs_src/docs/installation.md @@ -1,6 +1,6 @@ --- title: Installation -description: Install the AgentDiff 0.1.0 beta from source on Python 3.12 or newer. +description: Install the AgentDiff 0.2.x beta from source on Python 3.12 or newer. --- Getting started diff --git a/docs_src/docs/integrations/custom.md b/docs_src/docs/integrations/custom.md index 7f3139c..6611676 100644 --- a/docs_src/docs/integrations/custom.md +++ b/docs_src/docs/integrations/custom.md @@ -55,4 +55,4 @@ with AgentDiffSession("Fix authentication", config=config) as session: evaluation = session.evaluate() ``` -The compatibility session is separate from the transaction/recovery core. There is no plugin entry-point discovery or built-in CrewAI/AutoGen adapter in `0.1.0`. +The compatibility session is separate from the transaction/recovery core. There is no plugin entry-point discovery or built-in CrewAI/AutoGen adapter in `0.2.x`. diff --git a/docs_src/docs/quickstart.md b/docs_src/docs/quickstart.md index d9aa6ac..1068193 100644 --- a/docs_src/docs/quickstart.md +++ b/docs_src/docs/quickstart.md @@ -93,9 +93,31 @@ Skipped: 1 `src/parser.py` stays because policy allowed it. If a person or later process changed a collateral path, AgentDiff records a conflict and preserves the current path. +## 7. Prove and promote (trust pipeline) + +The full trust pipeline isolates the agent, reproduces the patch in a clean +room, and promotes only proven work: + +```bash +agentdiff run --runtime docker --task "Update the parser" -- python agent_task.py +agentdiff inspect +agentdiff prove +agentdiff promote --dry-run --safe-only +agentdiff promote --safe-only +agentdiff verify +``` + +`prove` rebuilds the base-plus-patch workspace in a clean Docker container and +runs both the patched tests and the trusted baseline tests. `promote` applies +only proof-PROVEN changes through a write-ahead journal; `--safe-only` selects +only `ALLOW` changes. Without Docker, `--runtime local` observes a host +subprocess — lower isolation, as described in the runtime concept. + ## Next steps - [Understand the runtime and limits](concepts/runtime.md) +- [Trust pipeline and promotion](concepts/trust-pipeline.md) +- [Prove and promote commands](cli/prove-promote.md) - [Customize mutation policy](concepts/policy.md) - [Connect Cortex to Claude, Codex, or Ollama](concepts/cortex.md) - [Review every CLI command](cli/index.md) diff --git a/docs_src/docs/sdk-reference.md b/docs_src/docs/sdk-reference.md index 4954c23..c81818f 100644 --- a/docs_src/docs/sdk-reference.md +++ b/docs_src/docs/sdk-reference.md @@ -1,11 +1,11 @@ --- title: Python API -description: Implemented Python interfaces for AgentDiff 0.1.0. +description: Implemented Python interfaces for AgentDiff 0.2.x. --- # Python API -This page lists the implemented `0.1.0` interfaces. The transaction API is **Beta**; the MCP hook and evaluator integrations are **Experimental**. +This page lists the implemented `0.2.x` interfaces. The transaction API is **Beta**; the MCP hook and evaluator integrations are **Experimental**. ## Run a transaction @@ -103,4 +103,4 @@ decision = hook.authorize("filesystem.write_file", {"path": "src/app.py"}) | `agentdiff.cortex` | Experimental evidence memory, skill-card generation, provider routing, and remediation advice | | `agentdiff.integrations` | MCP hook plus legacy evaluator session helpers | -There is no `AgentDiffRuntime`, async runtime, HTTP server, `serve` command, or `__version_info__` attribute in `0.1.0`. +There is no `AgentDiffRuntime`, async runtime, HTTP server, `serve` command, or `__version_info__` attribute in `0.2.x`. diff --git a/examples/trust_pipeline.py b/examples/trust_pipeline.py index 7628188..70b378d 100644 --- a/examples/trust_pipeline.py +++ b/examples/trust_pipeline.py @@ -43,7 +43,8 @@ def main() -> int: [ "python", "-c", - "from pathlib import Path; Path('src/result.txt').write_text('ok\\n')", + "from pathlib import Path; " + "Path('src/result.txt').write_text('ok\\n', encoding='utf-8')", ], timeout_seconds=60, ) diff --git a/pyproject.toml b/pyproject.toml index f0c3838..13f7c4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agentdiff" -version = "0.2.0" +version = "0.2.1" description = "Runtime evidence, mutation policy, and conflict-safe recovery for AI agents" readme = "README.md" requires-python = ">=3.12" @@ -90,6 +90,10 @@ warn_unused_ignores = true module = ["langchain_core.*"] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ["watchdog.*"] +ignore_missing_imports = true + [tool.ruff.lint.per-file-ignores] "src/agentdiff/evaluator.py" = ["TC001", "TC003"] "src/agentdiff/trajectory.py" = ["TC001", "TC003"] diff --git a/src/agentdiff/__init__.py b/src/agentdiff/__init__.py index 928360b..686b02e 100644 --- a/src/agentdiff/__init__.py +++ b/src/agentdiff/__init__.py @@ -3,7 +3,7 @@ The transaction API and trust pipeline are the primary product surfaces. """ -__version__ = "0.2.0" +__version__ = "0.2.1" from .analyzers import FutureBlastEngine, FutureBlastResult from .cortex import ( @@ -59,6 +59,7 @@ DockerRuntime, LocalRuntime, RuntimeBackend, + RuntimeCapabilities, RuntimeCapability, RuntimeControlLevel, RuntimeResult, @@ -134,6 +135,7 @@ "RunInspector", "RunStore", "RuntimeBackend", + "RuntimeCapabilities", "RuntimeCapability", "RuntimeControlLevel", "RuntimeResult", diff --git a/src/agentdiff/cli.py b/src/agentdiff/cli.py index d69d6e5..c4ebb24 100644 --- a/src/agentdiff/cli.py +++ b/src/agentdiff/cli.py @@ -110,6 +110,20 @@ def _json(value: Any) -> str: return json.dumps(value, indent=2, sort_keys=True) +def _emit_report(payload: Any, *, machine_format: str) -> None: + """Emit a machine-readable report with recursive redaction applied. + + Machine JSON output is evidence that may be shared or uploaded, so it is + passed through :func:`agentdiff.redaction.redact_data` to redact + sensitive-looking fields before serialization. + """ + + if machine_format == "json": + from agentdiff.redaction import redact_data + + print(_json(redact_data(payload))) + + def _load_runtime_policy(root: Path, requested: str | None) -> Policy: if requested is not None: return load_policy_file(requested) @@ -544,6 +558,100 @@ def cmd_verify(args: argparse.Namespace) -> int: return 0 if report.ok else 6 +def cmd_prove(args: argparse.Namespace) -> int: + """Reproduce the patch in a clean room and verify it deterministically.""" + + from agentdiff.proof import ProofEngine + + proof = ProofEngine(args.root, args.run_id).prove(timeout_seconds=args.timeout) + if args.format == "json": + _emit_report(proof.to_dict(), machine_format="json") + return 0 if proof.verdict.value == "PROVEN" else 7 + + tests_phase = next((phase for phase in proof.phases if phase.phase == "tests"), None) + baseline_phase = next( + (phase for phase in proof.phases if phase.phase == "baseline_tests"), None + ) + baseline_tests = ( + f"{baseline_phase.tests_passed} / {baseline_phase.tests_total}" + if baseline_phase is not None and baseline_phase.tests_total is not None + else "-" + ) + patched_tests = ( + f"{tests_phase.tests_passed} / {tests_phase.tests_total}" + if tests_phase is not None and tests_phase.tests_total is not None + else "-" + ) + print("AgentDiff Trust Report") + print(f" Runtime {proof.verification_source or 'n/a'}") + print(f" Policy {proof.policy}") + print( + f" Immediate Blast {proof.immediate_blast_radius} / " + f"{blast_level(proof.immediate_blast_radius)}" + ) + print( + f" Future Blast {proof.future_blast_radius} / " + f"{blast_level(proof.future_blast_radius)}" + ) + print(f" Trusted Plan {'YES' if proof.verification_confirmed else 'NO'}") + print(f" Baseline Tests {baseline_tests}") + print(f" Patched Tests {patched_tests}") + print(f" Verifier Changes {proof.verifier_files_changed}") + print(f" Hidden State {proof.hidden_state_dependency}") + print(f" Proof Strength {proof.proof_strength} / {proof.proof_strength_label}") + print(f" Promotion {proof.promotion}") + print() + verdict = "PROVEN" if proof.verdict.value == "PROVEN" else "NOT_PROVEN" + print(f" VERDICT: {verdict}") + for reason in proof.reasons: + print(f" reason: {safe_display(reason)}") + return 0 if proof.verdict.value == "PROVEN" else 7 + + +def blast_level(score: int) -> str: + if score >= 60: + return "CRITICAL" + if score >= 30: + return "HIGH" + if score >= 10: + return "MEDIUM" + return "LOW" + + +def cmd_promote(args: argparse.Namespace) -> int: + """Apply only proven, policy-selected patch changes to the real repository.""" + + from agentdiff.promotion import PromotionEngine, PromotionRecoveryError + + try: + report = PromotionEngine(args.root, args.run_id).promote( + dry_run=args.dry_run, + safe_only=args.safe_only, + paths=args.path, + ) + except PromotionRecoveryError as error: + print(f"agentdiff: promotion blocked: {safe_display(str(error))}", file=sys.stderr) + return 9 + except PermissionError as error: + print(f"agentdiff: promotion blocked: {safe_display(str(error))}", file=sys.stderr) + return 9 + if args.format == "json": + _emit_report(report.to_dict(), machine_format="json") + else: + print(f"Promotion run: {report.run_id}") + print(f"Status: {report.status}") + print(f"Actions: {len(report.actions)}") + for action in report.actions: + print(f" {action.action:8} {safe_display(action.path)}") + print(f"Conflicts: {len(report.conflicts)}") + for conflict in report.conflicts: + print(f" {safe_display(conflict.path)}: {safe_display(conflict.reason)}") + print(f"Skipped: {len(report.skipped)}") + if report.conflicts: + return 4 + return 0 if report.ok else 8 + + def cmd_rollback(args: argparse.Namespace) -> int: report = RollbackEngine.open(args.root, args.run_id).rollback( safe_only=args.safe_only, @@ -668,6 +776,27 @@ def build_parser() -> argparse.ArgumentParser: p_verify.add_argument("--format", choices=["json", "summary"], default="summary") p_verify.set_defaults(func=cmd_verify) + p_prove = subparsers.add_parser( + "prove", help="Reproduce the patch in a clean room and verify it deterministically" + ) + p_prove.add_argument("run_id") + p_prove.add_argument("--root", default=".") + p_prove.add_argument("--timeout", type=float, default=900.0) + p_prove.add_argument("--format", choices=["json", "summary"], default="summary") + p_prove.set_defaults(func=cmd_prove) + + p_promote = subparsers.add_parser( + "promote", + help="Apply only proven, policy-selected changes through the crash-consistent gate", + ) + p_promote.add_argument("run_id") + p_promote.add_argument("--root", default=".") + p_promote.add_argument("--dry-run", action="store_true", help="Plan without applying") + p_promote.add_argument("--safe-only", action="store_true", help="Select only ALLOW changes") + p_promote.add_argument("--path", action="append", help="Limit promotion to a relative path") + p_promote.add_argument("--format", choices=["json", "summary"], default="summary") + p_promote.set_defaults(func=cmd_promote) + p_rollback = subparsers.add_parser("rollback", help="Conflict-check and recover run changes") p_rollback.add_argument("run_id") p_rollback.add_argument("--root", default=".") diff --git a/src/agentdiff/diff_engine.py b/src/agentdiff/diff_engine.py index 7c5270a..a13754a 100644 --- a/src/agentdiff/diff_engine.py +++ b/src/agentdiff/diff_engine.py @@ -20,7 +20,7 @@ from .redaction import fingerprint warnings.warn( - "agentdiff.diff_engine is deprecated and will be removed in a future release. " + "agentdiff.diff_engine is deprecated since 0.2 and will be removed in 0.4 or later. " "Use agentdiff.state.FilesystemScanner and diff_manifests instead.", DeprecationWarning, stacklevel=2, diff --git a/src/agentdiff/evaluator.py b/src/agentdiff/evaluator.py index 7f32a4e..5cb8efd 100644 --- a/src/agentdiff/evaluator.py +++ b/src/agentdiff/evaluator.py @@ -24,7 +24,7 @@ from .trajectory import TrajectoryRecord warnings.warn( - "agentdiff.evaluator is deprecated and will be removed in a future release. " + "agentdiff.evaluator is deprecated since 0.2 and will be removed in 0.4 or later. " "Use agentdiff.scoring and agentdiff.analyzers instead.", DeprecationWarning, stacklevel=2, diff --git a/src/agentdiff/evidence/__init__.py b/src/agentdiff/evidence/__init__.py index a6dde6f..c5c68b2 100644 --- a/src/agentdiff/evidence/__init__.py +++ b/src/agentdiff/evidence/__init__.py @@ -1,6 +1,7 @@ """Normalized source and patch evidence used by proof and promotion.""" from .capsule import BlobReference, CapsuleReader +from .objects import ObjectRef, ObjectStore, ObjectStoreError, validate_object_digest from .patch import ( PatchBundle, PatchEntry, @@ -14,11 +15,15 @@ __all__ = [ "BlobReference", "CapsuleReader", + "ObjectRef", + "ObjectStore", + "ObjectStoreError", "PatchBundle", "PatchEntry", "PatchManifest", "SourceSnapshot", "capture_patch", "capture_source_snapshot", + "validate_object_digest", "validate_source_snapshot", ] diff --git a/src/agentdiff/evidence/capsule.py b/src/agentdiff/evidence/capsule.py index 6edd4dd..2d04278 100644 --- a/src/agentdiff/evidence/capsule.py +++ b/src/agentdiff/evidence/capsule.py @@ -54,8 +54,15 @@ def read_manifest(self) -> dict[str, Any]: raise FileNotFoundError("capsule integrity manifest not found") return json.loads(target.read_text(encoding="utf-8")) - def compute_merkle_root(self) -> str: - """Compute the deterministic Merkle root digest over the sealed manifest entries.""" + def compute_root_digest(self) -> str: + """Compute the deterministic Capsule Root Digest over sealed manifest entries. + + This is a flat aggregate digest over ``path:digest:size`` entries — + deliberately NOT a Merkle tree. It provides tamper evidence against + accidental or modest modification, not inclusion proofs and not + authentication (an attacker who can rewrite the artifact and the + manifest can produce a new self-consistent capsule). + """ manifest = self.read_manifest() files = manifest.get("files", {}) hasher = hashlib.sha256() @@ -64,6 +71,14 @@ def compute_merkle_root(self) -> str: hasher.update(f"{relpath}:{entry.get('sha256')}:{entry.get('size')}\n".encode("utf-8")) return hasher.hexdigest() + def compute_merkle_root(self) -> str: + """Deprecated alias for :meth:`compute_root_digest`. + + The digest is a flat aggregate, not a Merkle tree; the old name was + inaccurate and is retained only for compatibility. + """ + return self.compute_root_digest() + def get_artifact_path(self, relative_path: str) -> Path: normalized = normalize_relative_path(relative_path) target = self.run_dir.joinpath(*normalized.split("/")) diff --git a/src/agentdiff/evidence/objects.py b/src/agentdiff/evidence/objects.py new file mode 100644 index 0000000..22c89e1 --- /dev/null +++ b/src/agentdiff/evidence/objects.py @@ -0,0 +1,232 @@ +"""Content-addressed immutable object storage. + +Objects live under ``/.agentdiff/objects//`` and are +written once: content is streamed while its SHA-256 is computed, written to a +temp file, fsynced, digest-validated, and atomically renamed into place. An +existing object with the same digest is never rewritten (immutability). + +Layout notes +------------ + +Run capsules remain self-contained file trees today (spec v2). This object +store is the foundation for the incremental migration to content-addressed +artifact references (spec v3 planning) and for future export/import +hydration. Nothing in the sealed-capsule format changes as a result of this +module. + +Objects are tamper-evident, not authenticated: anyone able to write the +object store can add or replace objects, but digest-addressed reads fail +closed if the content does not match the requested digest. +""" + +from __future__ import annotations + +import hashlib +import io +import os +import re +import stat +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Iterable, cast + +_CHUNK_SIZE = 1024 * 1024 +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + + +class ObjectStoreError(RuntimeError): + """Raised for invalid digests, corrupted objects, or storage failures.""" + + +@dataclass(frozen=True, slots=True) +class ObjectRef: + sha256: str + size: int + + def to_dict(self) -> dict[str, object]: + return {"sha256": self.sha256, "size": self.size} + + +def validate_object_digest(value: str) -> str: + """Return the lowercase hex digest or raise.""" + if not isinstance(value, str) or not _SHA256_PATTERN.fullmatch(value): + raise ObjectStoreError(f"invalid object digest: {value!r}") + return value + + +class ObjectStore: + """Content-addressed immutable object storage below one AgentDiff root.""" + + def __init__(self, root: str | Path) -> None: + self.root = Path(root).resolve() + self.objects_dir = self.root / ".agentdiff" / "objects" + self.objects_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + self._assert_real_directory(self.objects_dir) + + # ------------------------------------------------------------------ + # Writing + # ------------------------------------------------------------------ + + def put(self, source: str | Path | BinaryIO | bytes) -> ObjectRef: + """Store one stream of bytes and return its content address. + + The object is written immutably: if an object with the same digest + already exists, its stored content is re-verified and nothing is + rewritten. + """ + hasher = hashlib.sha256() + size = 0 + temporary: Path | None = None + close_source = False + try: + if isinstance(source, (bytes, bytearray)): + stream: BinaryIO = io.BytesIO(bytes(source)) + elif hasattr(source, "read"): + stream = cast("BinaryIO", source) + else: + path = Path(cast("str | os.PathLike[str]", source)) + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise ObjectStoreError("object source must be a regular file") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + stream = os.fdopen(descriptor, "rb", closefd=False) + close_source = True + temp_fd, temp_name = tempfile.mkstemp( + prefix="object-", suffix=".tmp", dir=str(self.objects_dir) + ) + temporary = Path(temp_name) + try: + with os.fdopen(temp_fd, "wb") as output: + while chunk := stream.read(_CHUNK_SIZE): + hasher.update(chunk) + size += len(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + finally: + if close_source: + stream.close() + except OSError as error: + if temporary is not None: + temporary.unlink(missing_ok=True) + raise ObjectStoreError(f"object write failed: {error}") from error + + digest = hasher.hexdigest() + target = self.path_for(digest) + if target.exists(): + self._verify_existing(digest, size) + temporary.unlink(missing_ok=True) + return ObjectRef(sha256=digest, size=size) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + os.replace(temporary, target) + except OSError as error: + temporary.unlink(missing_ok=True) + raise ObjectStoreError(f"object commit failed: {error}") from error + return ObjectRef(sha256=digest, size=size) + + # ------------------------------------------------------------------ + # Reading + # ------------------------------------------------------------------ + + def path_for(self, digest: str) -> Path: + """Return the on-disk path for a digest without touching the filesystem.""" + validated = validate_object_digest(digest) + return self.objects_dir / validated[:2] / validated + + def has(self, digest: str) -> bool: + try: + path = self.path_for(digest) + except ObjectStoreError: + return False + try: + info = path.lstat() + except FileNotFoundError: + return False + return stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode) + + def open(self, digest: str) -> BinaryIO: + """Open an object for reading after verifying its identity.""" + path = self.path_for(digest) + try: + info = path.lstat() + except FileNotFoundError as error: + raise ObjectStoreError(f"object not found: {digest}") from error + if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_nlink != 1: + raise ObjectStoreError(f"object is not a single-link regular file: {digest}") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_dev != info.st_dev + or opened.st_ino != info.st_ino + ): + os.close(descriptor) + raise ObjectStoreError(f"object identity changed while opening: {digest}") + return os.fdopen(descriptor, "rb") + + def read_bytes(self, digest: str) -> bytes: + """Read an object and fail closed if its content does not match the digest.""" + validated = validate_object_digest(digest) + with self.open(digest) as stream: + hasher = hashlib.sha256() + chunks: list[bytes] = [] + while chunk := stream.read(_CHUNK_SIZE): + hasher.update(chunk) + chunks.append(chunk) + if hasher.hexdigest() != validated: + raise ObjectStoreError(f"object content does not match digest: {digest}") + return b"".join(chunks) + + def verify(self, digest: str, expected_size: int | None = None) -> int: + """Recompute the digest of one stored object; raises on mismatch.""" + validated = validate_object_digest(digest) + with self.open(digest) as stream: + hasher = hashlib.sha256() + size = 0 + while chunk := stream.read(_CHUNK_SIZE): + hasher.update(chunk) + size += len(chunk) + if hasher.hexdigest() != validated: + raise ObjectStoreError(f"object content does not match digest: {digest}") + if expected_size is not None and size != expected_size: + raise ObjectStoreError(f"object size mismatch: {digest}") + return size + + def iter_all(self) -> Iterable[ObjectRef]: + """Yield every stored object reference in deterministic order.""" + references: list[ObjectRef] = [] + for prefix_dir in sorted(self.objects_dir.iterdir()): + if not prefix_dir.is_dir() or prefix_dir.is_symlink(): + continue + for candidate in sorted(prefix_dir.iterdir()): + if candidate.is_symlink() or not candidate.is_file(): + continue + name = candidate.name + if not _SHA256_PATTERN.fullmatch(name): + continue + try: + size = self.verify(name) + except ObjectStoreError: + continue + references.append(ObjectRef(sha256=name, size=size)) + return references + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _verify_existing(self, digest: str, expected_size: int) -> None: + actual = self.verify(digest, expected_size=expected_size) + if actual != expected_size: + raise ObjectStoreError(f"existing object size mismatch: {digest}") + + @staticmethod + def _assert_real_directory(path: Path) -> None: + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise ObjectStoreError(f"object store path is not a real directory: {path}") diff --git a/src/agentdiff/evidence/patch.py b/src/agentdiff/evidence/patch.py index e8a14a0..e32a943 100644 --- a/src/agentdiff/evidence/patch.py +++ b/src/agentdiff/evidence/patch.py @@ -341,6 +341,7 @@ def materialize_source(self, destination: str | Path) -> None: f"source/files/{normalize_relative_path(path)}", target_root, path, + mode=int(expected.get("mode", 0o644)), ) def apply(self, destination: str | Path) -> None: @@ -358,9 +359,17 @@ def apply(self, destination: str | Path) -> None: if os.name != "nt" and entry.result_mode is not None: target.chmod(entry.result_mode) - def _copy_capsule_file(self, artifact: str, root: Path, relative: str) -> None: + def _copy_capsule_file( + self, + artifact: str, + root: Path, + relative: str, + *, + mode: int | None = None, + ) -> None: target = self._safe_target(root, relative, create_parents=True) source = self.store.artifact_path(artifact) + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) info = source.lstat() if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise RuntimeError("capsule payload is not a single-link regular file") @@ -369,13 +378,28 @@ def _copy_capsule_file(self, artifact: str, root: Path, relative: str) -> None: temporary = target.parent / ( f".agentdiff-{hashlib.sha256(relative.encode()).hexdigest()}.tmp" ) + source_fd = os.open(source, flags) try: - with source.open("rb") as input_stream, temporary.open("xb") as output_stream: + opened = os.fstat(source_fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_dev != info.st_dev + or opened.st_ino != info.st_ino + ): + raise RuntimeError("capsule payload changed while opening") + with ( + os.fdopen(source_fd, "rb", closefd=False) as input_stream, + temporary.open("xb") as output_stream, + ): shutil.copyfileobj(input_stream, output_stream, length=1024 * 1024) output_stream.flush() os.fsync(output_stream.fileno()) os.replace(temporary, target) + if os.name != "nt" and mode is not None: + target.chmod(stat.S_IMODE(mode)) finally: + os.close(source_fd) temporary.unlink(missing_ok=True) @staticmethod diff --git a/src/agentdiff/policy/loader.py b/src/agentdiff/policy/loader.py index bbd8d67..d2886c8 100644 --- a/src/agentdiff/policy/loader.py +++ b/src/agentdiff/policy/loader.py @@ -246,16 +246,16 @@ def load_policy_file(path: str | Path) -> Policy: if yaml is None: raise ImportError("PyYAML not installed") parsed = yaml.safe_load(raw) - except (ImportError, ModuleNotFoundError): + except (ImportError, ModuleNotFoundError) as exc: import json try: parsed = json.loads(raw) - except (json.JSONDecodeError, ValueError): + except (TypeError, ValueError, json.JSONDecodeError): raise PolicyLoadError( "PyYAML is required to load policy files; install it with 'pip install PyYAML'" - ) from None - except (OSError, ValueError, TypeError) as exc: + ) from exc + except (OSError, TypeError, ValueError) as exc: raise PolicyLoadError(f"failed to parse policy file {path}: {exc}") from exc if not isinstance(parsed, dict): diff --git a/src/agentdiff/promotion/__init__.py b/src/agentdiff/promotion/__init__.py index aa9fb41..f9de39e 100644 --- a/src/agentdiff/promotion/__init__.py +++ b/src/agentdiff/promotion/__init__.py @@ -1,7 +1,14 @@ """Fail-closed, crash-consistent host promotion.""" from .engine import PromotionEngine -from .journal import JournalEntry, JournalState, PromotionJournal +from .journal import ( + EntryState, + JournalEntry, + JournalLoadOutcome, + JournalLoadResult, + JournalState, + PromotionJournal, +) from .lock import PromotionLockError, WorkspaceLease from .models import ( PromotionAction, @@ -10,11 +17,14 @@ PromotionPlanEntry, PromotionReport, ) -from .recovery import PromotionRecovery, RecoveryReport +from .recovery import PromotionRecovery, PromotionRecoveryError, RecoveryReport from .staging import PromotionStager __all__ = [ + "EntryState", "JournalEntry", + "JournalLoadOutcome", + "JournalLoadResult", "JournalState", "PromotionAction", "PromotionConflict", @@ -24,6 +34,7 @@ "PromotionPlan", "PromotionPlanEntry", "PromotionRecovery", + "PromotionRecoveryError", "PromotionReport", "PromotionStager", "RecoveryReport", diff --git a/src/agentdiff/promotion/engine.py b/src/agentdiff/promotion/engine.py index 52972b6..e79cbba 100644 --- a/src/agentdiff/promotion/engine.py +++ b/src/agentdiff/promotion/engine.py @@ -1,4 +1,11 @@ -"""Fail-closed host promotion for proven, policy-selected patch entries.""" +"""Fail-closed host promotion for proven, policy-selected patch entries. + +The application phase is crash-consistent: every mutation is preceded by a +persisted ``APPLY_INTENT`` journal state (write-ahead), followed by a +post-state verification and a persisted ``APPLIED`` state. Recovery never +guesses between "mutated" and "not mutated" -- it compares the current host +state against the recorded base and result and fails closed on ambiguity. +""" from __future__ import annotations @@ -14,7 +21,7 @@ from agentdiff.state import FileRecord, FilesystemScanner from agentdiff.transaction.store import RunStore -from .journal import JournalEntry, JournalState, PromotionJournal +from .journal import EntryState, JournalEntry, JournalState, PromotionJournal from .lock import WorkspaceLease from .models import ( PromotionAction, @@ -23,7 +30,7 @@ PromotionPlanEntry, PromotionReport, ) -from .recovery import PromotionRecovery +from .recovery import PromotionRecovery, PromotionRecoveryError from .staging import PromotionStager @@ -49,7 +56,8 @@ def promote( paths: list[str] | None = None, ) -> PromotionReport: with self.lease.hold(): - # Pre-flight: recover any stale crashed journal if present + # Pre-flight: recover any stale crashed journal if present. A + # corrupt, ambiguous, or failed recovery raises and blocks. PromotionRecovery(self.root).check_and_recover() integrity = self.store.verify_integrity() @@ -118,6 +126,7 @@ def promote( ) # 2. Stage all modifications and backups + base_sizes = self._base_sizes() for planned in ready: patch_entry = entries_by_path[planned.path] staged_rel = None @@ -138,37 +147,45 @@ def promote( result_sha256=patch_entry.result_sha256, base_mode=patch_entry.base_mode, result_mode=patch_entry.result_mode, + base_size=base_sizes.get(patch_entry.path), + result_size=patch_entry.size, staged_relpath=staged_rel, backup_relpath=backup_rel, - applied=False, ) ) - # 3. Write-ahead log journal commit + # 3. Write-ahead log: transaction is about to mutate the host. journal.state = JournalState.APPLYING journal.persist() - # 4. Atomic application phase + # 4. Atomic application phase with per-entry write-ahead states. + report.status = "PROMOTED" for journal_entry, planned in zip(journal.entries, ready, strict=True): patch_entry = entries_by_path[planned.path] - try: + # APPLY_INTENT: persist intent, then mutate, then verify. + journal_entry.state = EntryState.APPLY_INTENT + journal.persist() action = self._apply_one(bundle, patch_entry) - journal_entry.applied = True + self._verify_post_state(patch_entry) + journal_entry.state = EntryState.APPLIED journal.persist() except (OSError, RuntimeError, ValueError) as error: report.conflicts.append( PromotionConflict(patch_entry.path, str(error) or type(error).__name__) ) report.status = "PARTIAL_CONFLICT" if report.actions else "CONFLICT" - # Interrupted -> trigger recovery journal.state = JournalState.RECOVERY_REQUIRED journal.persist() - PromotionRecovery(self.root).check_and_recover() + try: + PromotionRecovery(self.root).check_and_recover() + except PromotionRecoveryError: + report.status = "RECOVERY_FAILED" + self._persist_result(report) + raise break report.actions.append(action) else: - report.status = "PROMOTED" journal.state = JournalState.COMMITTED journal.persist() journal.clean() @@ -177,6 +194,20 @@ def promote( self._persist_result(report) return report + def _base_sizes(self) -> dict[str, int]: + """Map patch paths to their recorded pre-run file sizes for backup checks.""" + sizes: dict[str, int] = {} + try: + before = self.store.read_json("before.json") + except (OSError, ValueError, TypeError, KeyError, FileNotFoundError): + return sizes + if not isinstance(before, dict) or not isinstance(before.get("files"), dict): + return sizes + for path, record in before["files"].items(): + if isinstance(record, dict) and isinstance(record.get("size"), int): + sizes[str(path)] = int(record["size"]) + return sizes + def _persist_result(self, report: PromotionReport) -> None: self.store.write_json_path("promotion/result.json", report.to_dict()) self.store.seal_extension("promotion", ("plan.json", "result.json")) @@ -312,6 +343,17 @@ def _apply_one(self, bundle: PatchBundle, entry: PatchEntry) -> PromotionAction: self._atomic_replace(source, target, entry, current) return PromotionAction(entry.path, "modified", "proven file replaced") + def _verify_post_state(self, entry: PatchEntry) -> None: + """Verify the promoted host file exactly equals the proven result.""" + if entry.change_type == "deleted": + target = self._resolve_target(entry.path, create_parents=False) + if target.exists() or target.is_symlink(): + raise RuntimeError("deleted file reappeared after promotion") + return + current = self.scanner.capture_one(entry.path) + if not self._matches_result(current, entry): + raise RuntimeError("promoted file does not equal the proven result") + def _atomic_create(self, source: Path, target: Path, entry: PatchEntry) -> None: descriptor, temporary_name = tempfile.mkstemp( prefix=".agentdiff-promote-", dir=target.parent @@ -370,21 +412,37 @@ def _atomic_replace( @staticmethod def _copy_payload(source: Path, descriptor: int, entry: PatchEntry) -> None: - info = source.lstat() - if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: - raise RuntimeError("patch payload is not a single-link regular file") - digest = hashlib.sha256() - size = 0 - with ( - source.open("rb") as input_stream, - os.fdopen(descriptor, "wb", closefd=False) as output_stream, - ): - while chunk := input_stream.read(1024 * 1024): - digest.update(chunk) - size += len(chunk) - output_stream.write(chunk) - output_stream.flush() - os.fsync(output_stream.fileno()) + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + source_fd = os.open(source, flags) + try: + opened = os.fstat(source_fd) + info = source.lstat() + if not stat.S_ISREG(opened.st_mode) or opened.st_nlink != 1: + raise RuntimeError("patch payload is not a single-link regular file") + if opened.st_dev != info.st_dev or opened.st_ino != info.st_ino: + raise RuntimeError("patch payload changed while opening") + digest = hashlib.sha256() + size = 0 + with ( + os.fdopen(source_fd, "rb", closefd=False) as input_stream, + os.fdopen(descriptor, "wb", closefd=False) as output_stream, + ): + while chunk := input_stream.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + output_stream.write(chunk) + output_stream.flush() + os.fsync(output_stream.fileno()) + finished = os.fstat(source_fd) + if ( + finished.st_dev != opened.st_dev + or finished.st_ino != opened.st_ino + or finished.st_size != opened.st_size + or finished.st_mtime_ns != opened.st_mtime_ns + ): + raise RuntimeError("patch payload changed while copying") + finally: + os.close(source_fd) if digest.hexdigest() != entry.result_sha256 or size != entry.size: raise RuntimeError("patch payload digest mismatch") diff --git a/src/agentdiff/promotion/journal.py b/src/agentdiff/promotion/journal.py index f4797b1..86ed4e9 100644 --- a/src/agentdiff/promotion/journal.py +++ b/src/agentdiff/promotion/journal.py @@ -1,7 +1,35 @@ -"""Write-ahead promotion journal for crash-consistent multi-file promotion.""" +"""Write-ahead promotion journal for crash-consistent multi-file promotion. + +Crash consistency model +----------------------- + +A promotion entry moves through an explicit state machine. Every state +transition that precedes a filesystem mutation is persisted and fsynced +**before** the mutation happens (write-ahead), and every post-mutation state +is persisted **after** the mutation and its verification: + + PREPARED + ↓ (persist + fsync) + APPLY_INTENT ← mutation may now occur; recovery must disambiguate + ↓ (mutate → verify) + APPLIED + ↓ + RECOVER_INTENT ← recovery may now mutate; recovery disambiguates + ↓ + RECOVERED + +``AMBIGUOUS`` is a terminal fail-closed state: the journal entry's expected +base/result state cannot be distinguished from the current host state, so +automatic recovery refuses to overwrite anything. + +A journal file that exists but cannot be parsed safely is **corrupt**, not +absent: promotion and recovery must fail closed instead of treating it like +"no journal". +""" from __future__ import annotations +import contextlib import json import os import tempfile @@ -11,8 +39,14 @@ from pathlib import Path from typing import Any +_JOURNAL_SCHEMA_VERSION = 3 +_LEGACY_SCHEMA_VERSION = 2 +_LEGACY_CHANGE_TYPES = frozenset({"created", "modified", "deleted"}) + class JournalState(str, Enum): + """Transaction-level promotion state.""" + IDLE = "IDLE" PLAN_RECORDED = "PLAN_RECORDED" STAGED = "STAGED" @@ -20,35 +54,109 @@ class JournalState(str, Enum): COMMITTED = "COMMITTED" ROLLED_BACK = "ROLLED_BACK" RECOVERY_REQUIRED = "RECOVERY_REQUIRED" + RECOVERY_FAILED = "RECOVERY_FAILED" + + +class EntryState(str, Enum): + """Per-entry write-ahead and recovery state.""" + + PREPARED = "PREPARED" + APPLY_INTENT = "APPLY_INTENT" + APPLIED = "APPLIED" + RECOVER_INTENT = "RECOVER_INTENT" + RECOVERED = "RECOVERED" + AMBIGUOUS = "AMBIGUOUS" + + +class JournalLoadOutcome(str, Enum): + """Explicit outcome of loading a promotion journal. + + ``CORRUPT_JOURNAL`` is distinct from ``NO_JOURNAL``: a corrupt journal + must block promotion while an absent journal must not. + """ + + NO_JOURNAL = "NO_JOURNAL" + VALID_JOURNAL = "VALID_JOURNAL" + CORRUPT_JOURNAL = "CORRUPT_JOURNAL" + + +@dataclass(frozen=True, slots=True) +class JournalLoadResult: + """Parsed journal plus its explicit load outcome.""" + + outcome: JournalLoadOutcome + journal: "PromotionJournal | None" = None + error: str = "" + + @property + def ok(self) -> bool: + return self.outcome is JournalLoadOutcome.VALID_JOURNAL @dataclass class JournalEntry: + """One planned filesystem mutation with write-ahead state.""" + path: str change_type: str base_sha256: str | None result_sha256: str | None base_mode: int | None result_mode: int | None + state: EntryState = EntryState.PREPARED + base_size: int | None = None + result_size: int | None = None staged_relpath: str | None = None backup_relpath: str | None = None - applied: bool = False def to_dict(self) -> dict[str, Any]: return asdict(self) @classmethod def from_dict(cls, data: dict[str, Any]) -> JournalEntry: + path = data.get("path") + change_type = data.get("change_type") + if not isinstance(path, str) or not isinstance(change_type, str): + raise ValueError("journal entry path/change_type must be strings") + if change_type not in _LEGACY_CHANGE_TYPES: + raise ValueError(f"invalid journal change type: {change_type!r}") + # Legacy schema 2 journals only carried ``applied: bool``. Map that + # onto the new state machine conservatively: applied=True means the + # mutation was recorded as complete (APPLIED); applied=False means + # the entry was never confirmed applied (PREPARED). Legacy journals + # cannot express APPLY_INTENT, which is exactly the crash window this + # state machine closes for new journals. + raw_state = data.get("state") + if raw_state is None and "applied" in data: + raw_state = ( + EntryState.APPLIED.value if bool(data["applied"]) else EntryState.PREPARED.value + ) + if raw_state is None: + raw_state = EntryState.PREPARED.value + try: + state = EntryState(str(raw_state)) + except ValueError as error: + raise ValueError(f"invalid journal entry state: {raw_state!r}") from error + for key in ("base_sha256", "result_sha256", "staged_relpath", "backup_relpath"): + value = data.get(key) + if value is not None and not isinstance(value, str): + raise ValueError(f"journal entry {key} must be a string or null") + for key in ("base_mode", "result_mode", "base_size", "result_size"): + value = data.get(key) + if value is not None and not isinstance(value, int): + raise ValueError(f"journal entry {key} must be an integer or null") return cls( - path=str(data["path"]), - change_type=str(data["change_type"]), + path=path, + change_type=change_type, base_sha256=data.get("base_sha256"), result_sha256=data.get("result_sha256"), base_mode=data.get("base_mode"), result_mode=data.get("result_mode"), + state=state, + base_size=data.get("base_size"), + result_size=data.get("result_size"), staged_relpath=data.get("staged_relpath"), backup_relpath=data.get("backup_relpath"), - applied=bool(data.get("applied", False)), ) @@ -63,6 +171,7 @@ class PromotionJournal: entries: list[JournalEntry] = field(default_factory=list) created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) updated_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + schema_version: int = _JOURNAL_SCHEMA_VERSION @property def path(self) -> Path: @@ -70,7 +179,7 @@ def path(self) -> Path: def to_dict(self) -> dict[str, Any]: return { - "schema_version": 2, + "schema_version": self.schema_version, "run_id": self.run_id, "patch_digest": self.patch_digest, "state": self.state.value, @@ -80,7 +189,7 @@ def to_dict(self) -> dict[str, Any]: } def persist(self) -> None: - """Atomically persist journal state with fsync.""" + """Atomically persist journal state with fsync of file and directory.""" self.updated_at = datetime.now(timezone.utc).isoformat() journal_dir = self.path.parent journal_dir.mkdir(mode=0o700, parents=True, exist_ok=True) @@ -97,34 +206,122 @@ def persist(self) -> None: file.flush() os.fsync(file.fileno()) os.replace(temp_path, str(self.path)) + _fsync_directory(journal_dir) except BaseException: if os.path.exists(temp_path): os.unlink(temp_path) raise @classmethod - def load(cls, root: str | Path) -> PromotionJournal | None: + def load(cls, root: str | Path) -> JournalLoadResult: + """Load the promotion journal with an explicit load outcome. + + Returns ``NO_JOURNAL`` only when no journal file exists. Any journal + file that exists but cannot be parsed and validated returns + ``CORRUPT_JOURNAL`` with a reason. Callers must fail closed on + ``CORRUPT_JOURNAL``. + """ target = Path(root).resolve() / ".agentdiff" / "promotion-journal.json" - if not target.is_file(): - return None + if not target.is_file() or target.is_symlink(): + if target.is_symlink(): + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error="promotion journal is a symlink", + ) + return JournalLoadResult(JournalLoadOutcome.NO_JOURNAL) try: data = json.loads(target.read_text(encoding="utf-8")) - return cls( - root=Path(root).resolve(), - run_id=str(data["run_id"]), - patch_digest=str(data["patch_digest"]), - state=JournalState(data.get("state", "IDLE")), - created_at=str(data.get("created_at", "")), - updated_at=str(data.get("updated_at", "")), - entries=[JournalEntry.from_dict(item) for item in data.get("entries", [])], + if not isinstance(data, dict): + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error="promotion journal root must be an object", + ) + schema = data.get("schema_version") + if schema not in {_LEGACY_SCHEMA_VERSION, _JOURNAL_SCHEMA_VERSION}: + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error=f"unsupported promotion journal schema: {schema!r}", + ) + run_id = data.get("run_id") + patch_digest = data.get("patch_digest") + state_value = data.get("state") + if not isinstance(run_id, str) or not isinstance(patch_digest, str): + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error="promotion journal run_id/patch_digest must be strings", + ) + try: + state = JournalState(str(state_value)) + except (TypeError, ValueError): + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error=f"invalid promotion journal state: {state_value!r}", + ) + raw_entries = data.get("entries") + if not isinstance(raw_entries, list): + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error="promotion journal entries must be a list", + ) + entries: list[JournalEntry] = [] + for index, item in enumerate(raw_entries): + if not isinstance(item, dict): + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error=f"journal entry {index} must be an object", + ) + try: + entries.append(JournalEntry.from_dict(item)) + except (TypeError, ValueError) as error: + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error=f"journal entry {index} is invalid: {error}", + ) + created_at = data.get("created_at", "") + updated_at = data.get("updated_at", "") + if not isinstance(created_at, str) or not isinstance(updated_at, str): + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error="promotion journal timestamps must be strings", + ) + return JournalLoadResult( + JournalLoadOutcome.VALID_JOURNAL, + journal=cls( + root=Path(root).resolve(), + run_id=run_id, + patch_digest=patch_digest, + state=state, + entries=entries, + created_at=created_at, + updated_at=updated_at, + schema_version=schema, + ), + ) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + return JournalLoadResult( + JournalLoadOutcome.CORRUPT_JOURNAL, + error=f"promotion journal is unreadable: {type(error).__name__}", ) - except (OSError, ValueError, TypeError, json.JSONDecodeError): - return None def clean(self) -> None: - """Remove completed journal file.""" + """Remove a completed journal file (only safe after COMMITTED/ROLLED_BACK).""" if self.path.is_file(): - import contextlib - with contextlib.suppress(OSError): self.path.unlink(missing_ok=True) + + +def _fsync_directory(directory: Path) -> None: + """Best-effort directory fsync so the journal rename is durable.""" + if os.name == "nt": + return + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + try: + descriptor = os.open(directory, flags) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) diff --git a/src/agentdiff/promotion/lock.py b/src/agentdiff/promotion/lock.py index 5c6f35e..67cb313 100644 --- a/src/agentdiff/promotion/lock.py +++ b/src/agentdiff/promotion/lock.py @@ -1,18 +1,27 @@ -"""Advisory workspace lease to prevent concurrent promotion races.""" +"""Advisory workspace lease to prevent concurrent promotion races. + +The lease coordinates only AgentDiff-aware promotion processes. The lock +file (``.agentdiff/promotion.lock``) is created once and **never unlinked**: +deleting the pathname while another process holds a lock on the old inode +would let two processes believe they both hold the lease. The lock is an +OS-level exclusive advisory lock (``flock`` on POSIX, ``LockFileEx``-style +byte locking on Windows) that is released automatically when the owning +process exits, so crashed promotions never leave a stale lease. +""" from __future__ import annotations -import contextlib import json import os import platform -import sys import time from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path from typing import Generator +_LOCK_BYTE = b"\x00" + class PromotionLockError(RuntimeError): """Raised when the repository lease cannot be acquired.""" @@ -29,69 +38,88 @@ def __init__(self, root: str | Path, run_id: str) -> None: self._fd: int | None = None def acquire(self, timeout_seconds: float = 5.0) -> None: - """Acquire an exclusive advisory lock with timeout.""" + """Acquire an exclusive advisory lock with timeout. + + The lock file is created once and intentionally left in place on + release; only the OS-level lock is dropped. + """ self.lock_dir.mkdir(mode=0o700, parents=True, exist_ok=True) deadline = time.monotonic() + timeout_seconds while True: + descriptor = None try: - flags = os.O_RDWR | os.O_CREAT - self._fd = os.open(str(self.lock_file), flags, 0o600) - if sys.platform == "win32": - import msvcrt - - msvcrt.locking(self._fd, msvcrt.LK_NBLCK, 1) - else: - import fcntl - - fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - - # Write lease metadata + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) + descriptor = os.open(str(self.lock_file), flags, 0o600) + self._lock_descriptor(descriptor) + # Lock acquired: write lease metadata while the lock is held. metadata = { "run_id": self.run_id, "pid": os.getpid(), "platform": platform.platform(), "acquired_at": datetime.now(timezone.utc).isoformat(), } - os.ftruncate(self._fd, 0) - os.lseek(self._fd, 0, os.SEEK_SET) - os.write(self._fd, json.dumps(metadata, indent=2).encode("utf-8")) + payload = json.dumps(metadata, indent=2).encode("utf-8") + os.ftruncate(descriptor, 0) + os.lseek(descriptor, 0, os.SEEK_SET) + os.write(descriptor, payload) + os.fsync(descriptor) + self._fd = descriptor return - except (BlockingIOError, OSError, PermissionError) as exc: - if self._fd is not None: + except (BlockingIOError, OSError, PermissionError): + if descriptor is not None: + import contextlib + with contextlib.suppress(OSError): - os.close(self._fd) - self._fd = None + os.close(descriptor) if time.monotonic() >= deadline: - msg = ( + raise PromotionLockError( f"could not acquire promotion lease on {self.lock_file}: " "another process is promoting" - ) - raise PromotionLockError(msg) from exc + ) from None time.sleep(0.1) + @staticmethod + def _lock_descriptor(descriptor: int) -> None: + """Apply the platform's exclusive advisory lock (non-blocking).""" + if os.name == "nt": + import msvcrt + + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) # type: ignore[attr-defined] + else: + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + def release(self) -> None: - """Release the advisory lock and clean up lease metadata.""" - if self._fd is not None: - try: - if sys.platform == "win32": - import msvcrt + """Unlock and close; the lock file itself is never removed. + + Removing the pathname would open the inode-reuse race described in + the module docstring, so the file is left in place as the stable + lock object for every future promotion. + """ + descriptor = self._fd + self._fd = None + if descriptor is None: + return + import contextlib - os.lseek(self._fd, 0, os.SEEK_SET) - with contextlib.suppress(OSError): - msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) - else: - import fcntl + try: + if os.name == "nt": + import msvcrt - with contextlib.suppress(OSError): - fcntl.flock(self._fd, fcntl.LOCK_UN) + os.lseek(descriptor, 0, os.SEEK_SET) + with contextlib.suppress(OSError): + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) # type: ignore[attr-defined] + else: + import fcntl - os.close(self._fd) - finally: - self._fd = None with contextlib.suppress(OSError): - if self.lock_file.is_file(): - self.lock_file.unlink(missing_ok=True) + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + with contextlib.suppress(OSError): + os.close(descriptor) @contextmanager def hold(self, timeout_seconds: float = 5.0) -> Generator[WorkspaceLease, None, None]: diff --git a/src/agentdiff/promotion/recovery.py b/src/agentdiff/promotion/recovery.py index 3fa6e68..96d4a4f 100644 --- a/src/agentdiff/promotion/recovery.py +++ b/src/agentdiff/promotion/recovery.py @@ -1,30 +1,77 @@ -"""Crash-recovery engine for interrupted promotion transactions.""" +"""Crash-recovery engine for interrupted promotion transactions. + +The promotion journal is untrusted persisted input. Every path it names is +validated before use: normalized relative paths that stay below the project +root (and below the approved backup directory for backups), with no symlink +parents and no special files. Backups are re-verified against the expected +SHA-256 / size / mode before any host file is overwritten, and restore copies +are written and fsynced from a validated descriptor so a concurrent swap of +the backup cannot be promoted into the host tree. + +Recovery follows the write-ahead entry state machine: + +- ``PREPARED`` → nothing was applied. +- ``APPLY_INTENT`` → the mutation may have happened; disambiguate by + comparing current host state to the expected base + and result, then restore (or confirm no-op). +- ``APPLIED`` → the mutation happened; restore. +- ``RECOVER_INTENT`` → a previous recovery was interrupted; re-run or + confirm the restore. +- ``RECOVERED`` → done. +- ``AMBIGUOUS`` → terminal fail-closed state; nothing is overwritten. + +If the journal cannot be parsed or host state cannot be disambiguated, a +``PromotionRecoveryError`` is raised and promotion is blocked. +""" from __future__ import annotations +import hashlib import os -from dataclasses import dataclass +import stat +import tempfile +from dataclasses import dataclass, field from pathlib import Path from typing import Any -from .journal import JournalState, PromotionJournal +from agentdiff.pathing import normalize_relative_path +from agentdiff.state import FilesystemScanner + +from .journal import ( + EntryState, + JournalEntry, + JournalLoadOutcome, + JournalState, + PromotionJournal, +) + +_CHUNK_SIZE = 1024 * 1024 +_PROMOTE_TEMP_PREFIX = ".agentdiff-promote-" + + +class PromotionRecoveryError(RuntimeError): + """Raised when recovery cannot establish or restore a safe state.""" @dataclass class RecoveryReport: + """Result of one deterministic recovery pass.""" + run_id: str status: str - restored: list[str] - cleaned: list[str] - errors: list[str] + restored: list[str] = field(default_factory=list) + cleaned: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + ambiguous: list[str] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return { "run_id": self.run_id, "status": self.status, - "restored": self.restored, - "cleaned": self.cleaned, - "errors": self.errors, + "restored": list(self.restored), + "cleaned": list(self.cleaned), + "errors": list(self.errors), + "ambiguous": list(self.ambiguous), } @@ -33,56 +80,450 @@ class PromotionRecovery: def __init__(self, root: str | Path) -> None: self.root = Path(root).resolve() + self.scanner = FilesystemScanner(self.root) def check_and_recover(self) -> RecoveryReport | None: - """Check for an interrupted journal and restore host state if needed.""" - journal = PromotionJournal.load(self.root) - if journal is None or journal.state in {JournalState.COMMITTED, JournalState.ROLLED_BACK}: - return None + """Check for an interrupted journal and restore host state if needed. - report = RecoveryReport( - run_id=journal.run_id, - status="IN_PROGRESS", - restored=[], - cleaned=[], - errors=[], - ) - - if journal.state in {JournalState.APPLYING, JournalState.RECOVERY_REQUIRED}: - for entry in journal.entries: - if not entry.applied: - continue + Raises :class:`PromotionRecoveryError` when the journal is corrupt, + host state is ambiguous, or a restore cannot be completed safely. + Returns ``None`` when no journal exists, and a report otherwise. + """ + loaded = PromotionJournal.load(self.root) + if loaded.outcome is JournalLoadOutcome.NO_JOURNAL: + return None + if loaded.outcome is JournalLoadOutcome.CORRUPT_JOURNAL: + raise PromotionRecoveryError( + f"promotion journal exists but recovery state cannot be established: {loaded.error}" + ) + journal = loaded.journal + assert journal is not None - host_path = self.root / entry.path - if entry.change_type == "created": - try: - if host_path.is_file(): - host_path.unlink() - report.cleaned.append(entry.path) - except OSError as exc: - report.errors.append(f"failed to remove created file {entry.path}: {exc}") - - elif entry.change_type in {"modified", "deleted"} and entry.backup_relpath: - backup_path = self.root / entry.backup_relpath - if backup_path.is_file(): - try: - host_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - os.replace(str(backup_path), str(host_path)) - report.restored.append(entry.path) - except OSError as exc: - report.errors.append(f"failed to restore {entry.path}: {exc}") - - if report.errors: - journal.state = JournalState.RECOVERY_REQUIRED - report.status = "RECOVERY_FAILED" - else: - journal.state = JournalState.ROLLED_BACK - report.status = "RECOVERED" - journal.persist() + if journal.state in {JournalState.COMMITTED, JournalState.ROLLED_BACK}: + self._validate_terminal_consistency(journal) + journal.clean() + return RecoveryReport( + run_id=journal.run_id, + status="NOTHING_TO_DO", + ) - elif journal.state in {JournalState.PLAN_RECORDED, JournalState.STAGED}: - # Safe to discard unapplied staging + if journal.state in {JournalState.PLAN_RECORDED, JournalState.STAGED}: + if any(entry.state is not EntryState.PREPARED for entry in journal.entries): + raise PromotionRecoveryError( + "promotion journal is internally inconsistent: " + "unapplied transaction contains progressed entries" + ) journal.clean() - report.status = "CLEANED_UNAPPLIED" + return RecoveryReport( + run_id=journal.run_id, + status="CLEANED_UNAPPLIED", + ) + + if journal.state not in {JournalState.APPLYING, JournalState.RECOVERY_REQUIRED}: + raise PromotionRecoveryError( + f"promotion journal is in an unrecoverable state: {journal.state.value}" + ) + report = RecoveryReport(run_id=journal.run_id, status="IN_PROGRESS") + try: + self._recover_entries(journal, report) + except PromotionRecoveryError: + journal.state = JournalState.RECOVERY_FAILED + journal.persist() + raise + if report.ambiguous: + journal.state = JournalState.RECOVERY_FAILED + journal.persist() + raise PromotionRecoveryError( + "promotion recovery is ambiguous for: " + + ", ".join(report.ambiguous) + + "; refusing to overwrite host state" + ) + if report.errors: + journal.state = JournalState.RECOVERY_FAILED + journal.persist() + raise PromotionRecoveryError( + "promotion recovery failed for: " + ", ".join(report.errors) + ) + journal.state = JournalState.ROLLED_BACK + journal.persist() + report.status = "RECOVERED" return report + + # ------------------------------------------------------------------ + # Entry state machine + # ------------------------------------------------------------------ + + def _recover_entries(self, journal: PromotionJournal, report: RecoveryReport) -> None: + for entry in journal.entries: + self._validate_entry_paths(journal, entry) + if entry.state is EntryState.PREPARED or entry.state is EntryState.RECOVERED: + continue + if entry.state is EntryState.AMBIGUOUS: + report.ambiguous.append(entry.path) + continue + current = self._capture_host(entry.path) + if entry.change_type == "created": + self._recover_created(entry, current, report) + elif entry.change_type in {"modified", "deleted"}: + self._recover_present_or_deleted(entry, current, report) + else: # pragma: no cover - guarded by journal validation + raise PromotionRecoveryError( + f"unsupported journal change type: {entry.change_type}" + ) + + def _recover_created(self, entry: JournalEntry, current: Any, report: RecoveryReport) -> None: + """A created file is rolled back by unlinking the promoted result. + + Content match is sufficient to decide removal: a crash between the + ``os.link`` and the follow-up ``chmod`` can leave result content with + a temporary mode, and the file must be removed either way. + """ + expected_result = self._content_matches(current, entry.result_sha256) + if entry.state is EntryState.APPLY_INTENT: + if current is None: + # Host is absent -> the mutation never happened. + entry.state = EntryState.PREPARED + return + if expected_result: + entry.state = EntryState.RECOVER_INTENT + else: + report.ambiguous.append(entry.path) + entry.state = EntryState.AMBIGUOUS + return + if entry.state in {EntryState.APPLIED, EntryState.RECOVER_INTENT}: + if current is None: + entry.state = EntryState.RECOVERED + return + if not expected_result: + report.ambiguous.append(entry.path) + entry.state = EntryState.AMBIGUOUS + return + self._remove_created_file(entry) + self._cleanup_promote_temps(entry) + entry.state = EntryState.RECOVERED + report.cleaned.append(entry.path) + + def _recover_present_or_deleted( + self, entry: JournalEntry, current: Any, report: RecoveryReport + ) -> None: + """A modified or deleted file is rolled back by restoring the base copy. + + Content-only base matches are treated as a partially completed + restore (``os.replace`` done, ``chmod`` pending): the restore is + repeated, which is idempotent, instead of declaring ambiguity. + """ + expected_base_content = self._content_matches(current, entry.base_sha256) + expected_base = self._file_matches(current, entry.base_sha256, entry.base_mode) + expected_result = self._content_matches(current, entry.result_sha256) + host_absent = current is None + result_absent = host_absent if entry.change_type == "deleted" else False + + if entry.state is EntryState.APPLY_INTENT: + if expected_base: + entry.state = EntryState.PREPARED + return + if result_absent or expected_result or expected_base_content: + # Mutation occurred (or a restore is partially complete): + # restore the verified base copy. + entry.state = EntryState.RECOVER_INTENT + else: + report.ambiguous.append(entry.path) + entry.state = EntryState.AMBIGUOUS + return + if entry.state in {EntryState.APPLIED, EntryState.RECOVER_INTENT}: + if expected_base: + # Restore already complete; re-apply the mode in case a crash + # interrupted the chmod between replace and verification. + if not self._file_matches(current, entry.base_sha256, entry.base_mode): + self._apply_base_mode(entry) + rechecked = self._capture_host(entry.path) + if not self._file_matches(rechecked, entry.base_sha256, entry.base_mode): + raise PromotionRecoveryError( + f"restored mode does not match expected base state: {entry.path}" + ) + entry.state = EntryState.RECOVERED + return + if entry.change_type == "deleted" and not host_absent and not expected_result: + report.ambiguous.append(entry.path) + entry.state = EntryState.AMBIGUOUS + return + if entry.change_type == "modified" and host_absent: + report.ambiguous.append(entry.path) + entry.state = EntryState.AMBIGUOUS + return + self._restore_from_backup(entry) + entry.state = EntryState.RECOVERED + report.restored.append(entry.path) + + # ------------------------------------------------------------------ + # Host capture and identity helpers + # ------------------------------------------------------------------ + + def _capture_host(self, relative: str) -> Any: + try: + return self.scanner.capture_one(relative) + except (OSError, ValueError): + return None + + @staticmethod + def _file_matches(record: Any, sha256: str | None, mode: int | None) -> bool: + if record is None or record.kind != "file" or record.link_count != 1: + return False + if record.sha256 is None or sha256 is None: + return False + if record.sha256 != sha256: + return False + # Mode is only meaningful/restorable on POSIX; Windows st_mode is not + # a security property and is not preserved by restore. + if mode is None or os.name == "nt": + return True + return record.mode == mode + + @staticmethod + def _content_matches(record: Any, sha256: str | None) -> bool: + if record is None or record.kind != "file" or record.link_count != 1: + return False + return record.sha256 is not None and record.sha256 == sha256 + + def _apply_base_mode(self, entry: JournalEntry) -> None: + """Restore only the base mode on an already-restored host file.""" + if os.name == "nt" or entry.base_mode is None: + return + host_path = self._validate_target_path(self.root, entry.path) + try: + host_path.chmod(stat.S_IMODE(entry.base_mode)) + except OSError as error: + raise PromotionRecoveryError( + f"failed to restore mode for {entry.path}: {error}" + ) from error + + # ------------------------------------------------------------------ + # Path validation (journal content is untrusted) + # ------------------------------------------------------------------ + + def _validate_entry_paths(self, journal: PromotionJournal, entry: JournalEntry) -> None: + """Reject unsafe journal paths before touching the host filesystem.""" + try: + normalized = normalize_relative_path(entry.path) + except ValueError as error: + raise PromotionRecoveryError( + f"unsafe promotion journal path: {entry.path!r}" + ) from error + if normalized != entry.path: + raise PromotionRecoveryError(f"unsafe promotion journal path: {entry.path!r}") + if entry.path == ".agentdiff" or entry.path.startswith(".agentdiff/"): + raise PromotionRecoveryError( + f"promotion journal path targets AgentDiff internal state: {entry.path!r}" + ) + # Validate the host target and every parent now, so a symlink parent + # is rejected deterministically instead of surfacing as ambiguity. + self._validate_target_path(self.root, entry.path) + if entry.change_type in {"modified", "deleted"}: + if entry.backup_relpath is None: + raise PromotionRecoveryError( + f"promotion journal entry {entry.path!r} has no backup path" + ) + self._validate_backup_path(journal, entry) + + def _validate_backup_path(self, journal: PromotionJournal, entry: JournalEntry) -> None: + backup = entry.backup_relpath + assert backup is not None + try: + normalized = normalize_relative_path(backup) + except ValueError as error: + raise PromotionRecoveryError(f"unsafe backup path in journal: {backup!r}") from error + if normalized != backup: + raise PromotionRecoveryError(f"unsafe backup path in journal: {backup!r}") + approved_prefix = f".agentdiff/backups/{journal.run_id}/" + if not backup.startswith(approved_prefix): + raise PromotionRecoveryError( + f"backup path escapes the approved backup directory: {backup!r}" + ) + backup_root = self.root / ".agentdiff" / "backups" / journal.run_id + self._validate_target_path(self.root, backup, approved_root=backup_root) + + def _validate_target_path( + self, + root: Path, + relative: str, + *, + approved_root: Path | None = None, + ) -> Path: + try: + normalized = normalize_relative_path(relative) + except ValueError as error: + raise PromotionRecoveryError(f"unsafe path in journal: {relative!r}") from error + if normalized != relative: + raise PromotionRecoveryError(f"unsafe path in journal: {relative!r}") + target = root.joinpath(*normalized.split("/")) + if approved_root is not None: + try: + target.relative_to(approved_root) + except ValueError as error: + raise PromotionRecoveryError( + f"path escapes the approved directory: {relative!r}" + ) from error + current = root + for part in normalized.split("/")[:-1]: + current /= part + try: + info = current.lstat() + except FileNotFoundError as error: + raise PromotionRecoveryError( + f"journal parent directory is missing: {current}" + ) from error + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise PromotionRecoveryError(f"journal parent is not a real directory: {current}") + return target + + # ------------------------------------------------------------------ + # Mutations + # ------------------------------------------------------------------ + + def _remove_created_file(self, entry: JournalEntry) -> None: + target = self._validate_target_path(self.root, entry.path) + info = target.lstat() + if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise PromotionRecoveryError(f"created target is not a regular file: {entry.path}") + if info.st_nlink != 1: + raise PromotionRecoveryError(f"created target has unexpected link count: {entry.path}") + try: + target.unlink() + except OSError as error: + raise PromotionRecoveryError( + f"failed to remove created file {entry.path}: {error}" + ) from error + + def _cleanup_promote_temps(self, entry: JournalEntry) -> None: + """Remove leftover promotion temp files next to a recovered created file. + + ``_atomic_create`` links a fully-written temp file into place and then + unlinks the temp. A crash between the two leaves the temp behind with + the same inode; it is only removed here, after the created target has + been removed, and only for files matching our exact temp prefix. + """ + target = self._validate_target_path(self.root, entry.path) + self._cleanup_restore_temps(target.parent) + + @staticmethod + def _cleanup_restore_temps(directory: Path) -> None: + """Remove stale ``.agentdiff-promote-*`` temp files inside one directory.""" + try: + for candidate in directory.iterdir(): + if not candidate.name.startswith(_PROMOTE_TEMP_PREFIX): + continue + info = candidate.lstat() + if stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode): + candidate.unlink(missing_ok=True) + except OSError: + raise PromotionRecoveryError( + f"failed to clean promotion temp files in {directory}" + ) from None + + def _restore_from_backup(self, entry: JournalEntry) -> None: + """Restore base content and mode from a re-verified backup copy. + + The backup is copied through a validated descriptor while its digest + and size are recomputed; the verified bytes are written to an fsynced + temp file that is then atomically replaced onto the host path, so a + concurrent swap of the backup can never be moved into the tree. + """ + backup = entry.backup_relpath + assert backup is not None + backup_path = self._validate_target_path(self.root, backup) + host_path = self._validate_target_path(self.root, entry.path) + + backup_info = backup_path.lstat() + if not stat.S_ISREG(backup_info.st_mode) or stat.S_ISLNK(backup_info.st_mode): + raise PromotionRecoveryError(f"backup is not a regular file: {entry.path}") + if backup_info.st_nlink != 1: + raise PromotionRecoveryError(f"backup has unexpected link count: {entry.path}") + + host_parent = host_path.parent + if not host_parent.is_dir() or host_parent.is_symlink(): + raise PromotionRecoveryError(f"host parent is not a real directory: {entry.path}") + self._cleanup_restore_temps(host_parent) + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + source_fd = os.open(backup_path, flags) + except OSError as error: + raise PromotionRecoveryError(f"backup is unreadable: {entry.path}") from error + temp_path: Path | None = None + try: + opened = os.fstat(source_fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_dev != backup_info.st_dev + or opened.st_ino != backup_info.st_ino + ): + raise PromotionRecoveryError(f"backup identity changed while opening: {entry.path}") + temp_descriptor, temp_name = tempfile.mkstemp( + prefix=_PROMOTE_TEMP_PREFIX, + suffix=".restore", + dir=str(host_parent), + ) + temp_path = Path(temp_name) + digest = hashlib.sha256() + size = 0 + with ( + os.fdopen(source_fd, "rb", closefd=False) as source, + os.fdopen(temp_descriptor, "wb") as destination, + ): + while chunk := source.read(_CHUNK_SIZE): + digest.update(chunk) + size += len(chunk) + destination.write(chunk) + destination.flush() + os.fsync(destination.fileno()) + finished = os.fstat(source_fd) + if ( + finished.st_dev != opened.st_dev + or finished.st_ino != opened.st_ino + or finished.st_size != opened.st_size + or finished.st_mtime_ns != opened.st_mtime_ns + ): + raise PromotionRecoveryError(f"backup changed while restoring: {entry.path}") + if entry.base_sha256 is not None and digest.hexdigest() != entry.base_sha256: + raise PromotionRecoveryError( + f"backup digest mismatch for {entry.path}; refusing to overwrite host state" + ) + if entry.base_size is not None and size != entry.base_size: + raise PromotionRecoveryError( + f"backup size mismatch for {entry.path}; refusing to overwrite host state" + ) + os.replace(temp_path, host_path) + temp_path = None + if os.name != "nt" and entry.base_mode is not None: + host_path.chmod(stat.S_IMODE(entry.base_mode)) + except OSError as error: + raise PromotionRecoveryError(f"failed to restore {entry.path}: {error}") from error + finally: + os.close(source_fd) + if temp_path is not None: + temp_path.unlink(missing_ok=True) + restored = self._capture_host(entry.path) + if not self._file_matches(restored, entry.base_sha256, entry.base_mode): + raise PromotionRecoveryError( + f"restored file does not match expected base state: {entry.path}" + ) + + def _validate_terminal_consistency(self, journal: PromotionJournal) -> None: + """A COMMITTED/ROLLED_BACK journal must be internally consistent.""" + if journal.state is JournalState.COMMITTED: + progressed = [ + entry.path + for entry in journal.entries + if entry.state not in {EntryState.APPLIED, EntryState.RECOVERED} + ] + else: # ROLLED_BACK + progressed = [ + entry.path for entry in journal.entries if entry.state is not EntryState.RECOVERED + ] + if progressed: + raise PromotionRecoveryError( + "promotion journal is internally inconsistent: " + f"{journal.state.value} contains unconfirmed entries: " + ", ".join(progressed) + ) diff --git a/src/agentdiff/promotion/staging.py b/src/agentdiff/promotion/staging.py index 90dcc0b..082c935 100644 --- a/src/agentdiff/promotion/staging.py +++ b/src/agentdiff/promotion/staging.py @@ -9,6 +9,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from agentdiff.pathing import normalize_relative_path + if TYPE_CHECKING: from agentdiff.evidence import PatchBundle, PatchEntry @@ -35,56 +37,174 @@ def stage_entry(self, bundle: PatchBundle, entry: PatchEntry) -> Path: raise ValueError("deleted entries have no staged content") source_path = bundle.entry_path(entry) - if not source_path.is_file(): + if not source_path.is_file() or source_path.is_symlink(): raise FileNotFoundError(f"missing patch artifact for {entry.path}") - target_staged = self.staging_dir / entry.path + normalized = normalize_relative_path(entry.path) + if normalized != entry.path: + raise ValueError(f"unsafe staging path: {entry.path!r}") + target_staged = self.staging_dir.joinpath(*normalized.split("/")) target_staged.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + self._assert_real_directory(target_staged.parent, self.staging_dir) + source_info = source_path.lstat() + if not stat.S_ISREG(source_info.st_mode) or source_info.st_nlink != 1: + raise RuntimeError("staging source is not a single-link regular file") hasher = hashlib.sha256() - with open(source_path, "rb") as src, open(target_staged, "wb") as dst: - while chunk := src.read(_CHUNK_SIZE): - dst.write(chunk) - hasher.update(chunk) - dst.flush() - os.fsync(dst.fileno()) + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + source_fd = os.open(source_path, flags) + try: + opened = os.fstat(source_fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_dev != source_info.st_dev + or opened.st_ino != source_info.st_ino + ): + raise RuntimeError("staging source changed while opening") + with ( + os.fdopen(source_fd, "rb", closefd=False) as src, + target_staged.open("wb") as dst, + ): + while chunk := src.read(_CHUNK_SIZE): + dst.write(chunk) + hasher.update(chunk) + dst.flush() + os.fsync(dst.fileno()) + finished = os.fstat(source_fd) + if ( + finished.st_dev != opened.st_dev + or finished.st_ino != opened.st_ino + or finished.st_size != opened.st_size + or finished.st_mtime_ns != opened.st_mtime_ns + ): + raise RuntimeError("staging source changed while reading") + finally: + os.close(source_fd) digest = hasher.hexdigest() if entry.result_sha256 is not None and digest != entry.result_sha256: - msg = ( + raise ValueError( f"staged digest mismatch for {entry.path}: " f"expected {entry.result_sha256}, got {digest}" ) - raise ValueError(msg) + if entry.size is not None and target_staged.stat().st_size != entry.size: + raise ValueError(f"staged size mismatch for {entry.path}") mode = entry.result_mode if entry.result_mode is not None else 0o644 - import contextlib - - with contextlib.suppress(OSError): - target_staged.chmod(stat.S_IMODE(mode)) + target_staged.chmod(stat.S_IMODE(mode)) + _fsync_directory(target_staged.parent) return target_staged def backup_host_file(self, relpath: str) -> Path | None: - """Backup existing host file before mutation.""" - host_path = self.root / relpath - if not host_path.is_file() or host_path.is_symlink(): + """Backup an existing host regular file before mutation. + + The source is opened without following links and its opened identity + (device, inode, file type) must match the pre-open ``lstat`` so a + symlink substitution during the copy cannot be recorded as the base. + """ + normalized = normalize_relative_path(relpath) + if normalized != relpath: + raise ValueError(f"unsafe backup path: {relpath!r}") + host_path = self.root.joinpath(*normalized.split("/")) + if self._has_symlink_parent(host_path): + raise RuntimeError(f"unsafe backup parent for {relpath}") + try: + host_info = host_path.lstat() + except FileNotFoundError: return None + if stat.S_ISLNK(host_info.st_mode) or not stat.S_ISREG(host_info.st_mode): + return None + if host_info.st_nlink != 1: + raise RuntimeError(f"host file has unexpected link count: {relpath}") - backup_path = self.backup_dir / relpath + backup_path = self.backup_dir.joinpath(*normalized.split("/")) backup_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - - with open(host_path, "rb") as src, open(backup_path, "wb") as dst: - while chunk := src.read(_CHUNK_SIZE): - dst.write(chunk) - dst.flush() - os.fsync(dst.fileno()) + self._assert_real_directory(backup_path.parent, self.backup_dir) + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + source_fd = os.open(host_path, flags) + try: + opened = os.fstat(source_fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_dev != host_info.st_dev + or opened.st_ino != host_info.st_ino + ): + raise RuntimeError("host file changed while opening backup") + with ( + os.fdopen(source_fd, "rb", closefd=False) as src, + backup_path.open("xb") as dst, + ): + while chunk := src.read(_CHUNK_SIZE): + dst.write(chunk) + dst.flush() + os.fsync(dst.fileno()) + finished = os.fstat(source_fd) + if ( + finished.st_dev != opened.st_dev + or finished.st_ino != opened.st_ino + or finished.st_size != opened.st_size + or finished.st_mtime_ns != opened.st_mtime_ns + ): + raise RuntimeError("host file changed while backing up") + finally: + os.close(source_fd) + if os.name != "nt": + backup_path.chmod(stat.S_IMODE(host_info.st_mode)) + _fsync_directory(backup_path.parent) return backup_path + @staticmethod + def _has_symlink_parent(target: Path) -> bool: + current = target.parent + while True: + try: + info = current.lstat() + except FileNotFoundError: + return False + if stat.S_ISLNK(info.st_mode): + return True + if current == current.parent: + return False + current = current.parent + + @staticmethod + def _assert_real_directory(path: Path, approved_root: Path) -> None: + try: + path.relative_to(approved_root) + except ValueError as error: + raise RuntimeError("staging path escapes the approved directory") from error + try: + info = path.lstat() + except FileNotFoundError as error: + raise RuntimeError("staging directory disappeared") from error + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise RuntimeError("staging path is not a real directory") + def clean(self) -> None: """Clean staging and backup temporary artifacts.""" if self.staging_dir.exists(): shutil.rmtree(self.staging_dir, ignore_errors=True) if self.backup_dir.exists(): shutil.rmtree(self.backup_dir, ignore_errors=True) + + +def _fsync_directory(directory: Path) -> None: + """Best-effort directory fsync so staged renames are durable.""" + if os.name == "nt": + return + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + try: + descriptor = os.open(directory, flags) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) diff --git a/src/agentdiff/proof/__init__.py b/src/agentdiff/proof/__init__.py index 7331aa6..4248c2c 100644 --- a/src/agentdiff/proof/__init__.py +++ b/src/agentdiff/proof/__init__.py @@ -1,6 +1,32 @@ """Deterministic clean-room proof for sealed AgentDiff patches.""" -from .engine import ProofEngine -from .models import ProofPhaseResult, ProofResult, ProofVerdict +from .engine import ProofEngine, compute_proof_strength +from .models import ( + ProofPhaseResult, + ProofResult, + ProofStrengthLabel, + ProofStrengthLevel, + ProofVerdict, + VerifierIndependence, + strength_label, +) +from .verifier_files import ( + VerifierMutationReport, + analyze_verifier_mutations, + is_verifier_related, +) -__all__ = ["ProofEngine", "ProofPhaseResult", "ProofResult", "ProofVerdict"] +__all__ = [ + "ProofEngine", + "ProofPhaseResult", + "ProofResult", + "ProofStrengthLabel", + "ProofStrengthLevel", + "ProofVerdict", + "VerifierIndependence", + "VerifierMutationReport", + "analyze_verifier_mutations", + "compute_proof_strength", + "is_verifier_related", + "strength_label", +] diff --git a/src/agentdiff/proof/engine.py b/src/agentdiff/proof/engine.py index c7521f9..85c9c90 100644 --- a/src/agentdiff/proof/engine.py +++ b/src/agentdiff/proof/engine.py @@ -1,7 +1,26 @@ -"""Clean-room proof orchestration bound to sealed run evidence.""" +"""Clean-room proof orchestration bound to sealed run evidence. + +Two distinguishable verification concepts are produced: + +- **Patched verification**: the complete patched project tests run in the + clean room, exactly as today. +- **Baseline verification**: after the patched run, verifier-related files + (tests, fixtures, runner config, manifests, lockfiles, CI workflows) are + deterministically restored to their sealed **pre-run** versions, and the + trusted test commands run again. This tests patched product code against + the original trusted tests, so an agent that weakened the test suite to + make it pass cannot hide behind its own modifications. + +The baseline result, verifier-mutation report, and proof strength are +explanatory metadata; the PROVEN / NOT_PROVEN verdict stays deterministic +and fails closed whenever the baseline cannot confirm the patched run. +""" from __future__ import annotations +import hashlib +import os +import stat import tempfile from pathlib import Path from typing import Any, Callable @@ -13,11 +32,56 @@ from .environment import DockerProofEnvironment from .hidden_state import hidden_state_result -from .models import ProofPhaseResult, ProofResult, ProofVerdict +from .models import ( + ProofPhaseResult, + ProofResult, + ProofStrengthLabel, + ProofStrengthLevel, + ProofVerdict, + VerifierIndependence, + strength_label, +) from .plan import select_trusted_verification_plan +from .verifier_files import analyze_verifier_mutations, is_verifier_related EnvironmentFactory = Callable[..., Any] +_CHUNK_SIZE = 1024 * 1024 + + +def compute_proof_strength( + *, + clean_environment: str, + verification_confirmed: bool, + baseline_verifier: str, + baseline_available: bool, + verifier_files_changed: int, +) -> tuple[ProofStrengthLevel, ProofStrengthLabel, VerifierIndependence]: + """Deterministically derive proof-strength metadata from recorded evidence. + + Levels are cumulative: L3 implies L2/L1/L0. The label is a human summary + of the level; neither the level nor the verdict is an LLM decision. + """ + level = ProofStrengthLevel.L0_EXECUTION_ONLY + if clean_environment == "PASS": + level = ProofStrengthLevel.L1_CLEAN_ROOM + if verification_confirmed: + level = ProofStrengthLevel.L2_TRUSTED_COMMAND + baseline_confirms = baseline_available and baseline_verifier == "PASS" + if baseline_confirms: + level = ProofStrengthLevel.L3_BASELINE_VERIFIER + if not baseline_available: + independence = VerifierIndependence.WEAK + elif baseline_verifier == "PASS": + independence = VerifierIndependence.STRONG + elif verifier_files_changed == 0: + # No verifier file was touched and the baseline still could not run; + # treat the verification as unconfirmed rather than independent. + independence = VerifierIndependence.REVIEW + else: + independence = VerifierIndependence.REVIEW + return level, strength_label(level), independence + class ProofEngine: """Rebuild a base-plus-patch workspace and verify it without LLM decisions.""" @@ -60,6 +124,10 @@ def prove(self, *, timeout_seconds: float = 900.0) -> ProofResult: "clean_environment": False, } clean_environment = "FAIL" + verifier_mutations = analyze_verifier_mutations( + (entry.path, entry.change_type) for entry in bundle.manifest.entries + ) + baseline_available = self._base_has_verifier_files() with tempfile.TemporaryDirectory(prefix="agentdiff-proof-") as temporary: workspace = Path(temporary) / "workspace" @@ -92,6 +160,7 @@ def prove(self, *, timeout_seconds: float = 900.0) -> ProofResult: "build": [redact_argv(command) for command in plan.build], "tests": [redact_argv(command) for command in plan.tests], } + environment_payload["verifier_mutations"] = verifier_mutations.to_dict() clean_environment = "PASS" commands = ( *(("dependency_setup", command) for command in plan.setup), @@ -114,6 +183,25 @@ def prove(self, *, timeout_seconds: float = 900.0) -> ProofResult: f"{phase_name} failed with return code {phase.returncode}" ) break + + # 4. Baseline verification: patched product code against + # the ORIGINAL trusted tests, independent of any + # agent-modified verifier files. + baseline_verifier, baseline_phases, baseline_reasons, overlay_digest = ( + self._run_baseline_verification( + environment, + workspace, + bundle, + plan.tests, + timeout_seconds=timeout_seconds, + baseline_available=baseline_available, + ) + ) + phases.extend(baseline_phases) + reasons.extend(baseline_reasons) + environment_payload["baseline_verifier"] = baseline_verifier + environment_payload["baseline_available"] = baseline_available + environment_payload["baseline_overlay_sha256"] = overlay_digest finally: environment.close() except (OSError, RuntimeError, TypeError, ValueError) as error: @@ -127,6 +215,22 @@ def prove(self, *, timeout_seconds: float = 900.0) -> ProofResult: reasons.append("sealed patch evidence is incomplete") all_phases_pass = bool(phases) and all(phase.passed for phase in phases) has_test_phase = any(phase.phase == "tests" for phase in phases) + baseline_verifier = str(environment_payload.get("baseline_verifier", "SKIPPED")) + baseline_required = verifier_mutations.any_modification + baseline_confirms = not baseline_required or ( + baseline_available and baseline_verifier == "PASS" + ) + if baseline_required and not baseline_confirms: + if not baseline_available: + reasons.append( + "verifier-related files were modified but no baseline verifier " + "is available from the pre-run state" + ) + else: + reasons.append( + "baseline verifier did not pass; patched tests cannot be " + "trusted against modified test/verifier code" + ) proven = ( original_passed and policy_allowed @@ -135,12 +239,22 @@ def prove(self, *, timeout_seconds: float = 900.0) -> ProofResult: and all_phases_pass and has_test_phase and plan.trusted + and baseline_confirms and not reasons ) hidden_state = hidden_state_result( original_passed=original_passed, phases=tuple(phases), ) + tests_phase = next((phase for phase in phases if phase.phase == "tests"), None) + baseline_phase = next((phase for phase in phases if phase.phase == "baseline_tests"), None) + level, label, independence = compute_proof_strength( + clean_environment=clean_environment, + verification_confirmed=plan.trusted, + baseline_verifier=baseline_verifier, + baseline_available=baseline_available, + verifier_files_changed=verifier_mutations.modified_count, + ) proof = ProofResult( run_id=self.store.run_id, verdict=ProofVerdict.PROVEN if proven else ProofVerdict.NOT_PROVEN, @@ -161,14 +275,180 @@ def prove(self, *, timeout_seconds: float = 900.0) -> ProofResult: reasons=tuple(dict.fromkeys(reasons)), verification_source=plan.source, verification_digest=plan.plan_digest, - trusted_plan=plan.trusted, + verification_confirmed=plan.trusted, + baseline_verifier=baseline_verifier, + baseline_tests_passed=( + baseline_phase.tests_passed if baseline_phase is not None else None + ), + baseline_tests_total=( + baseline_phase.tests_total if baseline_phase is not None else None + ), + patched_tests_passed=tests_phase.tests_passed if tests_phase is not None else None, + patched_tests_total=tests_phase.tests_total if tests_phase is not None else None, + verifier_files_changed=verifier_mutations.modified_count, + verifier_changes=( + *verifier_mutations.existing_changed, + *verifier_mutations.added, + *verifier_mutations.removed, + ), + baseline_available=baseline_available, + verifier_independence=independence.value, + proof_strength=level.value, + proof_strength_label=label.value, ) self.store.write_json_path("proof/environment.json", environment_payload) self.store.write_json_path("proof/result.json", proof.to_dict()) self.store.seal_extension("proof", ("environment.json", "result.json")) return proof + def _run_baseline_verification( + self, + environment: Any, + workspace: Path, + bundle: PatchBundle, + test_commands: tuple[tuple[str, ...], ...], + *, + timeout_seconds: float, + baseline_available: bool, + ) -> tuple[str, list[ProofPhaseResult], list[str], str]: + """Overlay base verifier files and re-run the trusted test commands. + + Returns ``(status, phases, reasons, overlay_digest)`` where status is + PASS, FAIL, or SKIPPED. The overlay is applied in place: + verifier-related files are restored to their sealed pre-run versions + (including files the patch deleted or created), so the baseline run + executes the original tests against the patched product code. + """ + if not baseline_available or not test_commands: + return "SKIPPED", [], [], "" + try: + overlay_digest = self._overlay_baseline_verifier_files(workspace, bundle) + except (OSError, RuntimeError, ValueError) as error: + return "FAIL", [], [f"baseline verifier overlay failed: {error}"], "" + phases: list[ProofPhaseResult] = [] + reasons: list[str] = [] + for command in test_commands: + phase = environment.run_phase( + "baseline_tests", + command, + timeout_seconds=timeout_seconds, + ) + phases.append(phase) + if not phase.passed: + reasons.append(f"baseline_tests failed with return code {phase.returncode}") + break + status = "PASS" if phases and all(phase.passed for phase in phases) else "FAIL" + return status, phases, reasons, overlay_digest + + def _overlay_baseline_verifier_files( + self, + workspace: Path, + bundle: PatchBundle, + ) -> str: + """Restore sealed pre-run verifier files over the patched workspace. + + Returns a digest of the restored file list for evidence. Only paths + classified as verifier-related are touched; all writes are fsynced + and atomic replacements, and every parent directory is validated. + """ + raw = self.store.read_json_path("source/manifest.json") + captured = raw.get("captured", []) if isinstance(raw, dict) else [] + base_verifier_paths = sorted( + path for path in captured if isinstance(path, str) and is_verifier_related(path) + ) + hasher = hashlib.sha256() + for path in base_verifier_paths: + source = self.store.artifact_path(f"source/files/{path}") + info = source.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise RuntimeError(f"baseline source is not a regular file: {path}") + target = _safe_workspace_target(workspace, path, create_parents=True) + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + source_fd = os.open(source, flags) + temporary = target.parent / ( + f".agentdiff-baseline-{hashlib.sha256(path.encode()).hexdigest()[:16]}.tmp" + ) + try: + opened = os.fstat(source_fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_dev != info.st_dev + or opened.st_ino != info.st_ino + ): + raise RuntimeError(f"baseline source changed while opening: {path}") + digest = hashlib.sha256() + with ( + os.fdopen(source_fd, "rb", closefd=False) as src, + temporary.open("wb") as dst, + ): + while chunk := src.read(_CHUNK_SIZE): + digest.update(chunk) + dst.write(chunk) + dst.flush() + os.fsync(dst.fileno()) + finished = os.fstat(source_fd) + if ( + finished.st_dev != opened.st_dev + or finished.st_ino != opened.st_ino + or finished.st_size != opened.st_size + or finished.st_mtime_ns != opened.st_mtime_ns + ): + raise RuntimeError(f"baseline source changed while copying: {path}") + os.replace(temporary, target) + if os.name != "nt": + target.chmod(stat.S_IMODE(info.st_mode)) + finally: + os.close(source_fd) + temporary.unlink(missing_ok=True) + hasher.update(path.encode("utf-8")) + # Remove verifier files the patch added (they are not part of the + # original trusted verifier and must not run during baseline tests). + base_set = set(base_verifier_paths) + for entry in bundle.manifest.entries: + if entry.change_type != "created" or not is_verifier_related(entry.path): + continue + if entry.path in base_set: + continue + target = _safe_workspace_target(workspace, entry.path, create_parents=False) + if target.is_file() and not target.is_symlink(): + target.unlink(missing_ok=True) + hasher.update(f"removed:{entry.path}".encode("utf-8")) + return hasher.hexdigest() + + def _base_has_verifier_files(self) -> bool: + try: + raw = self.store.read_json_path("source/manifest.json") + except (OSError, ValueError, TypeError, FileNotFoundError): + return False + captured = raw.get("captured", []) if isinstance(raw, dict) else [] + return any(isinstance(path, str) and is_verifier_related(path) for path in captured) + @staticmethod def _score(result: dict[str, Any], name: str) -> int: raw = result.get(name, {}) return int(raw.get("score", 0)) if isinstance(raw, dict) else 0 + + +def _safe_workspace_target(root: Path, relative: str, *, create_parents: bool) -> Path: + """Validate a workspace-relative target without following symlinks.""" + from agentdiff.pathing import normalize_relative_path + + normalized = normalize_relative_path(relative) + if normalized != relative: + raise ValueError(f"unsafe workspace path: {relative!r}") + target = root.joinpath(*normalized.split("/")) + current = root + for part in normalized.split("/")[:-1]: + current /= part + if current.exists() or current.is_symlink(): + info = current.lstat() + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise ValueError(f"unsafe workspace parent: {current}") + elif create_parents: + current.mkdir(mode=0o700) + else: + raise ValueError(f"missing workspace parent: {current}") + if target.is_symlink(): + raise ValueError(f"unsafe workspace target: {relative}") + return target diff --git a/src/agentdiff/proof/environment.py b/src/agentdiff/proof/environment.py index 061ddbd..d9842d0 100644 --- a/src/agentdiff/proof/environment.py +++ b/src/agentdiff/proof/environment.py @@ -56,7 +56,7 @@ def start(self) -> dict[str, Any]: user, "--read-only", "--tmpfs", - "/tmp:rw,noexec,nosuid,nodev,size=128m", # nosec B108 + "/tmp:rw,noexec,nosuid,nodev,size=128m", # nosec B108 hardened container tmpfs, never host /tmp "--cap-drop", "ALL", "--security-opt", @@ -137,6 +137,7 @@ def run_phase( ) if process.stdout is None: # pragma: no cover - guaranteed by PIPE raise RuntimeError("proof output pipe is unavailable") + stdout_pipe: Any = process.stdout digest = hashlib.sha256() bounded = bytearray() output_bytes = 0 @@ -144,16 +145,15 @@ def run_phase( def drain_output() -> None: nonlocal output_bytes - if process.stdout is not None: - with process.stdout: - while chunk := process.stdout.read(64 * 1024): - digest.update(chunk) - output_bytes += len(chunk) - remaining = _MAX_OUTPUT_BYTES - len(bounded) - if remaining > 0: - bounded.extend(chunk[:remaining]) - if output_bytes > _MAX_OUTPUT_BYTES: - output_limit_reached.set() + with stdout_pipe: + while chunk := stdout_pipe.read(64 * 1024): + digest.update(chunk) + output_bytes += len(chunk) + remaining = _MAX_OUTPUT_BYTES - len(bounded) + if remaining > 0: + bounded.extend(chunk[:remaining]) + if output_bytes > _MAX_OUTPUT_BYTES: + output_limit_reached.set() reader = threading.Thread(target=drain_output, daemon=True) reader.start() @@ -213,7 +213,9 @@ def _phase_result( detail: str, ) -> ProofPhaseResult: text = bounded_output.decode("utf-8", "replace") - tests_passed, tests_total = parse_test_counts(text) if phase == "tests" else (None, None) + tests_passed, tests_total = ( + parse_test_counts(text) if phase in {"tests", "baseline_tests"} else (None, None) + ) return ProofPhaseResult( phase=phase, command=tuple(command), diff --git a/src/agentdiff/proof/models.py b/src/agentdiff/proof/models.py index 9c1a218..096c54e 100644 --- a/src/agentdiff/proof/models.py +++ b/src/agentdiff/proof/models.py @@ -14,6 +14,52 @@ class ProofVerdict(str, Enum): NOT_PROVEN = "NOT_PROVEN" +class ProofStrengthLevel(str, Enum): + """Deterministic proof-strength metadata (never an LLM decision). + + L0 — EXECUTION_ONLY: the agent process exited successfully. + L1 — CLEAN_ROOM: the patch was reproduced in a fresh environment. + L2 — TRUSTED_COMMAND: verification commands came from trusted pre-run + evidence, never from the patched state. + L3 — BASELINE_VERIFIER: the original trusted tests were run against the + patched product code independently of any + agent-modified tests. + L4 — EXTERNAL_VERIFIER: an independent external CI/signed verifier also + passed (reserved; not currently produced). + """ + + L0_EXECUTION_ONLY = "L0" + L1_CLEAN_ROOM = "L1" + L2_TRUSTED_COMMAND = "L2" + L3_BASELINE_VERIFIER = "L3" + L4_EXTERNAL_VERIFIER = "L4" + + +class ProofStrengthLabel(str, Enum): + WEAK = "WEAK" + REVIEW = "REVIEW" + STRONG = "STRONG" + + +class VerifierIndependence(str, Enum): + """How independent the verification was from agent-modified test code.""" + + STRONG = "STRONG" + REVIEW = "REVIEW" + WEAK = "WEAK" + + +def strength_label(level: ProofStrengthLevel) -> ProofStrengthLabel: + if level in { + ProofStrengthLevel.L0_EXECUTION_ONLY, + ProofStrengthLevel.L1_CLEAN_ROOM, + }: + return ProofStrengthLabel.WEAK + if level is ProofStrengthLevel.L2_TRUSTED_COMMAND: + return ProofStrengthLabel.REVIEW + return ProofStrengthLabel.STRONG + + @dataclass(frozen=True, slots=True) class ProofPhaseResult: """One exact-argv verification phase without raw log persistence.""" @@ -58,8 +104,19 @@ class ProofResult: reasons: tuple[str, ...] verification_source: str = "unconfigured" verification_digest: str = "" - trusted_plan: bool = True - schema_version: int = 1 + verification_confirmed: bool = True + baseline_verifier: str = "SKIPPED" + baseline_tests_passed: int | None = None + baseline_tests_total: int | None = None + patched_tests_passed: int | None = None + patched_tests_total: int | None = None + verifier_files_changed: int = 0 + verifier_changes: tuple[str, ...] = () + baseline_available: bool = False + verifier_independence: str = VerifierIndependence.WEAK.value + proof_strength: str = ProofStrengthLevel.L0_EXECUTION_ONLY.value + proof_strength_label: str = ProofStrengthLabel.WEAK.value + schema_version: int = 2 def to_dict(self) -> dict[str, Any]: return { @@ -79,5 +136,16 @@ def to_dict(self) -> dict[str, Any]: "reasons": list(self.reasons), "verification_source": self.verification_source, "verification_digest": self.verification_digest, - "trusted_plan": self.trusted_plan, + "verification_confirmed": self.verification_confirmed, + "baseline_verifier": self.baseline_verifier, + "baseline_tests_passed": self.baseline_tests_passed, + "baseline_tests_total": self.baseline_tests_total, + "patched_tests_passed": self.patched_tests_passed, + "patched_tests_total": self.patched_tests_total, + "verifier_files_changed": self.verifier_files_changed, + "verifier_changes": list(self.verifier_changes), + "baseline_available": self.baseline_available, + "verifier_independence": self.verifier_independence, + "proof_strength": self.proof_strength, + "proof_strength_label": self.proof_strength_label, } diff --git a/src/agentdiff/proof/plan.py b/src/agentdiff/proof/plan.py index 21d9218..c1dd649 100644 --- a/src/agentdiff/proof/plan.py +++ b/src/agentdiff/proof/plan.py @@ -7,6 +7,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Iterable +from .verifier_files import is_verifier_related + if TYPE_CHECKING: from pathlib import Path @@ -14,43 +16,6 @@ from agentdiff.policy import Policy -_PYTHON_CONFIG_NAMES = frozenset( - { - "pyproject.toml", - "setup.py", - "setup.cfg", - "tox.ini", - "pytest.ini", - "conftest.py", - "requirements.txt", - "requirements-dev.txt", - "Pipfile", - "Pipfile.lock", - "poetry.lock", - "uv.lock", - } -) - -_NODE_CONFIG_NAMES = frozenset( - { - "package.json", - "package-lock.json", - "yarn.lock", - "pnpm-lock.yaml", - "tsconfig.json", - ".npmrc", - } -) - -_CI_BUILD_CONFIG_NAMES = frozenset( - { - "Makefile", - "Dockerfile", - "CMakeLists.txt", - } -) - - @dataclass(frozen=True, slots=True) class TrustedVerificationPlan: """A deterministic verification plan with provenance and tamper-detection.""" @@ -82,7 +47,9 @@ def to_dict(self) -> dict[str, object]: def _compute_plan_digest( - setup: Iterable[Iterable[str]], build: Iterable[Iterable[str]], tests: Iterable[Iterable[str]] + setup: Iterable[Iterable[str]], + build: Iterable[Iterable[str]], + tests: Iterable[Iterable[str]], ) -> str: payload = { "setup": [list(cmd) for cmd in setup], @@ -93,6 +60,11 @@ def _compute_plan_digest( return f"sha256:{hashlib.sha256(encoded).hexdigest()}" +def _tampered_verifier_files(modified_paths: Iterable[str]) -> tuple[str, ...]: + """Return the sorted verifier-related paths among the modified paths.""" + return tuple(sorted(path for path in modified_paths if is_verifier_related(path))) + + def select_trusted_verification_plan( base_root: Path, policy: Policy, @@ -103,8 +75,11 @@ def select_trusted_verification_plan( Trust hierarchy: 1. Explicit policy verification commands (always trusted). - 2. Auto-discovery from pre-run base source if patch did NOT tamper with config files. - 3. If patch modified auto-discovered test/build configs without explicit policy, mark UNTRUSTED. + 2. Auto-discovery from pre-run base source if the patch did NOT modify + any verifier-related file (tests, fixtures, runner config, manifests, + lockfiles, CI workflows). + 3. If the patch modified verifier-related files without an explicit + policy plan, mark UNTRUSTED. 4. If no tests configured or discoverable, mark UNTRUSTED. """ configured = bool(policy.proof.setup or policy.proof.build or policy.proof.tests) @@ -126,108 +101,77 @@ def select_trusted_verification_plan( else "policy specifies no test commands", ) - # Auto-discovery inspects the base_root (pre-run state) + # Auto-discovery inspects the base_root (pre-run state). modified_paths: set[str] = set() if patch_bundle is not None: modified_paths.update(entry.path for entry in patch_bundle.manifest.entries) if patch_entries is not None: modified_paths.update(entry.path for entry in patch_entries) + tampered = _tampered_verifier_files(modified_paths) + trusted = len(tampered) == 0 + tamper_reason = ( + f"patch modified test/build infrastructure without policy override: {', '.join(tampered)}" + if tampered + else "" + ) if (base_root / "uv.lock").is_file(): - relevant_tampered = tuple( - sorted( - p for p in modified_paths if p in _PYTHON_CONFIG_NAMES or p.startswith(".github/") - ) - ) - trusted = len(relevant_tampered) == 0 - setup_cmds: tuple[tuple[str, ...], ...] = (("uv", "sync", "--frozen", "--no-cache"),) - build_cmds: tuple[tuple[str, ...], ...] = (("uv", "build"),) - test_cmds: tuple[tuple[str, ...], ...] = (("uv", "run", "pytest", "-q"),) - tampered_msg = ", ".join(relevant_tampered) - digest = _compute_plan_digest(setup_cmds, build_cmds, test_cmds) - reason = ( - "discovered from base uv.lock" - if trusted - else f"patch modified test/build infrastructure without policy override: {tampered_msg}" - ) + setup: tuple[tuple[str, ...], ...] = (("uv", "sync", "--frozen", "--no-cache"),) + build: tuple[tuple[str, ...], ...] = (("uv", "build"),) + tests: tuple[tuple[str, ...], ...] = (("uv", "run", "pytest", "-q"),) return TrustedVerificationPlan( image=policy.proof.image or "ghcr.io/astral-sh/uv:python3.12-bookworm-slim", network=policy.proof.network, - setup=setup_cmds, - build=build_cmds, - tests=test_cmds, + setup=setup, + build=build, + tests=tests, source="auto:uv", trusted=trusted, - plan_digest=digest, - tampered_files=relevant_tampered, - reason=reason, + plan_digest=_compute_plan_digest(setup, build, tests), + tampered_files=tampered, + reason="discovered from base uv.lock" if trusted else tamper_reason, ) if (base_root / "pyproject.toml").is_file(): - relevant_tampered = tuple( - sorted( - p for p in modified_paths if p in _PYTHON_CONFIG_NAMES or p.startswith(".github/") - ) - ) - trusted = len(relevant_tampered) == 0 python = ".agentdiff-proof/venv/bin/python" - py_setup: tuple[tuple[str, ...], ...] = ( + setup = ( ("python", "-m", "venv", ".agentdiff-proof/venv"), (python, "-m", "pip", "install", "--no-cache-dir", "-e", "."), ) - py_build: tuple[tuple[str, ...], ...] = ((python, "-m", "compileall", "-q", "src"),) - py_tests: tuple[tuple[str, ...], ...] = ((python, "-m", "pytest", "-q"),) - digest = _compute_plan_digest(py_setup, py_build, py_tests) - tampered_msg = ", ".join(relevant_tampered) - reason = ( - "discovered from base pyproject.toml" - if trusted - else f"patch modified test/build infrastructure without policy override: {tampered_msg}" - ) + build = ((python, "-m", "compileall", "-q", "src"),) + tests = ((python, "-m", "pytest", "-q"),) return TrustedVerificationPlan( image=policy.proof.image or "python:3.12-slim", network=policy.proof.network, - setup=py_setup, - build=py_build, - tests=py_tests, + setup=setup, + build=build, + tests=tests, source="auto:python", trusted=trusted, - plan_digest=digest, - tampered_files=relevant_tampered, - reason=reason, + plan_digest=_compute_plan_digest(setup, build, tests), + tampered_files=tampered, + reason="discovered from base pyproject.toml" if trusted else tamper_reason, ) if (base_root / "package-lock.json").is_file() or (base_root / "package.json").is_file(): - relevant_tampered = tuple( - sorted(p for p in modified_paths if p in _NODE_CONFIG_NAMES or p.startswith(".github/")) - ) - trusted = len(relevant_tampered) == 0 - npm_setup: tuple[tuple[str, ...], ...] = ( + setup = ( (("npm", "ci"),) if (base_root / "package-lock.json").is_file() else (("npm", "install"),) ) - npm_build: tuple[tuple[str, ...], ...] = (("npm", "run", "build", "--if-present"),) - npm_tests: tuple[tuple[str, ...], ...] = (("npm", "test"),) - digest = _compute_plan_digest(npm_setup, npm_build, npm_tests) - tampered_msg = ", ".join(relevant_tampered) - reason = ( - "discovered from base package manifest" - if trusted - else f"patch modified test/build infrastructure without policy override: {tampered_msg}" - ) - + build = (("npm", "run", "build", "--if-present"),) + tests = (("npm", "test"),) return TrustedVerificationPlan( image=policy.proof.image or "node:22-slim", network=policy.proof.network, - setup=npm_setup, - build=npm_build, - tests=npm_tests, + setup=setup, + build=build, + tests=tests, source="auto:npm", trusted=trusted, - plan_digest=digest, - tampered_files=relevant_tampered, - reason=reason, + plan_digest=_compute_plan_digest(setup, build, tests), + tampered_files=tampered, + reason="discovered from base package manifest" if trusted else tamper_reason, ) return TrustedVerificationPlan( diff --git a/src/agentdiff/proof/verification.py b/src/agentdiff/proof/verification.py index 78f171f..e2df1e3 100644 --- a/src/agentdiff/proof/verification.py +++ b/src/agentdiff/proof/verification.py @@ -5,6 +5,9 @@ import re from typing import TYPE_CHECKING +if TYPE_CHECKING: + from agentdiff.policy import Policy + from .plan import ( TrustedVerificationPlan, select_trusted_verification_plan, @@ -15,7 +18,6 @@ from agentdiff.policy import Policy - # Compatibility alias VerificationPlan = TrustedVerificationPlan diff --git a/src/agentdiff/proof/verifier_files.py b/src/agentdiff/proof/verifier_files.py new file mode 100644 index 0000000..cb5ecd0 --- /dev/null +++ b/src/agentdiff/proof/verifier_files.py @@ -0,0 +1,144 @@ +"""Deterministic classification of verifier-related files. + +A verifier-related file is any file whose content can change what a +verification command executes or how it is interpreted: test sources, +fixtures, test runners, package/build manifests, lockfiles, CI workflows, +and tool configuration. The classifier is deliberately conservative +(inclusion-biased): under-inclusion would let an agent silently weaken the +verifier, while over-inclusion only reports a change that the baseline +verifier must then independently confirm. + +The classifier never decides intent. It feeds a mutation *report*; policy +and proof verdicts remain deterministic. +""" + +from __future__ import annotations + +import fnmatch +from dataclasses import dataclass +from typing import Iterable + +# Directories that contain test sources or fixtures anywhere in the tree. +_VERIFIER_DIR_PREFIXES = ("tests/", "test/", "__tests__/") + +# Well-known CI workflow locations. +_VERIFIER_PATH_PREFIXES = (".github/", ".circleci/", ".travis.yml", ".gitlab-ci.yml") + +# Exact configuration/test-runner filenames. +_VERIFIER_BASENAMES = frozenset( + { + "conftest.py", + "pytest.ini", + "tox.ini", + "noxfile.py", + "setup.py", + "setup.cfg", + "Makefile", + "Dockerfile", + "package.json", + "package-lock.json", + "yarn.lock", + "pnpm-lock.yaml", + "uv.lock", + "poetry.lock", + "Pipfile", + "Pipfile.lock", + "pyproject.toml", + } +) + +# Glob patterns for runner configs and dependency manifests. +_VERIFIER_PATTERNS = ( + "vitest.config.*", + "jest.config.*", + "playwright.config.*", + "karma.conf.*", + "requirements*.txt", + "requirements*.in", +) + +# Common test-file naming conventions (path-level, not content-level). +_TEST_FILE_PATTERNS = ( + "test_*.py", + "*_test.py", + "*.test.js", + "*.test.jsx", + "*.test.ts", + "*.test.tsx", + "*.spec.js", + "*.spec.jsx", + "*.spec.ts", + "*.spec.tsx", + "*_test.go", + "*.test.rs", +) + + +def is_verifier_related(relative_path: str) -> bool: + """Return whether ``relative_path`` can influence verification behavior.""" + path = relative_path.replace("\\", "/") + if not path or path.startswith("/") or ".." in path.split("/"): + return False + basename = path.rsplit("/", 1)[-1] + if basename in _VERIFIER_BASENAMES: + return True + if any(path.startswith(prefix) for prefix in _VERIFIER_DIR_PREFIXES): + return True + if any(path.startswith(prefix) for prefix in _VERIFIER_PATH_PREFIXES): + return True + if any(fnmatch.fnmatchcase(basename, pattern) for pattern in _VERIFIER_PATTERNS): + return True + return any(fnmatch.fnmatchcase(basename, pattern) for pattern in _TEST_FILE_PATTERNS) + + +@dataclass(frozen=True, slots=True) +class VerifierMutationReport: + """Deterministic summary of patch changes to verifier-related files.""" + + existing_changed: tuple[str, ...] = () + added: tuple[str, ...] = () + removed: tuple[str, ...] = () + + @property + def modified_count(self) -> int: + return len(self.existing_changed) + + @property + def total_changes(self) -> int: + return self.modified_count + len(self.added) + len(self.removed) + + @property + def any_modification(self) -> bool: + return self.total_changes > 0 + + def to_dict(self) -> dict[str, object]: + return { + "existing_changed": list(self.existing_changed), + "added": list(self.added), + "removed": list(self.removed), + "modified_count": self.modified_count, + "total_changes": self.total_changes, + } + + +def analyze_verifier_mutations( + changed_paths: Iterable[tuple[str, str]], +) -> VerifierMutationReport: + """Classify ``(path, change_type)`` pairs against verifier-related files.""" + existing_changed: list[str] = [] + added: list[str] = [] + removed: list[str] = [] + for path, change_type in changed_paths: + if not is_verifier_related(path): + continue + if change_type == "modified": + existing_changed.append(path) + elif change_type == "created": + added.append(path) + elif change_type == "deleted": + removed.append(path) + return VerifierMutationReport( + existing_changed=tuple(sorted(existing_changed)), + added=tuple(sorted(added)), + removed=tuple(sorted(removed)), + ) diff --git a/src/agentdiff/runtime/__init__.py b/src/agentdiff/runtime/__init__.py index 17ca49c..37bf8c8 100644 --- a/src/agentdiff/runtime/__init__.py +++ b/src/agentdiff/runtime/__init__.py @@ -7,6 +7,7 @@ PortEndpoint, PortObservation, RuntimeBackend, + RuntimeCapabilities, RuntimeCapability, RuntimeControlLevel, RuntimeResult, @@ -31,6 +32,7 @@ "PortEndpoint", "PortObservation", "RuntimeBackend", + "RuntimeCapabilities", "RuntimeCapability", "RuntimeControlLevel", "RuntimeResult", diff --git a/src/agentdiff/runtime/base.py b/src/agentdiff/runtime/base.py index 27056e0..1805ea7 100644 --- a/src/agentdiff/runtime/base.py +++ b/src/agentdiff/runtime/base.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Sequence + from pathlib import Path class RuntimeControlLevel(str, Enum): @@ -39,6 +40,41 @@ def to_dict(self) -> dict[str, Any]: } +@dataclass(frozen=True) +class RuntimeCapabilities: + """Explicit, pre-execution runtime capability metadata. + + Every backend reports exactly the controls it actually provides, before + any command runs, so callers never have to guess from post-hoc result + strings or attribute presence. + """ + + backend: str + filesystem: RuntimeControlLevel + host_repository: RuntimeControlLevel + network: RuntimeControlLevel + processes: RuntimeControlLevel + resources: RuntimeControlLevel + privileges: RuntimeControlLevel + private_workspace: bool + supports_live_safety: bool + supports_source_snapshot: bool + + def to_dict(self) -> dict[str, Any]: + return { + "backend": self.backend, + "filesystem": self.filesystem.value, + "host_repository": self.host_repository.value, + "network": self.network.value, + "processes": self.processes.value, + "resources": self.resources.value, + "privileges": self.privileges.value, + "private_workspace": self.private_workspace, + "supports_live_safety": self.supports_live_safety, + "supports_source_snapshot": self.supports_source_snapshot, + } + + @dataclass(frozen=True) class OwnedProcess: """PID identity observed in the launched process tree. @@ -202,6 +238,11 @@ def to_dict(self) -> dict[str, Any]: class RuntimeBackend(Protocol): """Execution backend implemented by local and future isolated runtimes.""" + @property + def capabilities(self) -> RuntimeCapabilities: + """Explicit capability metadata available before execution.""" + ... + def run( self, argv: Sequence[str], @@ -222,3 +263,15 @@ def cleanup( ) -> CleanupReport: """Clean up only process identities proven to belong to a prior run.""" ... + + def configure_source(self, source_dir: str | Path) -> None: + """Supply a sealed source snapshot for backends that use a private copy.""" + ... + + def configure_safety(self, controller: Any) -> None: + """Attach the live safety controller/watcher used during execution.""" + ... + + def close(self) -> None: + """Release backend resources after evidence collection.""" + ... diff --git a/src/agentdiff/runtime/docker.py b/src/agentdiff/runtime/docker.py index 82da009..4d064d4 100644 --- a/src/agentdiff/runtime/docker.py +++ b/src/agentdiff/runtime/docker.py @@ -22,16 +22,17 @@ from .base import ( CleanupReport, OwnedProcess, + RuntimeCapabilities, RuntimeCapability, RuntimeControlLevel, RuntimeResult, ) from .local import LocalRuntime +from .materialize import WorkspaceMaterializer if TYPE_CHECKING: from collections.abc import Iterable, Sequence - from agentdiff.safety import SafetyController _ENV_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$") @@ -83,8 +84,28 @@ def __init__( self._source_dir: Path | None = None self._workspace: Path | None = None self._temporary_root: Path | None = None - self._safety_controller: SafetyController | None = None + self._safety_controller: Any | None = None self._container_id: str | None = None + self._materialization_report: Any | None = None + + @property + def capabilities(self) -> RuntimeCapabilities: + """Explicit capability metadata available before any execution.""" + network_level = ( + RuntimeControlLevel.BLOCKED if self.network == "none" else RuntimeControlLevel.SANDBOXED + ) + return RuntimeCapabilities( + backend="docker", + filesystem=RuntimeControlLevel.SANDBOXED, + host_repository=RuntimeControlLevel.SANDBOXED, + network=network_level, + processes=RuntimeControlLevel.SANDBOXED, + resources=RuntimeControlLevel.SANDBOXED, + privileges=RuntimeControlLevel.SANDBOXED, + private_workspace=True, + supports_live_safety=True, + supports_source_snapshot=True, + ) def configure_source(self, source_dir: str | Path) -> None: """Use the transaction's sealed pre-run source copy as container input.""" @@ -94,7 +115,7 @@ def configure_source(self, source_dir: str | Path) -> None: raise ValueError("Docker source snapshot must be a real directory") self._source_dir = candidate - def configure_safety(self, controller: SafetyController) -> None: + def configure_safety(self, controller: Any) -> None: self._safety_controller = controller def run( @@ -119,7 +140,11 @@ def run( temporary_root = Path(tempfile.mkdtemp(prefix="agentdiff-docker-")) workspace = temporary_root / "workspace" workspace.mkdir(mode=0o700) - shutil.copytree(self._source_dir, workspace, dirs_exist_ok=True, symlinks=False) + # Materialize the private writable source copy through the validated + # WorkspaceMaterializer instead of a raw copytree: modes are preserved + # and unsafe entries are rejected, never silently followed or dropped. + materialized = WorkspaceMaterializer().materialize(self._source_dir, workspace) + self._materialization_report = materialized self._temporary_root = temporary_root self._workspace = workspace container_name = f"agentdiff-{secrets.token_hex(8)}" @@ -225,7 +250,7 @@ def _create_argv( user, "--read-only", "--tmpfs", - "/tmp:rw,noexec,nosuid,nodev,size=64m", # nosec B108 + "/tmp:rw,noexec,nosuid,nodev,size=64m", # nosec B108 hardened container tmpfs, never host /tmp "--cap-drop", "ALL", "--security-opt", @@ -253,7 +278,7 @@ def _create_argv( return argv def _runtime_config(self, user: str) -> dict[str, Any]: - return { + config: dict[str, Any] = { "schema_version": 1, "image": self.image, "user": user, @@ -269,7 +294,17 @@ def _runtime_config(self, user: str) -> dict[str, Any]: "pids_limit": self.pids_limit, "environment_allowlist": list(self.environment_allowlist), "ephemeral_container": True, + "capabilities": self.capabilities.to_dict(), } + if self._materialization_report is not None: + report = self._materialization_report + config["materialization"] = { + "strategy_used": report.strategy_used, + "files_materialized": report.files_materialized, + "bytes_materialized": report.bytes_materialized, + "duration_seconds": report.duration_seconds, + } + return config def _docker_call( self, diff --git a/src/agentdiff/runtime/local.py b/src/agentdiff/runtime/local.py index 3bdf3ff..3098c80 100644 --- a/src/agentdiff/runtime/local.py +++ b/src/agentdiff/runtime/local.py @@ -17,14 +17,14 @@ OwnedProcess, PortEndpoint, PortObservation, + RuntimeCapabilities, + RuntimeControlLevel, RuntimeResult, ) if TYPE_CHECKING: from collections.abc import Iterable, Sequence - from agentdiff.safety import SafetyController - _TIMEOUT_RETURN_CODE = 124 @@ -37,7 +37,7 @@ def __init__( *, poll_interval_seconds: float = 0.02, observe_ports: bool = True, - safety_controller: SafetyController | None = None, + safety_controller: Any = None, ) -> None: self.root = Path(root).resolve() if not self.root.is_dir(): @@ -47,10 +47,32 @@ def __init__( self.poll_interval_seconds = poll_interval_seconds self.observe_ports = observe_ports self._safety_controller = safety_controller + self._root_session: int | None = None + + @property + def capabilities(self) -> RuntimeCapabilities: + return RuntimeCapabilities( + backend="local-observe", + filesystem=RuntimeControlLevel.OBSERVED, + host_repository=RuntimeControlLevel.OBSERVED, + network=RuntimeControlLevel.UNCONTROLLED, + processes=RuntimeControlLevel.OBSERVED, + resources=RuntimeControlLevel.UNCONTROLLED, + privileges=RuntimeControlLevel.UNCONTROLLED, + private_workspace=False, + supports_live_safety=True, + supports_source_snapshot=False, + ) - def configure_safety(self, controller: SafetyController) -> None: + def configure_source(self, source_dir: str | os.PathLike[str]) -> None: + """Local runs execute against the live host root; source snapshots are unused.""" + + def configure_safety(self, controller: Any) -> None: self._safety_controller = controller + def close(self) -> None: + """Local runtime holds no backend resources to release.""" + def run( self, argv: Sequence[str], @@ -82,6 +104,9 @@ def run( start_new_session=os.name == "posix", ) owned: dict[tuple[int, float], OwnedProcess] = {} + # Capture the execution session immediately: after the root exits, + # getsid on its pid fails, but reparented descendants keep the sid. + self._root_session = self._session_id(process.pid) deadline = started + timeout_seconds if timeout_seconds is not None else None try: return self._monitor_process( @@ -247,6 +272,10 @@ def _observe_execution_domain( owned: dict[tuple[int, float], OwnedProcess], ) -> None: self._observe_process_tree(root_pid, owned) + root_created = next( + (created for owned_pid, created in owned if owned_pid == root_pid), None + ) + root_session = self._root_session or self._session_id(root_pid) for process in psutil.process_iter(["pid", "create_time", "ppid"]): with suppress(psutil.NoSuchProcess, psutil.AccessDenied): pid = int(process.info["pid"]) @@ -256,11 +285,16 @@ def _observe_execution_domain( owned_pid == parent_pid and create_time >= owned_created for owned_pid, owned_created in owned ) - matching_session = False - if os.name == "posix" and hasattr(os, "getsid"): - with suppress(OSError): - matching_session = os.getsid(pid) == root_pid - if matching_parent or matching_session: + # A descendant that was reparented (its parent exited) keeps + # the execution session id, so it can still be attributed to + # the run even though its ppid no longer points at us. + same_session = ( + root_session is not None + and self._session_id(pid) == root_session + and root_created is not None + and create_time >= root_created + ) + if matching_parent or same_session: owned.setdefault( (pid, create_time), OwnedProcess( @@ -271,6 +305,17 @@ def _observe_execution_domain( ), ) + @staticmethod + def _session_id(pid: int) -> int | None: + """Return the POSIX session id of ``pid``, or None when unavailable.""" + getsid = getattr(os, "getsid", None) + if getsid is None: + return None + try: + return int(getsid(pid)) + except (OSError, ProcessLookupError): + return None + def cleanup( self, processes: Iterable[OwnedProcess], @@ -326,7 +371,7 @@ def cleanup( wait_fn = getattr(psutil, "wait_procs", None) alive: list[Any] = [] if wait_fn is not None: - _gone, alive = wait_fn([proc for _, proc in signaled], timeout=grace_period_seconds) + _, alive = wait_fn([proc for _, proc in signaled], timeout=grace_period_seconds) else: deadline = time.monotonic() + max(0.0, grace_period_seconds) while time.monotonic() < deadline: diff --git a/src/agentdiff/runtime/materialize.py b/src/agentdiff/runtime/materialize.py index 32cf0d3..183966f 100644 --- a/src/agentdiff/runtime/materialize.py +++ b/src/agentdiff/runtime/materialize.py @@ -1,9 +1,30 @@ -"""Fast copy-on-write / reflink / copy workspace materialization for clean-room runtimes.""" +"""Workspace materialization for clean-room and isolated runtimes. + +Strategies are named accurately: + +- ``CLONE`` / CoW: platform-native clone only where actually supported + (Linux ``FICLONE``). This is a true reflink clone on supporting + filesystems. +- ``FAST_COPY``: ``copy_file_range`` / platform accelerated copy. This is a + fast copy primitive, **not** a guaranteed reflink or CoW guarantee. +- ``STREAM_COPY``: portable streaming fallback. + +Every strategy preserves regular-file content, size, and POSIX mode +(including the executable bit), verifies the opened source identity, and +rejects symlinks and special files instead of silently dropping or following +them. This is a trust-boundary component: a source swap during materialization +must never leak into the private workspace. + +The materialized workspace is an ephemeral copy derived from durable sealed +evidence (the run capsule), so destination writes are flushed but not fsynced; +durability is owned by the capsule and by promotion staging, which fsync. +""" from __future__ import annotations +import contextlib +import hashlib import os -import shutil import stat import time from dataclasses import dataclass @@ -13,11 +34,20 @@ _CHUNK_SIZE = 1024 * 1024 +# Linux FICLONE ioctl (linux/fs.h) — true reflink where the filesystem +# supports it. Request code 0x40049409 (direction IOW, size 4, type 0x94). +_FICLONE = 0x40049409 + class MaterializationStrategy(str, Enum): AUTO = "auto" - REFLINK = "reflink" - COPY = "copy" + CLONE = "clone" + FAST_COPY = "fast_copy" + STREAM_COPY = "stream_copy" + + # Deprecated aliases kept for callers written against the earlier names. + REFLINK = "clone" + COPY = "stream_copy" @dataclass(frozen=True, slots=True) @@ -29,7 +59,7 @@ class MaterializationReport: class WorkspaceMaterializer: - """High-speed workspace materializer for container and clean-room sandboxes.""" + """High-speed, validated workspace materializer for container and clean-room sandboxes.""" def __init__( self, @@ -46,20 +76,31 @@ def materialize( *, filter_fn: Callable[[str], bool] | None = None, ) -> MaterializationReport: - src = Path(source_dir).resolve(strict=True) - dst = Path(target_dir).resolve() - dst.mkdir(parents=True, exist_ok=True, mode=0o700) + src_raw = Path(source_dir) + if src_raw.is_symlink(): + raise ValueError("materializer source must be a real directory") + src = src_raw.resolve(strict=True) + dst_raw = Path(target_dir) + if dst_raw.is_symlink(): + raise ValueError("materializer target must be a real directory") + dst = dst_raw.resolve() + if dst.exists() and not dst.is_dir(): + raise ValueError("materializer target must be a real directory") + dst.mkdir(parents=True, exist_ok=True) + self._assert_real_directory(dst) started = time.monotonic() files_count = 0 total_bytes = 0 + strategies_seen: list[str] = [] for root, dirs, files in os.walk(src, followlinks=False): - # Exclude ignored directories dirs[:] = [d for d in dirs if d not in self.ignored_names] rel_root = Path(root).relative_to(src) current_dst = dst / rel_root - current_dst.mkdir(mode=0o700, parents=True, exist_ok=True) + current_dst.mkdir(parents=True, exist_ok=True) + self._assert_real_directory(current_dst) + self._copy_directory_mode(Path(root), current_dst) for filename in files: if filename in self.ignored_names: @@ -67,62 +108,178 @@ def materialize( relpath = (rel_root / filename).as_posix() if filter_fn is not None and not filter_fn(relpath): continue - source_file = Path(root) / filename target_file = current_dst / filename - - size = self._copy_file(source_file, target_file) + if target_file.is_symlink() or target_file.exists(): + raise RuntimeError(f"materializer target already exists: {relpath}") + size, strategy = self._copy_file(source_file, target_file) files_count += 1 total_bytes += size + strategies_seen.append(strategy) elapsed = time.monotonic() - started + actual = _dominant_strategy(self.strategy, strategies_seen) return MaterializationReport( - strategy_used=self.strategy.value, + strategy_used=actual, files_materialized=files_count, bytes_materialized=total_bytes, duration_seconds=elapsed, ) - def _copy_file(self, src: Path, dst: Path) -> int: + def _copy_file(self, src: Path, dst: Path) -> tuple[int, str]: + """Copy one file preserving content, size, and mode on every path.""" info = src.lstat() + if stat.S_ISLNK(info.st_mode): + raise RuntimeError( + f"materializer refuses symlink {src}: unsafe entries are rejected, not followed" + ) if not stat.S_ISREG(info.st_mode): - return 0 + raise RuntimeError( + f"materializer refuses special file {src}: unsupported entry types are rejected" + ) + if info.st_nlink != 1: + raise RuntimeError( + f"materializer refuses hardlinked file {src}: hardlink ambiguity is rejected" + ) - # Reflink attempt on POSIX if requested/auto - if self.strategy in { - MaterializationStrategy.AUTO, - MaterializationStrategy.REFLINK, - } and hasattr(os, "copy_file_range"): + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + source_fd = os.open(src, flags) + try: + opened = os.fstat(source_fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_nlink != 1 + or opened.st_dev != info.st_dev + or opened.st_ino != info.st_ino + ): + raise RuntimeError("materializer source changed while opening") + strategy = self._copy_descriptor(source_fd, dst, opened.st_size) + finished = os.fstat(source_fd) + if ( + finished.st_dev != opened.st_dev + or finished.st_ino != opened.st_ino + or finished.st_size != opened.st_size + or finished.st_mtime_ns != opened.st_mtime_ns + ): + raise RuntimeError("materializer source changed while copying") + finally: + os.close(source_fd) + if os.name != "nt": + dst.chmod(stat.S_IMODE(info.st_mode)) + return info.st_size, strategy + + def _copy_descriptor(self, source_fd: int, dst: Path, size: int) -> str: + """Copy from an open source descriptor using the configured strategy.""" + requested = self.strategy + if requested is MaterializationStrategy.CLONE: + return self._try_clone(source_fd, dst, size) or self._stream_copy(source_fd, dst, size) + if requested is MaterializationStrategy.FAST_COPY: try: - src_fd = os.open(src, os.O_RDONLY | getattr(os, "O_BINARY", 0)) - dst_fd = os.open( - dst, - os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0), - 0o644, - ) - try: - total = 0 - while total < info.st_size: - copied = os.copy_file_range(src_fd, dst_fd, info.st_size - total) - if copied == 0: - break - total += copied - return total - finally: - os.close(src_fd) - os.close(dst_fd) + return self._fast_copy(source_fd, dst, size) except OSError: - pass + # copy_file_range may be unavailable (e.g. removed in + # Python 3.14) or unsupported by the filesystem; the report + # records the actual strategy used. + return self._stream_copy(source_fd, dst, size) + if requested is MaterializationStrategy.STREAM_COPY: + return self._stream_copy(source_fd, dst, size) + # AUTO: clone when supported, then fast copy, then streaming. + cloned = self._try_clone(source_fd, dst, size) + if cloned is not None: + return cloned + try: + return self._fast_copy(source_fd, dst, size) + except OSError: + return self._stream_copy(source_fd, dst, size) - # Robust streaming fallback - with open(src, "rb") as input_f, open(dst, "wb") as output_f: - shutil.copyfileobj(input_f, output_f, length=_CHUNK_SIZE) + def _try_clone(self, source_fd: int, dst: Path, size: int) -> str | None: + """Linux FICLONE reflink; returns None when unsupported.""" + if os.name == "nt" or not hasattr(os, "O_NOFOLLOW"): + return None + try: + dst_fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except OSError: + raise RuntimeError(f"materializer target exists: {dst}") from None + try: + import fcntl - mode = stat.S_IMODE(info.st_mode) - if os.name != "nt": - import contextlib + fcntl.ioctl(dst_fd, _FICLONE, source_fd) + os.fsync(dst_fd) + return "clone" + except OSError: + # Clone unsupported: remove the empty probe file so the + # fallback strategy can create the destination itself. + with contextlib.suppress(OSError): + dst.unlink(missing_ok=True) + return None + finally: + os.close(dst_fd) + def _fast_copy(self, source_fd: int, dst: Path, size: int) -> str: + """copy_file_range: an accelerated copy primitive, not a guaranteed reflink.""" + if not hasattr(os, "copy_file_range"): + raise OSError("copy_file_range is unavailable") + try: + dst_fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except OSError: + raise RuntimeError(f"materializer target exists: {dst}") from None + try: + total = 0 + while total < size: + copied = os.copy_file_range(source_fd, dst_fd, size - total) + if copied == 0: + break + total += copied + if total != size: + raise OSError("copy_file_range produced a short copy") + os.fsync(dst_fd) + return "fast_copy" + except OSError: with contextlib.suppress(OSError): - dst.chmod(mode) + dst.unlink(missing_ok=True) + raise + finally: + os.close(dst_fd) + + def _stream_copy(self, source_fd: int, dst: Path, size: int) -> str: + dst_fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + digest = hashlib.sha256() + copied = 0 + try: + with ( + os.fdopen(source_fd, "rb", closefd=False) as src, + os.fdopen(dst_fd, "wb", closefd=False) as output, + ): + while chunk := src.read(_CHUNK_SIZE): + output.write(chunk) + digest.update(chunk) + copied += len(chunk) + output.flush() + if copied != size: + raise OSError("streaming copy produced a short copy") + return "stream_copy" + finally: + if dst_fd >= 0: + os.close(dst_fd) + + def _copy_directory_mode(self, src_dir: Path, dst_dir: Path) -> None: + if os.name == "nt": + return + with contextlib.suppress(OSError): + mode = stat.S_IMODE(src_dir.lstat().st_mode) + dst_dir.chmod(mode) + + @staticmethod + def _assert_real_directory(path: Path) -> None: + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise RuntimeError(f"materializer path is not a real directory: {path}") + - return info.st_size +def _dominant_strategy(requested: MaterializationStrategy, seen: list[str]) -> str: + if not seen: + return requested.value + counts: dict[str, int] = {} + for name in seen: + counts[name] = counts.get(name, 0) + 1 + return max(counts, key=lambda name: counts[name]) diff --git a/src/agentdiff/runtime/sandbox.py b/src/agentdiff/runtime/sandbox.py index 0da2799..abec598 100644 --- a/src/agentdiff/runtime/sandbox.py +++ b/src/agentdiff/runtime/sandbox.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import IO, TYPE_CHECKING, Any +from .base import RuntimeCapabilities, RuntimeControlLevel from .local import LocalRuntime if TYPE_CHECKING: @@ -60,6 +61,36 @@ def __init__( poll_interval_seconds=poll_interval_seconds, ) + @property + def capabilities(self) -> RuntimeCapabilities: + """Report only guarantees this adapter actually provides. + + Isolation is delegated to the external Sandbox Runtime; AgentDiff + itself observes the host around the wrapper and does not reimplement + or silently emulate the external OS controls. + """ + return RuntimeCapabilities( + backend="anthropic-sandbox-runtime", + filesystem=RuntimeControlLevel.OBSERVED, + host_repository=RuntimeControlLevel.OBSERVED, + network=RuntimeControlLevel.OBSERVED, + processes=RuntimeControlLevel.OBSERVED, + resources=RuntimeControlLevel.UNCONTROLLED, + privileges=RuntimeControlLevel.UNCONTROLLED, + private_workspace=False, + supports_live_safety=True, + supports_source_snapshot=False, + ) + + def configure_source(self, source_dir: str | os.PathLike[str]) -> None: + """The Sandbox Runtime executes on the host root; no source copy is used.""" + + def configure_safety(self, controller: Any) -> None: + self._local.configure_safety(controller) + + def close(self) -> None: + self._local.close() + def run( self, argv: Sequence[str], diff --git a/src/agentdiff/safety/__init__.py b/src/agentdiff/safety/__init__.py index a21331e..136b7da 100644 --- a/src/agentdiff/safety/__init__.py +++ b/src/agentdiff/safety/__init__.py @@ -2,7 +2,7 @@ from .controller import SafetyController from .models import ControlLevel, SafetyEvent, SafetyReport -from .watcher import HybridSafetyWatcher, WatcherStats +from .watcher import HybridSafetyWatcher, WatcherStats, WatcherStatus __all__ = [ "ControlLevel", @@ -11,4 +11,5 @@ "SafetyEvent", "SafetyReport", "WatcherStats", + "WatcherStatus", ] diff --git a/src/agentdiff/safety/controller.py b/src/agentdiff/safety/controller.py index be85036..0357590 100644 --- a/src/agentdiff/safety/controller.py +++ b/src/agentdiff/safety/controller.py @@ -4,14 +4,21 @@ that observation syscall interception. A backend may terminate the owned execution domain when :attr:`terminated` becomes true; Docker additionally keeps observed mutations away from the host repository. + +The controller exposes cheap budget checks separately from the expensive +full filesystem reconciliation so a hybrid watcher can run targeted dirty-path +checks between authoritative full captures. """ from __future__ import annotations import time -from pathlib import Path +from typing import TYPE_CHECKING from agentdiff.policy import Policy, PolicyAction, PolicyEngine + +if TYPE_CHECKING: + from pathlib import Path from agentdiff.state import FilesystemManifest, FilesystemScanner, diff_manifests from .models import ControlLevel, SafetyEvent, SafetyReport @@ -61,9 +68,29 @@ def observe( runtime_active: bool = True, ) -> bool: """Record current state and return whether execution must terminate.""" - if self.report.terminated: return True + self.check_budgets( + duration_seconds=duration_seconds, + processes_spawned=processes_spawned, + runtime_active=runtime_active, + ) + now = time.monotonic() + if force_filesystem or now - self._last_filesystem_poll >= self.filesystem_poll_interval: + self._last_filesystem_poll = now + self.check_filesystem(root, runtime_active=runtime_active) + return self.report.terminated + + def check_budgets( + self, + *, + duration_seconds: float, + processes_spawned: int, + runtime_active: bool, + ) -> None: + """Cheap duration/process budget checks (no filesystem work).""" + if self.report.terminated: + return self._check_limit( "duration_seconds", duration_seconds, @@ -76,13 +103,13 @@ def observe( self.policy.limits.processes_spawned, runtime_active=runtime_active, ) - now = time.monotonic() - if force_filesystem or now - self._last_filesystem_poll >= self.filesystem_poll_interval: - self._last_filesystem_poll = now - self._observe_filesystem(Path(root), runtime_active=runtime_active) - return self.report.terminated - def _observe_filesystem(self, root: Path, *, runtime_active: bool) -> None: + def check_filesystem(self, root: str | Path, *, runtime_active: bool) -> None: + """Full authoritative reconciliation against the pre-run manifest. + + This is the only place filesystem budgets and protected-path verdicts + are computed: event-driven dirty-path checks are hints, never truth. + """ if self.policy.version < 2: return try: @@ -131,6 +158,28 @@ def _observe_filesystem(self, root: Path, *, runtime_active: bool) -> None: ) return + def check_path(self, relative: str, *, runtime_active: bool) -> None: + """Targeted policy check for one dirty path (acceleration hint only). + + This never replaces :meth:`check_filesystem`; it only fails fast on + clearly protected paths between authoritative reconciliations. + """ + if self.report.terminated or self.policy.version < 2: + return + decision = self.engine.decide_path(relative, phase="intercept") + if decision.action is PolicyAction.DENY: + self._terminate( + metric="protected_path", + observed="targeted_check", + limit=None, + detail=( + "protected path was modified; the owned runtime is terminated, " + "but the write was not syscall-intercepted" + ), + path=relative, + runtime_active=runtime_active, + ) + def _check_limit( self, name: str, diff --git a/src/agentdiff/safety/watcher.py b/src/agentdiff/safety/watcher.py index e68cd9a..21c47eb 100644 --- a/src/agentdiff/safety/watcher.py +++ b/src/agentdiff/safety/watcher.py @@ -1,4 +1,31 @@ -"""Hybrid event-driven and deterministic polling safety watcher.""" +"""Hybrid event-driven and deterministic polling safety watcher. + +The watcher uses fast event hints to accelerate deterministic safety audits: + + OS filesystem event source + ↓ + dirty path/directory queue + ↓ + targeted policy checks (acceleration hints, never verdicts) + + + periodic full reconciliation (authoritative) + ↓ + SafetyController + +Rules enforced here: + +- Watcher events are strictly hints. Security verdicts and budget + enforcement always come from authoritative full captures via + :class:`SafetyController`. +- Targeted dirty-path checks fail fast on clearly protected paths but never + compute budgets. +- Full reconciliation runs on a configurable cadence, after a configurable + number of events, on overflow, and whenever a targeted check cannot + resolve the dirty state. +- If the event backend fails or is unsupported, the watcher degrades to + polling and records ``status: degraded`` with the reason; it never + silently stops live observation. +""" from __future__ import annotations @@ -7,26 +34,38 @@ from pathlib import Path from typing import TYPE_CHECKING, Callable +from agentdiff.state import FilesystemScanner + +from .controller import SafetyController +from .watchers.polling import PollingEventSource + if TYPE_CHECKING: from agentdiff.policy import Policy from agentdiff.state import FilesystemManifest -from .controller import SafetyController + from .watchers.base import EventSource @dataclass class WatcherStats: hints_received: int = 0 + targeted_checks_performed: int = 0 full_scans_performed: int = 0 last_scan_duration: float = 0.0 -class HybridSafetyWatcher: - """Hybrid watcher that uses fast event hints to accelerate deterministic safety audits. +@dataclass +class WatcherStatus: + backend: str = "polling" + status: str = "active" + reason: str = "" - Watcher events are strictly acceleration hints: security verdicts and budget - enforcement decisions are always computed from ground-truth filesystem snapshots. - """ + def to_dict(self) -> dict[str, str]: + return {"backend": self.backend, "status": self.status, "reason": self.reason} + + +class HybridSafetyWatcher: + """Hybrid watcher that uses fast event hints to accelerate deterministic safety audits.""" def __init__( self, @@ -38,7 +77,15 @@ def __init__( isolated_workspace: bool = False, controller: SafetyController | None = None, on_terminate: Callable[[], None] | None = None, + event_source: EventSource | None = None, + reconcile_interval_seconds: float = 2.0, + max_dirty_before_reconcile: int = 100, + reconcile_after_event_seconds: float = 0.1, ) -> None: + if reconcile_interval_seconds <= 0: + raise ValueError("reconcile_interval_seconds must be greater than zero") + if max_dirty_before_reconcile <= 0: + raise ValueError("max_dirty_before_reconcile must be greater than zero") self.root = Path(root).resolve() self.policy = policy self.before = before @@ -50,12 +97,80 @@ def __init__( ) self.on_terminate = on_terminate self.stats = WatcherStats() - self._dirty = False + self.reconcile_interval_seconds = reconcile_interval_seconds + self.max_dirty_before_reconcile = max_dirty_before_reconcile + self.reconcile_after_event_seconds = reconcile_after_event_seconds + self._dirty_paths: set[str] = set() + self._dirty_directories: set[str] = set() + self._last_full_scan = time.monotonic() + self._last_event = 0.0 + self._event_source: EventSource | None = event_source + self._observe_root = self.root + self._scanner_cache: dict[Path, FilesystemScanner] = {} + self.status = WatcherStatus(backend=event_source.backend if event_source else "polling") + self._started = False + + @property + def report(self): + """Delegated safety report of the wrapped controller.""" + return self.controller.report + + # ------------------------------------------------------------------ + # Event lifecycle + # ------------------------------------------------------------------ + + def start(self) -> None: + """Start the event source; degrade to polling on failure.""" + if self._started: + return + self._started = True + if self._event_source is None: + return + try: + self._event_source.start() + self.status = WatcherStatus(backend=self._event_source.backend, status="active") + except (OSError, RuntimeError) as error: + self._event_source = PollingEventSource() + self.status = WatcherStatus( + backend=self._event_source.backend, + status="degraded", + reason=f"event backend failed to start: {error}", + ) + + def stop(self) -> None: + if self._event_source is not None and self._started: + try: + self._event_source.stop() + finally: + self._started = False def notify_event(self, path: str | None = None) -> None: """Receive an OS or runtime filesystem event hint.""" self.stats.hints_received += 1 - self._dirty = True + self._last_event = time.monotonic() + if not path: + self._dirty_directories.add("") + return + normalized = path.replace("\\", "/") + try: + relative = str(Path(normalized).relative_to(self.root)).replace("\\", "/") + except ValueError: + relative = normalized + self._dirty_paths.add(relative) + self._dirty_directories.add(str(Path(relative).parent) if relative else "") + + def drain_events(self) -> None: + if self._event_source is None: + return + try: + for event_path in self._event_source.drain(): + self.notify_event(event_path) + except (OSError, RuntimeError): + self._degrade("event drain failed") + + # ------------------------------------------------------------------ + # Evaluation + # ------------------------------------------------------------------ def poll( self, @@ -64,22 +179,124 @@ def poll( processes_spawned: int, force: bool = False, ) -> bool: - """Evaluate safety state using dirty hints or periodic schedule.""" - force_fs = force or self._dirty - start = time.monotonic() - should_terminate = self.controller.observe( - root=self.root, + """Evaluate safety using dirty hints, targeted checks, and scheduled scans.""" + self.drain_events() + if self.controller.report.terminated: + return True + self.controller.check_budgets( duration_seconds=duration_seconds, processes_spawned=processes_spawned, - force_filesystem=force_fs, runtime_active=True, ) - if force_fs: - self.stats.full_scans_performed += 1 - self.stats.last_scan_duration = time.monotonic() - start - self._dirty = False + if self.controller.report.terminated: + self._signal_terminate() + return True - if should_terminate and self.on_terminate is not None: + now = time.monotonic() + due_for_reconcile = ( + force + or now - self._last_full_scan >= self.reconcile_interval_seconds + or len(self._dirty_paths) >= self.max_dirty_before_reconcile + or (self._dirty_paths and now - self._last_event >= self.reconcile_after_event_seconds) + ) + if due_for_reconcile: + self._full_reconcile(runtime_active=True) + else: + self._targeted_check(runtime_active=True) + if self.controller.report.terminated: + self._signal_terminate() + return True + return False + + def observe( + self, + *, + root: str | Path, + duration_seconds: float, + processes_spawned: int, + force_filesystem: bool = False, + runtime_active: bool = True, + ) -> bool: + """Controller-compatible entry point used by runtime backends. + + The per-call ``root`` is authoritative for this observation window: + isolated backends observe their private workspace, local backends + observe the live host root. + """ + self._observe_root = Path(root).resolve() + if not runtime_active: + self._full_reconcile(runtime_active=False) + return self.controller.report.terminated + return self.poll( + duration_seconds=duration_seconds, + processes_spawned=processes_spawned, + force=force_filesystem, + ) + + @property + def terminated(self) -> bool: + return self.controller.report.terminated + + # ------------------------------------------------------------------ + # Checks + # ------------------------------------------------------------------ + + def _scanner_for(self) -> FilesystemScanner: + scanner = self._scanner_cache.get(self._observe_root) + if scanner is None: + scanner = FilesystemScanner( + self._observe_root, + protected_patterns=list(self.policy.filesystem.deny), + ) + self._scanner_cache[self._observe_root] = scanner + return scanner + + def _targeted_check(self, *, runtime_active: bool) -> None: + """Fail fast on dirty paths that clearly hit a protected pattern.""" + if self.policy.version < 2: + self._dirty_paths.clear() + self._dirty_directories.clear() + return + scanner = self._scanner_for() + for relative in sorted(self._dirty_paths): + self.stats.targeted_checks_performed += 1 + try: + record = scanner.capture_one(relative) + except (OSError, ValueError): + # Cannot resolve dirty state cheaply: force a full reconcile. + self._full_reconcile(runtime_active=runtime_active) + return + before_record = self.before.files.get(relative) + changed = record is not None and ( + before_record is None + or record.sha256 != before_record.sha256 + or record.size != before_record.size + or record.mode != before_record.mode + ) + if changed: + self.controller.check_path(relative, runtime_active=runtime_active) + if self.controller.report.terminated: + return + self._dirty_paths.clear() + self._dirty_directories.clear() + + def _full_reconcile(self, *, runtime_active: bool) -> None: + start = time.monotonic() + self.controller.check_filesystem(self._observe_root, runtime_active=runtime_active) + self.stats.full_scans_performed += 1 + self.stats.last_scan_duration = time.monotonic() - start + self._last_full_scan = time.monotonic() + self._dirty_paths.clear() + self._dirty_directories.clear() + + def _signal_terminate(self) -> None: + if self.on_terminate is not None: self.on_terminate() - return should_terminate + def _degrade(self, reason: str) -> None: + self._event_source = PollingEventSource() + self.status = WatcherStatus( + backend=self._event_source.backend, + status="degraded", + reason=reason, + ) diff --git a/src/agentdiff/safety/watchers/base.py b/src/agentdiff/safety/watchers/base.py new file mode 100644 index 0000000..43d80ce --- /dev/null +++ b/src/agentdiff/safety/watchers/base.py @@ -0,0 +1,28 @@ +"""Event-source backend contracts for the hybrid safety watcher.""" + +from __future__ import annotations + +from typing import Protocol + + +class EventSource(Protocol): + """A filesystem event source feeding dirty-path hints. + + Events are strictly acceleration hints. A failing or unsupported source + must raise from :meth:`start` so the watcher can degrade to polling; + it must never silently stop observing. + """ + + backend: str + + def start(self) -> None: + """Begin delivering events to :meth:`drain`.""" + ... + + def drain(self) -> list[str | None]: + """Return paths modified since the previous drain (or ``None`` events).""" + ... + + def stop(self) -> None: + """Stop event delivery and release resources.""" + ... diff --git a/src/agentdiff/safety/watchers/polling.py b/src/agentdiff/safety/watchers/polling.py new file mode 100644 index 0000000..25dee50 --- /dev/null +++ b/src/agentdiff/safety/watchers/polling.py @@ -0,0 +1,23 @@ +"""Pure-polling fallback event source (no OS event backend).""" + +from __future__ import annotations + + +class PollingEventSource: + """Event source that never produces events. + + Used when no platform event backend is available or when a backend + failed to start. The hybrid watcher then relies on its scheduled full + reconciliation cadence, which is always authoritative. + """ + + backend = "polling" + + def start(self) -> None: + return None + + def drain(self) -> list[str | None]: + return [] + + def stop(self) -> None: + return None diff --git a/src/agentdiff/safety/watchers/watchdog_backend.py b/src/agentdiff/safety/watchers/watchdog_backend.py new file mode 100644 index 0000000..0896019 --- /dev/null +++ b/src/agentdiff/safety/watchers/watchdog_backend.py @@ -0,0 +1,69 @@ +"""Optional watchdog-based filesystem event source. + +``watchdog`` is imported lazily and kept optional: when it is not installed, +or its observer cannot start on the platform, :class:`WatchdogEventSource` +raises and the hybrid watcher degrades to polling instead of pretending it +has live event hints. Events remain hints only; every security verdict is +computed from authoritative full captures. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +class WatchdogEventSource: + """Drain ``watchdog`` filesystem events into dirty-path hints.""" + + backend = "watchdog" + + def __init__(self, root: str | Path) -> None: + try: + from watchdog.events import FileSystemEventHandler + from watchdog.observers import Observer + except ImportError as error: # pragma: no cover - depends on environment + raise RuntimeError("watchdog is not installed") from error + self._observer_class = Observer + self._handler_class = FileSystemEventHandler + self.root = Path(root).resolve() + self._observer: Any | None = None + self._pending: list[str | None] = [] + + def _make_handler(self, sink): + handler_class = self._handler_class + + class _Handler(handler_class): + def on_any_event(self, event) -> None: + source = getattr(event, "src_path", None) + if isinstance(source, str): + sink.append(source) + else: + sink.append(None) + + return _Handler() + + def start(self) -> None: + if self._observer is not None: + return + handler = self._make_handler(self._pending) + observer = self._observer_class(timeout=0.2) + observer.schedule(handler, str(self.root), recursive=True) + try: + observer.start() + except OSError as error: + raise RuntimeError(f"watchdog observer failed to start: {error}") from error + self._observer = observer + + def drain(self) -> list[str | None]: + pending = self._pending + self._pending = [] + return pending + + def stop(self) -> None: + if self._observer is not None: + try: + self._observer.stop() + self._observer.join(timeout=2.0) + finally: + self._observer = None diff --git a/src/agentdiff/trajectory.py b/src/agentdiff/trajectory.py index fa2ac8a..70d9348 100644 --- a/src/agentdiff/trajectory.py +++ b/src/agentdiff/trajectory.py @@ -19,7 +19,7 @@ from typing_extensions import Self warnings.warn( - "agentdiff.trajectory is deprecated and will be removed in a future release. " + "agentdiff.trajectory is deprecated since 0.2 and will be removed in 0.4 or later. " "Use agentdiff.transaction.RunStore for structured run evidence and event logging.", DeprecationWarning, stacklevel=2, diff --git a/src/agentdiff/transaction/runner.py b/src/agentdiff/transaction/runner.py index c03313c..cb6ed73 100644 --- a/src/agentdiff/transaction/runner.py +++ b/src/agentdiff/transaction/runner.py @@ -22,7 +22,7 @@ policy_to_dict, ) from agentdiff.runtime import LocalRuntime, RuntimeBackend, RuntimeResult -from agentdiff.safety import SafetyController +from agentdiff.safety import HybridSafetyWatcher from agentdiff.scoring import ( BlastRadiusResult, BlastRadiusScorer, @@ -37,7 +37,7 @@ diff_manifests, ) -from .store import RunStore +from .store import InvalidRunIdError, RunStore if TYPE_CHECKING: import os @@ -192,14 +192,13 @@ def _validate_recovery_backups( size=record.size, ) verified_files[path] = record - except (OSError, ValueError, TypeError) as error: + except (InvalidRunIdError, OSError, ValueError, TypeError, KeyError) as error: verified_unsupported[path] = f"backup validation failed: {error}" verified_files[path] = replace( record, backup_path=None, backup_error=f"backup validation failed: {error}", ) - return FilesystemManifest( schema_version=manifest.schema_version, captured_at=manifest.captured_at, @@ -273,7 +272,7 @@ def run( runtime_result: RuntimeResult | None = None execution_error: dict[str, Any] | None = None blocked = command_decision.action is PolicyAction.DENY - safety_controller: SafetyController | None = None + safety_watcher: HybridSafetyWatcher | None = None selected_runtime: RuntimeBackend | None = None if not blocked: @@ -281,20 +280,34 @@ def run( self.root, observe_ports=self.policy.network.mode is NetworkMode.OBSERVE, ) - if hasattr(selected_runtime, "configure_source"): + if selected_runtime.capabilities.supports_source_snapshot: selected_runtime.configure_source(store.artifact_path("source/files")) - isolated_workspace = ( - getattr(selected_runtime, "enforcement", "") == "isolated_private_workspace" - ) - safety_controller = SafetyController( + isolated_workspace = selected_runtime.capabilities.private_workspace + backend = selected_runtime.capabilities.backend + event_source: Any = None + if not isolated_workspace: + # Local runs can accelerate with OS event hints; isolated + # backends observe their private workspace, which does not + # exist until the backend creates it, so they poll. + from agentdiff.safety.watchers.polling import PollingEventSource + from agentdiff.safety.watchers.watchdog_backend import WatchdogEventSource + + try: + event_source = WatchdogEventSource(self.root) + except RuntimeError: + event_source = PollingEventSource() + watcher = HybridSafetyWatcher( + root=self.root, policy=self.policy, before=before, - backend=getattr(selected_runtime, "backend", "local-observe"), + backend=backend, isolated_workspace=isolated_workspace, + event_source=event_source, ) - if hasattr(selected_runtime, "configure_safety"): - selected_runtime.configure_safety(safety_controller) + safety_watcher = watcher + selected_runtime.configure_safety(safety_watcher) + watcher.start() effective_timeout = timeout_seconds policy_timeout = self.policy.limits.duration_seconds @@ -401,12 +414,12 @@ def run( unsupported_paths = sorted(set(before.unsupported) | set(after.unsupported)) observation_warnings = [ ObservationWarning( - path=unsupported_path, - reason=after.unsupported.get(unsupported_path) - or before.unsupported.get(unsupported_path) + path=path, + reason=after.unsupported.get(path) + or before.unsupported.get(path) or "unsupported entry", ) - for unsupported_path in unsupported_paths + for path in unsupported_paths ] files_deleted = sum(change.change_type == "deleted" for change in changes) processes_spawned = len(runtime_result.owned_processes) if runtime_result is not None else 0 @@ -417,7 +430,6 @@ def run( processes_spawned=processes_spawned, duration_seconds=duration, ) - orphan_processes = 0 opened_ports = 0 if runtime_result is not None: @@ -450,7 +462,7 @@ def run( actions.append(PolicyAction.REVIEW) safety_outcome = _highest_action(actions) - is_terminated = bool(safety_controller is not None and safety_controller.terminated) + is_terminated = bool(safety_watcher is not None and safety_watcher.terminated) if blocked: status = "blocked" elif is_terminated: @@ -468,9 +480,15 @@ def run( else: status = "passed" - safety_report = ( - safety_controller.report.to_dict() if safety_controller is not None else None - ) + safety_report = None + if safety_watcher is not None: + safety_report = safety_watcher.report.to_dict() + safety_report["watcher"] = safety_watcher.status.to_dict() + safety_report["watcher_stats"] = { + "hints_received": safety_watcher.stats.hints_received, + "targeted_checks_performed": safety_watcher.stats.targeted_checks_performed, + "full_scans_performed": safety_watcher.stats.full_scans_performed, + } result = TransactionResult( run_id=store.run_id, status=status, @@ -498,6 +516,8 @@ def run( }, ) store.finalize_integrity() - if selected_runtime is not None and hasattr(selected_runtime, "close"): + if safety_watcher is not None: + safety_watcher.stop() + if selected_runtime is not None: selected_runtime.close() return result diff --git a/src/agentdiff/transaction/store.py b/src/agentdiff/transaction/store.py index 562042c..72d61fe 100644 --- a/src/agentdiff/transaction/store.py +++ b/src/agentdiff/transaction/store.py @@ -64,6 +64,7 @@ class IntegrityReport: ok: bool files_checked: int issues: tuple[IntegrityIssue, ...] = () + version: int = 0 def to_dict(self) -> dict[str, Any]: return { @@ -71,6 +72,7 @@ def to_dict(self) -> dict[str, Any]: "present": self.present, "ok": self.ok, "files_checked": self.files_checked, + "version": self.version, "issues": [issue.to_dict() for issue in self.issues], } @@ -234,17 +236,33 @@ def finalize_integrity(self) -> dict[str, Any]: return manifest def verify_integrity(self) -> IntegrityReport: - """Verify a sealed capsule without following links.""" - + """Verify a sealed capsule without following links. + + Verification is routed by capsule version: spec v2 capsules + (``integrity/manifest.json``) are verified with the structured + manifest, and legacy v1 capsules (``integrity.json`` only, schema + version 1) are verified with their original guarantees. A schema-2 + ``integrity.json`` mirror without the ``integrity/`` directory is + treated as an incomplete seal and fails. + """ self._ensure_run_dir_identity() - v2_integrity = self.run_dir / "integrity" / "manifest.json" - v1_integrity = self.run_dir / "integrity.json" - if not v2_integrity.exists() and not v1_integrity.exists(): - return IntegrityReport(present=False, ok=False, files_checked=0) - issues: list[IntegrityIssue] = [] - if not v2_integrity.exists(): - issues.append(IntegrityIssue("integrity/manifest.json", "missing integrity manifest")) - return IntegrityReport(present=True, ok=False, files_checked=0, issues=tuple(issues)) + if self.capsule_version == 2: + return self._verify_v2_capsule() + if self.capsule_version == 1: + return self._verify_v1_capsule() + return IntegrityReport(present=False, ok=False, files_checked=0, version=0) + + @property + def capsule_version(self) -> int: + """Return the sealed capsule spec version (0 when unsealed).""" + if (self.run_dir / "integrity" / "manifest.json").is_file(): + return 2 + if (self.run_dir / "integrity.json").is_file(): + return 1 + return 0 + + def _verify_v2_capsule(self) -> IntegrityReport: + """Verify a spec v2 capsule: structured manifest + required artifacts.""" try: manifest = self.read_json_path("integrity/manifest.json") except (OSError, ValueError, TypeError, json.JSONDecodeError) as error: @@ -252,6 +270,7 @@ def verify_integrity(self) -> IntegrityReport: present=True, ok=False, files_checked=0, + version=2, issues=(IntegrityIssue("integrity/manifest.json", type(error).__name__),), ) if not isinstance(manifest, dict) or manifest.get("algorithm") != "sha256": @@ -259,6 +278,7 @@ def verify_integrity(self) -> IntegrityReport: present=True, ok=False, files_checked=0, + version=2, issues=(IntegrityIssue("integrity/manifest.json", "invalid integrity manifest"),), ) raw_files = manifest.get("files") @@ -267,18 +287,94 @@ def verify_integrity(self) -> IntegrityReport: present=True, ok=False, files_checked=0, + version=2, issues=(IntegrityIssue("integrity/manifest.json", "files must be an object"),), ) + report = self._verify_manifest_files(raw_files, version=2) + return IntegrityReport( + present=True, + ok=bool(report[0]), + files_checked=report[1], + version=2, + issues=tuple(report[2]), + ) + def _verify_v1_capsule(self) -> IntegrityReport: + """Verify a legacy v1 capsule under its original guarantees. + + V1 manifests are the flat ``integrity.json`` with schema version 1. + A schema-2 mirror without the ``integrity/`` directory is an + incomplete spec-v2 seal, not a v1 capsule, and fails closed. + """ try: - actual_files = dict(self._discover_sealed_files()) - except (OSError, ValueError) as error: + manifest = self.read_json("integrity.json") + except (OSError, ValueError, TypeError, json.JSONDecodeError) as error: + return IntegrityReport( + present=True, + ok=False, + files_checked=0, + version=1, + issues=(IntegrityIssue("integrity.json", type(error).__name__),), + ) + if not isinstance(manifest, dict): + return IntegrityReport( + present=True, + ok=False, + files_checked=0, + version=1, + issues=(IntegrityIssue("integrity.json", "invalid integrity manifest"),), + ) + if int(manifest.get("schema_version", -1)) != 1: + return IntegrityReport( + present=True, + ok=False, + files_checked=0, + version=1, + issues=( + IntegrityIssue( + "integrity/manifest.json", + "incomplete spec-v2 seal: missing structured integrity manifest", + ), + ), + ) + if manifest.get("algorithm") != "sha256": return IntegrityReport( present=True, ok=False, files_checked=0, - issues=(IntegrityIssue("", str(error)),), + version=1, + issues=(IntegrityIssue("integrity.json", "invalid integrity manifest"),), ) + raw_files = manifest.get("files") + if not isinstance(raw_files, dict): + return IntegrityReport( + present=True, + ok=False, + files_checked=0, + version=1, + issues=(IntegrityIssue("integrity.json", "files must be an object"),), + ) + report = self._verify_manifest_files(raw_files, version=1) + return IntegrityReport( + present=True, + ok=bool(report[0]), + files_checked=report[1], + version=1, + issues=tuple(report[2]), + ) + + def _verify_manifest_files( + self, + raw_files: dict[str, Any], + *, + version: int, + ) -> tuple[bool, int, list[IntegrityIssue]]: + """Shared digest/size verification against one manifest files mapping.""" + issues: list[IntegrityIssue] = [] + try: + actual_files = dict(self._discover_sealed_files()) + except (OSError, ValueError) as error: + return True, 0, [IntegrityIssue("", str(error))] expected_names = {str(name) for name in raw_files} required = set(_REQUIRED_SEALED_ARTIFACTS) try: @@ -309,12 +405,7 @@ def verify_integrity(self) -> IntegrityReport: checked += 1 if expected.get("sha256") != digest or expected.get("size") != size: issues.append(IntegrityIssue(relative, "digest or size mismatch")) - return IntegrityReport( - present=True, - ok=not issues, - files_checked=checked, - issues=tuple(issues), - ) + return not issues, checked, issues def _required_sealed_paths(self) -> set[str]: """Return principal artifacts plus every backup referenced by before-state.""" @@ -357,14 +448,7 @@ def verify_backup(self, relative: str, *, sha256: str, size: int) -> None: def _discover_sealed_files(self) -> list[tuple[str, Path]]: self._ensure_run_dir_identity() _EXTENSION_DIRS = frozenset( - { - "proof", - "promotion", - "recovery", - "staging", - "backups", - ".agentdiff", - } + {"proof", "promotion", "recovery", "staging", "backups", ".agentdiff"} ) discovered: list[tuple[str, Path]] = [] for directory, directory_names, file_names in os.walk(self.run_dir, followlinks=False): @@ -390,7 +474,6 @@ def _discover_sealed_files(self) -> list[tuple[str, Path]]: if parts[0] in _EXTENSION_DIRS: continue discovered.append((relative, path)) - return sorted(discovered) @staticmethod @@ -727,7 +810,7 @@ def verify_extension(self, name: str) -> IntegrityReport: issues: list[IntegrityIssue] = [] try: manifest = self.read_json_path(f"{normalized_name}/integrity.json") - except (OSError, ValueError, TypeError) as error: + except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: return IntegrityReport( present=True, ok=False, @@ -782,11 +865,8 @@ def verify_extension(self, name: str) -> IntegrityReport: or expected.get("size") != size ): issues.append(IntegrityIssue(f"{name}/{filename}", "digest or size mismatch")) - except (OSError, ValueError, TypeError) as error: + except (OSError, ValueError, TypeError, RuntimeError, InvalidRunIdError) as error: issues.append(IntegrityIssue(f"{name}/{filename}", str(error))) return IntegrityReport( - present=True, - ok=not issues, - files_checked=checked, - issues=tuple(issues), + present=True, ok=not issues, files_checked=checked, issues=tuple(issues) ) diff --git a/tests/test_capsule_versions.py b/tests/test_capsule_versions.py new file mode 100644 index 0000000..100d4e1 --- /dev/null +++ b/tests/test_capsule_versions.py @@ -0,0 +1,117 @@ +"""Capsule spec v1/v2 verification separation and root-digest terminology.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path +import json + +from agentdiff.evidence import CapsuleReader +from agentdiff.transaction import RunStore + + +def build_capsule(tmp_path: Path) -> RunStore: + """Create a sealed store with the principal artifacts written.""" + store = RunStore.create(tmp_path, task="t", command=["agent"]) + for name in ("metadata.json", "policy.json", "after.json", "runtime.json"): + store.write_json(name, {"schema_version": 1, "x": 1}) + store.write_json("before.json", {"schema_version": 1, "files": {}}) + store.write_json("result.json", {"schema_version": 1, "status": "passed"}) + store.append_event("created", {}) + return store + + +def v1_manifest(store: RunStore) -> dict[str, object]: + files: dict[str, dict[str, object]] = {} + for name in ( + "metadata.json", + "policy.json", + "before.json", + "after.json", + "runtime.json", + "result.json", + "events.jsonl", + ): + digest, size = store.artifact_digest(name) + files[name] = {"sha256": digest, "size": size} + return {"schema_version": 1, "algorithm": "sha256", "files": files} + + +def test_v1_capsule_verifies_under_original_guarantees(tmp_path: Path) -> None: + store = build_capsule(tmp_path) + (store.run_dir / "integrity.json").write_text(json.dumps(v1_manifest(store)), encoding="utf-8") + + report = store.verify_integrity() + assert report.present is True + assert report.ok is True + assert report.version == 1 + assert report.files_checked == 7 + + reader = CapsuleReader(store.run_dir) + assert reader.version == 1 + assert reader.read_manifest()["schema_version"] == 1 + + +def test_v2_capsule_verifies_with_structured_manifest(tmp_path: Path) -> None: + store = build_capsule(tmp_path) + store.finalize_integrity() + + report = store.verify_integrity() + assert report.ok is True + assert report.version == 2 + + reader = CapsuleReader(store.run_dir) + assert reader.version == 2 + assert (store.run_dir / "integrity" / "manifest.json").is_file() + + +def test_v1_capsule_tampering_is_detected(tmp_path: Path) -> None: + store = build_capsule(tmp_path) + (store.run_dir / "integrity.json").write_text(json.dumps(v1_manifest(store)), encoding="utf-8") + (store.run_dir / "result.json").write_text('{"status": "tampered"}\n', encoding="utf-8") + + report = store.verify_integrity() + assert report.ok is False + assert any(issue.path == "result.json" for issue in report.issues) + + +def test_schema2_mirror_without_integrity_dir_is_incomplete(tmp_path: Path) -> None: + """Deleting integrity/manifest.json must not downgrade a v2 capsule to v1.""" + store = build_capsule(tmp_path) + store.finalize_integrity() + (store.run_dir / "integrity" / "manifest.json").unlink() + + report = store.verify_integrity() + assert report.ok is False + assert any(issue.path == "integrity/manifest.json" for issue in report.issues) + assert any("incomplete spec-v2 seal" in issue.reason for issue in report.issues) + + +def test_root_digest_is_flat_aggregate_not_merkle(tmp_path: Path) -> None: + store = build_capsule(tmp_path) + store.finalize_integrity() + reader = CapsuleReader(store.run_dir) + + digest = reader.compute_root_digest() + assert isinstance(digest, str) and len(digest) == 64 + # The deprecated alias must agree. + assert reader.compute_merkle_root() == digest + # Deterministic across reads. + assert reader.compute_root_digest() == digest + + +def test_root_digest_changes_when_manifest_changes(tmp_path: Path) -> None: + store = build_capsule(tmp_path) + store.finalize_integrity() + reader = CapsuleReader(store.run_dir) + before = reader.compute_root_digest() + + manifest = reader.read_manifest() + manifest["files"]["result.json"] = {"sha256": "0" * 64, "size": 1} + (store.run_dir / "integrity" / "manifest.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + + assert reader.compute_root_digest() != before diff --git a/tests/test_cas.py b/tests/test_cas.py new file mode 100644 index 0000000..1e5cffc --- /dev/null +++ b/tests/test_cas.py @@ -0,0 +1,115 @@ +"""Content-addressed object storage tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path +import hashlib +import os + +import pytest + +from agentdiff.evidence import ObjectRef, ObjectStore, ObjectStoreError + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def test_object_put_and_read_back(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + payload = b"hello object store" + ref = store.put(payload) + assert isinstance(ref, ObjectRef) + assert ref.sha256 == sha256(payload) + assert ref.size == len(payload) + assert store.read_bytes(ref.sha256) == payload + assert store.verify(ref.sha256) == len(payload) + + +def test_object_store_deduplicates_identical_content(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + payload = b"shared content" + first = store.put(payload) + second = store.put(payload) + assert first.sha256 == second.sha256 + assert first.sha256 == sha256(payload) + # One object on disk. + count = sum(1 for _ in store.iter_all()) + assert count == 1 + + +def test_object_store_keeps_objects_immutable(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + payload = b"immutable" + ref = store.put(payload) + path = store.path_for(ref.sha256) + before = path.stat().st_mtime_ns + # Re-putting the same content must not rewrite the object. + store.put(payload) + assert path.stat().st_mtime_ns == before + + +def test_object_store_rejects_invalid_digest(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + with pytest.raises(ObjectStoreError, match="invalid object digest"): + store.path_for("../../etc/passwd") + with pytest.raises(ObjectStoreError, match="invalid object digest"): + store.path_for("abc") + with pytest.raises(ObjectStoreError, match="invalid object digest"): + store.path_for("A" * 64) + assert store.has("../../etc/passwd") is False + with pytest.raises(ObjectStoreError): + store.open("deadbeef") + + +def test_object_store_path_stays_under_objects_dir(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + ref = store.put(b"x") + resolved = store.path_for(ref.sha256).resolve() + assert resolved.parent.parent == store.objects_dir.resolve() + assert str(resolved).startswith(str(store.objects_dir.resolve())) + + +def test_object_store_corruption_detected(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + ref = store.put(b"original content") + path = store.path_for(ref.sha256) + path.write_bytes(b"tampered content") + with pytest.raises(ObjectStoreError, match="does not match digest"): + store.verify(ref.sha256) + with pytest.raises(ObjectStoreError): + store.read_bytes(ref.sha256) + + +def test_object_store_rejects_symlinked_object(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + ref = store.put(b"data") + target = store.path_for(ref.sha256) + target.unlink() + outside = tmp_path / "outside" + outside.write_bytes(b"outside data") + os.symlink(outside, target) + assert store.has(ref.sha256) is False + with pytest.raises(ObjectStoreError, match="single-link regular file"): + store.open(ref.sha256) + + +def test_object_store_put_from_path(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + source = tmp_path / "source.bin" + source.write_bytes(b"from path") + ref = store.put(source) + assert store.read_bytes(ref.sha256) == b"from path" + + +def test_object_store_iter_all_deterministic(tmp_path: Path) -> None: + store = ObjectStore(tmp_path) + payloads = [b"one", b"two", b"three"] + refs = [store.put(payload) for payload in payloads] + found = list(store.iter_all()) + assert {item.sha256 for item in found} == {ref.sha256 for ref in refs} + # Deterministic order. + assert [item.sha256 for item in found] == sorted(item.sha256 for item in found) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0219462..d8d9b7d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,9 +33,14 @@ def test_cli_help_only_lists_implemented_commands(tmp_path: Path) -> None: assert result.returncode == 0 assert "snapshot" not in result.stdout assert "eval" not in result.stdout - assert "{run,inspect,runs,verify,rollback,cleanup,doctor,policy,cortex}" in result.stdout + assert ( + "{run,inspect,runs,verify,prove,promote,rollback,cleanup,doctor,policy,cortex}" + in result.stdout + ) assert "run" in result.stdout assert "inspect" in result.stdout + assert "prove" in result.stdout + assert "promote" in result.stdout assert "rollback" in result.stdout assert "cleanup" in result.stdout assert "doctor" in result.stdout @@ -388,3 +393,45 @@ def test_cli_doctor_reports_port_observation_without_network_control(tmp_path: P assert report["network_observation"] is False assert report["network_enforcement"] is False assert report["sandboxed"] is False + + +def test_cli_promote_blocks_without_sealed_capsule(tmp_path: Path) -> None: + result = run_cli("promote", "missing-run", cwd=tmp_path) + assert result.returncode == 2 + assert "error" in result.stderr.lower() or "not found" in result.stderr.lower() + + +def test_cli_promote_fails_closed_without_proof(tmp_path: Path) -> None: + """A sealed run without PROVEN proof must be blocked by the promotion gate.""" + (tmp_path / "base.txt").write_text("old", encoding="utf-8") + task = tmp_path / "task.py" + task.write_text( + "from pathlib import Path\nPath('base.txt').write_text('new', encoding='utf-8')\n", + encoding="utf-8", + ) + run = run_cli( + "run", + "--task", + "cli promote gate", + "--", + sys.executable, + "-c", + "from pathlib import Path; Path('base.txt').write_text('new', encoding='utf-8')", + cwd=tmp_path, + ) + assert run.returncode == 0, run.stdout + run.stderr + run_id = run.stdout.split("Run: ")[1].split()[0] + + promoted = run_cli("promote", run_id, "--safe-only", cwd=tmp_path) + assert promoted.returncode == 9 + assert "blocked" in promoted.stderr + + # A dry-run must also fail closed: promotion requires PROVEN proof. + dry = run_cli("promote", run_id, "--dry-run", "--safe-only", cwd=tmp_path) + assert dry.returncode == 9 + + +def test_cli_prove_requires_sealed_capsule(tmp_path: Path) -> None: + result = run_cli("prove", "missing-run", cwd=tmp_path) + assert result.returncode == 2 + assert "error" in result.stderr.lower() or "not found" in result.stderr.lower() diff --git a/tests/test_docker_integration.py b/tests/test_docker_integration.py index 0ca03ff..3e71f78 100644 --- a/tests/test_docker_integration.py +++ b/tests/test_docker_integration.py @@ -13,7 +13,6 @@ if TYPE_CHECKING: from pathlib import Path - from agentdiff.policy import load_policy from agentdiff.promotion import PromotionEngine from agentdiff.proof import ProofEngine, ProofVerdict diff --git a/tests/test_materializer_security.py b/tests/test_materializer_security.py new file mode 100644 index 0000000..81d3348 --- /dev/null +++ b/tests/test_materializer_security.py @@ -0,0 +1,144 @@ +"""Workspace materializer correctness and trust-boundary tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path +import hashlib +import os +import stat + +import pytest + +from agentdiff.runtime import MaterializationStrategy, WorkspaceMaterializer + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def make_tree(root: Path) -> None: + (root / "src").mkdir() + (root / "src" / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (root / "script.sh").write_text("#!/bin/sh\necho hi\n", encoding="utf-8") + (root / "script.sh").chmod(0o755) + (root / "empty.txt").write_text("", encoding="utf-8") + (root / "src" / "nested").mkdir() + (root / "src" / "nested" / "deep.txt").write_text("x" * 2_000_000, encoding="utf-8") + + +@pytest.mark.parametrize( + "strategy", + [ + MaterializationStrategy.AUTO, + MaterializationStrategy.FAST_COPY, + MaterializationStrategy.STREAM_COPY, + ], +) +def test_materializer_preserves_content_size_and_mode(tmp_path: Path, strategy: object) -> None: + src = tmp_path / "source" + dst = tmp_path / "target" + src.mkdir() + make_tree(src) + + report = WorkspaceMaterializer(strategy=strategy).materialize(src, dst) + + assert report.files_materialized == 4 + assert (dst / "script.sh").read_text(encoding="utf-8") == "#!/bin/sh\necho hi\n" + assert (dst / "empty.txt").read_text(encoding="utf-8") == "" + assert len((dst / "src" / "nested" / "deep.txt").read_bytes()) == 2_000_000 + assert (dst / "src" / "app.py").read_text(encoding="utf-8") == "VALUE = 1\n" + if os.name != "nt": + assert stat.S_IMODE((dst / "script.sh").stat().st_mode) == 0o755 + assert stat.S_IMODE((dst / "src" / "app.py").stat().st_mode) == 0o644 + # No live-host or sealed-source mutation. + assert sha256_bytes((src / "script.sh").read_bytes()) == sha256_bytes( + (dst / "script.sh").read_bytes() + ) + assert (src / "src" / "nested" / "deep.txt").stat().st_size == 2_000_000 + + +def test_materializer_rejects_symlink(tmp_path: Path) -> None: + src = tmp_path / "source" + dst = tmp_path / "target" + src.mkdir() + (src / "app.py").write_text("x", encoding="utf-8") + os.symlink(tmp_path / "outside.txt", src / "link.txt") + + with pytest.raises(RuntimeError, match="symlink"): + WorkspaceMaterializer().materialize(src, dst) + # The offending symlink is never materialized or followed. The copy is + # not transactional across files (walk order is filesystem-dependent); + # the caller discards the failed private workspace. + assert not (dst / "link.txt").exists() + + +def test_materializer_rejects_hardlink(tmp_path: Path) -> None: + src = tmp_path / "source" + dst = tmp_path / "target" + src.mkdir() + first = src / "first.txt" + first.write_text("shared", encoding="utf-8") + os.link(first, src / "second.txt") + + with pytest.raises(RuntimeError, match="hardlink"): + WorkspaceMaterializer().materialize(src, dst) + + +@pytest.mark.skipif(os.name == "nt", reason="named pipes are POSIX-only") +def test_materializer_rejects_special_file(tmp_path: Path) -> None: + import os as _os + + src = tmp_path / "source" + dst = tmp_path / "target" + src.mkdir() + _os.mkfifo(src / "pipe.fifo") + + with pytest.raises(RuntimeError, match="special file"): + WorkspaceMaterializer().materialize(src, dst) + + +def test_materializer_rejects_target_symlink(tmp_path: Path) -> None: + src = tmp_path / "source" + src.mkdir() + (src / "app.py").write_text("x", encoding="utf-8") + outside = tmp_path / "outside" + outside.mkdir() + dst = tmp_path / "target" + os.symlink(outside, dst) + + with pytest.raises(ValueError, match="real directory"): + WorkspaceMaterializer().materialize(src, dst) + assert not (outside / "app.py").exists() + + +def test_materializer_fast_copy_reports_actual_strategy(tmp_path: Path) -> None: + src = tmp_path / "source" + dst = tmp_path / "target" + src.mkdir() + (src / "app.py").write_text("x" * 100, encoding="utf-8") + + report = WorkspaceMaterializer(strategy=MaterializationStrategy.STREAM_COPY).materialize( + src, dst + ) + assert report.strategy_used in {"stream_copy", "clone", "fast_copy"} + assert report.files_materialized == 1 + assert report.bytes_materialized == 100 + + +def test_materializer_ignores_git_and_agentdiff_dirs(tmp_path: Path) -> None: + src = tmp_path / "source" + dst = tmp_path / "target" + src.mkdir() + (src / "app.py").write_text("x", encoding="utf-8") + (src / ".git").mkdir() + (src / ".git" / "HEAD").write_text("ref", encoding="utf-8") + (src / ".agentdiff").mkdir() + (src / ".agentdiff" / "secret").write_text("hidden", encoding="utf-8") + + report = WorkspaceMaterializer().materialize(src, dst) + assert report.files_materialized == 1 + assert not (dst / ".git").exists() + assert not (dst / ".agentdiff").exists() diff --git a/tests/test_promotion_fault_injection.py b/tests/test_promotion_fault_injection.py new file mode 100644 index 0000000..f750a94 --- /dev/null +++ b/tests/test_promotion_fault_injection.py @@ -0,0 +1,626 @@ +"""Adversarial promotion recovery and fail-closed tests. + +These tests simulate every crash point in the promotion write-ahead +state machine by constructing the exact on-disk journal/host state a crash +would leave behind, then asserting that recovery is deterministic, +convergent, and fails closed on ambiguity or corruption. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path +import hashlib +import json +import os +import stat +import subprocess +import sys + +import pytest + +from agentdiff.promotion import ( + EntryState, + JournalEntry, + JournalLoadOutcome, + JournalState, + PromotionJournal, + PromotionLockError, + PromotionRecovery, + PromotionRecoveryError, + WorkspaceLease, +) +from agentdiff.promotion.lock import PromotionLockError as _LockError # noqa: F401 + + +def sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def make_journal( + root: Path, + *, + state: JournalState, + entries: list[JournalEntry], + run_id: str = "run-crash", + schema_version: int = 3, +) -> PromotionJournal: + journal = PromotionJournal( + root=root, + run_id=run_id, + patch_digest="patch-digest", + state=state, + entries=entries, + ) + journal.schema_version = schema_version + journal.persist() + return journal + + +def base_entry( + path: str, + change_type: str, + *, + base_content: str | None = None, + result_content: str | None = None, + base_mode: int | None = 0o644, + result_mode: int | None = 0o644, + state: EntryState = EntryState.APPLIED, + with_backup: bool = True, + run_id: str = "run-crash", +) -> JournalEntry: + backup_rel = ( + f".agentdiff/backups/{run_id}/{path}" + if with_backup and change_type in {"modified", "deleted"} + else None + ) + return JournalEntry( + path=path, + change_type=change_type, + base_sha256=sha256(base_content) if base_content is not None else None, + result_sha256=sha256(result_content) if result_content is not None else None, + base_mode=base_mode, + result_mode=result_mode, + base_size=len(base_content.encode()) if base_content is not None else None, + result_size=len(result_content.encode()) if result_content is not None else None, + state=state, + staged_relpath=None, + backup_relpath=backup_rel, + ) + + +def write_backup(root: Path, relpath: str, content: str, run_id: str = "run-crash") -> Path: + target = root / ".agentdiff" / "backups" / run_id / relpath + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + +# --------------------------------------------------------------------------- +# Corrupt / malformed journal must fail closed +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "payload", + [ + "{ not json", + "[]", + '{"schema_version": 3}', + '{"schema_version": 3, "run_id": 1, "patch_digest": "x", "state": "APPLYING", ' + '"entries": []}', + '{"schema_version": 3, "run_id": "r", "patch_digest": "x", "state": "BOGUS", ' + '"entries": []}', + '{"schema_version": 3, "run_id": "r", "patch_digest": "x", "state": "APPLYING", ' + '"entries": [{"path": "a.txt"}]}', + '{"schema_version": 99, "run_id": "r", "patch_digest": "x", "state": "APPLYING", ' + '"entries": []}', + '{"schema_version": 3, "run_id": "r", "patch_digest": "x", "state": "APPLYING", ' + '"entries": [{"path": "a.txt", "change_type": "explode", "state": "PREPARED"}]}', + ], +) +def test_corrupt_journal_fails_closed(tmp_path: Path, payload: str) -> None: + (tmp_path / ".agentdiff").mkdir() + (tmp_path / ".agentdiff" / "promotion-journal.json").write_text(payload, encoding="utf-8") + + loaded = PromotionJournal.load(tmp_path) + assert loaded.outcome is JournalLoadOutcome.CORRUPT_JOURNAL + + with pytest.raises(PromotionRecoveryError, match=r"recovery state cannot be established"): + PromotionRecovery(tmp_path).check_and_recover() + + +def test_no_journal_returns_none(tmp_path: Path) -> None: + assert PromotionRecovery(tmp_path).check_and_recover() is None + + +def test_journal_symlink_fails_closed(tmp_path: Path) -> None: + (tmp_path / ".agentdiff").mkdir() + (tmp_path / "target.json").write_text("{}", encoding="utf-8") + os.symlink(tmp_path / "target.json", tmp_path / ".agentdiff" / "promotion-journal.json") + + loaded = PromotionJournal.load(tmp_path) + assert loaded.outcome is JournalLoadOutcome.CORRUPT_JOURNAL + with pytest.raises(PromotionRecoveryError): + PromotionRecovery(tmp_path).check_and_recover() + + +# --------------------------------------------------------------------------- +# Path traversal and symlink attacks in journal content +# --------------------------------------------------------------------------- + + +def test_journal_path_traversal_fails_closed(tmp_path: Path) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("do not touch", encoding="utf-8") + entry = base_entry( + "../outside.txt", "created", result_content="x", state=EntryState.APPLY_INTENT + ) + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + with pytest.raises(PromotionRecoveryError, match=r"unsafe|path"): + PromotionRecovery(tmp_path).check_and_recover() + assert outside.read_text(encoding="utf-8") == "do not touch" + + +def test_journal_targets_agentdiff_internal_state_fails_closed(tmp_path: Path) -> None: + entry = base_entry(".agentdiff/promotion-journal.json", "created", result_content="x") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + with pytest.raises(PromotionRecoveryError, match="internal state"): + PromotionRecovery(tmp_path).check_and_recover() + + +def test_journal_backup_escapes_approved_dir_fails_closed(tmp_path: Path) -> None: + entry = JournalEntry( + path="target.txt", + change_type="modified", + base_sha256=sha256("base"), + result_sha256=sha256("result"), + base_mode=0o644, + result_mode=0o644, + state=EntryState.APPLIED, + backup_relpath=".agentdiff/backups/other-run/target.txt", + ) + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + with pytest.raises(PromotionRecoveryError, match="approved backup directory"): + PromotionRecovery(tmp_path).check_and_recover() + + +def test_journal_backup_parent_symlink_fails_closed(tmp_path: Path) -> None: + outside_dir = tmp_path / "outside" + outside_dir.mkdir() + (outside_dir / "target.txt").write_text("base", encoding="utf-8") + os.symlink(outside_dir, tmp_path / "evil") + entry = JournalEntry( + path="target.txt", + change_type="modified", + base_sha256=sha256("base"), + result_sha256=sha256("result"), + base_mode=0o644, + result_mode=0o644, + state=EntryState.APPLIED, + backup_relpath=".agentdiff/backups/run-crash/evil/target.txt", + ) + (tmp_path / ".agentdiff" / "backups" / "run-crash" / "evil").mkdir(parents=True) + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + with pytest.raises(PromotionRecoveryError): + PromotionRecovery(tmp_path).check_and_recover() + + +def test_host_parent_symlink_fails_closed(tmp_path: Path) -> None: + outside = tmp_path / "outside" + outside.mkdir() + (outside / "target.txt").write_text("result", encoding="utf-8") + os.symlink(outside, tmp_path / "link") + entry = base_entry("link/target.txt", "modified", base_content="base", result_content="result") + write_backup(tmp_path, "link/target.txt", "base") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + with pytest.raises(PromotionRecoveryError, match=r"parent|directory"): + PromotionRecovery(tmp_path).check_and_recover() + + +# --------------------------------------------------------------------------- +# Backup integrity validation +# --------------------------------------------------------------------------- + + +def test_backup_digest_mismatch_fails_closed(tmp_path: Path) -> None: + host = tmp_path / "target.txt" + host.write_text("result", encoding="utf-8") + entry = base_entry("target.txt", "modified", base_content="base", result_content="result") + write_backup(tmp_path, "target.txt", "TAMPERED BACKUP CONTENT") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + with pytest.raises(PromotionRecoveryError, match="digest mismatch"): + PromotionRecovery(tmp_path).check_and_recover() + assert host.read_text(encoding="utf-8") == "result" + + +def test_backup_symlink_fails_closed(tmp_path: Path) -> None: + host = tmp_path / "target.txt" + host.write_text("result", encoding="utf-8") + entry = base_entry("target.txt", "modified", base_content="base", result_content="result") + backup_dir = tmp_path / ".agentdiff" / "backups" / "run-crash" + backup_dir.mkdir(parents=True) + outside = tmp_path / "outside.txt" + outside.write_text("base", encoding="utf-8") + os.symlink(outside, backup_dir / "target.txt") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + with pytest.raises(PromotionRecoveryError): + PromotionRecovery(tmp_path).check_and_recover() + assert host.read_text(encoding="utf-8") == "result" + + +def test_backup_hardlink_fails_closed(tmp_path: Path) -> None: + host = tmp_path / "target.txt" + host.write_text("result", encoding="utf-8") + entry = base_entry("target.txt", "modified", base_content="base", result_content="result") + backup = write_backup(tmp_path, "target.txt", "base") + os.link(backup, tmp_path / "extra-link") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + with pytest.raises(PromotionRecoveryError, match="link count"): + PromotionRecovery(tmp_path).check_and_recover() + + +# --------------------------------------------------------------------------- +# Crash-point matrix +# --------------------------------------------------------------------------- + + +def test_crash_before_any_apply_cleans_up(tmp_path: Path) -> None: + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + state=EntryState.PREPARED, + ) + write_backup(tmp_path, "target.txt", "base") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + report = PromotionRecovery(tmp_path).check_and_recover() + assert report is not None + assert report.status == "RECOVERED" + assert report.restored == [] + assert (tmp_path / "target.txt").exists() is False # never created + + +def test_crash_after_first_file_recovers_all(tmp_path: Path) -> None: + first = tmp_path / "first.txt" + second = tmp_path / "second.txt" + first.write_text("first-result", encoding="utf-8") + second.write_text("second-result", encoding="utf-8") + entries = [ + base_entry( + "first.txt", "modified", base_content="first-base", result_content="first-result" + ), + base_entry( + "second.txt", "modified", base_content="second-base", result_content="second-result" + ), + base_entry("third.txt", "created", result_content="third-result"), + ] + write_backup(tmp_path, "first.txt", "first-base") + write_backup(tmp_path, "second.txt", "second-base") + # Crash state: first applied, second intent, third prepared. + entries[0].state = EntryState.APPLIED + entries[1].state = EntryState.APPLY_INTENT + entries[2].state = EntryState.PREPARED + make_journal(tmp_path, state=JournalState.APPLYING, entries=entries) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report is not None + assert report.status == "RECOVERED" + assert first.read_text(encoding="utf-8") == "first-base" + assert second.read_text(encoding="utf-8") == "second-base" + assert not (tmp_path / "third.txt").exists() + assert "first.txt" in report.restored + assert "second.txt" in report.restored + # The PREPARED created entry was never applied, so nothing to clean. + assert "third.txt" not in report.cleaned + + +def test_crash_after_mutation_before_journal_update_is_recovered(tmp_path: Path) -> None: + """APPLY_INTENT with host == result: recovery must detect the mutation.""" + host = tmp_path / "target.txt" + host.write_text("result", encoding="utf-8") + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + state=EntryState.APPLY_INTENT, + ) + write_backup(tmp_path, "target.txt", "base") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert host.read_text(encoding="utf-8") == "base" + + +def test_crash_after_mutation_before_journal_update_noop(tmp_path: Path) -> None: + """APPLY_INTENT with host == base: recovery must NOT restore anything.""" + host = tmp_path / "target.txt" + host.write_text("base", encoding="utf-8") + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + state=EntryState.APPLY_INTENT, + ) + write_backup(tmp_path, "target.txt", "base") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert report.restored == [] + assert host.read_text(encoding="utf-8") == "base" + + +def test_ambiguous_state_fails_closed(tmp_path: Path) -> None: + """Host matches neither base nor result: recovery must refuse to overwrite.""" + host = tmp_path / "target.txt" + host.write_text("UNRELATED HOST EDIT", encoding="utf-8") + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + state=EntryState.APPLY_INTENT, + ) + write_backup(tmp_path, "target.txt", "base") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + with pytest.raises(PromotionRecoveryError, match="ambiguous"): + PromotionRecovery(tmp_path).check_and_recover() + assert host.read_text(encoding="utf-8") == "UNRELATED HOST EDIT" + + +def test_modified_file_recovery_preserves_mode(tmp_path: Path) -> None: + host = tmp_path / "script.sh" + host.write_text("result", encoding="utf-8") + host.chmod(0o755) + entry = base_entry( + "script.sh", + "modified", + base_content="base", + result_content="result", + base_mode=0o755, + result_mode=0o644, + state=EntryState.APPLIED, + ) + backup = write_backup(tmp_path, "script.sh", "base") + backup.chmod(0o755) + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert host.read_text(encoding="utf-8") == "base" + if os.name != "nt": + assert stat.S_IMODE(host.stat().st_mode) == 0o755 + + +def test_deleted_file_recovery_restores_backup(tmp_path: Path) -> None: + entry = base_entry( + "gone.txt", "deleted", base_content="base", result_content=None, state=EntryState.APPLIED + ) + write_backup(tmp_path, "gone.txt", "base") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert (tmp_path / "gone.txt").read_text(encoding="utf-8") == "base" + + +def test_created_file_recovery_removes_result(tmp_path: Path) -> None: + created = tmp_path / "created.txt" + created.write_text("result", encoding="utf-8") + entry = base_entry("created.txt", "created", result_content="result", state=EntryState.APPLIED) + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert not created.exists() + + +def test_created_file_recovery_cleans_promote_temps(tmp_path: Path) -> None: + created = tmp_path / "created.txt" + created.write_text("result", encoding="utf-8") + leftover = tmp_path / ".agentdiff-promote-abc123.tmp" + leftover.write_text("result", encoding="utf-8") + entry = base_entry( + "created.txt", "created", result_content="result", state=EntryState.APPLY_INTENT + ) + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert not created.exists() + assert not leftover.exists() + + +def test_recover_intent_retry_after_crash_between_replace_and_chmod(tmp_path: Path) -> None: + """A crash between os.replace and chmod leaves content==base with a temp + mode; retry must converge instead of declaring ambiguity.""" + host = tmp_path / "target.txt" + host.write_text("base", encoding="utf-8") + if os.name != "nt": + host.chmod(0o600) # temp-file mode from mkstemp, chmod never ran + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + base_mode=0o755, + result_mode=0o644, + state=EntryState.RECOVER_INTENT, + ) + write_backup(tmp_path, "target.txt", "base") + make_journal(tmp_path, state=JournalState.APPLYING, entries=[entry]) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert host.read_text(encoding="utf-8") == "base" + if os.name != "nt": + assert stat.S_IMODE(host.stat().st_mode) == 0o755 + + +def test_stale_committed_journal_is_cleaned(tmp_path: Path) -> None: + entry = base_entry("target.txt", "modified", base_content="base", result_content="result") + journal = make_journal(tmp_path, state=JournalState.COMMITTED, entries=[entry]) + report = PromotionRecovery(tmp_path).check_and_recover() + assert report is not None + assert report.status == "NOTHING_TO_DO" + assert not journal.path.exists() + + +def test_committed_journal_with_unconfirmed_entry_fails_closed(tmp_path: Path) -> None: + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + state=EntryState.APPLY_INTENT, + ) + make_journal(tmp_path, state=JournalState.COMMITTED, entries=[entry]) + with pytest.raises(PromotionRecoveryError, match="inconsistent"): + PromotionRecovery(tmp_path).check_and_recover() + + +def test_unrecoverable_journal_state_fails_closed(tmp_path: Path) -> None: + entry = base_entry("target.txt", "modified", base_content="base", result_content="result") + make_journal(tmp_path, state=JournalState.RECOVERY_FAILED, entries=[entry]) + with pytest.raises(PromotionRecoveryError, match="unrecoverable"): + PromotionRecovery(tmp_path).check_and_recover() + + +def test_unapplied_staged_journal_is_cleaned(tmp_path: Path) -> None: + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + state=EntryState.PREPARED, + ) + journal = make_journal(tmp_path, state=JournalState.STAGED, entries=[entry]) + report = PromotionRecovery(tmp_path).check_and_recover() + assert report is not None + assert report.status == "CLEANED_UNAPPLIED" + assert not journal.path.exists() + + +def test_staged_journal_with_progressed_entry_fails_closed(tmp_path: Path) -> None: + entry = base_entry( + "target.txt", + "modified", + base_content="base", + result_content="result", + state=EntryState.APPLIED, + ) + make_journal(tmp_path, state=JournalState.STAGED, entries=[entry]) + with pytest.raises(PromotionRecoveryError, match="inconsistent"): + PromotionRecovery(tmp_path).check_and_recover() + + +def test_legacy_schema2_journal_is_recoverable(tmp_path: Path) -> None: + """Schema-2 journals (applied: bool) remain recoverable under old semantics.""" + host = tmp_path / "target.txt" + host.write_text("result", encoding="utf-8") + write_backup(tmp_path, "target.txt", "base") + payload = { + "schema_version": 2, + "run_id": "run-crash", + "patch_digest": "d", + "state": "APPLYING", + "created_at": "", + "updated_at": "", + "entries": [ + { + "path": "target.txt", + "change_type": "modified", + "base_sha256": sha256("base"), + "result_sha256": sha256("result"), + "base_mode": 0o644, + "result_mode": 0o644, + "applied": True, + "backup_relpath": ".agentdiff/backups/run-crash/target.txt", + } + ], + } + (tmp_path / ".agentdiff").mkdir(exist_ok=True) + (tmp_path / ".agentdiff" / "promotion-journal.json").write_text( + json.dumps(payload), encoding="utf-8" + ) + + report = PromotionRecovery(tmp_path).check_and_recover() + assert report.status == "RECOVERED" + assert host.read_text(encoding="utf-8") == "base" + + +# --------------------------------------------------------------------------- +# Workspace lease: inode race and cross-process exclusion +# --------------------------------------------------------------------------- + + +def test_lock_file_is_never_unlinked(tmp_path: Path) -> None: + lease = WorkspaceLease(tmp_path, run_id="run-1") + with lease.hold(): + assert lease.lock_file.is_file() + assert lease.lock_file.is_file(), "lock file must persist after release" + # A second lease on the same persistent file must still work. + with lease.hold(): + pass + + +def test_lease_excludes_second_holder(tmp_path: Path) -> None: + lease2 = WorkspaceLease(tmp_path, run_id="run-2") + with ( + WorkspaceLease(tmp_path, run_id="run-1").hold(), + pytest.raises(PromotionLockError, match=r"another process is promoting"), + lease2.hold(), + ): + pass + + +def test_lease_cross_process_exclusion(tmp_path: Path) -> None: + """Two OS processes must not both hold the promotion lease.""" + script = ( + "import sys, time\n" + "from agentdiff.promotion import WorkspaceLease, PromotionLockError\n" + "root, marker = sys.argv[1], sys.argv[2]\n" + "try:\n" + " with WorkspaceLease(root, 'run-x').hold(timeout_seconds=2.0):\n" + " open(marker, 'w').write('acquired')\n" + " time.sleep(3.0)\n" + "except PromotionLockError:\n" + " open(marker, 'w').write('denied')\n" + ) + marker = tmp_path / "marker" + first = subprocess.Popen( + [sys.executable, "-c", script, str(tmp_path), str(marker)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = 10.0 + while not marker.exists() and deadline > 0: + import time + + time.sleep(0.1) + deadline -= 0.1 + assert marker.exists() + # While the first process holds the lock, a second process must be denied. + second = subprocess.run( + [sys.executable, "-c", script, str(tmp_path), str(tmp_path / "marker2")], + capture_output=True, + timeout=10, + text=True, + ) + assert second.returncode == 0 + assert (tmp_path / "marker2").read_text(encoding="utf-8") == "denied" + finally: + first.terminate() + first.wait(timeout=5) diff --git a/tests/test_proof_strength.py b/tests/test_proof_strength.py new file mode 100644 index 0000000..531293a --- /dev/null +++ b/tests/test_proof_strength.py @@ -0,0 +1,439 @@ +"""Adversarial proof tests: verifier tampering, baseline verifier, strength.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agentdiff.policy import load_policy +from agentdiff.proof import ( + ProofEngine, + ProofPhaseResult, + ProofStrengthLabel, + ProofStrengthLevel, + ProofVerdict, + VerifierIndependence, + analyze_verifier_mutations, + compute_proof_strength, + is_verifier_related, +) +from agentdiff.proof.plan import select_trusted_verification_plan +from agentdiff.runtime import CleanupReport, RuntimeControlLevel, RuntimeResult +from agentdiff.transaction import AgentRunTransaction + + +def trust_policy() -> object: + return load_policy( + { + "version": 2, + "filesystem": {"allow_write": ["**"], "default": "allow"}, + "process": {"allow": ["agent"], "default": "deny"}, + "network": {"mode": "off"}, + "proof": {"network": False, "tests": [["tests"]]}, + } + ) + + +class IsolatedRuntime: + """Test backend with a private writable copy like DockerRuntime.""" + + def __init__(self, mutator) -> None: + self.mutator = mutator + self.source: Path | None = None + self.temporary = None + self.workspace: Path | None = None + + @property + def capabilities(self): + from agentdiff.runtime import RuntimeCapabilities + + return RuntimeCapabilities( + backend="test-isolated", + filesystem=RuntimeControlLevel.SANDBOXED, + host_repository=RuntimeControlLevel.SANDBOXED, + network=RuntimeControlLevel.BLOCKED, + processes=RuntimeControlLevel.SANDBOXED, + resources=RuntimeControlLevel.SANDBOXED, + privileges=RuntimeControlLevel.SANDBOXED, + private_workspace=True, + supports_live_safety=True, + supports_source_snapshot=True, + ) + + def configure_source(self, source: Path) -> None: + self.source = source + + def configure_safety(self, _watcher) -> None: + return None + + def run(self, argv, **_kwargs) -> RuntimeResult: + import shutil + import tempfile + + assert self.source is not None + self.temporary = Path(tempfile.mkdtemp(prefix="agentdiff-test-isolated-")) + self.workspace = self.temporary / "workspace" + shutil.copytree(self.source, self.workspace) + self.mutator(self.workspace) + return RuntimeResult( + argv=tuple(argv), + cwd="/workspace", + returncode=0, + timed_out=False, + duration_seconds=0.01, + backend="test-isolated", + enforcement="isolated_private_workspace", + capabilities=(), + observation_root=str(self.workspace), + ) + + def cleanup(self, _processes, **_kwargs) -> CleanupReport: + return CleanupReport() + + def close(self) -> None: + if self.temporary is not None: + import shutil + + shutil.rmtree(self.temporary) + self.temporary = None + + +class RecordingProofEnvironment: + """Fake clean room that asserts baseline overlay state during run.""" + + def __init__( + self, + *, + workspace: Path, + image: str, + network: bool, + baseline_passes: bool = True, + expect_overlay: str | None = None, + expected_overlay_content: str = "", + ) -> None: + self.workspace = workspace + self.baseline_passes = baseline_passes + self.expect_overlay = expect_overlay + self.expected_overlay_content = expected_overlay_content + self.baseline_seen = False + + def start(self) -> dict[str, object]: + return {"schema_version": 1, "backend": "test", "clean_environment": True} + + def run_phase(self, phase: str, command, *, timeout_seconds: float) -> ProofPhaseResult: + assert timeout_seconds > 0 + if phase == "tests": + return ProofPhaseResult( + phase=phase, + command=tuple(command), + status="PASS", + returncode=0, + duration_seconds=0.01, + tests_passed=2, + tests_total=2, + ) + if phase == "baseline_tests": + self.baseline_seen = True + if self.expect_overlay is not None: + assert (self.workspace / self.expect_overlay).read_text( + encoding="utf-8" + ) == self.expected_overlay_content, ( + f"baseline did not restore {self.expect_overlay}" + ) + status = "PASS" if self.baseline_passes else "FAIL" + return ProofPhaseResult( + phase=phase, + command=tuple(command), + status=status, + returncode=0 if self.baseline_passes else 1, + duration_seconds=0.01, + tests_passed=2 if self.baseline_passes else 0, + tests_total=2, + ) + return ProofPhaseResult( + phase=phase, + command=tuple(command), + status="PASS", + returncode=0, + duration_seconds=0.01, + ) + + def close(self) -> None: + return None + + +def run_and_prove( + tmp_path: Path, + mutator, + *, + baseline_passes: bool = True, + expect_overlay: str | None = None, + expected_overlay_content: str = "", +) -> tuple[str, object]: + runtime = IsolatedRuntime(mutator) + result = AgentRunTransaction( + root=tmp_path, + policy=trust_policy(), + runtime=runtime, + task="proof test", + ).run(["agent"]) + assert result.status == "passed" + return result.run_id, ProofEngine( + tmp_path, + result.run_id, + environment_factory=lambda workspace, image, network: RecordingProofEnvironment( + workspace=workspace, + image=image, + network=network, + baseline_passes=baseline_passes, + expect_overlay=expect_overlay, + expected_overlay_content=expected_overlay_content, + ), + ).prove(timeout_seconds=5) + + +# --------------------------------------------------------------------------- +# Verifier-file classification +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "path", + [ + "tests/test_login.py", + "tests/conftest.py", + "test/unit_test.py", + "__tests__/app.test.js", + "conftest.py", + "pytest.ini", + "tox.ini", + "noxfile.py", + "setup.py", + "setup.cfg", + "Makefile", + "Dockerfile", + "package.json", + "package-lock.json", + "yarn.lock", + "pnpm-lock.yaml", + "uv.lock", + "poetry.lock", + "pyproject.toml", + "vitest.config.ts", + "jest.config.js", + "playwright.config.ts", + "requirements-dev.txt", + ".github/workflows/ci.yml", + "src/app.test.tsx", + "src/test_util.py", + ], +) +def test_verifier_related_detected(path: str) -> None: + assert is_verifier_related(path) + + +@pytest.mark.parametrize( + "path", + [ + "src/app.py", + "README.md", + "data/input.csv", + "docs/index.md", + "src/util.rs", + "assets/logo.png", + ], +) +def test_verifier_related_negative(path: str) -> None: + assert not is_verifier_related(path) + + +def test_verifier_mutation_report_classifies_changes() -> None: + report = analyze_verifier_mutations( + [ + ("tests/test_login.py", "modified"), + ("conftest.py", "modified"), + ("tests/test_new.py", "created"), + ("tests/test_deleted.py", "deleted"), + ("src/app.py", "modified"), + ] + ) + assert report.modified_count == 2 + assert report.existing_changed == ("conftest.py", "tests/test_login.py") + assert report.added == ("tests/test_new.py",) + assert report.removed == ("tests/test_deleted.py",) + assert report.any_modification + + +def test_plan_untrusted_when_test_files_modified(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text("[project]\nname='x'\n", encoding="utf-8") + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_app.py").write_text("def test_x(): pass\n", encoding="utf-8") + policy = load_policy({"version": 2}) + tampered_entry = { + "path": "tests/test_app.py", + "change_type": "modified", + "decision": "allow", + "base_sha256": "a", + "result_sha256": "b", + "base_mode": 0o644, + "result_mode": 0o644, + "size": 5, + "materialized": True, + "reason": "", + } + from agentdiff.evidence import PatchEntry + + plan = select_trusted_verification_plan( + tmp_path, + policy, + patch_entries=(PatchEntry.from_dict(tampered_entry),), + ) + assert plan.trusted is False + assert "tests/test_app.py" in plan.tampered_files + + +# --------------------------------------------------------------------------- +# Proof-strength model +# --------------------------------------------------------------------------- + + +def test_proof_strength_matrix() -> None: + assert compute_proof_strength( + clean_environment="FAIL", + verification_confirmed=False, + baseline_verifier="SKIPPED", + baseline_available=False, + verifier_files_changed=0, + ) == ( + ProofStrengthLevel.L0_EXECUTION_ONLY, + ProofStrengthLabel.WEAK, + VerifierIndependence.WEAK, + ) + assert compute_proof_strength( + clean_environment="PASS", + verification_confirmed=False, + baseline_verifier="SKIPPED", + baseline_available=False, + verifier_files_changed=0, + ) == (ProofStrengthLevel.L1_CLEAN_ROOM, ProofStrengthLabel.WEAK, VerifierIndependence.WEAK) + assert compute_proof_strength( + clean_environment="PASS", + verification_confirmed=True, + baseline_verifier="SKIPPED", + baseline_available=False, + verifier_files_changed=0, + ) == ( + ProofStrengthLevel.L2_TRUSTED_COMMAND, + ProofStrengthLabel.REVIEW, + VerifierIndependence.WEAK, + ) + assert compute_proof_strength( + clean_environment="PASS", + verification_confirmed=True, + baseline_verifier="PASS", + baseline_available=True, + verifier_files_changed=0, + ) == ( + ProofStrengthLevel.L3_BASELINE_VERIFIER, + ProofStrengthLabel.STRONG, + VerifierIndependence.STRONG, + ) + + +# --------------------------------------------------------------------------- +# Baseline verifier end-to-end +# --------------------------------------------------------------------------- + + +def test_baseline_verifier_runs_against_restored_base_tests(tmp_path: Path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_app.py").write_text( + "def test_value():\n assert True\n", encoding="utf-8" + ) + + def mutator(workspace: Path) -> None: + (workspace / "src" / "app.py").write_text("VALUE = 2\n", encoding="utf-8") + (workspace / "tests" / "test_app.py").write_text( + "def test_value():\n assert True # weakened?\n", encoding="utf-8" + ) + + _, proof = run_and_prove( + tmp_path, + mutator, + expect_overlay="tests/test_app.py", + expected_overlay_content="def test_value():\n assert True\n", + ) + assert proof.verdict is ProofVerdict.PROVEN + assert proof.verifier_files_changed >= 1 + assert "tests/test_app.py" in proof.verifier_changes + assert proof.baseline_available is True + assert proof.baseline_verifier == "PASS" + assert proof.patched_tests_total == 2 + assert proof.baseline_tests_total == 2 + assert proof.proof_strength == ProofStrengthLevel.L3_BASELINE_VERIFIER.value + assert proof.proof_strength_label == ProofStrengthLabel.STRONG.value + assert proof.verifier_independence == VerifierIndependence.STRONG.value + + +def test_baseline_failure_blocks_proven_when_tests_tampered(tmp_path: Path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_app.py").write_text( + "def test_value():\n assert True\n", encoding="utf-8" + ) + + def mutator(workspace: Path) -> None: + (workspace / "tests" / "test_app.py").write_text( + "def test_value():\n assert True # weakened\n", encoding="utf-8" + ) + + _, proof = run_and_prove(tmp_path, mutator, baseline_passes=False) + assert proof.verdict is ProofVerdict.NOT_PROVEN + assert proof.promotion == "BLOCKED" + assert any("baseline verifier" in reason for reason in proof.reasons) + assert proof.baseline_verifier == "FAIL" + assert proof.verifier_independence == VerifierIndependence.REVIEW.value + + +def test_unmodified_verifier_files_still_run_baseline(tmp_path: Path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_app.py").write_text( + "def test_value():\n assert True\n", encoding="utf-8" + ) + + def mutator(workspace: Path) -> None: + (workspace / "src" / "app.py").write_text("VALUE = 2\n", encoding="utf-8") + + _, proof = run_and_prove(tmp_path, mutator) + assert proof.verdict is ProofVerdict.PROVEN + assert proof.verifier_files_changed == 0 + assert proof.baseline_verifier == "PASS" + assert proof.proof_strength == ProofStrengthLevel.L3_BASELINE_VERIFIER.value + + +def test_patch_added_verifier_file_removed_for_baseline(tmp_path: Path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + + def mutator(workspace: Path) -> None: + (workspace / "src" / "app.py").write_text("VALUE = 2\n", encoding="utf-8") + (workspace / "tests").mkdir() + (workspace / "tests" / "fake_extra_test.py").write_text( + "def test_fake():\n assert True\n", encoding="utf-8" + ) + + # Baseline is unavailable because the base had no verifier files; the + # added test still must not be promoted silently (reported + NOT_PROVEN + # because baseline cannot confirm). + _, proof = run_and_prove(tmp_path, mutator) + assert proof.baseline_available is False + assert proof.verifier_files_changed == 0 + assert "tests/fake_extra_test.py" in proof.verifier_changes + assert proof.verdict is ProofVerdict.NOT_PROVEN + assert any("no baseline verifier" in reason for reason in proof.reasons) diff --git a/tests/test_run_transaction.py b/tests/test_run_transaction.py index faa7c46..c598842 100644 --- a/tests/test_run_transaction.py +++ b/tests/test_run_transaction.py @@ -7,7 +7,13 @@ import pytest from agentdiff.policy import Policy, PolicyAction, load_policy -from agentdiff.runtime import OwnedProcess, PortObservation, RuntimeResult +from agentdiff.runtime import ( + OwnedProcess, + PortObservation, + RuntimeCapabilities, + RuntimeControlLevel, + RuntimeResult, +) from agentdiff.transaction import AgentRunTransaction, RollbackEngine, RunStore @@ -32,6 +38,30 @@ def _policy(*, allow: list[str], deny: list[str]) -> Policy: def test_missing_cleanup_report_is_scored_as_residual_process_risk(tmp_path: Path) -> None: class RuntimeWithoutCleanup: + @property + def capabilities(self): + return RuntimeCapabilities( + backend="test-no-cleanup", + filesystem=RuntimeControlLevel.OBSERVED, + host_repository=RuntimeControlLevel.OBSERVED, + network=RuntimeControlLevel.UNCONTROLLED, + processes=RuntimeControlLevel.OBSERVED, + resources=RuntimeControlLevel.UNCONTROLLED, + privileges=RuntimeControlLevel.UNCONTROLLED, + private_workspace=False, + supports_live_safety=True, + supports_source_snapshot=False, + ) + + def configure_source(self, source): + return None + + def configure_safety(self, _watcher): + return None + + def close(self): + return None + def run(self, argv, **_kwargs): return RuntimeResult( argv=tuple(argv), diff --git a/tests/test_trust_pipeline.py b/tests/test_trust_pipeline.py index 603e69d..d70a1f2 100644 --- a/tests/test_trust_pipeline.py +++ b/tests/test_trust_pipeline.py @@ -18,6 +18,7 @@ from agentdiff.runtime import ( CleanupReport, DockerRuntime, + RuntimeCapabilities, RuntimeCapability, RuntimeControlLevel, RuntimeResult, @@ -54,6 +55,21 @@ def __init__(self, mutator) -> None: self.temporary: Path | None = None self.closed = False + @property + def capabilities(self): + return RuntimeCapabilities( + backend="test-isolated", + filesystem=RuntimeControlLevel.SANDBOXED, + host_repository=RuntimeControlLevel.SANDBOXED, + network=RuntimeControlLevel.BLOCKED, + processes=RuntimeControlLevel.SANDBOXED, + resources=RuntimeControlLevel.SANDBOXED, + privileges=RuntimeControlLevel.SANDBOXED, + private_workspace=True, + supports_live_safety=True, + supports_source_snapshot=True, + ) + def configure_source(self, source: Path) -> None: self.source = source diff --git a/tests/test_trust_pipeline_hardened.py b/tests/test_trust_pipeline_hardened.py index 66dd9b2..715dfc5 100644 --- a/tests/test_trust_pipeline_hardened.py +++ b/tests/test_trust_pipeline_hardened.py @@ -2,18 +2,18 @@ from __future__ import annotations -import json from typing import TYPE_CHECKING -import pytest - if TYPE_CHECKING: from pathlib import Path +import json +import pytest from agentdiff.evidence import CapsuleReader, PatchEntry from agentdiff.policy import Policy, ProofPolicy, load_policy from agentdiff.promotion import ( + EntryState, JournalEntry, JournalState, PromotionJournal, @@ -92,6 +92,11 @@ def test_proof_plan_accepts_explicit_policy_override(tmp_path: Path) -> None: def test_promotion_write_ahead_journal_and_recovery(tmp_path: Path) -> None: """Test WAL journal persistence and crash recovery restores files.""" + import hashlib + + def sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + host_file = tmp_path / "target.txt" host_file.write_text("initial host state", encoding="utf-8") @@ -101,9 +106,10 @@ def test_promotion_write_ahead_journal_and_recovery(tmp_path: Path) -> None: backup_file.write_text("initial host state", encoding="utf-8") created_file = tmp_path / "created.txt" - created_file.write_text("partially created", encoding="utf-8") + created_file.write_text("created content", encoding="utf-8") - # Simulate an interrupted journal in APPLYING state + # Simulate an interrupted journal in APPLYING state with both entries + # recorded as APPLIED (mutation happened, then the process died). journal = PromotionJournal( root=tmp_path, run_id="run-123", @@ -113,21 +119,25 @@ def test_promotion_write_ahead_journal_and_recovery(tmp_path: Path) -> None: JournalEntry( path="target.txt", change_type="modified", - base_sha256="base", - result_sha256="target", + base_sha256=sha256("initial host state"), + result_sha256=sha256("target"), base_mode=0o644, result_mode=0o644, + base_size=len("initial host state"), + result_size=len("target"), backup_relpath=".agentdiff/backups/run-123/target.txt", - applied=True, + state=EntryState.APPLIED, ), JournalEntry( path="created.txt", change_type="created", base_sha256=None, - result_sha256="created", + result_sha256=sha256("created content"), base_mode=None, result_mode=0o644, - applied=True, + base_size=None, + result_size=len("created content"), + state=EntryState.APPLIED, ), ], ) @@ -151,13 +161,13 @@ def test_promotion_write_ahead_journal_and_recovery(tmp_path: Path) -> None: def test_workspace_lease_concurrency(tmp_path: Path) -> None: """Test advisory lock lease prevents overlapping promotion locks.""" lease1 = WorkspaceLease(tmp_path, run_id="run-1") - lease2 = WorkspaceLease(tmp_path, run_id="run-2") - with ( - lease1.hold(), - pytest.raises(PromotionLockError, match="another process is promoting"), - lease2.hold(), - ): - pass + with lease1.hold(): + lease2 = WorkspaceLease(tmp_path, run_id="run-2") + with ( + pytest.raises(PromotionLockError, match=r"another process is promoting"), + lease2.hold(), + ): + pass def test_capsule_reader_and_merkle_root(tmp_path: Path) -> None: @@ -219,4 +229,16 @@ def test_hybrid_safety_watcher(tmp_path: Path) -> None: terminated = watcher.poll(duration_seconds=1.0, processes_spawned=1) assert terminated is False + # Dirty hints accelerate with targeted checks; authoritative full + # reconciliation runs on schedule/overflow/force. + assert watcher.stats.targeted_checks_performed == 1 + assert ( + watcher.observe( + root=tmp_path, + duration_seconds=1.0, + processes_spawned=1, + force_filesystem=True, + ) + is False + ) assert watcher.stats.full_scans_performed == 1 diff --git a/tests/test_watcher_hybrid.py b/tests/test_watcher_hybrid.py new file mode 100644 index 0000000..d808342 --- /dev/null +++ b/tests/test_watcher_hybrid.py @@ -0,0 +1,154 @@ +"""Hybrid safety watcher: dirty-path targeting, reconciliation, degradation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path +from agentdiff.policy import load_policy +from agentdiff.safety import HybridSafetyWatcher +from agentdiff.state import FilesystemScanner + + +class FakeEventSource: + backend = "fake" + + def __init__(self, events: list[str | None]) -> None: + self._events = list(events) + self.started = False + self.stopped = False + + def start(self) -> None: + self.started = True + + def drain(self) -> list[str | None]: + pending, self._events = self._events, [] + return pending + + def stop(self) -> None: + self.stopped = True + + +class ExplodingEventSource: + backend = "exploding" + + def start(self) -> None: + raise RuntimeError("no event backend on this platform") + + def drain(self) -> list[str | None]: + return [] + + def stop(self) -> None: + return None + + +def watcher(tmp_path: Path, *, policy_version: int = 2, **kwargs): + policy = load_policy( + { + "version": policy_version, + "filesystem": {"deny": ["**.env", "secret.txt"], "default": "allow"}, + } + ) + before = FilesystemScanner(tmp_path).capture() + return HybridSafetyWatcher(root=tmp_path, policy=policy, before=before, **kwargs) + + +def test_watcher_tracks_dirty_paths_and_runs_targeted_check(tmp_path: Path) -> None: + source = FakeEventSource([]) + w = watcher(tmp_path, event_source=source, reconcile_interval_seconds=60) + w.start() + w.notify_event(str(tmp_path / "src" / "app.py")) + assert w.stats.hints_received == 1 + assert "src/app.py" in w._dirty_paths + + terminated = w.poll(duration_seconds=1.0, processes_spawned=1) + assert terminated is False + assert w.stats.targeted_checks_performed >= 1 + assert w.stats.full_scans_performed == 0 + w.stop() + assert source.stopped is True + + +def test_watcher_terminates_on_protected_dirty_path(tmp_path: Path) -> None: + (tmp_path / "secret.txt").write_text("do not touch", encoding="utf-8") + w = watcher(tmp_path, reconcile_interval_seconds=60) + # The protected file already exists in `before`, so mark it modified via + # a dirty hint and force the targeted check to see a change. + (tmp_path / "secret.txt").write_text("touched", encoding="utf-8") + w.notify_event("secret.txt") + terminated = w.poll(duration_seconds=1.0, processes_spawned=1) + assert terminated is True + assert w.controller.report.termination_reason == "protected_path" + + +def test_watcher_periodic_full_reconciliation(tmp_path: Path) -> None: + w = watcher(tmp_path, reconcile_interval_seconds=60) + w.poll(duration_seconds=1.0, processes_spawned=1, force=True) + w.poll(duration_seconds=1.0, processes_spawned=1, force=True) + assert w.stats.full_scans_performed >= 1 + + +def test_watcher_degrades_to_polling_on_backend_failure(tmp_path: Path) -> None: + source = ExplodingEventSource() + w = watcher(tmp_path, event_source=source) + w.start() + assert w.status.status == "degraded" + assert w.status.backend == "polling" + assert "failed to start" in w.status.reason + # Degraded watcher must still observe. + assert w.poll(duration_seconds=1.0, processes_spawned=1) is False + + +def test_watcher_overflow_triggers_full_reconcile(tmp_path: Path) -> None: + w = watcher(tmp_path, max_dirty_before_reconcile=3, reconcile_interval_seconds=60) + for name in ("a.py", "b.py", "c.py", "d.py"): + w.notify_event(name) + w.poll(duration_seconds=1.0, processes_spawned=1) + assert w.stats.full_scans_performed == 1 + + +def test_watcher_drains_events_from_source(tmp_path: Path) -> None: + source = FakeEventSource(["one.py", "two.py", None]) + w = watcher(tmp_path, event_source=source) + w.start() + w.poll(duration_seconds=1.0, processes_spawned=1) + assert w.stats.hints_received == 3 + assert w.stats.targeted_checks_performed >= 1 + w.stop() + + +def test_watcher_observe_interface_matches_controller(tmp_path: Path) -> None: + w = watcher(tmp_path) + assert ( + w.observe( + root=tmp_path, + duration_seconds=1.0, + processes_spawned=1, + force_filesystem=True, + runtime_active=True, + ) + is False + ) + assert w.terminated is False + assert w.report is w.controller.report + + +def test_watcher_force_reconcile_uses_authoritative_capture(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text("one", encoding="utf-8") + w = watcher(tmp_path, reconcile_interval_seconds=60) + (tmp_path / "app.py").write_text("two", encoding="utf-8") + # Without events or force, the dirty path is not known and no scan runs. + assert w.poll(duration_seconds=1.0, processes_spawned=1) is False + assert w.stats.full_scans_performed == 0 + # A forced observation performs the authoritative full capture. + assert ( + w.observe( + root=tmp_path, + duration_seconds=1.0, + processes_spawned=1, + force_filesystem=True, + ) + is False + ) + assert w.stats.full_scans_performed == 1 diff --git a/uv.lock b/uv.lock index 425483b..d4b79f7 100644 --- a/uv.lock +++ b/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ [[package]] name = "agentdiff" -version = "0.2.0" +version = "0.2.1" source = { editable = "." } dependencies = [ { name = "psutil" },