diff --git a/.gitignore b/.gitignore index b3aa041..9fd4712 100644 --- a/.gitignore +++ b/.gitignore @@ -49,7 +49,6 @@ dmypy.json # Generated documentation and Pages output /site/ -/docs/ /docs_build/ # AgentDiff local run artifacts diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..7b17a1a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,33 @@ +# AgentDiff contributor map + +AgentDiff is a deterministic trust layer around software-maintenance workers. A worker may propose code; only AgentDiff policy and proof may authorize the exact patch. + +Read these short documents before changing behavior: + +- [`docs/engineering/PRODUCT.md`](docs/engineering/PRODUCT.md) — product promise, scope, and capability labels. +- [`docs/engineering/ARCHITECTURE.md`](docs/engineering/ARCHITECTURE.md) — component and evidence flow. +- [`docs/engineering/SECURITY_INVARIANTS.md`](docs/engineering/SECURITY_INVARIANTS.md) — non-negotiable trust boundaries. +- [`docs/engineering/IMPLEMENTATION_STATUS.md`](docs/engineering/IMPLEMENTATION_STATUS.md) — what is available, foundational, or roadmap. +- [`docs/engineering/DEVELOPMENT.md`](docs/engineering/DEVELOPMENT.md) and [`TESTING.md`](docs/engineering/TESTING.md) — local workflow and release gates. +- [`docs/engineering/DESIGN_SYSTEM.md`](docs/engineering/DESIGN_SYSTEM.md) — website language and visual rules. + +Repository map: + +- `src/agentdiff/api/`: provider discovery, scan, migration, certificates, campaigns, and sealed PR delivery. +- `src/agentdiff/proof/`: authoritative clean-room proof plus composable proof-strength primitives. +- `src/agentdiff/maintenance/`: generic change, provenance graph, priority, and scheduling foundations. +- `src/agentdiff/runtime/`, `transaction/`, `policy/`, `evidence/`: execution observation and deterministic evidence core. +- `src/agentdiff/delivery/`, `outcome/`, `minimize/`, `repair/`: bounded lifecycle components. +- User-facing website and documentation are maintained in the private `agentdiff-web` repository and deployed separately to GitHub Pages. +- `demos/`: reproducible fixtures; never replace their output with invented success data. + +Hard rules: + +1. Remote content and generator output are untrusted. +2. The local runtime is observation, not a sandbox. +3. Custom generators fail closed when isolated execution is unavailable; never fall back to host execution. +4. Proof binds the exact base, patch, policy, plan, runtime, and evidence. Do not restamp or regenerate during delivery. +5. Incomplete, flaky, stale, or ambiguous evidence is not proof. +6. Do not add auto-merge, model self-approval, home-grown cryptography, hidden telemetry, fake metrics, or default production rollback. + +Use a `codex/` branch, preserve unrelated work, add refusal-path tests for security changes, and run the checks in `docs/engineering/TESTING.md` before handoff. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7f0839..0055e3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ uv sync --locked --all-groups --extra langchain 1. Create a focused branch from `main`. 2. Add a regression test that demonstrates the missing behavior. 3. Implement the smallest complete change. -4. Update `README.md`, examples, security guidance, and the separate React documentation project when public behavior changes. +4. Update `README.md`, examples, and security guidance when public behavior changes. Open a matching update in the private `agentdiff-web` repository when user-facing documentation changes. 5. Run the local quality gates documented in `README.md`. 6. Open a pull request that states behavior, trust-boundary impact, limitations, and real verification output. @@ -29,6 +29,6 @@ uv sync --locked --all-groups --extra langchain Scanner, policy, persistence, process, redaction, and rollback changes require tests for failure and refusal paths—not only success paths. Recovery must preserve current data when identity or post-run equality is uncertain. -Do not commit coverage, cache, virtual-environment, distribution, or `.agentdiff/` run artifacts. The product website and React documentation source live in the separate `agentdiff-web` project; this Python repository has no generated website tree. +Do not commit coverage, cache, virtual-environment, distribution, or `.agentdiff/` run artifacts. The canonical product website and React documentation source are maintained separately in the private `agentdiff-web` repository. By participating, you agree to follow the [Code of Conduct](CODE_OF_CONDUCT.md). diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index 6bd6432..e2fd1f4 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -1,113 +1,12 @@ # AgentDiff project plan -## Product direction +AgentDiff is evolving from verified API migration into a **Verified Maintenance Autopilot** while preserving one authority: a worker proposes a patch and AgentDiff independently proves the exact change. -**AgentDiff is the trust layer for self-maintaining APIs.** +The maintained product contract and roadmap boundaries now live in: -When an API changes, AgentDiff finds affected customer code, generates or supervises the migration, independently proves the patch in a clean room, and opens a reviewable PR with a cryptographic Migration Certificate. +- [`docs/engineering/PRODUCT.md`](docs/engineering/PRODUCT.md) +- [`docs/engineering/IMPLEMENTATION_STATUS.md`](docs/engineering/IMPLEMENTATION_STATUS.md) +- [`docs/engineering/ARCHITECTURE.md`](docs/engineering/ARCHITECTURE.md) +- [`docs/engineering/SECURITY_INVARIANTS.md`](docs/engineering/SECURITY_INVARIANTS.md) -The trust engine underneath — independent real-state observation, deterministic policy, explainable blast radius, conflict-safe promotion, bounded repair, evidence capsules — is what makes verified migrations possible. The coding-agent automation (`agentdiff wrap`) is a powerful byproduct of the same infrastructure. - -> Independent real-state observation + deterministic intent policy + explainable blast radius + conflict-safe selective recovery. - -The primary product wedge is `agentdiff api scan/check/migrate` for Self-Maintaining APIs. The primary automation path is `agentdiff wrap -- ` for zero-touch coding-agent trust. - -## Product principles - -1. Evidence is captured independently of agent self-reporting. -2. Policy and scoring stay deterministic and versioned. -3. Recovery fails closed when identity, backup integrity, or post-state equality is ambiguous. -4. Safe recovery preserves allowed work. -5. Local observation is never described as sandboxing or network enforcement. -6. Evidence is redacted, bounded, and private by default. -7. Isolation, tracing, and agent protocols are integration seams, not features to rebuild. -8. **The AI generates; AgentDiff verifies.** Probabilistic code generation is untrusted until deterministic proof passes. - -## Current `0.3.0` surface (main branch) - -### Beta - -- No-follow filesystem manifests and private run capsules. -- Deterministic `allow`, `review`, and `deny` policy with provenance. -- Local shell-free command execution with timeout and best-effort process evidence. -- Explainable, capped blast-radius scoring. -- Run listing, inspection, checksum verification, and exact-identity cleanup. -- Conflict-safe recovery for eligible regular files. -- Linux, macOS, and native Windows CI on Python 3.12–3.13. -- **Self-Maintaining APIs: AST scanner, SDK version detector, breaking-change matcher, blast radius integration, `agentdiff api scan/check`** - -### Experimental - -- Anthropic Sandbox Runtime argv adapter; enforcement belongs to the external runtime. -- Transport-neutral MCP-style pre-dispatch policy hook; no MCP server or proxy. -- LangChain callback and internal snapshot/diff/evaluator compatibility APIs. -- Five-case local recovery regression suite. -- Cortex evidence memory and provider routing. - -### Planned - -- Published PyPI and signed release artifacts. -- Authenticated or signed evidence capsules. -- OpenTelemetry/OpenInference evidence export. -- One maintained hosted/disposable sandbox integration. -- Artifact migration and compatibility tooling. -- Larger external-state benchmark coverage. -- **Migration pipeline completion: real ProofEngine execution for migrations, rollback verification, failure evidence, GitHub PR delivery** - -An HTTP API, hosted dashboard, Docker backend, bundled sandbox, universal network blocking, and arbitrary external-state rollback are not implemented. - -## Release gates - -### Completed for the current source release - -- [x] Tests on Linux, macOS, and Windows. -- [x] Ruff formatting/lint, mypy, CodeQL, Bandit, and dependency audit. -- [x] Package build and clean-wheel smoke test. -- [x] Strict docs build plus internal link and asset validation. -- [x] Deterministic local recovery regression artifact. -- [x] Protected `main`, least-privilege workflow permissions, secret scanning, and Dependabot configuration. -- [x] CI on stable Python 3.12, 3.13 (3.14 tracked separately). - -### Required before a stable release - -- [ ] Versioned policy and artifact migration strategy. -- [ ] Documented compatibility and deprecation policy. -- [ ] Independent security review of scanning, persistence, process handling, and recovery. -- [ ] Published-package ownership and provenance. -- [ ] Measured performance bounds on representative repositories. -- [ ] No documentation claim beyond tested behavior. -- [ ] **Self-Maintaining APIs MVP: one real provider change, one affected repo, one verified migration PR** - -## Focused roadmap - -### Self-Maintaining APIs (primary wedge) - -1. **API Change Manifest** — structured machine-readable upstream change format (YAML/JSON) for provider deprecations, SDK releases, model shutdowns. *(implemented)* -2. **Deterministic AST Transforms** — for known simple migrations (OpenAI Responses API, Stripe PaymentIntents, etc.); registry extensible by providers. *(implemented)* -3. **Migration Engine** — scan → match → plan → transform in private workspace → verify → certificate. *(implemented)* -4. **Provider Intelligence Layer** — parse changelogs, diff OpenAPI specs, analyze SDK releases, and accept AI suggestions as validated manifest candidates. AI output never touches code directly. *(implemented)* -5. **Provider Plugin System** — `agentdiff provider install/list`; providers ship `manifests/`, `transforms/`, `tests/`, `metadata.yaml` without core changes. *(implemented)* -6. **Verification Levels (V0–V5)** — syntax/type/build → targeted tests → full repo tests → API contract/mock tests → user-defined integration verification. -7. **Migration Certificate** — machine-readable artifact: provider change, affected usages, files changed, blast radius, policy result, tests executed, verification level, proof digest, capsule ID, rollback info. *(implemented)* -8. **GitHub PR Automation** — `--open-pr` delivers Migration Certificate in PR body; conflict-safe promotion; no auto-merge. -9. **API Knowledge Graph** — track Repository → API usage → SDK version → migration status; design scalable, no extra database yet. - -### Credibility and distribution - -1. Publish signed artifacts through PyPI Trusted Publishing after ownership and provenance are configured. -2. Keep the primary CLI limited to transaction, evidence, recovery, policy, diagnostics, and API migration. -3. Keep Cortex experimental, namespaced, and described as deterministic evidence tooling without autonomy claims. -4. Ship a thin GitHub Action that reports transaction evidence without creating a proprietary dashboard. - -### Differentiated safety core - -1. Clean-room proof by replaying a captured patch in a fresh worktree before promotion (implemented). -2. Detect future execution risk in package scripts and GitHub Actions changes, then extend to Dockerfiles, Makefiles, hooks, and editor tasks. -3. Experimental copy-on-write Docker runtime where the real repository is changed only by an explicit, policy-filtered promotion step. - -### Evidence moat - -1. Add signed, shareable capsule export and standardized telemetry. -2. Add run attribution for changed lines and evidence-based comparison of parallel agent attempts. -3. Keep adversarial race, path, hardlink, redaction, and rollback tests ahead of new claims. -4. **Migration Certificate as interoperability format** — providers can require it, customers can audit it, regulators can accept it. \ No newline at end of file +The next integration priorities are continuous bounded Change Radar inputs, repository exposure ingestion, opt-in composition of proof-strength foundations with `ProofEngine`, operational campaign scheduling, and explicit delivery/outcome adapters. None may add generator self-approval, host fallback, regeneration during delivery, default auto-merge, or default production rollback. diff --git a/README.md b/README.md index 94cb015..7c3fa2c 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,46 @@ # AgentDiff -**Let any coding agent write the patch. Independently prove the exact change before it ships.** +**Your software changes itself. AgentDiff proves every change before it ships.** [![CI](https://img.shields.io/github/actions/workflow/status/kam6l/agentdiff/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/kam6l/agentdiff/actions/workflows/ci.yml) [![Python 3.12–3.14](https://img.shields.io/badge/Python-3.12%E2%80%933.14-171916?style=flat-square&logo=python&logoColor=white)](pyproject.toml) [![MIT](https://img.shields.io/badge/license-MIT-78f2c2?style=flat-square)](LICENSE) -[Website](https://agentdiff.usernameort.chatgpt.site/) · [Documentation](https://agentdiff.usernameort.chatgpt.site/docs) · [Quick start](https://agentdiff.usernameort.chatgpt.site/docs/getting-started) · [Security](SECURITY.md) +[Website](https://kam6l.github.io/agentdiff-web/) · [Documentation](https://kam6l.github.io/agentdiff-web/docs/) · [Getting started](https://kam6l.github.io/agentdiff-web/docs/getting-started) · [Security](SECURITY.md) -AgentDiff is a local, deterministic trust layer for autonomous software changes. Its first complete product path is verified API migration: +AgentDiff is an open-source deterministic trust layer around coding agents and maintenance automation. A built-in recipe, Codex, Claude, Gemini, Copilot, or a custom worker may propose code. AgentDiff independently decides whether the exact patch is authorized, reproducible, sufficiently minimal, and proven. ```text -provider signal → usage scan → migration plan → untrusted patch - → policy → impact → clean-room proof → certificate → reviewable PR +upstream change → exposure → untrusted patch → policy + proof + → exact certificate → reviewable PR → outcome evidence ``` -The generator is replaceable. The trust decision is not delegated to the generator. +The generator is replaceable. The proof authority is not. -> [!IMPORTANT] -> The default `ProofEngine` uses Docker for clean-room verification. The general `agentdiff run` local backend observes a normal host subprocess and is not a security sandbox. Read [SECURITY.md](SECURITY.md) before running untrusted commands. +![A real AgentDiff rejection certificate: an unauthorized workflow change remains NOT_PROVEN](https://kam6l.github.io/agentdiff-web/product/unsafe-patch-proof.svg) + +This image is rendered from the checked-in `demos/openai-failure` artifact. The worker exited successfully; AgentDiff still rejected the patch because it changed `.github/workflows/deploy.yml` outside the authorized scope. + +## What ships today + +The complete product path is verified Python API migration: + +- bounded provider-source discovery and data-only plugin candidates; +- read-only SDK usage scan, change matching, and migration simulation; +- a narrow, fail-closed OpenAI Chat Completions → Responses AST transform; +- private generation, deterministic mutation policy, and impact analysis; +- authoritative Docker-backed clean-room proof; +- integrity-bound migration certificates; +- sealed GitHub PR delivery with no regeneration and no auto-merge; and +- explicit multi-repository Verified Campaigns with independent proof per repository. + +This repository also contains tested foundations for generic `ChangeSpec`, the Maintenance Graph, deterministic priority and scheduling, DeltaProof, adversarial proof, flake classification, bounded patch minimization, Proof Lease, PR reconciliation, and post-merge outcome aggregation. They are intentionally labeled **Foundation** until wired into the default operational loop. + +See the [implementation status](docs/engineering/IMPLEMENTATION_STATUS.md) for exact boundaries. -## Install from source +## Install -AgentDiff 0.4.0 requires Python 3.12–3.14. The package is not yet published on PyPI; install the current source revision explicitly. +AgentDiff 0.4.x requires Python 3.12–3.14. It is not yet published on PyPI; install the current source revision explicitly: ```bash uv tool install git+https://github.com/kam6l/agentdiff.git @@ -38,18 +56,18 @@ uv sync --locked --all-groups --extra langchain uv run agentdiff --help ``` -## Verified API migration +Docker is required for the default clean-room proof path and for custom/external generators. Read-only scan and simulation do not require it. -Start read-only: +## Start with a read-only assessment ```bash agentdiff api check --provider openai agentdiff api simulate --provider openai --change chat_to_responses ``` -`simulate` reports affected usages and files, the generation strategy, required verification level, test/build coverage, policy constraints, and review blockers without modifying the repository. +`simulate` reports exact affected usages and files, generation strategy, required verification level, test/build coverage, policy constraints, and review blockers. It does not create a generation workspace, certificate, branch, commit, or pull request. -Run the migration only after the simulation is reviewable: +When the assessment is reviewable: ```bash agentdiff api migrate \ @@ -57,45 +75,24 @@ agentdiff api migrate \ --change chat_to_responses ``` -The default OpenAI transform automates only a narrow, text-only shape: +The built-in transform automates only direct keyword-based, text-only calls with supported parameters and the known `choices[0].message.content` consumer. Tools/functions, streaming, structured output, `n`, multimodal input, dynamic message builders, wrappers, expanded keyword arguments, and unusual response consumers fail closed as `NEEDS_REVIEW`. -- keyword-only `client.chat.completions.create(...)` calls; -- literal system/user/assistant message dictionaries with text content; -- `model`, `store`, `temperature`, `top_p`, and token-limit parameters; and -- response values consumed as `choices[0].message.content`. +Chat Completions remains supported by OpenAI. AgentDiff treats this as an optional behavior change; it does not invent a provider deadline. -Tools/functions, streaming, structured output, `n`, multimodal content, dynamic message builders, wrappers, expanded keyword arguments, and unusual response consumers fail closed as `NEEDS_REVIEW`. Chat Completions is still supported by OpenAI; this migration is optional and classified as a behavior change. - -The generated patch remains in a private workspace and is captured as untrusted evidence. The host source tree is not changed by `api migrate`. - -## Verified Campaigns - -Coordinate the same provider change across an explicit set of local repositories: - -```bash -agentdiff fleet simulate --config fleet.yaml -agentdiff fleet migrate --config fleet.yaml -agentdiff fleet verify .agentdiff/campaigns/openai-responses-2026.json -``` +## Independent proof -Every affected repository runs through the existing `MigrationEngine` and authoritative `ProofEngine` independently. The campaign report records per-repository `PROVEN`, `NEEDS_REVIEW`, `REJECTED`, `UNAFFECTED`, or `ERROR` outcomes. Its SHA-256 digest binds each child certificate ID, certificate integrity digest, patch digest, and proof digest. +There is one normal verifier: `agentdiff.proof.ProofEngine`. A migration becomes `PROVEN` only when all configured conditions agree: -Campaigns accept only explicitly configured local directories. They do not discover repositories, clone URLs, create bulk pull requests, merge changes, or provide a hosted dashboard. A campaign cannot become `PROVEN` unless every affected repository is independently proven. - -## Proof and certificates - -There is one authoritative verifier: `agentdiff.proof.ProofEngine`. The final migration verdict is `PROVEN` only when all required conditions agree: - -- generation completed; -- actual files exactly match the expected scope; -- deterministic mutation policy allows the change; -- the requested proof level was actually achieved; -- build, type-check, affected tests, and full tests satisfy the proof plan; and -- the exact patch and evidence capsule remain integrity-valid. +1. generation completed in the permitted runtime; +2. actual paths exactly match authorized scope; +3. deterministic mutation policy allows the change; +4. the required proof level was actually achieved; +5. trusted build, type-check, affected-test, and full-test phases pass; and +6. the exact patch, base, plan, environment, and evidence remain integrity-valid. Anything weaker is `NOT_PROVEN`. -Migration certificates bind the provider source, repository base SHA, expected and actual files, generator, policy, blast radius, verification plan/results, patch digest, proof digest, and evidence capsule. They provide SHA-256 integrity and freshness checks; they are **not cryptographic signatures**. +Migration certificates bind source provenance, repository base, expected and actual paths, generator, policy, impact, verification plan/results, patch digest, proof digest, and evidence capsule. Their SHA-256 integrity detects local modification; it is not an authenticated signature. ```bash agentdiff api certificate verify .agentdiff/certificates/.json @@ -103,111 +100,107 @@ agentdiff api certificate verify .agentdiff/certificates/.json Verification reports `VALID`, `INVALID`, `STALE`, or `MISMATCH`. -## Verified pull requests +## Use a custom coding agent without trusting it -After a `PROVEN` result, `--open-pr` can replay the sealed patch into a temporary Git worktree and open a GitHub pull request: +Custom and external workers are exact-argv specifications. They do not have a host `generate()` path and never fall back to local execution: ```bash agentdiff api migrate \ --provider openai \ --change chat_to_responses \ - --open-pr \ - --base-branch main + --generator command \ + --generator-image your-reviewed-worker@sha256: \ + --generator-argv your-agent migrate-source ``` -This path requires a clean tracked worktree, an unchanged certified base SHA, an `origin` remote, and an authenticated `gh` CLI. AgentDiff re-verifies every base and result file digest, stages only sealed paths, pushes a dedicated branch, and creates a PR body from the certificate. It never regenerates during delivery and never auto-merges. +The Docker generator runtime uses a private working copy, exact argv, an empty inherited environment, non-root execution, a read-only root filesystem, dropped capabilities, no-new-privileges, no network by default, CPU/memory/PID limits, bounded output, and bounded lifecycle calls. It never mounts the live repository or Docker socket. Docker unavailable, weakened capability evidence, timeout, or output overflow is a refusal. -## Any coding agent as the worker +The private temporary copy is currently bind-mounted into a container, and containers share the host kernel. That is a strong container boundary, not a VM or zero-host-mount claim. -The built-in deterministic AST transform is the default. A custom coding-agent command can be used as an untrusted generator: +## Deliver the exact proven patch + +After a `PROVEN` result, `--open-pr` replays the sealed patch into a temporary Git worktree: ```bash agentdiff api migrate \ --provider openai \ --change chat_to_responses \ - --generator command \ - --generator-argv your-agent migrate-source -``` - -The worker receives a private copy of the sealed source snapshot. AgentDiff captures changes only from that copy, but a custom command still runs with the caller's host permissions: this is observation, not an OS sandbox. Use only trusted commands until a sandbox-backed generator runtime is configured. Exact argv, time/output bounds, generated files, and generator identity are recorded. Missing expected edits, extra files, or policy violations force `NOT_PROVEN` even when the raw test command exits successfully. - -## Provider intelligence - -Create a data-only provider definition and configure official HTTPS sources: - -```bash -agentdiff provider init acme -agentdiff provider discover acme + --open-pr \ + --base-branch main ``` -Remote fetches are bounded by scheme, redirect count, DNS/IP validation, time, size, and content type. Responses are cached with provenance, validators, and a SHA-256 digest. Provider discovery emits untrusted manifest candidates; it never applies them directly. - -Provider plugins default to `DATA_ONLY`. Python transform code is not imported during install or list operations. Executable provider code requires both `TRUSTED_CODE` metadata and explicit caller opt-in. - -## Reproducible demos +Delivery requires a clean tracked worktree, unchanged certified base, `origin`, and authenticated `gh`. It verifies base/result digests, stages only sealed paths, pushes a dedicated branch, and creates a certificate-backed PR body. It never regenerates or auto-merges. -The repository includes focused OpenAI and campaign fixtures: +## Coordinate Verified Campaigns ```bash -agentdiff api simulate \ - --root demos/openai-success \ - --provider openai \ - --change chat_to_responses +agentdiff fleet simulate --config demos/fleet/fleet.yaml +agentdiff fleet migrate --config demos/fleet/fleet.yaml +agentdiff fleet verify .agentdiff/campaigns/openai-responses-2026.json ``` -- `demos/openai-success` is the supported text-only migration with explicit Docker proof commands. -- `demos/openai-failure` includes a deliberately unsafe worker that also edits a deployment workflow. Policy rejects the unexpected file and the verdict stays `NOT_PROVEN`. -- `demos/fleet/fleet.yaml` combines one supported, one review-required, and one unaffected repository for a read-only Verified Campaigns demo. +Every affected repository runs its own migration and proof. The aggregate binds child certificate, certificate-integrity, patch, and proof digests; it cannot become `PROVEN` unless every affected repository is independently proven. -## Foundation commands +Campaigns accept explicit local directories. They do not discover an organization, clone URLs, bulk merge, or provide a hosted dashboard. -The same trust infrastructure also supports general coding-agent transactions: +## General trust commands | Command | Purpose | |---|---| | `agentdiff init` / `bootstrap` | Compile repository trust configuration and proof plans | -| `agentdiff wrap -- ` | Run an agent through the local sidecar pipeline | -| `agentdiff run -- ` | Record an explicit local command transaction | +| `agentdiff wrap -- ` | Run a worker through the local sidecar pipeline | +| `agentdiff run -- ` | Record a local command transaction | | `agentdiff prove ` | Run clean-room proof for a sealed patch | | `agentdiff promote ` | Conflict-check and promote proven evidence | | `agentdiff repair ` | Run the bounded general repair loop | | `agentdiff inspect` / `verify` / `runs` | Inspect and validate evidence capsules | | `agentdiff rollback --safe-only` | Recover eligible regular-file collateral | -| `agentdiff trust` / `impact` / `proof cache-status` | Inspect trust compilation and proof planning | -| `agentdiff workspace status/warm/prune` | Manage immutable warm workspace snapshots | +| `agentdiff trust` / `impact` / `proof cache-status` | Inspect trust and proof planning | +| `agentdiff workspace status/warm/prune` | Manage immutable warm snapshots | -The general `repair` loop is implemented, but `api migrate` does not invoke it automatically in 0.4.0. A failed API migration is preserved as evidence and returned for review. +> [!IMPORTANT] +> `agentdiff run` observes a normal host subprocess. It is not OS isolation, and local filesystem policy is primarily a post-condition check. Use a disposable unprivileged workspace or reviewed external sandbox for untrusted general commands. The API custom-generator path is separate and fails closed to Docker. -## Available now and coming next +## Reproducible demos -Available now: +```bash +agentdiff api simulate \ + --root demos/openai-success \ + --provider openai \ + --change chat_to_responses +``` -- OpenAI Python usage scanning and a fail-closed Chat Completions → Responses transform; -- read-only simulation, private generation, deterministic policy, impact analysis, clean-room proof, integrity certificates, and verified-PR delivery; -- explicit multi-repository Verified Campaigns with integrity-bound child evidence; -- data-only custom providers and bounded official-source discovery; -- general transaction, proof, promotion, repair, recovery, and workspace primitives; and -- Linux/macOS/Windows CI across Python 3.12–3.14, package validation, dependency auditing, Bandit, and CodeQL. +- `demos/openai-success`: supported text-only migration and explicit Docker proof commands. +- `demos/openai-failure`: unsafe worker that also edits a deployment workflow; policy rejects it. +- `demos/fleet/fleet.yaml`: one supported, one review-required, and one unaffected repository. -Coming next: +Website proof surfaces are captured from these fixtures and preserve their real simulation or rejection status. They are not fabricated customer results. -- broader deterministic OpenAI shapes and additional provider migrations; -- automatic API-specific repair-loop integration; -- authenticated certificate signatures and external transparency storage; and -- a maintained hosted isolation backend. +## Website and documentation -There is no hosted dashboard, telemetry service, or hidden approval system. Evidence stays under the repository's `.agentdiff/` directory unless the user deliberately shares it. +The canonical warm editorial site and complete docs IA are maintained in the private `agentdiff-web` repository and deploy to GitHub Pages at [kam6l.github.io/agentdiff-web](https://kam6l.github.io/agentdiff-web/). -## Development +```bash +cd agentdiff-web +pnpm install --frozen-lockfile +pnpm dev +pnpm run check +``` + +The production build creates crawlable deep-route HTML, route-specific metadata, sitemap, `.nojekyll`, and a Pages fallback under the `/agentdiff-web/` base path. + +## Development checks ```bash uv run pytest -p no:cacheprovider tests/ uv run ruff format --check src tests examples benchmarks demos uv run ruff check src tests examples benchmarks demos uv run mypy src/agentdiff -uv build -uv run twine check dist/* -uv run check-wheel-contents dist/*.whl + +cd agentdiff-web +pnpm run check ``` -See [CONTRIBUTING.md](CONTRIBUTING.md), [SECURITY.md](SECURITY.md), and [CHANGELOG.md](CHANGELOG.md). AgentDiff is MIT licensed pre-release software. +The public repository CI adds cross-platform Python coverage, Docker proof, dependency auditing, Bandit, Python CodeQL, package build/smoke testing, and the recovery safety benchmark. The website repository independently validates lint, types, production builds, dependency changes, and JavaScript/TypeScript CodeQL. + +Read [AGENTS.md](AGENTS.md) for the repository map, [SECURITY.md](SECURITY.md) for the full threat model, and [CONTRIBUTING.md](CONTRIBUTING.md) before changing trust boundaries. AgentDiff is MIT-licensed pre-release software. diff --git a/SECURITY.md b/SECURITY.md index eda9a6d..bc7f2ff 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -34,6 +34,23 @@ Command policy can prevent a denied executable from being launched through Agent `--runtime srt` is an optional adapter for a separately installed Anthropic Sandbox Runtime. In that mode, the external tool owns OS enforcement and its settings are a separate security boundary. AgentDiff checks that the executable and optional settings path exist, preserves argv boundaries, and labels the delegated backend in evidence; it does not audit upstream code, validate that settings are sufficiently restrictive, or prove that host prerequisites enforce the declared policy. Current project tests use a fake wrapper and do not certify upstream isolation. +### API generator isolation + +The API migration path treats the exact built-in `DeterministicASTGenerator` as trusted AgentDiff code. Command and external generators are inert exact-argv specifications and have no host execution method. They require `DockerGeneratorRuntime`; an unavailable or weakened Docker boundary never falls back to `LocalRuntime` or a host subprocess. + +The generator runtime verifies evidence for: + +- a private temporary source copy that is disjoint from the live repository; +- no live repository or Docker socket mount; +- non-root execution, read-only root filesystem, dropped capabilities, and `no-new-privileges`; +- no inherited caller environment and networking disabled by default; +- CPU, memory, PID, wall-clock, and per-stream output bounds; and +- exact argv without shell parsing or option-like image injection. + +Output above 1 MiB per stream fails closed. Docker control calls and container attachment have explicit time limits and stage-specific error evidence. + +The current implementation bind-mounts the private temporary copy into the container. Docker containers also share the host kernel. This is a constrained container boundary, not a virtual machine, kernel boundary, or strict zero-host-mount design. + ### Filesystem observation The transaction scanner: @@ -59,7 +76,7 @@ Do not commit `.agentdiff/`. Filesystem metadata, paths, process IDs, process cr AgentDiff redacts common credential-bearing command flags and recursively redacts values under common sensitive keys before persistence. Environment observations use stable SHA-256-derived fingerprints rather than raw values. Terminal display escapes control characters. -Redaction is defense in depth, not a data-loss-prevention guarantee. Secrets can appear in filenames, task descriptions, source files, unknown argument formats, child output, or fields with nonstandard names. Child stdout/stderr is passed through and is not stored by the transaction runner, but it may still reach the user's terminal or surrounding CI logs. +Redaction is defense in depth, not a data-loss-prevention guarantee. Secrets can appear in filenames, task descriptions, source files, unknown argument formats, child output, or fields with nonstandard names. Child stdout/stderr is passed through and is not stored by the general transaction runner, but it may still reach the user's terminal or surrounding CI logs. Generator output is bounded and represented by capture metadata/digests; generators receive no inherited caller environment, but source content itself may still be sensitive. Review capsules before sharing them. @@ -115,4 +132,4 @@ Use version control, disposable workspaces, and real sandboxing alongside AgentD ## Security testing -Security-sensitive changes should include regression tests for path confinement, symlink behavior, identity checks, redaction, backup integrity, and post-run divergence. Pull requests run CodeQL, Bandit, `pip-audit`, Dependency Review, cross-platform tests, and package validation. +Security-sensitive changes should include regression tests for path confinement, symlink behavior, identity checks, redaction, backup integrity, post-run divergence, runtime capability weakening, timeouts, and output overflow. Pull requests run CodeQL for Python and JavaScript/TypeScript, Bandit, `pip-audit`, Dependency Review, cross-platform tests, real Docker proof, frontend validation, and package validation. diff --git a/docs/engineering/ARCHITECTURE.md b/docs/engineering/ARCHITECTURE.md new file mode 100644 index 0000000..94e8947 --- /dev/null +++ b/docs/engineering/ARCHITECTURE.md @@ -0,0 +1,49 @@ +# Architecture + +## System shape + +```text +remote signal (untrusted) + → validated API manifest / ChangeSpec + → exposure and deterministic priority + → built-in recipe or isolated worker (untrusted patch) + → policy and impact + → authoritative ProofEngine + → exact certificate and sealed patch + → reviewable PR + → explicit post-merge outcome evidence +``` + +The trust boundary is intentionally asymmetric: detection and generation are replaceable inputs; deterministic policy and `ProofEngine` own authorization. A graph may connect evidence, a model may suggest a change, and a test runner may report output, but none can promote its own claim. + +## Authoritative components + +| Concern | Primary implementation | Authority | +|---|---|---| +| Transaction capture | `transaction/`, `state/`, `evidence/` | Records observed state; does not prove isolation | +| Policy | `policy/` | Deterministic write/process/network-observation decision | +| Impact | `impact/`, `scoring/`, `trust/` | Explains affected paths and future-execution risk | +| API maintenance | `api/` | Scan, simulate, privately generate, verify, certify, deliver | +| Clean-room proof | `proof/engine.py` | Sole normal proof authority | +| Proof strength | `proof/delta.py`, `proof/adversarial.py`, `proof/flake.py`, `proof/lease.py` | Composes with proof; never replaces it | +| Generic maintenance | `maintenance/` | Canonical change, provenance, priority, and scheduling foundations | +| Delivery lifecycle | `api/github_pr.py`, `delivery/` | Replays sealed evidence; no regeneration or merge authority | +| Outcome lifecycle | `outcome/` | Aggregates bounded checks; never performs default production rollback | + +## Evidence identity + +Stable identities use canonical JSON-compatible payloads and SHA-256 digests. The relevant base, patch, policy, proof plan, runtime image/environment, and evidence references must remain explicit. Stronger or newer evidence creates a new identity; it does not mutate an older certificate into a stronger claim. + +`MaintenanceGraph` links lifecycle nodes and evidence references. It is a provenance index, not a second proof engine or source of authorization. + +## Execution boundaries + +- `agentdiff run` uses a normal host subprocess. Policy around its filesystem effects is principally post-condition observation. +- The built-in deterministic API transform may execute in-process because its exact implementation is trusted with AgentDiff itself. +- Command/external generators are inert exact-argv specifications and require the hardened Docker generator runtime. Docker unavailability is a refusal, not a local downgrade. +- `ProofEngine` independently materializes the captured base plus sealed patch in its clean-room runtime. +- Docker is a container boundary sharing the host kernel, not a VM claim. + +## Extension rules + +Add new source adapters behind bounded fetch and validation interfaces. Add generator integrations as untrusted exact-argv workers. Add proof dimensions as deterministic evidence that composes beside `ProofResult`. Keep filesystem storage behind small interfaces so local operation never requires a service database. diff --git a/docs/engineering/DESIGN_SYSTEM.md b/docs/engineering/DESIGN_SYSTEM.md new file mode 100644 index 0000000..e3a8a15 --- /dev/null +++ b/docs/engineering/DESIGN_SYSTEM.md @@ -0,0 +1,32 @@ +# Website design system + +The canonical site is maintained in the private `agentdiff-web` repository and deploys at `https://kam6l.github.io/agentdiff-web/`. + +## Visual language + +- Warm editorial canvas, near-black text, quiet neutral rules, white artifact surfaces, restrained green for verified states, and amber/red only for attention or refusal. +- One legible sans-serif stack for prose and one system monospace stack for commands, identities, and evidence. +- Square or subtly rounded geometry. Avoid oversized pills, glassmorphism, neon gradients, floating glow, and generic “AI” ornament. +- Alternate dense proof surfaces with generous reading space. Real product artifacts should carry more visual weight than decorative illustration. +- Use the AgentDiff mark and local, attributed brand SVGs. Brand marks identify compatible workers, not partnerships. + +The implementation tokens and responsive rules live in `agentdiff-web/src/styles.css`. Reuse existing components before adding variants. + +## Content rules + +- Lead with the concrete promise and explain why generation is not proof. +- Separate `Available`, `Foundation`, and `Roadmap` visually and verbally. +- Every metric, digest, certificate, before/after patch, and screenshot must trace to a committed fixture or captured run. Label rejected and simulated artifacts accurately. +- Prefer “container boundary” to “perfect sandbox,” “integrity hash” to “signature,” and `NOT_PROVEN` to vague failure language. +- Do not imply vendor endorsement from logos. + +## Interaction and accessibility + +- Semantic landmarks, visible keyboard focus, a focus-only skip link, labeled icon buttons, dialog escape handling, and reduced-motion support are required. +- The installation command is a real button with stable-width copied feedback. +- Documentation must support persistent desktop navigation, a mobile drawer, search, previous/next navigation, and per-page status. +- Verify 375, 430, 768, 1024, 1440, and 1920 pixel widths with no horizontal overflow. Test deep-route refresh under the `/agentdiff-web/` base path. + +## Assets and metadata + +Keep public assets under `agentdiff-web/public/`; never depend on a runtime CDN for core visuals. Every indexable route receives a specific title, description, canonical URL, Open Graph metadata, and sitemap entry during `pnpm run build`. The Pages build must keep `.nojekyll` and `404.html`. diff --git a/docs/engineering/DEVELOPMENT.md b/docs/engineering/DEVELOPMENT.md new file mode 100644 index 0000000..d604cdd --- /dev/null +++ b/docs/engineering/DEVELOPMENT.md @@ -0,0 +1,38 @@ +# Development + +## Prerequisites + +- Python 3.12–3.14 and `uv` 0.12.5. +- Node 24.19 and pnpm 11.22 for the separate `agentdiff-web` repository. +- Docker for real clean-room proof and custom-generator integration tests. + +## Setup + +```bash +uv sync --locked --all-groups --extra langchain +``` + +Use a focused `codex/` branch from current `main`. Preserve unrelated work. Public behavior or trust-boundary changes need tests and matching README, security, website, and status updates. + +## Repository conventions + +- Python source lives under `src/agentdiff`; keep public exports intentional and package-local for foundations. +- Use immutable dataclasses/enums and canonical JSON-compatible identities for lifecycle evidence. +- Inject clocks, runners, runtime factories, and adapters in tests. Do not add nondeterministic network or process behavior to pure models. +- Commands are argv sequences, never shell strings. +- Reuse existing evidence, policy, proof, and delivery authorities rather than building parallel verdict paths. +- Caches, build artifacts, and `.agentdiff/` evidence are generated and must not be committed. + +## Website + +```bash +cd agentdiff-web +pnpm dev # http://127.0.0.1:5173/agentdiff-web/ +pnpm run check # lint, types, production build, static route preparation +``` + +`VITE_BASE_PATH` defaults to `/agentdiff-web/` for GitHub Pages. Set it explicitly in CI. The build script writes per-route HTML metadata, a sitemap, and a Pages fallback from the Vite artifact. + +## Documentation ownership + +This `docs/engineering/` directory is concise repository memory. User-facing documentation lives in the private `agentdiff-web` repository at `src/content/docs.ts`. Keep the two aligned but do not duplicate large manuals here. diff --git a/docs/engineering/IMPLEMENTATION_STATUS.md b/docs/engineering/IMPLEMENTATION_STATUS.md new file mode 100644 index 0000000..4605f90 --- /dev/null +++ b/docs/engineering/IMPLEMENTATION_STATUS.md @@ -0,0 +1,35 @@ +# Implementation status + +This table distinguishes integrated product behavior from isolated foundations and roadmap work. It is a claim-control document, not a schedule. + +| Capability | Status | Current boundary | +|---|---|---| +| Python provider scan/check/simulate | Available | OpenAI is the complete narrow path; Stripe generation is not launch-grade | +| Deterministic Chat Completions → Responses transform | Available | Text-only supported shapes; complex/dynamic shapes require review | +| Private generation, policy, impact, ProofEngine | Available | Clean-room proof requires Docker by default | +| Integrity certificates and exact PR delivery | Available | SHA-256 integrity, not identity signatures; no auto-merge | +| Explicit multi-repository campaigns | Available | Local configured repositories; no organization discovery or bulk merge | +| Provider discovery and data-only plugins | Available | Candidates stay untrusted until validation/activation | +| Generic ChangeSpec, Maintenance Graph, priority | Foundation | Deterministic local models/store; not the default migration schema or scheduler loop | +| Hardened custom generator runtime | Foundation | Docker only, exact argv, no host fallback; private copy is bind-mounted and shares host kernel | +| DeltaProof and adversarial proof | Foundation | Tested composable evidence; not yet a required default ProofEngine gate | +| Patch minimizer | Foundation | Bounded ddmin-style engine; not wired into default migration | +| Proof Lease | Foundation | Pure fail-closed evaluator; no certificate restamping or default integration | +| PR reconciliation | Foundation | Pure reuse/update/stale/supersede decision; never regenerates or merges | +| Post-merge outcome verification | Foundation | Pure/pluggable bounded checks; no production adapter or automatic rollback | +| Maintenance scheduler | Foundation | Deterministic resource/window decisions; no background service | +| Flake Firewall | Foundation | Deterministic classification; flaky evidence cannot count as proof | +| Continuous Change Radar | Roadmap | Bounded source adapters and activation workflow remain to be integrated | +| Organization Exposure Graph | Roadmap | Existing local scanner/impact evidence is the starting seam | +| Verified Recipe promotion | Roadmap | Candidate → fixtures → proof → trusted approval lifecycle is not implemented | +| Operational pause/resume and post-merge adapters | Roadmap | Requires explicit external capabilities and permission boundaries | +| Authenticated attestations | Roadmap | Use Sigstore/in-toto/OIDC rather than custom cryptography | + +## Should not be implemented + +- Generator/model self-approval or proof selection. +- Silent Docker-to-host fallback for untrusted execution. +- Regeneration during delivery or certificate restamping after base movement. +- Default auto-merge or default production rollback. +- Claims of a hosted dashboard, customer telemetry, partnerships, adoption, or benchmark gains without real evidence. +- A second verifier that can disagree with or bypass `ProofEngine`. diff --git a/docs/engineering/PRODUCT.md b/docs/engineering/PRODUCT.md new file mode 100644 index 0000000..88f82fb --- /dev/null +++ b/docs/engineering/PRODUCT.md @@ -0,0 +1,37 @@ +# Product + +## Promise + +**Your software changes itself. AgentDiff proves every change before it ships.** + +AgentDiff detects upstream changes, finds exposed code, generates the smallest safe candidate, proves the exact patch independently, and delivers reviewable evidence. It is not another coding agent. Deterministic transforms, Codex, Claude, Gemini, Copilot, or a custom command may write the patch; none owns the verdict. + +## Product pillars + +1. **Change intelligence** — preserve source provenance, normalize a generic `ChangeSpec`, map exposure, and make priority explainable. +2. **Replaceable generation** — prefer narrow deterministic recipes; isolate flexible workers and treat all output as untrusted. +3. **Independent proof** — bind policy, scope, impact, clean-room execution, causal behavior, adversarial strength, flake state, and exact identity. +4. **Minimal change** — reduce proven patches within explicit proof budgets without claiming global optimality. +5. **Controlled delivery** — replay sealed evidence into a reviewable PR and reconcile lifecycle state without regeneration or auto-merge. +6. **Outcome evidence** — record deployment/health results, detect regressions, and route human attention without default production rollback. + +## Capability labels + +- **Available**: integrated public product behavior with end-to-end tests. +- **Foundation**: deterministic model or engine implemented and tested, but not yet wired into the default operational loop. +- **Roadmap**: documented product direction with no shipping claim. + +The canonical status table is [`IMPLEMENTATION_STATUS.md`](IMPLEMENTATION_STATUS.md). Website copy and screenshots must use these labels and traceable demo artifacts. Do not invent adoption metrics, customer logos, testimonials, benchmark gains, incidents prevented, or proof outcomes. + +## Current wedge + +The complete path is verified Python API migration: read-only provider assessment, narrow fail-closed transformation, private generation, deterministic policy and impact, clean-room proof, integrity certificate, explicit multi-repository campaigns, and sealed GitHub PR delivery. + +## Deliberate non-goals + +- No hosted dashboard, hidden telemetry service, or proprietary approval plane. +- No universal language/provider claim. +- No model self-approval, automatic trust promotion, or auto-merge. +- No replacement for OS sandboxing, CI, code review, or version control. +- No home-grown signature scheme; use established attestation standards when identity signing is added. +- No default production rollback. diff --git a/docs/engineering/SECURITY_INVARIANTS.md b/docs/engineering/SECURITY_INVARIANTS.md new file mode 100644 index 0000000..089fc2e --- /dev/null +++ b/docs/engineering/SECURITY_INVARIANTS.md @@ -0,0 +1,38 @@ +# Security invariants + +These rules are release blockers. + +## Authority + +- A generator cannot approve its patch, choose weaker proof, expand its file scope, promote a recipe, or merge its PR. +- `ProofEngine` is the sole normal proof authority. Adapters must not run a parallel host verifier or invent another final digest. +- `PROVEN` requires every configured gate to agree. Missing infrastructure, incomplete evidence, unexpected paths, stale identity, flakiness, or a weaker achieved level yields `NOT_PROVEN`, review, or an explicit inconclusive state. + +## Exact identity + +- Certificates and delivery bind the repository base, exact patch bytes, expected and actual paths, policy, proof plan/result, runtime identity, and evidence capsule. +- Delivery replays the sealed patch. It never regenerates. +- A base change cannot be handled by restamping a certificate. A Proof Lease remains valid only when verified ancestry and a complete dependency closure establish that all changes are disjoint. +- Cache keys for high-assurance work must bind immutable runtime identity; mutable tags alone are insufficient. + +## Execution + +- Local execution is observation, not syscall enforcement or sandboxing. +- Untrusted custom/external generators never run as host subprocesses and never fall back to one. +- Generator containers receive only a private working copy, never the live repository or Docker socket; inherit no caller secrets; use exact argv, non-root execution, read-only rootfs, dropped capabilities, no-new-privileges, network disabled by default, resource limits, bounded output, and wall-clock limits. +- Container claims must describe the real boundary: Docker shares the host kernel and the current implementation bind-mounts a private temporary copy. + +## Remote input and storage + +- Remote release notes, schemas, packages, advisories, redirects, and AI summaries are untrusted data until bounded fetch, provenance capture, parsing, validation, and activation policy complete. +- Symlinks, path traversal, ambiguous file identity, malformed stored records, and unsupported filesystem entries fail closed. +- Evidence is private by default, written atomically with restrictive permissions where supported, and redacted before persistence. Redaction is defense in depth, not a DLP guarantee. +- Integrity hashes detect modification within a local artifact set; they are not signatures or identity authentication. + +## Recovery and production + +- Rollback changes a path only when current state equals the recorded post-run state. +- Conflicts preserve current data. +- A production regression may preserve evidence, pause future work, notify a human, and prepare a revert for review. Automatic production rollback is always false unless a future, explicit, safely supported policy adds separate authority. + +Every change to these boundaries needs tests for refusal, timeout, malformed evidence, weakened capability reporting, and unavailable infrastructure—not only a happy path. diff --git a/docs/engineering/TESTING.md b/docs/engineering/TESTING.md new file mode 100644 index 0000000..287d0f8 --- /dev/null +++ b/docs/engineering/TESTING.md @@ -0,0 +1,36 @@ +# Testing and release checks + +Run the smallest focused test while iterating, then the full applicable gates before handoff. + +## Python + +```bash +uv run pytest -p no:cacheprovider tests/ +uv run ruff format --check src tests examples benchmarks demos +uv run ruff check src tests examples benchmarks demos +uv run mypy src/agentdiff +``` + +The desktop filesystem sandbox can block loopback sockets and descendant-process observation. A failure caused by that restriction must be rerun with the required local permissions; never change product expectations to make a sandbox-only failure pass. + +Real Docker proof is opt-in locally: + +```bash +AGENTDIFF_DOCKER_TESTS=1 uv run pytest -p no:cacheprovider tests/test_docker_integration.py -v -rs +``` + +## Website + +```bash +cd website +pnpm peers check +pnpm run check +``` + +After the production build, verify that `dist/docs//index.html`, `dist/404.html`, `.nojekyll`, `sitemap.xml`, local brand assets, and the social card exist. Browser QA must cover keyboard navigation, copy feedback, search, mobile drawer, deep-route refresh, console errors, and horizontal overflow at the widths listed in `DESIGN_SYSTEM.md`. + +## Package and security + +CI additionally builds and inspects the wheel, smoke-tests the installed CLI, audits locked runtime dependencies, runs Bandit, Dependency Review, CodeQL for Python and JavaScript/TypeScript, the recovery safety benchmark, and real Docker proof. + +Changes to security invariants require explicit negative tests. A test that demonstrates only successful execution is insufficient. diff --git a/pyproject.toml b/pyproject.toml index 2c6529a..a0cdf57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,8 +36,8 @@ dependencies = [ ] [project.urls] -Homepage = "https://agentdiff.usernameort.chatgpt.site/" -Documentation = "https://agentdiff.usernameort.chatgpt.site/docs/" +Homepage = "https://kam6l.github.io/agentdiff-web/" +Documentation = "https://kam6l.github.io/agentdiff-web/docs/" Repository = "https://github.com/kam6l/agentdiff" Issues = "https://github.com/kam6l/agentdiff/issues" diff --git a/src/agentdiff/api/fleet.py b/src/agentdiff/api/fleet.py index 8269327..2db677e 100644 --- a/src/agentdiff/api/fleet.py +++ b/src/agentdiff/api/fleet.py @@ -15,6 +15,7 @@ import yaml from agentdiff.api.certificate import CertificateStatus, verify_certificate +from agentdiff.api.generators import CustomCommandGenerator from agentdiff.api.manifest import APIChangeManifest, get_builtin_manifest from agentdiff.api.migrate import MigrationEngine, MigrationSimulation from agentdiff.api.models import MigrationResult, MigrationStatus @@ -351,6 +352,7 @@ def migrate_fleet( config: FleetConfig, *, generator: MigrationGenerator, + generator_image: str = "python:3.12-slim", engine_factory: EngineFactory = MigrationEngine, ) -> FleetCampaignResult: """Run the authoritative migration pipeline independently for every repository.""" @@ -359,10 +361,15 @@ def migrate_fleet( results: list[FleetRepositoryResult] = [] for repository in config.repositories: try: + engine_kwargs: dict[str, Any] = { + "root": repository.path, + "manifest": manifest, + "generator": generator, + } + if isinstance(generator, CustomCommandGenerator): + engine_kwargs["generator_image"] = generator_image migration = engine_factory( - root=repository.path, - manifest=manifest, - generator=generator, + **engine_kwargs, ).run() results.append(_migration_result(repository, migration)) except (OSError, RuntimeError, TypeError, ValueError) as error: diff --git a/src/agentdiff/api/generation_runtime.py b/src/agentdiff/api/generation_runtime.py index 373b158..ff33481 100644 --- a/src/agentdiff/api/generation_runtime.py +++ b/src/agentdiff/api/generation_runtime.py @@ -1,38 +1,106 @@ -"""Private workspace runtime for untrusted migration generators.""" +"""Fail-closed runtimes for migration generators.""" from __future__ import annotations +import hashlib +import os import shutil import tempfile +import threading import time +from dataclasses import replace from pathlib import Path -from typing import IO, TYPE_CHECKING, Any +from typing import IO, TYPE_CHECKING, Any, Protocol, runtime_checkable +from agentdiff.api.generators import ( + CustomCommandGenerator, + DeterministicASTGenerator, + GenerationResult, +) from agentdiff.runtime import ( CleanupReport, + DockerLifecycleError, + DockerRuntime, RuntimeCapability, RuntimeControlLevel, RuntimeResult, ) if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Callable, Iterable, Sequence - from agentdiff.api.generators import GenerationResult, MigrationGenerator from agentdiff.api.models import MigrationPlan from agentdiff.runtime import OwnedProcess +@runtime_checkable +class GeneratorRuntime(Protocol): + """Runtime contract used by the migration transaction.""" + + backend: str + enforcement: str + generation_result: GenerationResult | None + + @property + def command_argv(self) -> tuple[str, ...]: + """Return the one exact argv sequence this runtime accepts.""" + ... + + def configure_source(self, source_dir: str | Path) -> None: + """Configure the transaction's sealed source snapshot.""" + ... + + def configure_safety(self, controller: Any) -> None: + """Configure live safety observation where supported.""" + ... + + def run( + self, + argv: Sequence[str], + *, + timeout_seconds: float | None = None, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + ) -> RuntimeResult: + """Run the configured generator and return runtime evidence.""" + ... + + def cleanup( + self, + processes: Iterable[OwnedProcess], + *, + grace_period_seconds: float = 1.0, + ) -> CleanupReport: + """Clean up runtime-owned process identities.""" + ... + + def close(self) -> None: + """Destroy private runtime state.""" + ... + + class PrivateGenerationRuntime: - """Run a generator against a private copy and expose only its observed result.""" + """Run only AgentDiff's trusted deterministic generator in process.""" + + backend = "agentdiff-private-generation" + enforcement = "private_workspace_observation" - def __init__(self, plan: MigrationPlan, generator: MigrationGenerator) -> None: + def __init__(self, plan: MigrationPlan, generator: DeterministicASTGenerator) -> None: + if type(generator) is not DeterministicASTGenerator: + raise TypeError("PrivateGenerationRuntime accepts only DeterministicASTGenerator") self.plan = plan self.generator = generator self._source_dir: Path | None = None self._temporary_root: Path | None = None self.generation_result: GenerationResult | None = None + @property + def command_argv(self) -> tuple[str, ...]: + """Return the synthetic command used for deterministic policy evidence.""" + + return (self.generator.command_label,) + def configure_source(self, source_dir: str | Path) -> None: unresolved = Path(source_dir) if unresolved.is_symlink(): @@ -56,7 +124,7 @@ def run( ) -> RuntimeResult: del stdin, stdout, stderr command = tuple(argv) - if command != (self.generator.command_label,): + if command != self.command_argv: raise ValueError("generation runtime command does not match the configured worker") if timeout_seconds is not None and timeout_seconds <= 0: raise ValueError("timeout_seconds must be greater than zero") @@ -119,3 +187,346 @@ def close(self) -> None: raise RuntimeError("refusing to remove an unexpected generation workspace") shutil.rmtree(resolved, ignore_errors=False) self._temporary_root = None + + +class _BoundedOutputCapture: + """Drain one binary pipe while retaining at most a fixed number of bytes.""" + + def __init__(self, limit_bytes: int) -> None: + read_fd, write_fd = os.pipe() + self.reader = os.fdopen(read_fd, "rb", buffering=0) + self.writer = os.fdopen(write_fd, "wb", buffering=0) + self.limit_bytes = limit_bytes + self.captured = bytearray() + self.total_bytes = 0 + self.digest = hashlib.sha256() + self.complete = False + self._thread = threading.Thread(target=self._drain, daemon=True) + + @property + def limited(self) -> bool: + return self.total_bytes > self.limit_bytes + + def start(self) -> None: + self._thread.start() + + def finish(self) -> None: + self.writer.close() + self._thread.join(timeout=2.0) + if self._thread.is_alive(): + self.reader.close() + self._thread.join(timeout=1.0) + raise OSError("Docker generator output capture did not close") + if not self.complete: + raise OSError("Docker generator output capture did not close") + + def to_dict(self) -> dict[str, Any]: + return { + "total_bytes": self.total_bytes, + "captured_bytes": len(self.captured), + "sha256": self.digest.hexdigest(), + "limited": self.limited, + "complete": self.complete, + } + + def _drain(self) -> None: + try: + with self.reader: + while chunk := self.reader.read(64 * 1024): + self.total_bytes += len(chunk) + self.digest.update(chunk) + remaining = self.limit_bytes - len(self.captured) + if remaining > 0: + self.captured.extend(chunk[:remaining]) + self.complete = True + except OSError: + return + + +class DockerGeneratorRuntime: + """Run an exact-argv generator in a hardened private Docker workspace.""" + + backend = "docker" + enforcement = "isolated_private_workspace" + + def __init__( + self, + root: str | Path, + plan: MigrationPlan, + generator: CustomCommandGenerator, + *, + image: str = "python:3.12-slim", + output_limit_bytes_per_stream: int = 1_000_000, + execution_timeout_seconds: float = 900.0, + control_timeout_seconds: float = 120.0, + docker_runtime_factory: Callable[..., DockerRuntime] = DockerRuntime, + ) -> None: + if not isinstance(generator, CustomCommandGenerator): + raise TypeError("DockerGeneratorRuntime requires a CustomCommandGenerator") + if ( + not image + or image.startswith("-") + or any(character in image for character in "\r\n\x00") + ): + raise ValueError( + "generator image must be a non-empty single-line value and cannot be an option" + ) + if output_limit_bytes_per_stream <= 0: + raise ValueError("output_limit_bytes_per_stream must be greater than zero") + if execution_timeout_seconds <= 0: + raise ValueError("execution_timeout_seconds must be greater than zero") + if control_timeout_seconds <= 0: + raise ValueError("control_timeout_seconds must be greater than zero") + self.root = Path(root).expanduser().resolve(strict=True) + self.image = image + self.output_limit_bytes_per_stream = output_limit_bytes_per_stream + self.execution_timeout_seconds = float(execution_timeout_seconds) + self.control_timeout_seconds = float(control_timeout_seconds) + self.plan = plan + self.generator = generator + self.generation_result: GenerationResult | None = None + self._runtime = docker_runtime_factory( + self.root, + image=image, + network="none", + environment_allowlist=(), + control_timeout_seconds=self.control_timeout_seconds, + ) + + @property + def command_argv(self) -> tuple[str, ...]: + """Return the user-supplied argv without shell parsing or rewriting.""" + + return self.generator.argv + + def configure_source(self, source_dir: str | Path) -> None: + self._runtime.configure_source(source_dir) + + def configure_safety(self, controller: Any) -> None: + self._runtime.configure_safety(controller) + + def run( + self, + argv: Sequence[str], + *, + timeout_seconds: float | None = None, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, + ) -> RuntimeResult: + command = tuple(argv) + if command != self.command_argv: + raise ValueError("generation runtime command does not match the configured argv") + if timeout_seconds is not None and timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + if stdout is not None or stderr is not None: + raise ValueError("Docker generator output redirection is not supported") + effective_timeout = ( + self.execution_timeout_seconds + if timeout_seconds is None + else min(timeout_seconds, self.execution_timeout_seconds) + ) + stdout_capture = _BoundedOutputCapture(self.output_limit_bytes_per_stream) + stderr_capture = _BoundedOutputCapture(self.output_limit_bytes_per_stream) + stdout_capture.start() + stderr_capture.start() + try: + try: + result = self._runtime.run( + command, + timeout_seconds=effective_timeout, + stdin=stdin, + stdout=stdout_capture.writer, + stderr=stderr_capture.writer, + ) + finally: + try: + stdout_capture.finish() + finally: + stderr_capture.finish() + except DockerLifecycleError as error: + self.generation_result = GenerationResult( + success=False, + generator=self.generator.name, + strategy=self.generator.strategy, + errors=(f"Docker generator {error.stage} timed out",), + returncode=124, + ) + raise + except OSError: + self.generation_result = GenerationResult( + success=False, + generator=self.generator.name, + strategy=self.generator.strategy, + errors=("Docker generator runtime failed",), + returncode=1, + ) + raise + + output_limited = stdout_capture.limited or stderr_capture.limited + runtime_config = { + **(result.runtime_config or {}), + "generator_execution_timeout_seconds": effective_timeout, + "generator_output": { + "limit_bytes_per_stream": self.output_limit_bytes_per_stream, + "limited": output_limited, + "stdout": stdout_capture.to_dict(), + "stderr": stderr_capture.to_dict(), + }, + } + result = replace(result, runtime_config=runtime_config) + if output_limited and not result.timed_out: + result = replace(result, returncode=125) + self._validate_isolation(result, command) + errors: tuple[str, ...] + if result.timed_out: + errors = ("Docker generator start/attach timed out",) + elif output_limited: + errors = ("Docker generator output exceeded the safety limit",) + elif result.returncode != 0: + errors = (f"custom generator exited with status {result.returncode}",) + else: + errors = () + self.generation_result = GenerationResult( + success=result.returncode == 0 and not result.timed_out, + generator=self.generator.name, + strategy=self.generator.strategy, + errors=errors, + returncode=result.returncode, + ) + return result + + def cleanup( + self, + processes: Iterable[OwnedProcess], + *, + grace_period_seconds: float = 1.0, + ) -> CleanupReport: + return self._runtime.cleanup( + processes, + grace_period_seconds=grace_period_seconds, + ) + + def close(self) -> None: + self._runtime.close() + + def _validate_isolation( + self, + result: RuntimeResult, + command: tuple[str, ...], + ) -> None: + config = result.runtime_config or {} + user = str(config.get("user", "")) + cap_drop = config.get("cap_drop") + cap_drop_all = cap_drop == ["ALL"] + observation_root: Path | None = None + if result.observation_root is not None: + try: + observation_root = Path(result.observation_root).resolve(strict=True) + except OSError: + observation_root = None + wrapper = result.wrapper_argv or () + image_index = len(wrapper) - len(command) - 1 + docker_options = wrapper[:image_index] if image_index >= 0 else () + mount = f"type=bind,src={result.observation_root},dst=/workspace" + + def has_pair(flag: str, value: str) -> bool: + return any( + docker_options[index : index + 2] == (flag, value) + for index in range(len(docker_options) - 1) + ) + + environment_values = tuple( + docker_options[index + 1] + for index, argument in enumerate(docker_options[:-1]) + if argument == "--env" + ) + non_root_user = user.split(":", maxsplit=1)[0] not in {"", "0", "root"} + output_evidence_valid = self._valid_output_evidence(config.get("generator_output")) + + isolated = ( + result.argv == command + and result.backend == "docker" + and result.enforcement == "isolated_private_workspace" + and result.image == self.image + and observation_root is not None + and observation_root != self.root + and self.root not in observation_root.parents + and observation_root not in self.root.parents + and config.get("private_workspace") is True + and config.get("host_repository_mounted") is False + and config.get("docker_socket_mounted") is False + and config.get("read_only_rootfs") is True + and config.get("no_new_privileges") is True + and config.get("network") == "none" + and config.get("ephemeral_container") is True + and config.get("control_timeout_seconds") == self.control_timeout_seconds + and isinstance(config.get("generator_execution_timeout_seconds"), (int, float)) + and float(config["generator_execution_timeout_seconds"]) > 0 + and float(config["generator_execution_timeout_seconds"]) + <= self.execution_timeout_seconds + and output_evidence_valid + and config.get("environment_allowlist") == [] + and isinstance(config.get("cpus"), (int, float)) + and float(config["cpus"]) > 0 + and isinstance(config.get("memory"), str) + and bool(config["memory"]) + and isinstance(config.get("pids_limit"), int) + and int(config["pids_limit"]) > 0 + and cap_drop_all + and non_root_user + and image_index >= 0 + and wrapper[image_index] == self.image + and "--read-only" in docker_options + and has_pair("--user", user) + and has_pair("--cap-drop", "ALL") + and has_pair("--security-opt", "no-new-privileges") + and has_pair("--network", "none") + and has_pair("--mount", mount) + and has_pair("--tmpfs", "/tmp:rw,noexec,nosuid,nodev,size=64m") + and has_pair("--cpus", str(config["cpus"])) + and has_pair("--memory", str(config["memory"])) + and has_pair("--pids-limit", str(config["pids_limit"])) + and environment_values == ("HOME=/tmp",) + and docker_options.count("--mount") == 1 + and "--volume" not in docker_options + and "-v" not in docker_options + and "--privileged" not in docker_options + and "--cap-add" not in docker_options + and "docker.sock" not in " ".join(docker_options) + and tuple(wrapper[-len(command) :]) == command + ) + if not isolated: + raise OSError("Docker generator runtime did not enforce the required isolation") + + def _valid_output_evidence(self, value: Any) -> bool: + if not isinstance(value, dict): + return False + if value.get("limit_bytes_per_stream") != self.output_limit_bytes_per_stream: + return False + streams: list[dict[str, Any]] = [] + for name in ("stdout", "stderr"): + stream = value.get(name) + if not isinstance(stream, dict): + return False + total = stream.get("total_bytes") + captured = stream.get("captured_bytes") + digest = stream.get("sha256") + limited = stream.get("limited") + complete = stream.get("complete") + if ( + type(total) is not int + or type(captured) is not int + or total < 0 + or captured < 0 + or captured > self.output_limit_bytes_per_stream + or captured > total + or not isinstance(digest, str) + or len(digest) != 64 + or type(limited) is not bool + or limited is not (total > self.output_limit_bytes_per_stream) + or complete is not True + ): + return False + streams.append(stream) + return value.get("limited") is any(stream["limited"] for stream in streams) diff --git a/src/agentdiff/api/generators.py b/src/agentdiff/api/generators.py index 59386ce..d9b2573 100644 --- a/src/agentdiff/api/generators.py +++ b/src/agentdiff/api/generators.py @@ -6,8 +6,6 @@ from __future__ import annotations -import subprocess # nosec B404 -- custom generators use exact argv without a shell -from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING @@ -31,8 +29,8 @@ class GenerationResult: returncode: int = 0 -class MigrationGenerator(ABC): - """Interface implemented by deterministic, agent, and custom workers.""" +class MigrationGenerator: + """Metadata shared by deterministic workers and isolated command specs.""" name: str strategy: MigrationStrategy @@ -43,10 +41,6 @@ def command_label(self) -> str: return self.name - @abstractmethod - def generate(self, plan: MigrationPlan, workspace: Path) -> GenerationResult: - """Generate a patch inside a private workspace.""" - class DeterministicASTGenerator(MigrationGenerator): """Apply registered AST transforms without model calls.""" @@ -154,12 +148,14 @@ def generate(self, plan: MigrationPlan, workspace: Path) -> GenerationResult: class CustomCommandGenerator(MigrationGenerator): - """Run a user-supplied exact argv sequence in the private workspace.""" + """Describe a user-supplied exact argv sequence for an isolated runtime.""" strategy = MigrationStrategy.CODING_AGENT def __init__(self, argv: tuple[str, ...], *, name: str = "custom-command") -> None: - if not argv or any(not argument or "\x00" in argument for argument in argv): + if not argv or any( + not isinstance(argument, str) or not argument or "\x00" in argument for argument in argv + ): raise ValueError("custom generator argv must contain valid arguments") self.argv = argv self.name = name @@ -168,35 +164,6 @@ def __init__(self, argv: tuple[str, ...], *, name: str = "custom-command") -> No def command_label(self) -> str: return Path(self.argv[0]).name - def generate(self, plan: MigrationPlan, workspace: Path) -> GenerationResult: - del plan - try: - completed = subprocess.run( # nosec B603 - self.argv, - cwd=workspace, - shell=False, - check=False, - capture_output=True, - text=True, - timeout=900, - ) - except (OSError, subprocess.SubprocessError) as error: - return GenerationResult( - success=False, - generator=self.name, - strategy=self.strategy, - errors=(f"custom generator failed: {type(error).__name__}",), - returncode=1, - ) - detail = (completed.stderr or completed.stdout)[-1000:].strip() - return GenerationResult( - success=completed.returncode == 0, - generator=self.name, - strategy=self.strategy, - errors=() if completed.returncode == 0 else (detail or "custom generator failed",), - returncode=completed.returncode, - ) - class ExternalCodingAgentGenerator(CustomCommandGenerator): """Named custom-command worker for Codex, Claude, Gemini, or another agent.""" diff --git a/src/agentdiff/api/migrate.py b/src/agentdiff/api/migrate.py index 9d45d28..32b94cc 100644 --- a/src/agentdiff/api/migrate.py +++ b/src/agentdiff/api/migrate.py @@ -2,14 +2,23 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any from agentdiff.api.certificate import create_certificate, write_certificate -from agentdiff.api.generation_runtime import PrivateGenerationRuntime -from agentdiff.api.generators import DeterministicASTGenerator, MigrationGenerator +from agentdiff.api.generation_runtime import ( + DockerGeneratorRuntime, + GeneratorRuntime, + PrivateGenerationRuntime, +) +from agentdiff.api.generators import ( + CustomCommandGenerator, + DeterministicASTGenerator, + MigrationGenerator, +) from agentdiff.api.manifest import APIChangeManifest, get_builtin_manifest from agentdiff.api.matcher import APIMatcher from agentdiff.api.models import ( @@ -30,6 +39,8 @@ from agentdiff.policy import load_policy, load_policy_file, policy_to_dict from agentdiff.transaction import AgentRunTransaction +GeneratorRuntimeFactory = Callable[[MigrationPlan, MigrationGenerator], GeneratorRuntime] + @dataclass(frozen=True, slots=True) class MigrationSimulation: @@ -78,6 +89,8 @@ def __init__( provider: str | None = None, change_id: str | None = None, generator: MigrationGenerator | None = None, + generator_image: str = "python:3.12-slim", + generator_runtime_factory: GeneratorRuntimeFactory | None = None, proof_environment_factory: Any | None = None, proof_cache: Any | None = None, proof_timeout_seconds: float = 900.0, @@ -87,6 +100,8 @@ def __init__( self.provider = provider self.change_id = change_id self.generator = generator or DeterministicASTGenerator() + self.generator_image = generator_image + self.generator_runtime_factory = generator_runtime_factory self.proof_environment_factory = proof_environment_factory self.proof_cache = proof_cache self.proof_timeout_seconds = proof_timeout_seconds @@ -242,13 +257,13 @@ def run(self) -> MigrationResult: ) migration_policy = self._migration_policy(plan) - runtime = PrivateGenerationRuntime(plan, self.generator) + runtime = self._generation_runtime(plan) transaction = AgentRunTransaction( self.root, migration_policy, task=f"api migration {plan.provider}:{plan.change_id}", runtime=runtime, - ).run([self.generator.command_label], timeout_seconds=900) + ).run(runtime.command_argv, timeout_seconds=900) actual_files = tuple(sorted(change.path for change in transaction.changes)) unexpected_files = tuple(sorted(set(actual_files) - set(plan.affected_files))) missing_files = tuple(sorted(set(plan.affected_files) - set(actual_files))) @@ -313,6 +328,32 @@ def _migration_policy(self, plan: MigrationPlan) -> Any: payload["process"] = process return load_policy(payload) + def _generation_runtime(self, plan: MigrationPlan) -> GeneratorRuntime: + if self.generator_runtime_factory is not None: + runtime = self.generator_runtime_factory(plan, self.generator) + if not isinstance(runtime, GeneratorRuntime): + raise TypeError("generator runtime factory returned an invalid runtime") + if ( + type(self.generator) is not DeterministicASTGenerator + and runtime.enforcement != "isolated_private_workspace" + ): + raise TypeError( + "non-deterministic generators require an isolated private workspace runtime" + ) + return runtime + if type(self.generator) is DeterministicASTGenerator: + return PrivateGenerationRuntime(plan, self.generator) + if isinstance(self.generator, CustomCommandGenerator): + return DockerGeneratorRuntime( + self.root, + plan, + self.generator, + image=self.generator_image, + ) + raise TypeError( + "unrecognized migration generators require an explicit isolated runtime factory" + ) + def _empty_result(self, manifest: APIChangeManifest) -> MigrationResult: plan = MigrationPlan( provider=manifest.provider, diff --git a/src/agentdiff/cli.py b/src/agentdiff/cli.py index 01c71e7..bb1cca3 100644 --- a/src/agentdiff/cli.py +++ b/src/agentdiff/cli.py @@ -1237,6 +1237,7 @@ def cmd_api_migrate(args: argparse.Namespace) -> int: policy_path=args.policy, manifest=manifest, generator=generator, + generator_image=args.generator_image, ) # Run migration @@ -1353,7 +1354,11 @@ def cmd_fleet_migrate(args: argparse.Namespace) -> int: """Run a proof-backed migration independently in each configured repository.""" config = FleetConfig.load(args.config) - result = migrate_fleet(config, generator=_fleet_generator(args)) + result = migrate_fleet( + config, + generator=_fleet_generator(args), + generator_image=args.generator_image, + ) requested_output = args.output or ( config.config_path.parent / ".agentdiff" / "campaigns" / f"{config.campaign}.json" ) @@ -1836,6 +1841,11 @@ def build_parser() -> argparse.ArgumentParser: p_fleet_migrate.add_argument( "--generator-argv", nargs="+", help="Exact argv for the custom-command generator" ) + p_fleet_migrate.add_argument( + "--generator-image", + default="python:3.12-slim", + help="Docker image for command generators (host execution is never used)", + ) p_fleet_migrate.add_argument("--format", choices=["json", "summary"], default="summary") p_fleet_migrate.set_defaults(func=cmd_fleet_migrate) p_fleet_verify = fleet_commands.add_parser( @@ -1890,6 +1900,11 @@ def build_parser() -> argparse.ArgumentParser: p_api_migrate.add_argument( "--generator-argv", nargs="+", help="Exact argv for the custom-command generator" ) + p_api_migrate.add_argument( + "--generator-image", + default="python:3.12-slim", + help="Docker image for command generators (host execution is never used)", + ) p_api_migrate.add_argument("--open-pr", action="store_true", help="Open a PR only if PROVEN") p_api_migrate.add_argument("--base-branch", default="main", help="GitHub PR base branch") p_api_migrate.add_argument("--branch", help="Explicit PR head branch") diff --git a/src/agentdiff/delivery/__init__.py b/src/agentdiff/delivery/__init__.py new file mode 100644 index 0000000..3a16219 --- /dev/null +++ b/src/agentdiff/delivery/__init__.py @@ -0,0 +1,21 @@ +"""Deterministic lifecycle decisions for already-published verified changes.""" + +from .reconciler import ( + PRIdentity, + PRReconciler, + PRReconciliationDecision, + PRReconciliationResult, + PullRequestSnapshot, + PullRequestState, + SealedDeliveryIdentity, +) + +__all__ = [ + "PRIdentity", + "PRReconciler", + "PRReconciliationDecision", + "PRReconciliationResult", + "PullRequestSnapshot", + "PullRequestState", + "SealedDeliveryIdentity", +] diff --git a/src/agentdiff/delivery/reconciler.py b/src/agentdiff/delivery/reconciler.py new file mode 100644 index 0000000..6b46d1a --- /dev/null +++ b/src/agentdiff/delivery/reconciler.py @@ -0,0 +1,287 @@ +"""Pure reconciliation for pull requests bound to sealed proof identities. + +This module deliberately does not call GitHub, rewrite branches, regenerate a +patch, or merge anything. It compares an existing pull-request snapshot with +the exact desired base, patch, and proof identities and emits the only safe +next disposition. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class PullRequestState(str, Enum): + """Explicit remote lifecycle state used by the reconciler.""" + + OPEN = "OPEN" + DRAFT = "DRAFT" + CLOSED = "CLOSED" + MERGED = "MERGED" + + +class PRReconciliationDecision(str, Enum): + """Safe disposition for an already-created pull request.""" + + REUSE = "REUSE" + UPDATE = "UPDATE" + STALE = "STALE" + CLOSE_OR_SUPERSEDE = "CLOSE_OR_SUPERSEDE" + + +@dataclass(frozen=True, slots=True) +class SealedDeliveryIdentity: + """Exact content identities that bind delivery to deterministic proof.""" + + base_sha: str + patch_digest: str + proof_digest: str + schema_version: int = 1 + + def __post_init__(self) -> None: + for field_name in ("base_sha", "patch_digest", "proof_digest"): + value = getattr(self, field_name) + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError(f"{field_name} must be a non-empty exact identity") + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "base_sha": self.base_sha, + "patch_digest": self.patch_digest, + "proof_digest": self.proof_digest, + } + + +@dataclass(frozen=True, slots=True) +class PullRequestSnapshot: + """Read-only facts about one existing pull request. + + ``identity=None`` represents a PR whose sealed identity metadata is absent + or could not be verified. Such a PR is never reused or updated. + """ + + number: int + state: PullRequestState + identity: SealedDeliveryIdentity | None + url: str = "" + schema_version: int = 1 + + def __post_init__(self) -> None: + if isinstance(self.number, bool) or not isinstance(self.number, int) or self.number < 1: + raise ValueError("pull request number must be a positive integer") + if self.identity is not None and not isinstance(self.identity, SealedDeliveryIdentity): + raise TypeError("pull request identity must be a SealedDeliveryIdentity") + state = ( + self.state + if isinstance(self.state, PullRequestState) + else PullRequestState(str(self.state).upper()) + ) + object.__setattr__(self, "state", state) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "number": self.number, + "state": self.state.value, + "identity": self.identity.to_dict() if self.identity is not None else None, + "url": self.url, + } + + +@dataclass(frozen=True, slots=True) +class PRReconciliationResult: + """Evidence-backed recommendation with deliberately narrow authority.""" + + decision: PRReconciliationDecision + pull_request: PullRequestSnapshot + desired_identity: SealedDeliveryIdentity + base_matches: bool + patch_matches: bool + proof_matches: bool + reason: str + schema_version: int = 1 + + @property + def exact_identity_match(self) -> bool: + return self.base_matches and self.patch_matches and self.proof_matches + + @property + def metadata_update_only(self) -> bool: + """UPDATE may only refresh proof metadata for the same base and patch.""" + + return self.decision is PRReconciliationDecision.UPDATE + + @property + def requires_reproof(self) -> bool: + return self.decision is PRReconciliationDecision.STALE + + @property + def regeneration_allowed(self) -> bool: + """A reconciler never authorizes patch regeneration.""" + + return False + + @property + def merge_allowed(self) -> bool: + """A reconciler never authorizes or performs a merge.""" + + return False + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "decision": self.decision.value, + "pull_request": self.pull_request.to_dict(), + "desired_identity": self.desired_identity.to_dict(), + "base_matches": self.base_matches, + "patch_matches": self.patch_matches, + "proof_matches": self.proof_matches, + "exact_identity_match": self.exact_identity_match, + "metadata_update_only": self.metadata_update_only, + "requires_reproof": self.requires_reproof, + "regeneration_allowed": self.regeneration_allowed, + "merge_allowed": self.merge_allowed, + "reason": self.reason, + } + + +class PRReconciler: + """Classify an existing PR using only state and exact sealed identities.""" + + def reconcile( + self, + desired_identity: SealedDeliveryIdentity, + pull_request: PullRequestSnapshot, + ) -> PRReconciliationResult: + observed = pull_request.identity + base_matches = observed is not None and observed.base_sha == desired_identity.base_sha + patch_matches = ( + observed is not None and observed.patch_digest == desired_identity.patch_digest + ) + proof_matches = ( + observed is not None and observed.proof_digest == desired_identity.proof_digest + ) + + if pull_request.state is PullRequestState.CLOSED: + return self._result( + PRReconciliationDecision.CLOSE_OR_SUPERSEDE, + pull_request, + desired_identity, + base_matches, + patch_matches, + proof_matches, + "closed pull requests are not reusable delivery targets", + ) + + if pull_request.state is PullRequestState.MERGED: + if base_matches and patch_matches and proof_matches: + return self._result( + PRReconciliationDecision.REUSE, + pull_request, + desired_identity, + base_matches, + patch_matches, + proof_matches, + "the exact sealed delivery identity is already merged", + ) + return self._result( + PRReconciliationDecision.CLOSE_OR_SUPERSEDE, + pull_request, + desired_identity, + base_matches, + patch_matches, + proof_matches, + "a merged pull request is immutable and must be superseded", + ) + + if observed is None: + return self._result( + PRReconciliationDecision.STALE, + pull_request, + desired_identity, + False, + False, + False, + "pull request has no verified sealed delivery identity", + ) + + if not patch_matches: + return self._result( + PRReconciliationDecision.CLOSE_OR_SUPERSEDE, + pull_request, + desired_identity, + base_matches, + patch_matches, + proof_matches, + "pull request contains a different sealed patch", + ) + + if not base_matches: + return self._result( + PRReconciliationDecision.STALE, + pull_request, + desired_identity, + base_matches, + patch_matches, + proof_matches, + "sealed base identity changed; rebase and independent reproof are required", + ) + + if not proof_matches: + return self._result( + PRReconciliationDecision.UPDATE, + pull_request, + desired_identity, + base_matches, + patch_matches, + proof_matches, + "base and patch match; only sealed proof metadata may be refreshed", + ) + + return self._result( + PRReconciliationDecision.REUSE, + pull_request, + desired_identity, + base_matches, + patch_matches, + proof_matches, + "open pull request exactly matches the sealed base, patch, and proof", + ) + + @staticmethod + def _result( + decision: PRReconciliationDecision, + pull_request: PullRequestSnapshot, + desired_identity: SealedDeliveryIdentity, + base_matches: bool, + patch_matches: bool, + proof_matches: bool, + reason: str, + ) -> PRReconciliationResult: + return PRReconciliationResult( + decision=decision, + pull_request=pull_request, + desired_identity=desired_identity, + base_matches=base_matches, + patch_matches=patch_matches, + proof_matches=proof_matches, + reason=reason, + ) + + +# Compact product-language alias. +PRIdentity = SealedDeliveryIdentity + + +__all__ = [ + "PRIdentity", + "PRReconciler", + "PRReconciliationDecision", + "PRReconciliationResult", + "PullRequestSnapshot", + "PullRequestState", + "SealedDeliveryIdentity", +] diff --git a/src/agentdiff/maintenance/__init__.py b/src/agentdiff/maintenance/__init__.py new file mode 100644 index 0000000..7f11357 --- /dev/null +++ b/src/agentdiff/maintenance/__init__.py @@ -0,0 +1,72 @@ +"""Generic foundations for verified maintenance workflows.""" + +from agentdiff.maintenance.graph import ( + MaintenanceEdge, + MaintenanceEntityKind, + MaintenanceGraph, + MaintenanceNode, +) +from agentdiff.maintenance.models import ( + ChangeDelivery, + ChangeProof, + ChangeSource, + ChangeSpec, + ChangeStrategy, + ChangeTargets, + MaintenanceSeverity, + ProofRequirement, + StrategyDisposition, +) +from agentdiff.maintenance.priority import ( + PriorityDecision, + PriorityEngine, + PriorityInputs, + PriorityResult, +) +from agentdiff.maintenance.scheduler import ( + ActiveMaintenanceRun, + MaintenanceQueueItem, + MaintenanceSchedule, + MaintenanceScheduleDecision, + MaintenanceScheduler, + MaintenanceSchedulerState, + SchedulerAction, + SchedulerLimits, + SchedulerMode, + SchedulerReason, + SchedulerResources, +) +from agentdiff.maintenance.store import LocalJSONMaintenanceStore, MaintenanceGraphStore + +__all__ = [ + "ActiveMaintenanceRun", + "ChangeDelivery", + "ChangeProof", + "ChangeSource", + "ChangeSpec", + "ChangeStrategy", + "ChangeTargets", + "LocalJSONMaintenanceStore", + "MaintenanceEdge", + "MaintenanceEntityKind", + "MaintenanceGraph", + "MaintenanceGraphStore", + "MaintenanceNode", + "MaintenanceQueueItem", + "MaintenanceSchedule", + "MaintenanceScheduleDecision", + "MaintenanceScheduler", + "MaintenanceSchedulerState", + "MaintenanceSeverity", + "PriorityDecision", + "PriorityEngine", + "PriorityInputs", + "PriorityResult", + "ProofRequirement", + "SchedulerAction", + "SchedulerLimits", + "SchedulerMode", + "SchedulerReason", + "SchedulerResources", + "StrategyDisposition", +] diff --git a/src/agentdiff/maintenance/_canonical.py b/src/agentdiff/maintenance/_canonical.py new file mode 100644 index 0000000..f295e06 --- /dev/null +++ b/src/agentdiff/maintenance/_canonical.py @@ -0,0 +1,135 @@ +"""Canonical serialization and strict validation helpers for maintenance data.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from datetime import UTC, datetime +from typing import Any + +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9_.-]{0,127}$") + + +def canonical_sha256(value: object) -> str: + """Return a stable digest for a JSON-compatible value.""" + + try: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + except (TypeError, ValueError) as error: + raise ValueError("value is not canonically JSON serializable") from error + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def require_mapping(value: object, *, field: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value): + raise ValueError(f"{field} must be an object with string keys") + return value + + +def reject_unknown( + value: Mapping[str, Any], + allowed: frozenset[str], + *, + field: str, +) -> None: + unknown = sorted(set(value) - allowed) + if unknown: + raise ValueError(f"{field} contains unknown fields: {', '.join(unknown)}") + + +def clean_text( + value: object, + *, + field: str, + required: bool = False, + max_length: int = 4096, +) -> str: + if not isinstance(value, str): + raise ValueError(f"{field} must be a string") + cleaned = value.strip() + if required and not cleaned: + raise ValueError(f"{field} is required") + if "\x00" in cleaned: + raise ValueError(f"{field} cannot contain NUL characters") + if len(cleaned) > max_length: + raise ValueError(f"{field} exceeds the {max_length} character limit") + return cleaned + + +def clean_identifier(value: object, *, field: str) -> str: + cleaned = clean_text(value, field=field, required=True, max_length=128) + if not _IDENTIFIER.fullmatch(cleaned): + raise ValueError( + f"{field} must start with a lowercase letter and contain only " + "lowercase letters, digits, '.', '_' or '-'" + ) + return cleaned + + +def clean_string_tuple( + value: object, + *, + field: str, + max_items: int = 1000, +) -> tuple[str, ...]: + if not isinstance(value, (list, tuple)): + raise ValueError(f"{field} must be a list of strings") + if len(value) > max_items: + raise ValueError(f"{field} exceeds the {max_items} item limit") + cleaned = tuple( + clean_text(item, field=f"{field} item", required=True, max_length=2048) for item in value + ) + return tuple(sorted(set(cleaned))) + + +def clean_confidence(value: object, *, field: str = "confidence") -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field} must be a number") + normalized = float(value) + if not 0.0 <= normalized <= 1.0: + raise ValueError(f"{field} must be between 0.0 and 1.0") + return 0.0 if normalized == 0.0 else normalized + + +def clean_int_range( + value: object, + *, + field: str, + minimum: int, + maximum: int | None = None, +) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{field} must be an integer") + if value < minimum or (maximum is not None and value > maximum): + if maximum is None: + raise ValueError(f"{field} must be at least {minimum}") + raise ValueError(f"{field} must be between {minimum} and {maximum}") + return value + + +def clean_bool(value: object, *, field: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{field} must be a boolean") + return value + + +def clean_timestamp(value: object, *, field: str, required: bool = True) -> str: + cleaned = clean_text(value, field=field, required=required, max_length=64) + if not cleaned: + return "" + candidate = cleaned[:-1] + "+00:00" if cleaned.endswith("Z") else cleaned + try: + parsed = datetime.fromisoformat(candidate) + except ValueError as error: + raise ValueError(f"{field} must be an ISO-8601 timestamp") from error + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError(f"{field} must include a timezone") + return parsed.astimezone(UTC).isoformat() diff --git a/src/agentdiff/maintenance/graph.py b/src/agentdiff/maintenance/graph.py new file mode 100644 index 0000000..b4e9fd1 --- /dev/null +++ b/src/agentdiff/maintenance/graph.py @@ -0,0 +1,401 @@ +"""Canonical provenance graph for the verified maintenance lifecycle.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from ._canonical import ( + canonical_sha256, + clean_confidence, + clean_identifier, + clean_string_tuple, + clean_text, + clean_timestamp, + reject_unknown, + require_mapping, +) + +_MAX_GRAPH_ITEMS = 100_000 + + +class MaintenanceEntityKind(str, Enum): + """Canonical entities in the maintenance lifecycle.""" + + UPSTREAM_EVENT = "upstream_event" + CHANGE_SPEC = "change_spec" + EXPOSURE = "exposure" + REPOSITORY = "repository" + CANDIDATE_PATCH = "candidate_patch" + BEHAVIOR_CONTRACT = "behavior_contract" + PROOF = "proof" + CERTIFICATE = "certificate" + PROOF_LEASE = "proof_lease" + PULL_REQUEST = "pull_request" + DEPLOYMENT = "deployment" + OUTCOME = "outcome" + + +@dataclass(frozen=True, slots=True) +class MaintenanceNode: + """One immutable entity reference with provenance and evidence bindings.""" + + kind: MaintenanceEntityKind + key: str + source: str + timestamp: str + status: str + provenance: tuple[str, ...] = () + confidence: float = 1.0 + evidence: tuple[str, ...] = () + content_digest: str = "" + + def __post_init__(self) -> None: + try: + kind = MaintenanceEntityKind(self.kind) + except ValueError as error: + raise ValueError("node.kind is invalid") from error + object.__setattr__(self, "kind", kind) + object.__setattr__( + self, + "key", + clean_text(self.key, field="node.key", required=True, max_length=512), + ) + object.__setattr__( + self, + "source", + clean_text(self.source, field="node.source", required=True, max_length=2048), + ) + object.__setattr__( + self, + "timestamp", + clean_timestamp(self.timestamp, field="node.timestamp"), + ) + object.__setattr__( + self, + "status", + clean_text(self.status, field="node.status", required=True, max_length=128), + ) + object.__setattr__( + self, + "provenance", + clean_string_tuple(self.provenance, field="node.provenance"), + ) + object.__setattr__( + self, + "confidence", + clean_confidence(self.confidence, field="node.confidence"), + ) + object.__setattr__( + self, + "evidence", + clean_string_tuple(self.evidence, field="node.evidence"), + ) + object.__setattr__( + self, + "content_digest", + clean_text(self.content_digest, field="node.content_digest", max_length=256), + ) + + @property + def id(self) -> str: + identity = {"kind": self.kind.value, "key": self.key} + return f"{self.kind.value}-{canonical_sha256(identity)[:24]}" + + @property + def digest(self) -> str: + return canonical_sha256(self._payload()) + + def _payload(self) -> dict[str, Any]: + return { + "kind": self.kind.value, + "key": self.key, + "source": self.source, + "timestamp": self.timestamp, + "status": self.status, + "provenance": list(self.provenance), + "confidence": self.confidence, + "evidence": list(self.evidence), + "content_digest": self.content_digest, + } + + def to_dict(self) -> dict[str, Any]: + return {**self._payload(), "id": self.id, "digest": self.digest} + + @classmethod + def from_dict(cls, value: object) -> MaintenanceNode: + data = require_mapping(value, field="node") + reject_unknown( + data, + frozenset( + { + "id", + "digest", + "kind", + "key", + "source", + "timestamp", + "status", + "provenance", + "confidence", + "evidence", + "content_digest", + } + ), + field="node", + ) + try: + kind = MaintenanceEntityKind(data.get("kind", "")) + except ValueError as error: + raise ValueError("node.kind is invalid") from error + node = cls( + kind=kind, + key=data.get("key", ""), + source=data.get("source", ""), + timestamp=data.get("timestamp", ""), + status=data.get("status", ""), + provenance=clean_string_tuple(data.get("provenance", []), field="node.provenance"), + confidence=clean_confidence(data.get("confidence", 1.0), field="node.confidence"), + evidence=clean_string_tuple(data.get("evidence", []), field="node.evidence"), + content_digest=data.get("content_digest", ""), + ) + if data.get("id", node.id) != node.id: + raise ValueError("node id does not match its canonical identity") + if data.get("digest", node.digest) != node.digest: + raise ValueError("node digest does not match its canonical payload") + return node + + +@dataclass(frozen=True, slots=True) +class MaintenanceEdge: + """One immutable lifecycle relation with its own provenance.""" + + source_id: str + target_id: str + relationship: str + source: str + timestamp: str + status: str + provenance: tuple[str, ...] = () + confidence: float = 1.0 + evidence: tuple[str, ...] = () + + def __post_init__(self) -> None: + object.__setattr__( + self, + "source_id", + clean_text( + self.source_id, + field="edge.source_id", + required=True, + max_length=256, + ), + ) + object.__setattr__( + self, + "target_id", + clean_text( + self.target_id, + field="edge.target_id", + required=True, + max_length=256, + ), + ) + if self.source_id == self.target_id: + raise ValueError("edge cannot connect a node to itself") + object.__setattr__( + self, + "relationship", + clean_identifier(self.relationship, field="edge.relationship"), + ) + object.__setattr__( + self, + "source", + clean_text(self.source, field="edge.source", required=True, max_length=2048), + ) + object.__setattr__( + self, + "timestamp", + clean_timestamp(self.timestamp, field="edge.timestamp"), + ) + object.__setattr__( + self, + "status", + clean_text(self.status, field="edge.status", required=True, max_length=128), + ) + object.__setattr__( + self, + "provenance", + clean_string_tuple(self.provenance, field="edge.provenance"), + ) + object.__setattr__( + self, + "confidence", + clean_confidence(self.confidence, field="edge.confidence"), + ) + object.__setattr__( + self, + "evidence", + clean_string_tuple(self.evidence, field="edge.evidence"), + ) + + @property + def id(self) -> str: + identity = { + "source_id": self.source_id, + "target_id": self.target_id, + "relationship": self.relationship, + } + return f"edge-{canonical_sha256(identity)[:24]}" + + @property + def digest(self) -> str: + return canonical_sha256(self._payload()) + + def _payload(self) -> dict[str, Any]: + return { + "source_id": self.source_id, + "target_id": self.target_id, + "relationship": self.relationship, + "source": self.source, + "timestamp": self.timestamp, + "status": self.status, + "provenance": list(self.provenance), + "confidence": self.confidence, + "evidence": list(self.evidence), + } + + def to_dict(self) -> dict[str, Any]: + return {**self._payload(), "id": self.id, "digest": self.digest} + + @classmethod + def from_dict(cls, value: object) -> MaintenanceEdge: + data = require_mapping(value, field="edge") + reject_unknown( + data, + frozenset( + { + "id", + "digest", + "source_id", + "target_id", + "relationship", + "source", + "timestamp", + "status", + "provenance", + "confidence", + "evidence", + } + ), + field="edge", + ) + edge = cls( + source_id=data.get("source_id", ""), + target_id=data.get("target_id", ""), + relationship=data.get("relationship", ""), + source=data.get("source", ""), + timestamp=data.get("timestamp", ""), + status=data.get("status", ""), + provenance=clean_string_tuple(data.get("provenance", []), field="edge.provenance"), + confidence=clean_confidence(data.get("confidence", 1.0), field="edge.confidence"), + evidence=clean_string_tuple(data.get("evidence", []), field="edge.evidence"), + ) + if data.get("id", edge.id) != edge.id: + raise ValueError("edge id does not match its canonical identity") + if data.get("digest", edge.digest) != edge.digest: + raise ValueError("edge digest does not match its canonical payload") + return edge + + +@dataclass(frozen=True, slots=True) +class MaintenanceGraph: + """A validated immutable snapshot of maintenance nodes and edges.""" + + nodes: tuple[MaintenanceNode, ...] = () + edges: tuple[MaintenanceEdge, ...] = () + schema_version: int = 1 + + def __post_init__(self) -> None: + if self.schema_version != 1: + raise ValueError("unsupported MaintenanceGraph schema version") + if not isinstance(self.nodes, tuple) or any( + not isinstance(node, MaintenanceNode) for node in self.nodes + ): + raise ValueError("graph.nodes must be a tuple of MaintenanceNode values") + if not isinstance(self.edges, tuple) or any( + not isinstance(edge, MaintenanceEdge) for edge in self.edges + ): + raise ValueError("graph.edges must be a tuple of MaintenanceEdge values") + if len(self.nodes) > _MAX_GRAPH_ITEMS or len(self.edges) > _MAX_GRAPH_ITEMS: + raise ValueError(f"graph exceeds the {_MAX_GRAPH_ITEMS} item limit") + nodes = tuple(sorted(self.nodes, key=lambda node: node.id)) + edges = tuple(sorted(self.edges, key=lambda edge: edge.id)) + node_ids = [node.id for node in nodes] + edge_ids = [edge.id for edge in edges] + if len(set(node_ids)) != len(node_ids): + raise ValueError("graph contains duplicate node identities") + if len(set(edge_ids)) != len(edge_ids): + raise ValueError("graph contains duplicate edge identities") + known_nodes = set(node_ids) + dangling = sorted( + { + endpoint + for edge in edges + for endpoint in (edge.source_id, edge.target_id) + if endpoint not in known_nodes + } + ) + if dangling: + raise ValueError("graph contains dangling edge endpoints: " + ", ".join(dangling)) + object.__setattr__(self, "nodes", nodes) + object.__setattr__(self, "edges", edges) + + @property + def id(self) -> str: + identity = { + "schema_version": self.schema_version, + "nodes": [node.id for node in self.nodes], + "edges": [edge.id for edge in self.edges], + } + return f"maintenance-graph-{canonical_sha256(identity)[:24]}" + + @property + def digest(self) -> str: + return canonical_sha256(self._payload()) + + def _payload(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "nodes": [node.to_dict() for node in self.nodes], + "edges": [edge.to_dict() for edge in self.edges], + } + + def to_dict(self) -> dict[str, Any]: + return {**self._payload(), "id": self.id, "digest": self.digest} + + @classmethod + def from_dict(cls, value: object) -> MaintenanceGraph: + data = require_mapping(value, field="MaintenanceGraph") + reject_unknown( + data, + frozenset({"schema_version", "id", "digest", "nodes", "edges"}), + field="MaintenanceGraph", + ) + version = data.get("schema_version", 0) + if isinstance(version, bool) or version != 1: + raise ValueError("unsupported MaintenanceGraph schema version") + raw_nodes = data.get("nodes") + raw_edges = data.get("edges") + if not isinstance(raw_nodes, list) or not isinstance(raw_edges, list): + raise ValueError("graph nodes and edges must be lists") + graph = cls( + nodes=tuple(MaintenanceNode.from_dict(node) for node in raw_nodes), + edges=tuple(MaintenanceEdge.from_dict(edge) for edge in raw_edges), + ) + if data.get("id", graph.id) != graph.id: + raise ValueError("graph id does not match its canonical identity") + if data.get("digest", graph.digest) != graph.digest: + raise ValueError("graph digest does not match its canonical payload") + return graph diff --git a/src/agentdiff/maintenance/models.py b/src/agentdiff/maintenance/models.py new file mode 100644 index 0000000..cd1fc94 --- /dev/null +++ b/src/agentdiff/maintenance/models.py @@ -0,0 +1,545 @@ +"""Generic immutable maintenance change specifications.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import TYPE_CHECKING, Any + +from ._canonical import ( + canonical_sha256, + clean_bool, + clean_confidence, + clean_identifier, + clean_string_tuple, + clean_text, + clean_timestamp, + reject_unknown, + require_mapping, +) + +if TYPE_CHECKING: + from agentdiff.api.manifest import APIChangeManifest + + +class MaintenanceSeverity(str, Enum): + """Provider-neutral maintenance severity.""" + + INFO = "info" + LOW = "low" + MODERATE = "moderate" + HIGH = "high" + CRITICAL = "critical" + + +class StrategyDisposition(str, Enum): + """How a patch-generation strategy may be used.""" + + REQUIRED = "required" + PREFERRED = "preferred" + FALLBACK = "fallback" + ALLOWED = "allowed" + DISABLED = "disabled" + + +class ProofRequirement(str, Enum): + """Minimum requested proof strength for a change.""" + + NONE = "none" + BASIC = "basic" + STRONG = "strong" + CAUSAL = "causal" + ADVERSARIAL = "adversarial" + + +@dataclass(frozen=True, slots=True) +class ChangeSource: + """Where an upstream maintenance signal came from.""" + + type: str + provider: str = "" + version: str = "" + uri: str = "" + retrieved_at: str = "" + content_digest: str = "" + provenance: tuple[str, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "type", clean_identifier(self.type, field="source.type")) + object.__setattr__( + self, + "provider", + clean_text(self.provider, field="source.provider", max_length=128), + ) + object.__setattr__( + self, + "version", + clean_text(self.version, field="source.version", max_length=256), + ) + object.__setattr__(self, "uri", clean_text(self.uri, field="source.uri")) + object.__setattr__( + self, + "retrieved_at", + clean_timestamp(self.retrieved_at, field="source.retrieved_at", required=False), + ) + object.__setattr__( + self, + "content_digest", + clean_text(self.content_digest, field="source.content_digest", max_length=256), + ) + object.__setattr__( + self, + "provenance", + clean_string_tuple(self.provenance, field="source.provenance"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "type": self.type, + "provider": self.provider, + "version": self.version, + "uri": self.uri, + "retrieved_at": self.retrieved_at, + "content_digest": self.content_digest, + "provenance": list(self.provenance), + } + + @classmethod + def from_dict(cls, value: object) -> ChangeSource: + data = require_mapping(value, field="source") + reject_unknown( + data, + frozenset( + { + "type", + "provider", + "version", + "uri", + "retrieved_at", + "content_digest", + "provenance", + } + ), + field="source", + ) + return cls( + type=data.get("type", ""), + provider=data.get("provider", ""), + version=data.get("version", ""), + uri=data.get("uri", ""), + retrieved_at=data.get("retrieved_at", ""), + content_digest=data.get("content_digest", ""), + provenance=clean_string_tuple(data.get("provenance", []), field="source.provenance"), + ) + + +@dataclass(frozen=True, slots=True) +class ChangeTargets: + """Provider-neutral selectors for affected software.""" + + symbols: tuple[str, ...] = () + parameters: tuple[str, ...] = () + models: tuple[str, ...] = () + packages: tuple[str, ...] = () + repositories: tuple[str, ...] = () + files: tuple[str, ...] = () + + def __post_init__(self) -> None: + for name in ("symbols", "parameters", "models", "packages", "repositories", "files"): + object.__setattr__( + self, + name, + clean_string_tuple(getattr(self, name), field=f"targets.{name}"), + ) + if not any( + ( + self.symbols, + self.parameters, + self.models, + self.packages, + self.repositories, + self.files, + ) + ): + raise ValueError("targets must contain at least one selector") + + def to_dict(self) -> dict[str, list[str]]: + return { + "symbols": list(self.symbols), + "parameters": list(self.parameters), + "models": list(self.models), + "packages": list(self.packages), + "repositories": list(self.repositories), + "files": list(self.files), + } + + @classmethod + def from_dict(cls, value: object) -> ChangeTargets: + data = require_mapping(value, field="targets") + allowed = frozenset( + {"symbols", "parameters", "models", "packages", "repositories", "files"} + ) + reject_unknown(data, allowed, field="targets") + return cls( + symbols=clean_string_tuple(data.get("symbols", []), field="targets.symbols"), + parameters=clean_string_tuple(data.get("parameters", []), field="targets.parameters"), + models=clean_string_tuple(data.get("models", []), field="targets.models"), + packages=clean_string_tuple(data.get("packages", []), field="targets.packages"), + repositories=clean_string_tuple( + data.get("repositories", []), field="targets.repositories" + ), + files=clean_string_tuple(data.get("files", []), field="targets.files"), + ) + + +@dataclass(frozen=True, slots=True) +class ChangeStrategy: + """Allowed deterministic, agent, and manual generation paths.""" + + deterministic: StrategyDisposition = StrategyDisposition.PREFERRED + agent: StrategyDisposition = StrategyDisposition.FALLBACK + manual_review: bool = True + transform_id: str = "" + + def __post_init__(self) -> None: + try: + deterministic = StrategyDisposition(self.deterministic) + agent = StrategyDisposition(self.agent) + except ValueError as error: + raise ValueError("strategy disposition is invalid") from error + object.__setattr__(self, "deterministic", deterministic) + object.__setattr__(self, "agent", agent) + object.__setattr__( + self, + "manual_review", + clean_bool(self.manual_review, field="strategy.manual_review"), + ) + object.__setattr__( + self, + "transform_id", + clean_text(self.transform_id, field="strategy.transform_id", max_length=256), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "deterministic": self.deterministic.value, + "agent": self.agent.value, + "manual_review": self.manual_review, + "transform_id": self.transform_id, + } + + @classmethod + def from_dict(cls, value: object) -> ChangeStrategy: + data = require_mapping(value, field="strategy") + reject_unknown( + data, + frozenset({"deterministic", "agent", "manual_review", "transform_id"}), + field="strategy", + ) + try: + deterministic = StrategyDisposition(data.get("deterministic", "preferred")) + agent = StrategyDisposition(data.get("agent", "fallback")) + except ValueError as error: + raise ValueError("strategy disposition is invalid") from error + return cls( + deterministic=deterministic, + agent=agent, + manual_review=clean_bool( + data.get("manual_review", True), field="strategy.manual_review" + ), + transform_id=data.get("transform_id", ""), + ) + + +@dataclass(frozen=True, slots=True) +class ChangeProof: + """Proof requested before delivery.""" + + required: ProofRequirement = ProofRequirement.STRONG + behavior_contracts: tuple[str, ...] = () + + def __post_init__(self) -> None: + try: + required = ProofRequirement(self.required) + except ValueError as error: + raise ValueError("proof.required is invalid") from error + object.__setattr__(self, "required", required) + object.__setattr__( + self, + "behavior_contracts", + clean_string_tuple(self.behavior_contracts, field="proof.behavior_contracts"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "required": self.required.value, + "behavior_contracts": list(self.behavior_contracts), + } + + @classmethod + def from_dict(cls, value: object) -> ChangeProof: + data = require_mapping(value, field="proof") + reject_unknown( + data, + frozenset({"required", "behavior_contracts"}), + field="proof", + ) + try: + required = ProofRequirement(data.get("required", "strong")) + except ValueError as error: + raise ValueError("proof.required is invalid") from error + return cls( + required=required, + behavior_contracts=clean_string_tuple( + data.get("behavior_contracts", []), field="proof.behavior_contracts" + ), + ) + + +@dataclass(frozen=True, slots=True) +class ChangeDelivery: + """Authorized delivery behavior; automatic merge is off by default.""" + + open_pr: bool = False + auto_merge: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "open_pr", clean_bool(self.open_pr, field="delivery.open_pr")) + object.__setattr__( + self, + "auto_merge", + clean_bool(self.auto_merge, field="delivery.auto_merge"), + ) + if self.auto_merge and not self.open_pr: + raise ValueError("delivery.auto_merge requires delivery.open_pr") + + def to_dict(self) -> dict[str, bool]: + return {"open_pr": self.open_pr, "auto_merge": self.auto_merge} + + @classmethod + def from_dict(cls, value: object) -> ChangeDelivery: + data = require_mapping(value, field="delivery") + reject_unknown(data, frozenset({"open_pr", "auto_merge"}), field="delivery") + return cls( + open_pr=clean_bool(data.get("open_pr", False), field="delivery.open_pr"), + auto_merge=clean_bool(data.get("auto_merge", False), field="delivery.auto_merge"), + ) + + +@dataclass(frozen=True, slots=True) +class ChangeSpec: + """A deterministic, provider-neutral request for one maintenance change.""" + + kind: str + external_id: str + title: str + source: ChangeSource + targets: ChangeTargets + severity: MaintenanceSeverity + strategy: ChangeStrategy = field(default_factory=ChangeStrategy) + proof: ChangeProof = field(default_factory=ChangeProof) + delivery: ChangeDelivery = field(default_factory=ChangeDelivery) + description: str = "" + deadline: str = "" + confidence: float = 1.0 + schema_version: int = 1 + + def __post_init__(self) -> None: + if self.schema_version != 1: + raise ValueError("unsupported ChangeSpec schema version") + object.__setattr__(self, "kind", clean_identifier(self.kind, field="kind")) + object.__setattr__( + self, + "external_id", + clean_text(self.external_id, field="external_id", required=True, max_length=256), + ) + object.__setattr__( + self, + "title", + clean_text(self.title, field="title", required=True, max_length=512), + ) + if not isinstance(self.source, ChangeSource): + raise ValueError("source must be a ChangeSource") + if not isinstance(self.targets, ChangeTargets): + raise ValueError("targets must be ChangeTargets") + try: + severity = MaintenanceSeverity(self.severity) + except ValueError as error: + raise ValueError("severity is invalid") from error + object.__setattr__(self, "severity", severity) + if not isinstance(self.strategy, ChangeStrategy): + raise ValueError("strategy must be a ChangeStrategy") + if not isinstance(self.proof, ChangeProof): + raise ValueError("proof must be a ChangeProof") + if not isinstance(self.delivery, ChangeDelivery): + raise ValueError("delivery must be a ChangeDelivery") + object.__setattr__( + self, + "description", + clean_text(self.description, field="description", max_length=16_384), + ) + object.__setattr__( + self, + "deadline", + clean_text(self.deadline, field="deadline", max_length=64), + ) + object.__setattr__(self, "confidence", clean_confidence(self.confidence)) + + @property + def id(self) -> str: + identity = { + "schema_version": self.schema_version, + "kind": self.kind, + "external_id": self.external_id, + "source": { + "type": self.source.type, + "provider": self.source.provider, + "version": self.source.version, + "uri": self.source.uri, + }, + "targets": self.targets.to_dict(), + } + return f"change-spec-{canonical_sha256(identity)[:24]}" + + @property + def digest(self) -> str: + return canonical_sha256(self._payload()) + + def _payload(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "kind": self.kind, + "external_id": self.external_id, + "title": self.title, + "description": self.description, + "source": self.source.to_dict(), + "targets": self.targets.to_dict(), + "severity": self.severity.value, + "strategy": self.strategy.to_dict(), + "proof": self.proof.to_dict(), + "delivery": self.delivery.to_dict(), + "deadline": self.deadline, + "confidence": self.confidence, + } + + def to_dict(self) -> dict[str, Any]: + return {**self._payload(), "id": self.id, "digest": self.digest} + + @classmethod + def from_dict(cls, value: object) -> ChangeSpec: + data = require_mapping(value, field="ChangeSpec") + reject_unknown( + data, + frozenset( + { + "schema_version", + "id", + "digest", + "kind", + "external_id", + "title", + "description", + "source", + "targets", + "severity", + "strategy", + "proof", + "delivery", + "deadline", + "confidence", + } + ), + field="ChangeSpec", + ) + version = data.get("schema_version", 0) + if isinstance(version, bool) or version != 1: + raise ValueError("unsupported ChangeSpec schema version") + try: + severity = MaintenanceSeverity(data.get("severity", "")) + except ValueError as error: + raise ValueError("severity is invalid") from error + spec = cls( + kind=data.get("kind", ""), + external_id=data.get("external_id", ""), + title=data.get("title", ""), + description=data.get("description", ""), + source=ChangeSource.from_dict(data.get("source")), + targets=ChangeTargets.from_dict(data.get("targets")), + severity=severity, + strategy=ChangeStrategy.from_dict(data.get("strategy", {})), + proof=ChangeProof.from_dict(data.get("proof", {})), + delivery=ChangeDelivery.from_dict(data.get("delivery", {})), + deadline=data.get("deadline", ""), + confidence=clean_confidence(data.get("confidence", 1.0)), + schema_version=1, + ) + recorded_id = data.get("id") + if recorded_id is not None and recorded_id != spec.id: + raise ValueError("ChangeSpec id does not match its canonical identity") + recorded_digest = data.get("digest") + if recorded_digest is not None and recorded_digest != spec.digest: + raise ValueError("ChangeSpec digest does not match its canonical payload") + return spec + + @classmethod + def from_api_manifest( + cls, + manifest: APIChangeManifest, + *, + proof: ChangeProof | None = None, + delivery: ChangeDelivery | None = None, + ) -> ChangeSpec: + """Adapt the existing API manifest without changing its public schema.""" + + primary = manifest.strategy.primary.value + fallback = manifest.strategy.fallback.value if manifest.strategy.fallback else "" + deterministic = _strategy_disposition(primary, fallback, "ast_transform") + agent = _strategy_disposition(primary, fallback, "coding_agent") + provenance = tuple( + value for value in (manifest.source.url, manifest.migration_guide_url) if value + ) + source_version = ( + manifest.source.version or manifest.breaking_version or manifest.minimum_sdk_version + ) + return cls( + kind="upstream_change", + external_id=manifest.change_id, + title=manifest.title, + description=manifest.description, + source=ChangeSource( + type=manifest.source.type.value, + provider=manifest.provider, + version=source_version, + uri=manifest.source.url, + retrieved_at=manifest.source.retrieved_at, + provenance=provenance, + ), + targets=ChangeTargets( + symbols=manifest.affected.symbols, + parameters=manifest.affected.parameters, + models=manifest.affected.models, + ), + severity=MaintenanceSeverity(manifest.severity.value), + strategy=ChangeStrategy( + deterministic=deterministic, + agent=agent, + manual_review=primary == "manual" or fallback == "manual", + transform_id=manifest.strategy.transform_id, + ), + proof=proof or ChangeProof(), + delivery=delivery or ChangeDelivery(), + deadline=manifest.deadline, + confidence=manifest.confidence, + ) + + +def _strategy_disposition( + primary: str, + fallback: str, + selected: str, +) -> StrategyDisposition: + if primary == selected: + return StrategyDisposition.PREFERRED + if fallback == selected: + return StrategyDisposition.FALLBACK + return StrategyDisposition.DISABLED diff --git a/src/agentdiff/maintenance/priority.py b/src/agentdiff/maintenance/priority.py new file mode 100644 index 0000000..8b4fe74 --- /dev/null +++ b/src/agentdiff/maintenance/priority.py @@ -0,0 +1,314 @@ +"""Deterministic maintenance urgency policy.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Any, ClassVar + +from ._canonical import ( + clean_bool, + clean_int_range, + clean_string_tuple, + clean_timestamp, + reject_unknown, + require_mapping, +) +from .models import MaintenanceSeverity + + +class PriorityDecision(str, Enum): + """Deterministic maintenance queue decision.""" + + FIX_NOW = "FIX_NOW" + SCHEDULE = "SCHEDULE" + REVIEW = "REVIEW" + IGNORE = "IGNORE" + + +@dataclass(frozen=True, slots=True) +class PriorityInputs: + """Policy inputs for one repository exposure.""" + + security_severity: MaintenanceSeverity + repository_criticality: int + usage_count: int + blast_radius: int + version_distance: int + deterministic_transform: bool + test_quality: int + proof_available: bool + owners: tuple[str, ...] + maintenance_window_open: bool + breaking_deadline: str = "" + evaluated_at: str = "" + explicit_ignore: bool = False + + def __post_init__(self) -> None: + try: + severity = MaintenanceSeverity(self.security_severity) + except ValueError as error: + raise ValueError("security_severity is invalid") from error + object.__setattr__(self, "security_severity", severity) + object.__setattr__( + self, + "repository_criticality", + clean_int_range( + self.repository_criticality, + field="repository_criticality", + minimum=0, + maximum=100, + ), + ) + object.__setattr__( + self, + "usage_count", + clean_int_range(self.usage_count, field="usage_count", minimum=0), + ) + object.__setattr__( + self, + "blast_radius", + clean_int_range( + self.blast_radius, + field="blast_radius", + minimum=0, + maximum=100, + ), + ) + object.__setattr__( + self, + "version_distance", + clean_int_range(self.version_distance, field="version_distance", minimum=0), + ) + object.__setattr__( + self, + "deterministic_transform", + clean_bool(self.deterministic_transform, field="deterministic_transform"), + ) + object.__setattr__( + self, + "test_quality", + clean_int_range( + self.test_quality, + field="test_quality", + minimum=0, + maximum=100, + ), + ) + object.__setattr__( + self, + "proof_available", + clean_bool(self.proof_available, field="proof_available"), + ) + object.__setattr__( + self, + "owners", + clean_string_tuple(self.owners, field="owners"), + ) + object.__setattr__( + self, + "maintenance_window_open", + clean_bool(self.maintenance_window_open, field="maintenance_window_open"), + ) + object.__setattr__( + self, + "breaking_deadline", + clean_timestamp( + self.breaking_deadline, + field="breaking_deadline", + required=False, + ), + ) + object.__setattr__( + self, + "evaluated_at", + clean_timestamp(self.evaluated_at, field="evaluated_at", required=False), + ) + object.__setattr__( + self, + "explicit_ignore", + clean_bool(self.explicit_ignore, field="explicit_ignore"), + ) + if bool(self.breaking_deadline) != bool(self.evaluated_at): + raise ValueError("breaking_deadline and evaluated_at must be provided together") + + def to_dict(self) -> dict[str, Any]: + return { + "security_severity": self.security_severity.value, + "repository_criticality": self.repository_criticality, + "usage_count": self.usage_count, + "blast_radius": self.blast_radius, + "version_distance": self.version_distance, + "deterministic_transform": self.deterministic_transform, + "test_quality": self.test_quality, + "proof_available": self.proof_available, + "owners": list(self.owners), + "maintenance_window_open": self.maintenance_window_open, + "breaking_deadline": self.breaking_deadline, + "evaluated_at": self.evaluated_at, + "explicit_ignore": self.explicit_ignore, + } + + @classmethod + def from_dict(cls, value: object) -> PriorityInputs: + data = require_mapping(value, field="PriorityInputs") + reject_unknown( + data, + frozenset( + { + "security_severity", + "repository_criticality", + "usage_count", + "blast_radius", + "version_distance", + "deterministic_transform", + "test_quality", + "proof_available", + "owners", + "maintenance_window_open", + "breaking_deadline", + "evaluated_at", + "explicit_ignore", + } + ), + field="PriorityInputs", + ) + try: + severity = MaintenanceSeverity(data.get("security_severity", "")) + except ValueError as error: + raise ValueError("security_severity is invalid") from error + return cls( + security_severity=severity, + repository_criticality=data.get("repository_criticality", -1), + usage_count=data.get("usage_count", -1), + blast_radius=data.get("blast_radius", -1), + version_distance=data.get("version_distance", -1), + deterministic_transform=clean_bool( + data.get("deterministic_transform"), field="deterministic_transform" + ), + test_quality=data.get("test_quality", -1), + proof_available=clean_bool(data.get("proof_available"), field="proof_available"), + owners=clean_string_tuple(data.get("owners", []), field="owners"), + maintenance_window_open=clean_bool( + data.get("maintenance_window_open"), field="maintenance_window_open" + ), + breaking_deadline=data.get("breaking_deadline", ""), + evaluated_at=data.get("evaluated_at", ""), + explicit_ignore=clean_bool(data.get("explicit_ignore", False), field="explicit_ignore"), + ) + + +@dataclass(frozen=True, slots=True) +class PriorityResult: + """Explainable deterministic priority outcome.""" + + decision: PriorityDecision + score: int + reasons: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "decision": self.decision.value, + "score": self.score, + "reasons": list(self.reasons), + } + + +class PriorityEngine: + """Apply fixed policy rules; no model participates in the decision.""" + + _SEVERITY_POINTS: ClassVar[dict[MaintenanceSeverity, int]] = { + MaintenanceSeverity.INFO: 0, + MaintenanceSeverity.LOW: 10, + MaintenanceSeverity.MODERATE: 25, + MaintenanceSeverity.HIGH: 45, + MaintenanceSeverity.CRITICAL: 70, + } + + def decide(self, inputs: PriorityInputs) -> PriorityResult: + if not isinstance(inputs, PriorityInputs): + raise TypeError("inputs must be PriorityInputs") + if inputs.explicit_ignore: + return PriorityResult( + PriorityDecision.IGNORE, 0, ("policy explicitly ignores exposure",) + ) + if inputs.usage_count == 0: + return PriorityResult(PriorityDecision.IGNORE, 0, ("repository has no matched usage",)) + + deadline_days = self._deadline_days(inputs) + score = self._score(inputs, deadline_days) + reasons = [f"security severity is {inputs.security_severity.value}"] + if deadline_days is not None: + reasons.append(f"breaking deadline is {deadline_days} day(s) away") + if inputs.repository_criticality >= 75: + reasons.append("repository criticality is high") + if inputs.blast_radius >= 60: + reasons.append("blast radius is high") + + if inputs.security_severity is MaintenanceSeverity.CRITICAL: + reasons.append("critical security maintenance cannot wait for a window") + return PriorityResult(PriorityDecision.FIX_NOW, score, tuple(reasons)) + if deadline_days is not None and deadline_days <= 0: + reasons.append("breaking deadline has arrived") + return PriorityResult(PriorityDecision.FIX_NOW, score, tuple(reasons)) + + if score >= 70: + weakness = self._review_reason(inputs) + if weakness: + reasons.append(weakness) + return PriorityResult(PriorityDecision.REVIEW, score, tuple(reasons)) + if not inputs.maintenance_window_open: + reasons.append("maintenance window is closed") + return PriorityResult(PriorityDecision.SCHEDULE, score, tuple(reasons)) + reasons.append("urgency and repository exposure exceed the fix-now threshold") + return PriorityResult(PriorityDecision.FIX_NOW, score, tuple(reasons)) + + if score >= 40: + weakness = self._review_reason(inputs) + if weakness: + reasons.append(weakness) + return PriorityResult(PriorityDecision.REVIEW, score, tuple(reasons)) + + window_reason = ( + "maintenance window is open" + if inputs.maintenance_window_open + else "maintenance window is closed" + ) + reasons.append(window_reason) + return PriorityResult(PriorityDecision.SCHEDULE, score, tuple(reasons)) + + def _score(self, inputs: PriorityInputs, deadline_days: int | None) -> int: + score = self._SEVERITY_POINTS[inputs.security_severity] + score += inputs.repository_criticality // 5 + score += min(15, inputs.usage_count) + score += inputs.blast_radius // 10 + score += min(10, inputs.version_distance * 2) + if deadline_days is not None: + if deadline_days <= 0: + score += 30 + elif deadline_days <= 7: + score += 25 + elif deadline_days <= 30: + score += 15 + elif deadline_days <= 90: + score += 5 + return min(100, score) + + @staticmethod + def _deadline_days(inputs: PriorityInputs) -> int | None: + if not inputs.breaking_deadline: + return None + deadline = datetime.fromisoformat(inputs.breaking_deadline) + evaluated = datetime.fromisoformat(inputs.evaluated_at) + return int((deadline - evaluated).total_seconds() // 86_400) + + @staticmethod + def _review_reason(inputs: PriorityInputs) -> str: + if not inputs.owners: + return "affected repository has no assigned owner" + if inputs.test_quality < 40: + return "test quality is below the automatic-priority threshold" + if not inputs.proof_available and not inputs.deterministic_transform: + return "neither proof nor a deterministic transform is available" + return "" diff --git a/src/agentdiff/maintenance/scheduler.py b/src/agentdiff/maintenance/scheduler.py new file mode 100644 index 0000000..92db366 --- /dev/null +++ b/src/agentdiff/maintenance/scheduler.py @@ -0,0 +1,639 @@ +"""Pure deterministic planning for maintenance work. + +The scheduler reserves no real resources and starts no work. It orders an +immutable queue, evaluates an immutable state snapshot, and returns auditable +decisions that an external orchestrator may choose to apply. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from enum import Enum +from typing import Any + +from ._canonical import ( + canonical_sha256, + clean_bool, + clean_int_range, + clean_text, + clean_timestamp, + reject_unknown, + require_mapping, +) + + +class SchedulerAction(str, Enum): + """One side-effect-free planning decision.""" + + SCHEDULE = "SCHEDULE" + RUN = "RUN" + DEFER = "DEFER" + + +class SchedulerMode(str, Enum): + """Explicit scheduler control state.""" + + RUNNING = "RUNNING" + PAUSED = "PAUSED" + + +class SchedulerReason(str, Enum): + """Stable audit reasons emitted by the scheduler policy.""" + + READY = "ready" + SCHEDULER_INPUTS_INCOMPLETE = "scheduler_inputs_incomplete" + ITEM_INPUTS_INCOMPLETE = "item_inputs_incomplete" + SCHEDULER_PAUSED = "scheduler_paused" + MAINTENANCE_WINDOW_CLOSED = "maintenance_window_closed" + CPU_REQUEST_EXCEEDS_LIMIT = "cpu_request_exceeds_limit" + MEMORY_REQUEST_EXCEEDS_LIMIT = "memory_request_exceeds_limit" + RATE_REQUEST_EXCEEDS_LIMIT = "rate_request_exceeds_limit" + GLOBAL_CONCURRENCY_EXHAUSTED = "global_concurrency_exhausted" + REPOSITORY_CONCURRENCY_EXHAUSTED = "repository_concurrency_exhausted" + CPU_CURRENTLY_EXHAUSTED = "cpu_currently_exhausted" + MEMORY_CURRENTLY_EXHAUSTED = "memory_currently_exhausted" + RATE_CURRENTLY_EXHAUSTED = "rate_currently_exhausted" + + +@dataclass(frozen=True, slots=True) +class SchedulerResources: + """Integer resource reservation requested by one maintenance item.""" + + cpu_millis: int + memory_mb: int + rate_units: int + + def __post_init__(self) -> None: + object.__setattr__( + self, + "cpu_millis", + clean_int_range(self.cpu_millis, field="cpu_millis", minimum=1), + ) + object.__setattr__( + self, + "memory_mb", + clean_int_range(self.memory_mb, field="memory_mb", minimum=1), + ) + object.__setattr__( + self, + "rate_units", + clean_int_range(self.rate_units, field="rate_units", minimum=0), + ) + + def to_dict(self) -> dict[str, int]: + return { + "cpu_millis": self.cpu_millis, + "memory_mb": self.memory_mb, + "rate_units": self.rate_units, + } + + @classmethod + def from_dict(cls, value: object) -> SchedulerResources: + data = require_mapping(value, field="SchedulerResources") + reject_unknown( + data, + frozenset({"cpu_millis", "memory_mb", "rate_units"}), + field="SchedulerResources", + ) + return cls( + cpu_millis=data.get("cpu_millis", 0), + memory_mb=data.get("memory_mb", 0), + rate_units=data.get("rate_units", -1), + ) + + +@dataclass(frozen=True, slots=True) +class SchedulerLimits: + """Global resource and concurrency policy limits.""" + + global_concurrency: int + per_repository_concurrency: int + cpu_millis: int + memory_mb: int + rate_units: int + + def __post_init__(self) -> None: + for field in ( + "global_concurrency", + "per_repository_concurrency", + "cpu_millis", + "memory_mb", + ): + object.__setattr__( + self, + field, + clean_int_range(getattr(self, field), field=field, minimum=1), + ) + object.__setattr__( + self, + "rate_units", + clean_int_range(self.rate_units, field="rate_units", minimum=0), + ) + + def to_dict(self) -> dict[str, int]: + return { + "global_concurrency": self.global_concurrency, + "per_repository_concurrency": self.per_repository_concurrency, + "cpu_millis": self.cpu_millis, + "memory_mb": self.memory_mb, + "rate_units": self.rate_units, + } + + @classmethod + def from_dict(cls, value: object) -> SchedulerLimits: + data = require_mapping(value, field="SchedulerLimits") + reject_unknown( + data, + frozenset( + { + "global_concurrency", + "per_repository_concurrency", + "cpu_millis", + "memory_mb", + "rate_units", + } + ), + field="SchedulerLimits", + ) + return cls( + global_concurrency=data.get("global_concurrency", 0), + per_repository_concurrency=data.get("per_repository_concurrency", 0), + cpu_millis=data.get("cpu_millis", 0), + memory_mb=data.get("memory_mb", 0), + rate_units=data.get("rate_units", -1), + ) + + +@dataclass(frozen=True, slots=True) +class MaintenanceQueueItem: + """One complete, resource-bounded maintenance request.""" + + stable_id: str + repository_id: str + priority: int + resources: SchedulerResources + maintenance_window_open: bool + inputs_complete: bool + deadline: str = "" + + def __post_init__(self) -> None: + object.__setattr__( + self, + "stable_id", + clean_text(self.stable_id, field="stable_id", required=True, max_length=256), + ) + object.__setattr__( + self, + "repository_id", + clean_text( + self.repository_id, + field="repository_id", + required=True, + max_length=512, + ), + ) + object.__setattr__( + self, + "priority", + clean_int_range(self.priority, field="priority", minimum=0, maximum=100), + ) + if not isinstance(self.resources, SchedulerResources): + raise ValueError("resources must be SchedulerResources") + object.__setattr__( + self, + "maintenance_window_open", + clean_bool(self.maintenance_window_open, field="maintenance_window_open"), + ) + object.__setattr__( + self, + "inputs_complete", + clean_bool(self.inputs_complete, field="inputs_complete"), + ) + object.__setattr__( + self, + "deadline", + clean_timestamp(self.deadline, field="deadline", required=False), + ) + + @property + def digest(self) -> str: + return canonical_sha256(self.to_dict()) + + def to_dict(self) -> dict[str, Any]: + return { + "stable_id": self.stable_id, + "repository_id": self.repository_id, + "priority": self.priority, + "resources": self.resources.to_dict(), + "maintenance_window_open": self.maintenance_window_open, + "inputs_complete": self.inputs_complete, + "deadline": self.deadline, + } + + @classmethod + def from_dict(cls, value: object) -> MaintenanceQueueItem: + data = require_mapping(value, field="MaintenanceQueueItem") + reject_unknown( + data, + frozenset( + { + "stable_id", + "repository_id", + "priority", + "resources", + "maintenance_window_open", + "inputs_complete", + "deadline", + } + ), + field="MaintenanceQueueItem", + ) + return cls( + stable_id=data.get("stable_id", ""), + repository_id=data.get("repository_id", ""), + priority=data.get("priority", -1), + resources=SchedulerResources.from_dict(data.get("resources")), + maintenance_window_open=clean_bool( + data.get("maintenance_window_open"), field="maintenance_window_open" + ), + inputs_complete=clean_bool(data.get("inputs_complete"), field="inputs_complete"), + deadline=data.get("deadline", ""), + ) + + +@dataclass(frozen=True, slots=True) +class ActiveMaintenanceRun: + """One already-running reservation included in a scheduler snapshot.""" + + stable_id: str + repository_id: str + resources: SchedulerResources + + def __post_init__(self) -> None: + object.__setattr__( + self, + "stable_id", + clean_text(self.stable_id, field="active.stable_id", required=True, max_length=256), + ) + object.__setattr__( + self, + "repository_id", + clean_text( + self.repository_id, + field="active.repository_id", + required=True, + max_length=512, + ), + ) + if not isinstance(self.resources, SchedulerResources): + raise ValueError("active resources must be SchedulerResources") + + def to_dict(self) -> dict[str, Any]: + return { + "stable_id": self.stable_id, + "repository_id": self.repository_id, + "resources": self.resources.to_dict(), + } + + @classmethod + def from_dict(cls, value: object) -> ActiveMaintenanceRun: + data = require_mapping(value, field="ActiveMaintenanceRun") + reject_unknown( + data, + frozenset({"stable_id", "repository_id", "resources"}), + field="ActiveMaintenanceRun", + ) + return cls( + stable_id=data.get("stable_id", ""), + repository_id=data.get("repository_id", ""), + resources=SchedulerResources.from_dict(data.get("resources")), + ) + + +@dataclass(frozen=True, slots=True) +class MaintenanceSchedulerState: + """One validated point-in-time scheduler state.""" + + mode: SchedulerMode + limits: SchedulerLimits + active_runs: tuple[ActiveMaintenanceRun, ...] + rate_units_used: int + inputs_complete: bool + + def __post_init__(self) -> None: + try: + mode = SchedulerMode(self.mode) + except ValueError as error: + raise ValueError("scheduler mode is invalid") from error + object.__setattr__(self, "mode", mode) + if not isinstance(self.limits, SchedulerLimits): + raise ValueError("limits must be SchedulerLimits") + if not isinstance(self.active_runs, tuple) or any( + not isinstance(run, ActiveMaintenanceRun) for run in self.active_runs + ): + raise ValueError("active_runs must be a tuple of ActiveMaintenanceRun values") + active_runs = tuple(sorted(self.active_runs, key=lambda run: run.stable_id)) + active_ids = [run.stable_id for run in active_runs] + if len(active_ids) != len(set(active_ids)): + raise ValueError("active_runs contains duplicate stable IDs") + object.__setattr__(self, "active_runs", active_runs) + object.__setattr__( + self, + "rate_units_used", + clean_int_range(self.rate_units_used, field="rate_units_used", minimum=0), + ) + object.__setattr__( + self, + "inputs_complete", + clean_bool(self.inputs_complete, field="inputs_complete"), + ) + self._validate_usage() + + def _validate_usage(self) -> None: + if len(self.active_runs) > self.limits.global_concurrency: + raise ValueError("active runs exceed global concurrency") + repository_counts: dict[str, int] = {} + for run in self.active_runs: + repository_counts[run.repository_id] = repository_counts.get(run.repository_id, 0) + 1 + if any( + count > self.limits.per_repository_concurrency for count in repository_counts.values() + ): + raise ValueError("active runs exceed per-repository concurrency") + used_cpu = sum(run.resources.cpu_millis for run in self.active_runs) + used_memory = sum(run.resources.memory_mb for run in self.active_runs) + active_rate = sum(run.resources.rate_units for run in self.active_runs) + if used_cpu > self.limits.cpu_millis: + raise ValueError("active runs exceed the CPU budget") + if used_memory > self.limits.memory_mb: + raise ValueError("active runs exceed the memory budget") + if self.rate_units_used > self.limits.rate_units: + raise ValueError("rate usage exceeds the rate budget") + if self.rate_units_used < active_rate: + raise ValueError("rate usage does not account for active reservations") + + @property + def used_cpu_millis(self) -> int: + return sum(run.resources.cpu_millis for run in self.active_runs) + + @property + def used_memory_mb(self) -> int: + return sum(run.resources.memory_mb for run in self.active_runs) + + @property + def digest(self) -> str: + return canonical_sha256(self.to_dict()) + + def pause(self) -> MaintenanceSchedulerState: + """Return an immutable paused snapshot.""" + + return ( + self if self.mode is SchedulerMode.PAUSED else replace(self, mode=SchedulerMode.PAUSED) + ) + + def resume(self) -> MaintenanceSchedulerState: + """Return an immutable running snapshot.""" + + return ( + self + if self.mode is SchedulerMode.RUNNING + else replace(self, mode=SchedulerMode.RUNNING) + ) + + def to_dict(self) -> dict[str, Any]: + return { + "mode": self.mode.value, + "limits": self.limits.to_dict(), + "active_runs": [run.to_dict() for run in self.active_runs], + "rate_units_used": self.rate_units_used, + "inputs_complete": self.inputs_complete, + } + + @classmethod + def from_dict(cls, value: object) -> MaintenanceSchedulerState: + data = require_mapping(value, field="MaintenanceSchedulerState") + reject_unknown( + data, + frozenset({"mode", "limits", "active_runs", "rate_units_used", "inputs_complete"}), + field="MaintenanceSchedulerState", + ) + raw_runs = data.get("active_runs") + if not isinstance(raw_runs, list): + raise ValueError("active_runs must be a list") + try: + mode = SchedulerMode(data.get("mode", "")) + except ValueError as error: + raise ValueError("scheduler mode is invalid") from error + return cls( + mode=mode, + limits=SchedulerLimits.from_dict(data.get("limits")), + active_runs=tuple(ActiveMaintenanceRun.from_dict(run) for run in raw_runs), + rate_units_used=data.get("rate_units_used", -1), + inputs_complete=clean_bool(data.get("inputs_complete"), field="inputs_complete"), + ) + + +@dataclass(frozen=True, slots=True) +class MaintenanceScheduleDecision: + """Auditable decision for one canonical queue position.""" + + queue_position: int + stable_id: str + repository_id: str + priority: int + deadline: str + resources: SchedulerResources + item_digest: str + action: SchedulerAction + reasons: tuple[SchedulerReason, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "queue_position": self.queue_position, + "stable_id": self.stable_id, + "repository_id": self.repository_id, + "priority": self.priority, + "deadline": self.deadline, + "resources": self.resources.to_dict(), + "item_digest": self.item_digest, + "action": self.action.value, + "reasons": [reason.value for reason in self.reasons], + } + + +@dataclass(frozen=True, slots=True) +class MaintenanceSchedule: + """One deterministic batch plan and its projected reservations.""" + + mode: SchedulerMode + state_digest: str + decisions: tuple[MaintenanceScheduleDecision, ...] + starting_active_runs: int + planned_active_runs: int + starting_cpu_millis: int + planned_cpu_millis: int + starting_memory_mb: int + planned_memory_mb: int + starting_rate_units: int + planned_rate_units: int + + @property + def digest(self) -> str: + return canonical_sha256(self.to_dict()) + + def to_dict(self) -> dict[str, Any]: + return { + "mode": self.mode.value, + "state_digest": self.state_digest, + "decisions": [decision.to_dict() for decision in self.decisions], + "starting_active_runs": self.starting_active_runs, + "planned_active_runs": self.planned_active_runs, + "starting_cpu_millis": self.starting_cpu_millis, + "planned_cpu_millis": self.planned_cpu_millis, + "starting_memory_mb": self.starting_memory_mb, + "planned_memory_mb": self.planned_memory_mb, + "starting_rate_units": self.starting_rate_units, + "planned_rate_units": self.planned_rate_units, + } + + +class MaintenanceScheduler: + """Order and plan maintenance items without performing side effects.""" + + def plan( + self, + items: tuple[MaintenanceQueueItem, ...], + state: MaintenanceSchedulerState, + ) -> MaintenanceSchedule: + if not isinstance(items, tuple) or any( + not isinstance(item, MaintenanceQueueItem) for item in items + ): + raise TypeError("items must be a tuple of MaintenanceQueueItem values") + if not isinstance(state, MaintenanceSchedulerState): + raise TypeError("state must be a MaintenanceSchedulerState") + item_ids = [item.stable_id for item in items] + if len(item_ids) != len(set(item_ids)): + raise ValueError("queue contains duplicate stable IDs") + active_ids = {run.stable_id for run in state.active_runs} + if active_ids.intersection(item_ids): + raise ValueError("queue contains an item that is already active") + + ordered = tuple(sorted(items, key=self._order_key)) + active_count = len(state.active_runs) + used_cpu = state.used_cpu_millis + used_memory = state.used_memory_mb + used_rate = state.rate_units_used + starting_active = active_count + starting_cpu = used_cpu + starting_memory = used_memory + starting_rate = used_rate + repository_counts: dict[str, int] = {} + for run in state.active_runs: + repository_counts[run.repository_id] = repository_counts.get(run.repository_id, 0) + 1 + + decisions: list[MaintenanceScheduleDecision] = [] + for position, item in enumerate(ordered, start=1): + defer_reasons = self._defer_reasons(item, state) + if defer_reasons: + decisions.append( + self._decision(position, item, SchedulerAction.DEFER, defer_reasons) + ) + continue + + schedule_reasons: list[SchedulerReason] = [] + if active_count >= state.limits.global_concurrency: + schedule_reasons.append(SchedulerReason.GLOBAL_CONCURRENCY_EXHAUSTED) + if ( + repository_counts.get(item.repository_id, 0) + >= state.limits.per_repository_concurrency + ): + schedule_reasons.append(SchedulerReason.REPOSITORY_CONCURRENCY_EXHAUSTED) + if used_cpu + item.resources.cpu_millis > state.limits.cpu_millis: + schedule_reasons.append(SchedulerReason.CPU_CURRENTLY_EXHAUSTED) + if used_memory + item.resources.memory_mb > state.limits.memory_mb: + schedule_reasons.append(SchedulerReason.MEMORY_CURRENTLY_EXHAUSTED) + if used_rate + item.resources.rate_units > state.limits.rate_units: + schedule_reasons.append(SchedulerReason.RATE_CURRENTLY_EXHAUSTED) + if schedule_reasons: + decisions.append( + self._decision( + position, + item, + SchedulerAction.SCHEDULE, + tuple(schedule_reasons), + ) + ) + continue + + active_count += 1 + repository_counts[item.repository_id] = repository_counts.get(item.repository_id, 0) + 1 + used_cpu += item.resources.cpu_millis + used_memory += item.resources.memory_mb + used_rate += item.resources.rate_units + decisions.append( + self._decision( + position, + item, + SchedulerAction.RUN, + (SchedulerReason.READY,), + ) + ) + + return MaintenanceSchedule( + mode=state.mode, + state_digest=state.digest, + decisions=tuple(decisions), + starting_active_runs=starting_active, + planned_active_runs=active_count, + starting_cpu_millis=starting_cpu, + planned_cpu_millis=used_cpu, + starting_memory_mb=starting_memory, + planned_memory_mb=used_memory, + starting_rate_units=starting_rate, + planned_rate_units=used_rate, + ) + + @staticmethod + def _order_key(item: MaintenanceQueueItem) -> tuple[int, int, str, str]: + has_no_deadline = 1 if not item.deadline else 0 + return (-item.priority, has_no_deadline, item.deadline, item.stable_id) + + @staticmethod + def _defer_reasons( + item: MaintenanceQueueItem, + state: MaintenanceSchedulerState, + ) -> tuple[SchedulerReason, ...]: + reasons: list[SchedulerReason] = [] + if not state.inputs_complete: + reasons.append(SchedulerReason.SCHEDULER_INPUTS_INCOMPLETE) + if not item.inputs_complete: + reasons.append(SchedulerReason.ITEM_INPUTS_INCOMPLETE) + if state.mode is SchedulerMode.PAUSED: + reasons.append(SchedulerReason.SCHEDULER_PAUSED) + if not item.maintenance_window_open: + reasons.append(SchedulerReason.MAINTENANCE_WINDOW_CLOSED) + if item.resources.cpu_millis > state.limits.cpu_millis: + reasons.append(SchedulerReason.CPU_REQUEST_EXCEEDS_LIMIT) + if item.resources.memory_mb > state.limits.memory_mb: + reasons.append(SchedulerReason.MEMORY_REQUEST_EXCEEDS_LIMIT) + if item.resources.rate_units > state.limits.rate_units: + reasons.append(SchedulerReason.RATE_REQUEST_EXCEEDS_LIMIT) + return tuple(reasons) + + @staticmethod + def _decision( + position: int, + item: MaintenanceQueueItem, + action: SchedulerAction, + reasons: tuple[SchedulerReason, ...], + ) -> MaintenanceScheduleDecision: + return MaintenanceScheduleDecision( + queue_position=position, + stable_id=item.stable_id, + repository_id=item.repository_id, + priority=item.priority, + deadline=item.deadline, + resources=item.resources, + item_digest=item.digest, + action=action, + reasons=reasons, + ) diff --git a/src/agentdiff/maintenance/store.py b/src/agentdiff/maintenance/store.py new file mode 100644 index 0000000..8b3ef37 --- /dev/null +++ b/src/agentdiff/maintenance/store.py @@ -0,0 +1,160 @@ +"""Symlink-safe atomic local storage for maintenance graph snapshots.""" + +from __future__ import annotations + +import contextlib +import json +import os +import stat +import tempfile +from pathlib import Path +from typing import Protocol, runtime_checkable + +from .graph import MaintenanceGraph + +_DEFAULT_MAX_BYTES = 16 * 1024 * 1024 + + +@runtime_checkable +class MaintenanceGraphStore(Protocol): + """Storage interface for one canonical maintenance graph snapshot.""" + + def load(self) -> MaintenanceGraph | None: + """Load the current graph, or return ``None`` when none exists.""" + ... + + def save(self, graph: MaintenanceGraph) -> Path: + """Atomically replace the current graph snapshot.""" + ... + + +class LocalJSONMaintenanceStore: + """Persist one graph as bounded, private, atomic JSON on local disk.""" + + def __init__( + self, path: str | os.PathLike[str], *, max_bytes: int = _DEFAULT_MAX_BYTES + ) -> None: + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0: + raise ValueError("max_bytes must be greater than zero") + unresolved = Path(path).expanduser() + self.path = unresolved if unresolved.is_absolute() else Path.cwd() / unresolved + self.path = self.path.absolute() + self.max_bytes = max_bytes + + def load(self) -> MaintenanceGraph | None: + self._assert_no_symlink_components(include_leaf=True) + try: + identity = self.path.lstat() + except FileNotFoundError: + return None + if not stat.S_ISREG(identity.st_mode): + raise ValueError("maintenance store must be a regular, non-symlink file") + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(self.path, flags) + except OSError as error: + raise ValueError("maintenance store could not be opened safely") from error + try: + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_dev != identity.st_dev + or opened.st_ino != identity.st_ino + ): + raise ValueError("maintenance store identity changed while opening") + with os.fdopen(descriptor, "rb", closefd=False) as stream: + payload = stream.read(self.max_bytes + 1) + finally: + os.close(descriptor) + if len(payload) > self.max_bytes: + raise ValueError("maintenance store exceeds its size limit") + try: + decoded = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("maintenance store contains invalid JSON") from error + return MaintenanceGraph.from_dict(decoded) + + def save(self, graph: MaintenanceGraph) -> Path: + if not isinstance(graph, MaintenanceGraph): + raise TypeError("graph must be a MaintenanceGraph") + self._assert_no_symlink_components(include_leaf=True) + self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + self._assert_no_symlink_components(include_leaf=False) + parent_identity = self.path.parent.lstat() + if not stat.S_ISDIR(parent_identity.st_mode): + raise ValueError("maintenance store parent must be a real directory") + self._assert_existing_destination() + + payload = ( + json.dumps( + graph.to_dict(), + indent=2, + sort_keys=True, + ensure_ascii=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + if len(payload) > self.max_bytes: + raise ValueError("maintenance graph exceeds the store size limit") + + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{self.path.name}.", + suffix=".tmp", + dir=str(self.path.parent), + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + self._assert_no_symlink_components(include_leaf=False) + current_parent = self.path.parent.lstat() + if ( + current_parent.st_dev != parent_identity.st_dev + or current_parent.st_ino != parent_identity.st_ino + ): + raise ValueError("maintenance store parent identity changed during write") + self._assert_existing_destination() + os.replace(temporary, self.path) + if os.name != "nt": + self.path.chmod(0o600) + self._fsync_parent() + finally: + temporary.unlink(missing_ok=True) + return self.path + + def _assert_existing_destination(self) -> None: + try: + identity = self.path.lstat() + except FileNotFoundError: + return + if not stat.S_ISREG(identity.st_mode): + raise ValueError("maintenance store must be a regular, non-symlink file") + + def _assert_no_symlink_components(self, *, include_leaf: bool) -> None: + candidates = [*reversed(self.path.parents)] + if include_leaf: + candidates.append(self.path) + for candidate in candidates: + try: + identity = candidate.lstat() + except FileNotFoundError: + continue + if stat.S_ISLNK(identity.st_mode): + raise ValueError("maintenance store path cannot contain symlinks") + + def _fsync_parent(self) -> None: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor: int | None = None + try: + descriptor = os.open(self.path.parent, flags) + os.fsync(descriptor) + except OSError: + pass + finally: + if descriptor is not None: + with contextlib.suppress(OSError): + os.close(descriptor) diff --git a/src/agentdiff/minimize/__init__.py b/src/agentdiff/minimize/__init__.py new file mode 100644 index 0000000..e9c3344 --- /dev/null +++ b/src/agentdiff/minimize/__init__.py @@ -0,0 +1,23 @@ +"""Budgeted proof-preserving patch minimization.""" + +from .engine import ( + MinimizationStatus, + PatchGroup, + PatchMinimizationResult, + PatchMinimizer, + PatchMinimizerResult, + ProofAttempt, + ProofCheckStatus, + ProvenPatchMinimizer, +) + +__all__ = [ + "MinimizationStatus", + "PatchGroup", + "PatchMinimizationResult", + "PatchMinimizer", + "PatchMinimizerResult", + "ProofAttempt", + "ProofCheckStatus", + "ProvenPatchMinimizer", +] diff --git a/src/agentdiff/minimize/engine.py b/src/agentdiff/minimize/engine.py new file mode 100644 index 0000000..1b6abdd --- /dev/null +++ b/src/agentdiff/minimize/engine.py @@ -0,0 +1,441 @@ +"""Budgeted practical delta-debugging for already-proven patch groups.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable, Iterable, Mapping + +ProofCallback = Callable[[tuple[Any, ...]], Any] + + +class ProofCheckStatus(str, Enum): + """Normalized response from the injected proof callback.""" + + PROVEN = "PROVEN" + NOT_PROVEN = "NOT_PROVEN" + TIMEOUT = "TIMEOUT" + INVALID = "INVALID" + + +class MinimizationStatus(str, Enum): + """Final state of one bounded minimization attempt.""" + + MINIMIZED = "MINIMIZED" + UNCHANGED = "UNCHANGED" + BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED" + INITIAL_NOT_PROVEN = "INITIAL_NOT_PROVEN" + INCONCLUSIVE = "INCONCLUSIVE" + + +@dataclass(frozen=True, slots=True) +class PatchGroup: + """Opaque immutable unit that may be removed and independently re-proven. + + The engine never interprets ``payload``. Callers should place immutable + hunk/change identifiers or immutable patch data in it and reconstruct the + candidate patch inside the proof callback. + """ + + id: str + payload: Any = None + paths: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.id.strip(): + raise ValueError("patch group id must not be empty") + object.__setattr__(self, "paths", tuple(self.paths)) + + @property + def group_id(self) -> str: + return self.id + + def to_dict(self) -> dict[str, Any]: + return {"id": self.id, "paths": list(self.paths)} + + +@dataclass(frozen=True, slots=True) +class ProofAttempt: + """One unique candidate subset evaluated during minimization.""" + + kept_group_ids: tuple[str, ...] + status: ProofCheckStatus + duration_seconds: float + detail: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "kept_group_ids": list(self.kept_group_ids), + "status": self.status.value, + "duration_seconds": self.duration_seconds, + "detail": self.detail, + } + + +@dataclass(frozen=True, slots=True) +class PatchMinimizationResult: + """Smallest patch subset found within the explicit deterministic budget.""" + + status: MinimizationStatus + original_groups: tuple[Any, ...] + minimized_groups: tuple[Any, ...] + removed_groups: tuple[Any, ...] + attempts: tuple[ProofAttempt, ...] + proof_runs: int + combinations: int + duration_seconds: float + complete: bool + reasons: tuple[str, ...] = () + schema_version: int = 1 + + @property + def changed(self) -> bool: + return len(self.minimized_groups) < len(self.original_groups) + + @property + def minimal_within_strategy(self) -> bool: + return self.complete and self.status in { + MinimizationStatus.MINIMIZED, + MinimizationStatus.UNCHANGED, + } + + @property + def kept_groups(self) -> tuple[Any, ...]: + return self.minimized_groups + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "status": self.status.value, + "original_group_ids": [_group_id(group) for group in self.original_groups], + "minimized_group_ids": [_group_id(group) for group in self.minimized_groups], + "removed_group_ids": [_group_id(group) for group in self.removed_groups], + "attempts": [attempt.to_dict() for attempt in self.attempts], + "proof_runs": self.proof_runs, + "combinations": self.combinations, + "duration_seconds": self.duration_seconds, + "complete": self.complete, + "changed": self.changed, + "minimal_within_strategy": self.minimal_within_strategy, + "reasons": list(self.reasons), + } + + +class PatchMinimizer: + """Find a 1-minimal practical patch using bounded ddmin-style removal.""" + + def __init__( + self, + proof_callback: ProofCallback, + *, + max_proof_runs: int = 32, + max_duration_seconds: float = 900.0, + max_combinations: int = 64, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if max_proof_runs < 1: + raise ValueError("max_proof_runs must be at least 1") + if max_duration_seconds <= 0: + raise ValueError("max_duration_seconds must be greater than zero") + if max_combinations < 1: + raise ValueError("max_combinations must be at least 1") + self.proof_callback = proof_callback + self.max_proof_runs = max_proof_runs + self.max_duration_seconds = max_duration_seconds + self.max_combinations = max_combinations + self.clock = clock + + def minimize(self, groups: Iterable[Any]) -> PatchMinimizationResult: + """Minimize an immutable ordered collection without exhaustive search.""" + + original = tuple(groups) + patch_groups = [group for group in original if isinstance(group, PatchGroup)] + if len({group.id for group in patch_groups}) != len(patch_groups): + raise ValueError("patch group ids must be unique") + + state = _MinimizationState( + original=original, + proof_callback=self.proof_callback, + max_proof_runs=self.max_proof_runs, + max_duration_seconds=self.max_duration_seconds, + max_combinations=self.max_combinations, + clock=self.clock, + ) + initial = state.check(tuple(range(len(original))), combination=False) + if initial is None: + return state.result( + tuple(range(len(original))), + status=MinimizationStatus.BUDGET_EXHAUSTED, + complete=False, + ) + if initial is ProofCheckStatus.NOT_PROVEN: + state.reasons.append("the original patch groups did not pass the proof callback") + return state.result( + tuple(range(len(original))), + status=MinimizationStatus.INITIAL_NOT_PROVEN, + complete=True, + ) + if initial is not ProofCheckStatus.PROVEN: + state.reasons.append("the original patch proof was inconclusive") + return state.result( + tuple(range(len(original))), + status=MinimizationStatus.INCONCLUSIVE, + complete=False, + ) + + current = tuple(range(len(original))) + granularity = 2 + exhausted = False + + while len(current) >= 2: + reduced = False + for chunk in _partition(current, granularity): + removed = set(chunk) + candidate = tuple(index for index in current if index not in removed) + checked = state.check(candidate, combination=True) + if checked is None: + exhausted = True + break + if checked is ProofCheckStatus.PROVEN: + current = candidate + granularity = max(2, granularity - 1) + reduced = True + break + if exhausted: + break + if reduced: + continue + if granularity >= len(current): + break + granularity = min(len(current), granularity * 2) + + # A deterministic single-group cleanup establishes practical 1-minimality + # when the budget allows it, including the one-group -> empty case. + changed = True + while changed and not exhausted: + changed = False + for index in current: + candidate = tuple(item for item in current if item != index) + checked = state.check(candidate, combination=True) + if checked is None: + exhausted = True + break + if checked is ProofCheckStatus.PROVEN: + current = candidate + changed = True + break + + if exhausted: + return state.result( + current, + status=MinimizationStatus.BUDGET_EXHAUSTED, + complete=False, + ) + if state.inconclusive: + return state.result( + current, + status=MinimizationStatus.INCONCLUSIVE, + complete=False, + ) + status = ( + MinimizationStatus.MINIMIZED + if len(current) < len(original) + else MinimizationStatus.UNCHANGED + ) + return state.result(current, status=status, complete=True) + + def run(self, groups: Iterable[Any]) -> PatchMinimizationResult: + """Alias for :meth:`minimize` for orchestration code.""" + + return self.minimize(groups) + + +@dataclass(slots=True) +class _MinimizationState: + original: tuple[Any, ...] + proof_callback: ProofCallback + max_proof_runs: int + max_duration_seconds: float + max_combinations: int + clock: Callable[[], float] + started: float = field(init=False) + proof_runs: int = field(init=False, default=0) + combinations: int = field(init=False, default=0) + inconclusive: bool = field(init=False, default=False) + attempts: list[ProofAttempt] = field(init=False, default_factory=list) + reasons: list[str] = field(init=False, default_factory=list) + cache: dict[tuple[int, ...], ProofCheckStatus] = field( + init=False, + default_factory=dict, + ) + + def __post_init__(self) -> None: + self.started = self.clock() + + def check( + self, + indices: tuple[int, ...], + *, + combination: bool, + ) -> ProofCheckStatus | None: + if indices in self.cache: + return self.cache[indices] + if self.proof_runs >= self.max_proof_runs: + self.reasons.append("proof-run budget reached") + return None + if combination and self.combinations >= self.max_combinations: + self.reasons.append("candidate-combination budget reached") + return None + elapsed = max(0.0, self.clock() - self.started) + if elapsed >= self.max_duration_seconds: + self.reasons.append("minimization duration budget reached") + return None + + candidate = tuple(self.original[index] for index in indices) + self.proof_runs += 1 + if combination: + self.combinations += 1 + started = self.clock() + detail = "" + try: + raw = self.proof_callback(candidate) + status, detail = _normalize_proof(raw) + except TimeoutError as error: + status = ProofCheckStatus.TIMEOUT + detail = f"proof callback timed out: {error}" + except (OSError, RuntimeError, TypeError, ValueError) as error: + status = ProofCheckStatus.INVALID + detail = f"proof callback failed: {type(error).__name__}: {error}" + duration = max(0.0, self.clock() - started) + if max(0.0, self.clock() - self.started) > self.max_duration_seconds: + status = ProofCheckStatus.TIMEOUT + detail = "proof callback exceeded the minimization duration budget" + if status in {ProofCheckStatus.TIMEOUT, ProofCheckStatus.INVALID}: + self.inconclusive = True + self.reasons.append("at least one candidate proof was inconclusive") + self.cache[indices] = status + self.attempts.append( + ProofAttempt( + kept_group_ids=tuple(_group_id(group) for group in candidate), + status=status, + duration_seconds=duration, + detail=detail, + ) + ) + return status + + def result( + self, + current: tuple[int, ...], + *, + status: MinimizationStatus, + complete: bool, + ) -> PatchMinimizationResult: + selected = set(current) + minimized = tuple(self.original[index] for index in current) + removed = tuple(group for index, group in enumerate(self.original) if index not in selected) + return PatchMinimizationResult( + status=status, + original_groups=self.original, + minimized_groups=minimized, + removed_groups=removed, + attempts=tuple(self.attempts), + proof_runs=self.proof_runs, + combinations=self.combinations, + duration_seconds=max(0.0, self.clock() - self.started), + complete=complete, + reasons=tuple(dict.fromkeys(self.reasons)), + ) + + +def _partition(values: tuple[int, ...], count: int) -> tuple[tuple[int, ...], ...]: + count = min(max(1, count), len(values)) + quotient, remainder = divmod(len(values), count) + chunks: list[tuple[int, ...]] = [] + start = 0 + for index in range(count): + size = quotient + (1 if index < remainder else 0) + chunks.append(values[start : start + size]) + start += size + return tuple(chunks) + + +def _normalize_proof(raw: Any) -> tuple[ProofCheckStatus, str]: + if isinstance(raw, ProofCheckStatus): + return raw, "" + if isinstance(raw, bool): + return (ProofCheckStatus.PROVEN if raw else ProofCheckStatus.NOT_PROVEN), "" + if isinstance(raw, int): + return (ProofCheckStatus.PROVEN if raw == 0 else ProofCheckStatus.NOT_PROVEN), "" + if isinstance(raw, Mapping): + if bool(raw.get("timed_out", False)): + return ProofCheckStatus.TIMEOUT, str(raw.get("detail", "")) + if not bool(raw.get("valid", True)): + return ProofCheckStatus.INVALID, str(raw.get("detail", "")) + if "proven" in raw: + proven = bool(raw["proven"]) + elif "passed" in raw: + proven = bool(raw["passed"]) + elif "verdict" in raw: + proven = _verdict_is_proven(raw["verdict"]) + elif "returncode" in raw: + proven = int(raw["returncode"]) == 0 + else: + raise ValueError("mapping has no proven, passed, verdict, or returncode field") + return ( + ProofCheckStatus.PROVEN if proven else ProofCheckStatus.NOT_PROVEN, + str(raw.get("detail", "")), + ) + + if hasattr(raw, "valid") and not bool(raw.valid): + return ProofCheckStatus.INVALID, str(getattr(raw, "detail", "")) + if hasattr(raw, "causally_proven"): + proven = bool(raw.causally_proven) + elif hasattr(raw, "passed"): + proven = bool(raw.passed) + elif hasattr(raw, "verdict"): + proven = _verdict_is_proven(raw.verdict) + elif hasattr(raw, "returncode"): + proven = int(raw.returncode) == 0 + else: + raise TypeError("unsupported proof callback result") + return ( + ProofCheckStatus.PROVEN if proven else ProofCheckStatus.NOT_PROVEN, + str(getattr(raw, "detail", "")), + ) + + +def _verdict_is_proven(value: Any) -> bool: + raw = value.value if hasattr(value, "value") else value + return str(raw).upper() in { + "PASS", + "PASSED", + "PROVEN", + "CAUSAL_DELTA_CONFIRMED", + "BEHAVIOR_EQUIVALENCE_CONFIRMED", + } + + +def _group_id(group: Any) -> str: + if isinstance(group, PatchGroup): + return group.id + candidate = getattr(group, "id", getattr(group, "group_id", None)) + return str(candidate) if candidate is not None else str(group) + + +# Product-language aliases. +ProvenPatchMinimizer = PatchMinimizer +PatchMinimizerResult = PatchMinimizationResult + + +__all__ = [ + "MinimizationStatus", + "PatchGroup", + "PatchMinimizationResult", + "PatchMinimizer", + "PatchMinimizerResult", + "ProofAttempt", + "ProofCheckStatus", + "ProvenPatchMinimizer", +] diff --git a/src/agentdiff/outcome/__init__.py b/src/agentdiff/outcome/__init__.py new file mode 100644 index 0000000..e26eac8 --- /dev/null +++ b/src/agentdiff/outcome/__init__.py @@ -0,0 +1,27 @@ +"""Evidence-preserving post-merge outcome verification.""" + +from .verifier import ( + OutcomeCheck, + OutcomeCheckResult, + OutcomeCheckStatus, + OutcomeEvidence, + OutcomeState, + OutcomeVerifier, + PostMergeContext, + PostMergeOutcome, + PostMergeOutcomeEvaluator, + PostMergeVerifier, +) + +__all__ = [ + "OutcomeCheck", + "OutcomeCheckResult", + "OutcomeCheckStatus", + "OutcomeEvidence", + "OutcomeState", + "OutcomeVerifier", + "PostMergeContext", + "PostMergeOutcome", + "PostMergeOutcomeEvaluator", + "PostMergeVerifier", +] diff --git a/src/agentdiff/outcome/verifier.py b/src/agentdiff/outcome/verifier.py new file mode 100644 index 0000000..28d35f2 --- /dev/null +++ b/src/agentdiff/outcome/verifier.py @@ -0,0 +1,539 @@ +"""Deterministic, pluggable post-merge outcome verification. + +Checks provide explicit results and evidence. The verifier aggregates those +facts without model judgment and never executes a production rollback. A +regression can optionally recommend preparing a revert for human review, but +that recommendation is intentionally distinct from applying one. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from enum import Enum +from typing import Any, Callable, Iterable, Mapping + +from agentdiff.delivery import SealedDeliveryIdentity + +OutcomeCheckRunner = Callable[["PostMergeContext", float], Any] + + +class OutcomeState(str, Enum): + """Final post-merge state.""" + + HEALTHY = "HEALTHY" + REGRESSION_DETECTED = "REGRESSION_DETECTED" + INCONCLUSIVE = "INCONCLUSIVE" + + +class OutcomeCheckStatus(str, Enum): + """Explicit status supplied by or derived from one check plugin.""" + + PASS = "PASS" + FAIL = "FAIL" + INCONCLUSIVE = "INCONCLUSIVE" + TIMEOUT = "TIMEOUT" + UNAVAILABLE = "UNAVAILABLE" + INVALID = "INVALID" + + +@dataclass(frozen=True, slots=True) +class OutcomeEvidence: + """Bounded reference or digest retained from a post-merge check.""" + + source: str + summary: str = "" + digest: str = "" + reference: str = "" + schema_version: int = 1 + + def __post_init__(self) -> None: + if not isinstance(self.source, str) or not self.source.strip(): + raise ValueError("outcome evidence source must not be empty") + for field_name in ("summary", "digest", "reference"): + if not isinstance(getattr(self, field_name), str): + raise TypeError(f"outcome evidence {field_name} must be text") + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "source": self.source, + "summary": self.summary, + "digest": self.digest, + "reference": self.reference, + } + + +@dataclass(frozen=True, slots=True) +class PostMergeContext: + """Exact merged subject against which outcome evidence is collected.""" + + merged_commit_sha: str + delivery_identity: SealedDeliveryIdentity + pull_request_number: int | None = None + schema_version: int = 1 + + def __post_init__(self) -> None: + if not isinstance(self.delivery_identity, SealedDeliveryIdentity): + raise TypeError("delivery_identity must be a SealedDeliveryIdentity") + if ( + not isinstance(self.merged_commit_sha, str) + or not self.merged_commit_sha + or self.merged_commit_sha != self.merged_commit_sha.strip() + ): + raise ValueError("merged_commit_sha must be a non-empty exact identity") + if self.pull_request_number is not None and ( + isinstance(self.pull_request_number, bool) + or not isinstance(self.pull_request_number, int) + or self.pull_request_number < 1 + ): + raise ValueError("pull_request_number must be a positive integer") + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "merged_commit_sha": self.merged_commit_sha, + "delivery_identity": self.delivery_identity.to_dict(), + "pull_request_number": self.pull_request_number, + } + + +@dataclass(frozen=True, slots=True) +class OutcomeCheckResult: + """Normalized result and preserved evidence from one configured check.""" + + check_id: str + kind: str + status: OutcomeCheckStatus + evidence: tuple[OutcomeEvidence, ...] = () + duration_seconds: float = 0.0 + detail: str = "" + schema_version: int = 1 + + def __post_init__(self) -> None: + if not isinstance(self.check_id, str) or not self.check_id.strip(): + raise ValueError("outcome check id must not be empty") + if not isinstance(self.kind, str) or not self.kind.strip(): + raise ValueError("outcome check kind must not be empty") + status = ( + self.status + if isinstance(self.status, OutcomeCheckStatus) + else OutcomeCheckStatus(str(self.status).upper()) + ) + evidence = tuple(self.evidence) + if any(not isinstance(item, OutcomeEvidence) for item in evidence): + raise TypeError("outcome check evidence must contain OutcomeEvidence values") + if self.duration_seconds < 0: + raise ValueError("outcome check duration must not be negative") + object.__setattr__(self, "status", status) + object.__setattr__(self, "evidence", evidence) + + @property + def passed(self) -> bool: + return self.status is OutcomeCheckStatus.PASS + + @property + def failed(self) -> bool: + return self.status is OutcomeCheckStatus.FAIL + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "check_id": self.check_id, + "kind": self.kind, + "status": self.status.value, + "evidence": [item.to_dict() for item in self.evidence], + "duration_seconds": self.duration_seconds, + "detail": self.detail, + } + + +@dataclass(frozen=True, slots=True) +class OutcomeCheck: + """One named post-merge plugin and its deterministic runner.""" + + check_id: str + runner: OutcomeCheckRunner + kind: str = "custom" + + def __post_init__(self) -> None: + if not isinstance(self.check_id, str) or not self.check_id.strip(): + raise ValueError("outcome check id must not be empty") + if not isinstance(self.kind, str) or not self.kind.strip(): + raise ValueError("outcome check kind must not be empty") + if not callable(self.runner): + raise TypeError("outcome check runner must be callable") + + +@dataclass(frozen=True, slots=True) +class PostMergeOutcome: + """Aggregated outcome with evidence and deliberately limited authority.""" + + context: PostMergeContext + state: OutcomeState + checks: tuple[OutcomeCheckResult, ...] + complete: bool + reasons: tuple[str, ...] = () + prepare_revert: bool = False + schema_version: int = 1 + + @property + def maintenance_complete(self) -> bool: + return self.state is OutcomeState.HEALTHY + + @property + def stop_remaining_campaign(self) -> bool: + return self.state is OutcomeState.REGRESSION_DETECTED + + @property + def notify_human(self) -> bool: + return self.state is not OutcomeState.HEALTHY + + @property + def automatic_production_rollback(self) -> bool: + """Outcome verification never authorizes an automatic rollback.""" + + return False + + @property + def recommended_actions(self) -> tuple[str, ...]: + if self.state is OutcomeState.HEALTHY: + return ("MAINTENANCE_COMPLETE", "PRESERVE_EVIDENCE") + if self.state is OutcomeState.REGRESSION_DETECTED: + actions = ["STOP_REMAINING_CAMPAIGN", "NOTIFY_HUMAN", "PRESERVE_EVIDENCE"] + if self.prepare_revert: + actions.append("PREPARE_REVERT_FOR_REVIEW") + return tuple(actions) + return ("REQUIRE_HUMAN_REVIEW", "PRESERVE_EVIDENCE") + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "context": self.context.to_dict(), + "state": self.state.value, + "checks": [check.to_dict() for check in self.checks], + "complete": self.complete, + "maintenance_complete": self.maintenance_complete, + "stop_remaining_campaign": self.stop_remaining_campaign, + "notify_human": self.notify_human, + "prepare_revert": self.prepare_revert, + "automatic_production_rollback": self.automatic_production_rollback, + "recommended_actions": list(self.recommended_actions), + "reasons": list(self.reasons), + } + + +class PostMergeOutcomeEvaluator: + """Aggregate already-collected check results without invoking plugins.""" + + def evaluate( + self, + context: PostMergeContext, + results: Iterable[OutcomeCheckResult], + *, + complete: bool = True, + reasons: Iterable[str] = (), + prepare_revert: bool = False, + ) -> PostMergeOutcome: + checks = tuple(results) + if len({check.check_id for check in checks}) != len(checks): + raise ValueError("outcome check ids must be unique") + result_reasons = list(reasons) + + if any(check.status is OutcomeCheckStatus.FAIL for check in checks): + state = OutcomeState.REGRESSION_DETECTED + result_reasons.append("one or more post-merge checks reported regression") + elif ( + not checks + or not complete + or any(check.status is not OutcomeCheckStatus.PASS for check in checks) + ): + state = OutcomeState.INCONCLUSIVE + result_reasons.append("post-merge evidence is incomplete or unavailable") + else: + state = OutcomeState.HEALTHY + + return PostMergeOutcome( + context=context, + state=state, + checks=checks, + complete=complete, + reasons=tuple(dict.fromkeys(result_reasons)), + prepare_revert=prepare_revert and state is OutcomeState.REGRESSION_DETECTED, + ) + + +class PostMergeVerifier: + """Run bounded check plugins and aggregate their explicit results.""" + + def __init__( + self, + checks: Iterable[OutcomeCheck], + *, + max_checks: int = 16, + max_duration_seconds: float = 300.0, + clock: Callable[[], float] = time.monotonic, + ) -> None: + configured = tuple(checks) + if max_checks < 1: + raise ValueError("max_checks must be at least 1") + if max_duration_seconds <= 0: + raise ValueError("max_duration_seconds must be greater than zero") + if len({check.check_id for check in configured}) != len(configured): + raise ValueError("outcome check ids must be unique") + self.checks = configured + self.max_checks = max_checks + self.max_duration_seconds = max_duration_seconds + self.clock = clock + + def verify( + self, + context: PostMergeContext, + *, + timeout_seconds: float = 60.0, + prepare_revert_on_regression: bool = False, + ) -> PostMergeOutcome: + """Collect check evidence without triggering deployment or rollback actions.""" + + if timeout_seconds <= 0: + raise ValueError("outcome check timeout must be greater than zero") + started = self.clock() + results: list[OutcomeCheckResult] = [] + reasons: list[str] = [] + complete = True + + for index, check in enumerate(self.checks): + elapsed = max(0.0, self.clock() - started) + if index >= self.max_checks: + complete = False + reasons.append("post-merge check-count budget reached") + break + if elapsed >= self.max_duration_seconds: + complete = False + reasons.append("post-merge duration budget reached") + break + results.append( + self._run_check( + check, + context, + timeout_seconds=min( + timeout_seconds, + self.max_duration_seconds - elapsed, + ), + ) + ) + + return PostMergeOutcomeEvaluator().evaluate( + context, + results, + complete=complete, + reasons=reasons, + prepare_revert=prepare_revert_on_regression, + ) + + def run( + self, + context: PostMergeContext, + *, + timeout_seconds: float = 60.0, + prepare_revert_on_regression: bool = False, + ) -> PostMergeOutcome: + """Alias for :meth:`verify` for lifecycle orchestration.""" + + return self.verify( + context, + timeout_seconds=timeout_seconds, + prepare_revert_on_regression=prepare_revert_on_regression, + ) + + def _run_check( + self, + check: OutcomeCheck, + context: PostMergeContext, + *, + timeout_seconds: float, + ) -> OutcomeCheckResult: + started = self.clock() + try: + raw = check.runner(context, timeout_seconds) + except TimeoutError as error: + return OutcomeCheckResult( + check_id=check.check_id, + kind=check.kind, + status=OutcomeCheckStatus.TIMEOUT, + duration_seconds=max(0.0, self.clock() - started), + detail=f"post-merge check timed out: {error}", + ) + except (OSError, RuntimeError, TypeError, ValueError) as error: + return OutcomeCheckResult( + check_id=check.check_id, + kind=check.kind, + status=OutcomeCheckStatus.INVALID, + duration_seconds=max(0.0, self.clock() - started), + detail=f"post-merge check failed: {type(error).__name__}: {error}", + ) + elapsed = max(0.0, self.clock() - started) + try: + normalized = _normalize_result(raw, check, elapsed) + except (OverflowError, TypeError, ValueError) as error: + return OutcomeCheckResult( + check_id=check.check_id, + kind=check.kind, + status=OutcomeCheckStatus.INVALID, + duration_seconds=elapsed, + detail=f"invalid post-merge check result: {error}", + ) + if normalized.check_id != check.check_id: + return OutcomeCheckResult( + check_id=check.check_id, + kind=check.kind, + status=OutcomeCheckStatus.INVALID, + evidence=normalized.evidence, + duration_seconds=max(elapsed, normalized.duration_seconds), + detail="post-merge check result id does not match configured check", + ) + effective_duration = max(elapsed, normalized.duration_seconds) + if effective_duration > timeout_seconds or normalized.status is OutcomeCheckStatus.TIMEOUT: + return OutcomeCheckResult( + check_id=check.check_id, + kind=check.kind, + status=OutcomeCheckStatus.TIMEOUT, + evidence=normalized.evidence, + duration_seconds=effective_duration, + detail=normalized.detail or "post-merge check exceeded its timeout", + ) + return OutcomeCheckResult( + check_id=check.check_id, + kind=check.kind, + status=normalized.status, + evidence=normalized.evidence, + duration_seconds=effective_duration, + detail=normalized.detail, + ) + + +def _normalize_result( + raw: Any, + check: OutcomeCheck, + elapsed: float, +) -> OutcomeCheckResult: + if isinstance(raw, OutcomeCheckResult): + return raw + if isinstance(raw, bool): + return OutcomeCheckResult( + check.check_id, + check.kind, + OutcomeCheckStatus.PASS if raw else OutcomeCheckStatus.FAIL, + duration_seconds=elapsed, + ) + if isinstance(raw, int): + return OutcomeCheckResult( + check.check_id, + check.kind, + OutcomeCheckStatus.PASS if raw == 0 else OutcomeCheckStatus.FAIL, + duration_seconds=elapsed, + ) + if isinstance(raw, Mapping): + status = _mapping_status(raw) + return OutcomeCheckResult( + check_id=str(raw.get("check_id", check.check_id)), + kind=str(raw.get("kind", check.kind)), + status=status, + evidence=_normalize_evidence(raw.get("evidence"), check.check_id), + duration_seconds=float(raw.get("duration_seconds", elapsed)), + detail=str(raw.get("detail", "")), + ) + + if hasattr(raw, "status"): + status = _coerce_status(raw.status) + elif hasattr(raw, "passed"): + status = OutcomeCheckStatus.PASS if bool(raw.passed) else OutcomeCheckStatus.FAIL + elif hasattr(raw, "healthy"): + status = OutcomeCheckStatus.PASS if bool(raw.healthy) else OutcomeCheckStatus.FAIL + elif hasattr(raw, "returncode"): + status = OutcomeCheckStatus.PASS if int(raw.returncode) == 0 else OutcomeCheckStatus.FAIL + else: + raise TypeError("unsupported post-merge check result") + return OutcomeCheckResult( + check_id=str(getattr(raw, "check_id", check.check_id)), + kind=str(getattr(raw, "kind", check.kind)), + status=status, + evidence=_normalize_evidence(getattr(raw, "evidence", None), check.check_id), + duration_seconds=float(getattr(raw, "duration_seconds", elapsed)), + detail=str(getattr(raw, "detail", "")), + ) + + +def _mapping_status(raw: Mapping[Any, Any]) -> OutcomeCheckStatus: + if bool(raw.get("timed_out", False)): + return OutcomeCheckStatus.TIMEOUT + if bool(raw.get("unavailable", False)): + return OutcomeCheckStatus.UNAVAILABLE + if not bool(raw.get("valid", True)): + return OutcomeCheckStatus.INVALID + if "status" in raw: + return _coerce_status(raw["status"]) + if "passed" in raw: + return OutcomeCheckStatus.PASS if bool(raw["passed"]) else OutcomeCheckStatus.FAIL + if "healthy" in raw: + return OutcomeCheckStatus.PASS if bool(raw["healthy"]) else OutcomeCheckStatus.FAIL + if "returncode" in raw: + return OutcomeCheckStatus.PASS if int(raw["returncode"]) == 0 else OutcomeCheckStatus.FAIL + raise ValueError("mapping has no explicit post-merge check status") + + +def _coerce_status(value: Any) -> OutcomeCheckStatus: + if isinstance(value, OutcomeCheckStatus): + return value + return OutcomeCheckStatus(str(value).upper()) + + +def _normalize_evidence(raw: Any, default_source: str) -> tuple[OutcomeEvidence, ...]: + if raw is None: + return () + if isinstance(raw, OutcomeEvidence): + return (raw,) + if isinstance(raw, str): + return (OutcomeEvidence(source=default_source, summary=raw),) + if isinstance(raw, Mapping): + return (_evidence_from_mapping(raw, default_source),) + if isinstance(raw, Iterable): + items: list[OutcomeEvidence] = [] + for item in raw: + if isinstance(item, OutcomeEvidence): + items.append(item) + elif isinstance(item, str): + items.append(OutcomeEvidence(source=default_source, summary=item)) + elif isinstance(item, Mapping): + items.append(_evidence_from_mapping(item, default_source)) + else: + raise TypeError("unsupported post-merge evidence item") + return tuple(items) + raise TypeError("unsupported post-merge evidence") + + +def _evidence_from_mapping(raw: Mapping[Any, Any], default_source: str) -> OutcomeEvidence: + return OutcomeEvidence( + source=str(raw.get("source", default_source)), + summary=str(raw.get("summary", raw.get("detail", ""))), + digest=str(raw.get("digest", "")), + reference=str(raw.get("reference", "")), + ) + + +# Product-language alias. +OutcomeVerifier = PostMergeVerifier + + +__all__ = [ + "OutcomeCheck", + "OutcomeCheckResult", + "OutcomeCheckStatus", + "OutcomeEvidence", + "OutcomeState", + "OutcomeVerifier", + "PostMergeContext", + "PostMergeOutcome", + "PostMergeOutcomeEvaluator", + "PostMergeVerifier", +] diff --git a/src/agentdiff/proof/__init__.py b/src/agentdiff/proof/__init__.py index 8530e5a..53ef5a7 100644 --- a/src/agentdiff/proof/__init__.py +++ b/src/agentdiff/proof/__init__.py @@ -1,6 +1,45 @@ """Deterministic clean-room proof for sealed AgentDiff patches.""" +from .adversarial import ( + AdversarialProofEngine, + AdversarialProofResult, + InvalidMutationError, + MutationCase, + MutationOutcome, + MutationVerdict, + MutationVerification, +) +from .delta import ( + BehaviorContract, + BehaviorExecution, + BehaviorExecutionStatus, + BehaviorExpectation, + DeltaProofEngine, + DeltaProofResult, + DeltaProofVerdict, + ExpectedBehavior, +) from .engine import ProofEngine +from .flake import ( + FlakeCheck, + FlakeClassification, + FlakeExecution, + FlakeFirewall, + FlakeFirewallEngine, + FlakeFirewallResult, + FlakeSample, + FlakeSampleStatus, + InfrastructureFailure, + QuarantineCandidate, +) +from .lease import ( + ProofLease, + ProofLeaseEvaluation, + ProofLeaseEvaluator, + ProofLeaseInvalidation, + ProofLeaseObservation, + ProofLeaseStatus, +) from .models import ( ProofPhaseResult, ProofResult, @@ -17,12 +56,43 @@ ) __all__ = [ + "AdversarialProofEngine", + "AdversarialProofResult", + "BehaviorContract", + "BehaviorExecution", + "BehaviorExecutionStatus", + "BehaviorExpectation", + "DeltaProofEngine", + "DeltaProofResult", + "DeltaProofVerdict", + "ExpectedBehavior", + "FlakeCheck", + "FlakeClassification", + "FlakeExecution", + "FlakeFirewall", + "FlakeFirewallEngine", + "FlakeFirewallResult", + "FlakeSample", + "FlakeSampleStatus", + "InfrastructureFailure", + "InvalidMutationError", + "MutationCase", + "MutationOutcome", + "MutationVerdict", + "MutationVerification", "ProofEngine", + "ProofLease", + "ProofLeaseEvaluation", + "ProofLeaseEvaluator", + "ProofLeaseInvalidation", + "ProofLeaseObservation", + "ProofLeaseStatus", "ProofPhaseResult", "ProofResult", "ProofStrengthLabel", "ProofStrengthLevel", "ProofVerdict", + "QuarantineCandidate", "VerifierIndependence", "VerifierMutationReport", "analyze_verifier_mutations", diff --git a/src/agentdiff/proof/_flake_models.py b/src/agentdiff/proof/_flake_models.py new file mode 100644 index 0000000..66b3e83 --- /dev/null +++ b/src/agentdiff/proof/_flake_models.py @@ -0,0 +1,335 @@ +"""Immutable evidence models for deterministic flake classification.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from agentdiff.redaction import redact_argv + + +class FlakeClassification(str, Enum): + """Aggregate stability classification for repeated samples.""" + + STABLE_PASS = "STABLE_PASS" + STABLE_FAIL = "STABLE_FAIL" + FLAKY = "FLAKY" + INCONCLUSIVE = "INCONCLUSIVE" + + +class FlakeSampleStatus(str, Enum): + """Truthful status of one repeated check invocation.""" + + PASS = "PASS" + FAIL = "FAIL" + INFRASTRUCTURE_FAILURE = "INFRASTRUCTURE_FAILURE" + TIMEOUT = "TIMEOUT" + INVALID = "INVALID" + + +class InfrastructureFailure(RuntimeError): + """Raised by a runner when the environment, not the checked code, failed.""" + + +@dataclass(frozen=True, slots=True) +class FlakeCheck: + """One exact command and its bounded repeated-sampling request.""" + + id: str + command: tuple[str, ...] + repetitions: int = 3 + compare_output: bool = False + schema_version: int = 1 + + def __post_init__(self) -> None: + command = tuple(self.command) + if not isinstance(self.id, str) or not self.id.strip(): + raise ValueError("flake check id must not be empty") + if not command or any(not isinstance(part, str) or not part for part in command): + raise ValueError("flake check command must be non-empty exact argv") + if ( + isinstance(self.repetitions, bool) + or not isinstance(self.repetitions, int) + or self.repetitions < 2 + ): + raise ValueError("flake check repetitions must be at least 2") + object.__setattr__(self, "command", command) + + @property + def command_digest(self) -> str: + return _canonical_sha256(list(self.command)) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "id": self.id, + "command": redact_argv(self.command), + "command_digest": self.command_digest, + "repetitions": self.repetitions, + "compare_output": self.compare_output, + } + + +@dataclass(frozen=True, slots=True) +class FlakeExecution: + """Runner response; raw output remains in memory and is never serialized.""" + + returncode: int | None + command: tuple[str, ...] | None = None + duration_seconds: float = 0.0 + output: str | bytes | None = None + output_sha256: str | None = None + output_bytes: int | None = None + infrastructure_failure: bool = False + timed_out: bool = False + valid: bool = True + detail: str = "" + + def __post_init__(self) -> None: + if self.returncode is not None and ( + isinstance(self.returncode, bool) or not isinstance(self.returncode, int) + ): + raise TypeError("flake execution returncode must be an integer or None") + if self.command is not None: + command = tuple(self.command) + if not command or any(not isinstance(part, str) or not part for part in command): + raise ValueError("flake execution command must be non-empty exact argv") + object.__setattr__(self, "command", command) + if self.duration_seconds < 0: + raise ValueError("flake execution duration must not be negative") + if self.output is not None and not isinstance(self.output, (str, bytes)): + raise TypeError("flake execution output must be text or bytes") + + payload = _output_bytes(self.output) + digest = self.output_sha256 + byte_count = self.output_bytes + if payload is not None: + computed_digest = hashlib.sha256(payload).hexdigest() + if digest is not None and digest != computed_digest: + raise ValueError("flake execution output digest does not match output") + if byte_count is not None and byte_count != len(payload): + raise ValueError("flake execution output byte count does not match output") + digest = computed_digest + byte_count = len(payload) + elif digest is not None and not _is_sha256(digest): + raise ValueError("flake execution output_sha256 must be a lowercase SHA-256 digest") + if byte_count is not None and ( + isinstance(byte_count, bool) or not isinstance(byte_count, int) or byte_count < 0 + ): + raise ValueError("flake execution output_bytes must be a non-negative integer") + object.__setattr__(self, "output_sha256", digest) + object.__setattr__(self, "output_bytes", byte_count or 0) + + @property + def status(self) -> FlakeSampleStatus: + if self.timed_out: + return FlakeSampleStatus.TIMEOUT + if self.infrastructure_failure: + return FlakeSampleStatus.INFRASTRUCTURE_FAILURE + if not self.valid or self.returncode is None: + return FlakeSampleStatus.INVALID + return FlakeSampleStatus.PASS if self.returncode == 0 else FlakeSampleStatus.FAIL + + +@dataclass(frozen=True, slots=True) +class FlakeSample: + """Digest-only evidence for one invocation.""" + + index: int + command: tuple[str, ...] + status: FlakeSampleStatus + returncode: int | None + duration_seconds: float + output_sha256: str | None = None + output_bytes: int = 0 + detail: str = "" + schema_version: int = 1 + + @property + def digest(self) -> str: + return _canonical_sha256( + { + "index": self.index, + "command_digest": _canonical_sha256(list(self.command)), + "status": self.status.value, + "returncode": self.returncode, + "output_sha256": self.output_sha256, + "output_bytes": self.output_bytes, + } + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "index": self.index, + "command": redact_argv(self.command), + "status": self.status.value, + "returncode": self.returncode, + "duration_seconds": self.duration_seconds, + "output_sha256": self.output_sha256, + "output_bytes": self.output_bytes, + "detail": self.detail, + "digest": self.digest, + } + + +@dataclass(frozen=True, slots=True) +class QuarantineCandidate: + """Review metadata only; this object never quarantines a check.""" + + check_id: str + fingerprint: str + pass_count: int + fail_count: int + sample_digests: tuple[str, ...] + reason: str + schema_version: int = 1 + + @property + def automatic_quarantine(self) -> bool: + return False + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "check_id": self.check_id, + "fingerprint": self.fingerprint, + "pass_count": self.pass_count, + "fail_count": self.fail_count, + "sample_digests": list(self.sample_digests), + "reason": self.reason, + "automatic_quarantine": self.automatic_quarantine, + } + + +@dataclass(frozen=True, slots=True) +class FlakeFirewallResult: + """Stability verdict with explicit proof, repair, and quarantine boundaries.""" + + check: FlakeCheck + classification: FlakeClassification + samples: tuple[FlakeSample, ...] + requested_samples: int + complete: bool + quarantine_candidate: QuarantineCandidate | None = None + reasons: tuple[str, ...] = () + schema_version: int = 1 + + @property + def pass_count(self) -> int: + return sum(sample.status is FlakeSampleStatus.PASS for sample in self.samples) + + @property + def fail_count(self) -> int: + return sum(sample.status is FlakeSampleStatus.FAIL for sample in self.samples) + + @property + def infrastructure_failure_count(self) -> int: + return sum( + sample.status is FlakeSampleStatus.INFRASTRUCTURE_FAILURE for sample in self.samples + ) + + @property + def timeout_count(self) -> int: + return sum(sample.status is FlakeSampleStatus.TIMEOUT for sample in self.samples) + + @property + def invalid_count(self) -> int: + return sum(sample.status is FlakeSampleStatus.INVALID for sample in self.samples) + + @property + def inconclusive_sample_count(self) -> int: + return self.infrastructure_failure_count + self.timeout_count + self.invalid_count + + @property + def sample_digests(self) -> tuple[str, ...]: + return tuple(sample.digest for sample in self.samples) + + @property + def counts_as_proof(self) -> bool: + return ( + self.classification is FlakeClassification.STABLE_PASS + and self.complete + and len(self.samples) == self.requested_samples + and self.pass_count == self.requested_samples + ) + + @property + def repair_eligible(self) -> bool: + return ( + self.classification is FlakeClassification.STABLE_FAIL + and self.complete + and len(self.samples) == self.requested_samples + and self.fail_count == self.requested_samples + ) + + @property + def may_enter_repair(self) -> bool: + return self.repair_eligible + + @property + def automatic_quarantine(self) -> bool: + return False + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "check": self.check.to_dict(), + "classification": self.classification.value, + "samples": [sample.to_dict() for sample in self.samples], + "requested_samples": self.requested_samples, + "complete": self.complete, + "pass_count": self.pass_count, + "fail_count": self.fail_count, + "infrastructure_failure_count": self.infrastructure_failure_count, + "timeout_count": self.timeout_count, + "invalid_count": self.invalid_count, + "inconclusive_sample_count": self.inconclusive_sample_count, + "sample_digests": list(self.sample_digests), + "counts_as_proof": self.counts_as_proof, + "repair_eligible": self.repair_eligible, + "may_enter_repair": self.may_enter_repair, + "quarantine_candidate": ( + self.quarantine_candidate.to_dict() + if self.quarantine_candidate is not None + else None + ), + "automatic_quarantine": self.automatic_quarantine, + "reasons": list(self.reasons), + } + + +def _output_bytes(output: str | bytes | None) -> bytes | None: + if output is None: + return None + return output if isinstance(output, bytes) else output.encode("utf-8") + + +def _is_sha256(value: str) -> bool: + return len(value) == 64 and all(character in "0123456789abcdef" for character in value) + + +def _canonical_sha256(value: Any) -> str: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +__all__ = [ + "FlakeCheck", + "FlakeClassification", + "FlakeExecution", + "FlakeFirewallResult", + "FlakeSample", + "FlakeSampleStatus", + "InfrastructureFailure", + "QuarantineCandidate", +] diff --git a/src/agentdiff/proof/adversarial.py b/src/agentdiff/proof/adversarial.py new file mode 100644 index 0000000..3986ad6 --- /dev/null +++ b/src/agentdiff/proof/adversarial.py @@ -0,0 +1,432 @@ +"""Bounded deterministic mutation checks for verifier sensitivity. + +Adversarial proof never decides whether a mutation is semantically meaningful. +Callers supply an immutable :class:`MutationCase`, a deterministic mutation +applier, and the verifier that already represents the trusted behavior check. +The engine records whether that verifier kills or accidentally accepts each +mutant and truthfully downgrades proof strength when important challenges are +inconclusive or survive. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from enum import Enum +from typing import Any, Callable, Iterable, Mapping + +from .models import ProofStrengthLabel + +MutationApplier = Callable[["MutationCase"], Any] +MutationVerifier = Callable[[Any, float], Any] + + +class MutationVerdict(str, Enum): + """Outcome of applying one mutation and running its verifier.""" + + KILLED = "KILLED" + SURVIVED = "SURVIVED" + TIMEOUT = "TIMEOUT" + INVALID = "INVALID" + + +@dataclass(frozen=True, slots=True) +class MutationCase: + """One bounded deterministic mutation request. + + ``payload`` is intentionally opaque to the engine. It may be a hunk ID, + a structural mutation description, or another immutable value understood + by the injected mutation applier. + """ + + id: str + description: str = "" + payload: Any = None + important: bool = True + + def __post_init__(self) -> None: + if not self.id.strip(): + raise ValueError("mutation case id must not be empty") + + @property + def case_id(self) -> str: + return self.id + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "description": self.description, + "important": self.important, + } + + +@dataclass(frozen=True, slots=True) +class MutationVerification: + """Normalized verifier response for a mutated candidate.""" + + passed: bool + timed_out: bool = False + duration_seconds: float = 0.0 + detail: str = "" + valid: bool = True + + +@dataclass(frozen=True, slots=True) +class MutationOutcome: + """Evidence for one adversarial mutation case.""" + + case: MutationCase + verdict: MutationVerdict + duration_seconds: float + detail: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "case": self.case.to_dict(), + "verdict": self.verdict.value, + "duration_seconds": self.duration_seconds, + "detail": self.detail, + } + + +@dataclass(frozen=True, slots=True) +class AdversarialProofResult: + """Mutation score plus a deterministic proof-strength recommendation.""" + + outcomes: tuple[MutationOutcome, ...] + score: float + original_strength: ProofStrengthLabel + strength: ProofStrengthLabel + complete: bool + total_cases: int + reasons: tuple[str, ...] = () + schema_version: int = 1 + + @property + def killed(self) -> int: + return sum(outcome.verdict is MutationVerdict.KILLED for outcome in self.outcomes) + + @property + def survived(self) -> int: + return sum(outcome.verdict is MutationVerdict.SURVIVED for outcome in self.outcomes) + + @property + def timed_out(self) -> int: + return sum(outcome.verdict is MutationVerdict.TIMEOUT for outcome in self.outcomes) + + @property + def invalid(self) -> int: + return sum(outcome.verdict is MutationVerdict.INVALID for outcome in self.outcomes) + + @property + def decisive(self) -> int: + return self.killed + self.survived + + @property + def score_percent(self) -> int: + return round(self.score * 100) + + @property + def score_label(self) -> str: + return f"{self.killed} / {self.decisive}" + + @property + def downgraded(self) -> bool: + return self.strength is not self.original_strength + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "outcomes": [outcome.to_dict() for outcome in self.outcomes], + "score": self.score, + "score_percent": self.score_percent, + "score_label": self.score_label, + "killed": self.killed, + "survived": self.survived, + "timed_out": self.timed_out, + "invalid": self.invalid, + "complete": self.complete, + "total_cases": self.total_cases, + "original_strength": self.original_strength.value, + "strength": self.strength.value, + "downgraded": self.downgraded, + "reasons": list(self.reasons), + } + + +class InvalidMutationError(ValueError): + """Raised by an applier when a mutation cannot be constructed safely.""" + + +class AdversarialProofEngine: + """Apply and verify a bounded, ordered set of deterministic mutations.""" + + def __init__( + self, + mutation_applier: MutationApplier, + verifier: MutationVerifier, + *, + max_cases: int = 16, + max_duration_seconds: float = 300.0, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if max_cases < 1: + raise ValueError("max_cases must be at least 1") + if max_duration_seconds <= 0: + raise ValueError("max_duration_seconds must be greater than zero") + self.mutation_applier = mutation_applier + self.verifier = verifier + self.max_cases = max_cases + self.max_duration_seconds = max_duration_seconds + self.clock = clock + + def prove( + self, + cases: Iterable[MutationCase], + *, + timeout_seconds: float = 60.0, + starting_strength: ProofStrengthLabel | str = ProofStrengthLabel.STRONG, + ) -> AdversarialProofResult: + """Challenge the verifier in input order within strict case/time budgets.""" + + if timeout_seconds <= 0: + raise ValueError("mutation verifier timeout must be greater than zero") + mutation_cases = tuple(cases) + if len({case.id for case in mutation_cases}) != len(mutation_cases): + raise ValueError("mutation case ids must be unique") + strength = _coerce_strength(starting_strength) + started = self.clock() + outcomes: list[MutationOutcome] = [] + reasons: list[str] = [] + complete = True + + for index, case in enumerate(mutation_cases): + elapsed = max(0.0, self.clock() - started) + if index >= self.max_cases: + complete = False + reasons.append("mutation case budget reached") + break + if elapsed >= self.max_duration_seconds: + complete = False + reasons.append("adversarial proof duration budget reached") + break + outcome = self._run_case( + case, + timeout_seconds=min(timeout_seconds, self.max_duration_seconds - elapsed), + ) + outcomes.append(outcome) + + decisive = sum( + outcome.verdict in {MutationVerdict.KILLED, MutationVerdict.SURVIVED} + for outcome in outcomes + ) + killed = sum(outcome.verdict is MutationVerdict.KILLED for outcome in outcomes) + score = killed / decisive if decisive else 0.0 + important_concern = any( + outcome.case.important and outcome.verdict is not MutationVerdict.KILLED + for outcome in outcomes + ) + if not mutation_cases: + reasons.append("no mutation cases were supplied") + important_concern = True + if not complete: + important_concern = True + if important_concern: + strength = _downgrade_strength(strength) + reasons.append("important mutation evidence was survived or inconclusive") + + return AdversarialProofResult( + outcomes=tuple(outcomes), + score=score, + original_strength=_coerce_strength(starting_strength), + strength=strength, + complete=complete, + total_cases=len(mutation_cases), + reasons=tuple(dict.fromkeys(reasons)), + ) + + def run( + self, + cases: Iterable[MutationCase], + *, + timeout_seconds: float = 60.0, + starting_strength: ProofStrengthLabel | str = ProofStrengthLabel.STRONG, + ) -> AdversarialProofResult: + """Alias for :meth:`prove` for orchestration code.""" + + return self.prove( + cases, + timeout_seconds=timeout_seconds, + starting_strength=starting_strength, + ) + + def _run_case(self, case: MutationCase, *, timeout_seconds: float) -> MutationOutcome: + started = self.clock() + try: + mutant = self.mutation_applier(case) + except (OSError, RuntimeError, TypeError, ValueError) as error: + return MutationOutcome( + case=case, + verdict=MutationVerdict.INVALID, + duration_seconds=max(0.0, self.clock() - started), + detail=f"mutation application failed: {type(error).__name__}: {error}", + ) + if mutant is None: + return MutationOutcome( + case=case, + verdict=MutationVerdict.INVALID, + duration_seconds=max(0.0, self.clock() - started), + detail="mutation applier returned no candidate", + ) + + elapsed = max(0.0, self.clock() - started) + remaining = timeout_seconds - elapsed + if remaining <= 0: + return MutationOutcome( + case=case, + verdict=MutationVerdict.TIMEOUT, + duration_seconds=elapsed, + detail="mutation application exhausted the case timeout", + ) + try: + raw = self.verifier(mutant, remaining) + except TimeoutError as error: + return MutationOutcome( + case=case, + verdict=MutationVerdict.TIMEOUT, + duration_seconds=max(0.0, self.clock() - started), + detail=f"mutation verifier timed out: {error}", + ) + except (OSError, RuntimeError, TypeError, ValueError) as error: + return MutationOutcome( + case=case, + verdict=MutationVerdict.INVALID, + duration_seconds=max(0.0, self.clock() - started), + detail=f"mutation verifier failed: {type(error).__name__}: {error}", + ) + duration = max(0.0, self.clock() - started) + try: + verification = _normalize_verification(raw, duration) + except (TypeError, ValueError) as error: + return MutationOutcome( + case=case, + verdict=MutationVerdict.INVALID, + duration_seconds=duration, + detail=f"invalid mutation verifier result: {error}", + ) + effective_duration = max(duration, verification.duration_seconds) + if effective_duration > timeout_seconds or verification.timed_out: + return MutationOutcome( + case=case, + verdict=MutationVerdict.TIMEOUT, + duration_seconds=effective_duration, + detail=verification.detail or "mutation verifier exceeded the case timeout", + ) + if not verification.valid: + return MutationOutcome( + case=case, + verdict=MutationVerdict.INVALID, + duration_seconds=effective_duration, + detail=verification.detail or "mutation verifier returned invalid evidence", + ) + return MutationOutcome( + case=case, + verdict=(MutationVerdict.SURVIVED if verification.passed else MutationVerdict.KILLED), + duration_seconds=effective_duration, + detail=verification.detail, + ) + + +def _normalize_verification(raw: Any, duration_seconds: float) -> MutationVerification: + if isinstance(raw, MutationVerification): + return MutationVerification( + passed=raw.passed, + timed_out=raw.timed_out, + duration_seconds=raw.duration_seconds or duration_seconds, + detail=raw.detail, + valid=raw.valid, + ) + if isinstance(raw, bool): + return MutationVerification(passed=raw, duration_seconds=duration_seconds) + if isinstance(raw, int): + return MutationVerification(passed=raw == 0, duration_seconds=duration_seconds) + if isinstance(raw, Mapping): + timed_out = bool(raw.get("timed_out", False)) + if "causally_proven" in raw: + passed = bool(raw["causally_proven"]) + elif "passed" in raw: + passed = bool(raw["passed"]) + elif "returncode" in raw: + passed = int(raw["returncode"]) == 0 + elif "verdict" in raw: + passed = _verdict_is_proven(raw["verdict"]) + elif timed_out or not bool(raw.get("valid", True)): + passed = False + else: + raise ValueError("mapping has no causally_proven, passed, returncode, or verdict field") + return MutationVerification( + passed=passed, + timed_out=timed_out, + duration_seconds=float(raw.get("duration_seconds", duration_seconds)), + detail=str(raw.get("detail", "")), + valid=bool(raw.get("valid", True)), + ) + + if hasattr(raw, "causally_proven"): + passed = bool(raw.causally_proven) + elif hasattr(raw, "passed"): + passed = bool(raw.passed) + elif hasattr(raw, "verdict"): + passed = _verdict_is_proven(raw.verdict) + elif hasattr(raw, "returncode"): + passed = int(raw.returncode) == 0 + else: + raise TypeError("unsupported mutation verifier result") + return MutationVerification( + passed=passed, + timed_out=bool(getattr(raw, "timed_out", False)), + duration_seconds=float(getattr(raw, "duration_seconds", duration_seconds)), + detail=str(getattr(raw, "detail", "")), + valid=bool(getattr(raw, "valid", True)), + ) + + +def _verdict_is_proven(value: Any) -> bool: + raw = value.value if hasattr(value, "value") else value + return str(raw).upper() in { + "PASS", + "PASSED", + "PROVEN", + "CAUSAL_DELTA_CONFIRMED", + "BEHAVIOR_EQUIVALENCE_CONFIRMED", + } + + +def _coerce_strength(value: ProofStrengthLabel | str) -> ProofStrengthLabel: + if isinstance(value, ProofStrengthLabel): + return value + return ProofStrengthLabel(str(value).upper()) + + +def _downgrade_strength(value: ProofStrengthLabel) -> ProofStrengthLabel: + if value is ProofStrengthLabel.STRONG: + return ProofStrengthLabel.REVIEW + return ProofStrengthLabel.WEAK + + +# Explicit aliases make the public vocabulary easy to discover. +AdversarialVerdict = MutationVerdict +MutationResult = MutationOutcome + + +__all__ = [ + "AdversarialProofEngine", + "AdversarialProofResult", + "AdversarialVerdict", + "InvalidMutationError", + "MutationCase", + "MutationOutcome", + "MutationResult", + "MutationVerdict", + "MutationVerification", +] diff --git a/src/agentdiff/proof/delta.py b/src/agentdiff/proof/delta.py new file mode 100644 index 0000000..edcc703 --- /dev/null +++ b/src/agentdiff/proof/delta.py @@ -0,0 +1,577 @@ +"""Deterministic causal checks over independently executed code states. + +``DeltaProofEngine`` is deliberately separate from :class:`ProofEngine`. The +authoritative proof engine still decides whether a sealed patch is safe to +promote; delta proof answers the narrower question of whether one declared +behavior contract changed in the expected direction. + +Executors are injected so callers can bind the exact same argv to independently +materialized baseline and patched workspaces. No shell command is assembled +and no model is asked to interpret the result. +""" + +from __future__ import annotations + +import hashlib +import time +from dataclasses import dataclass +from enum import Enum +from typing import Any, Callable, Mapping + +from agentdiff.redaction import redact_argv + +BehaviorExecutor = Callable[[tuple[str, ...], float], Any] + + +class BehaviorExpectation(str, Enum): + """Supported causal relationships between baseline and patched behavior.""" + + BUGFIX = "BUGFIX" + FEATURE = "FEATURE" + REFACTOR = "REFACTOR" + + +class ExpectedBehavior(str, Enum): + """Expected process-level result for one side of a behavior contract.""" + + PASS = "PASS" + FAIL = "FAIL" + + +class BehaviorExecutionStatus(str, Enum): + """Truthful classification of one executor invocation.""" + + PASS = "PASS" + FAIL = "FAIL" + TIMEOUT = "TIMEOUT" + INVALID = "INVALID" + + +class DeltaProofVerdict(str, Enum): + """Causal verdicts emitted without interpreting command output with an LLM.""" + + CAUSAL_DELTA_CONFIRMED = "CAUSAL_DELTA_CONFIRMED" + BEHAVIOR_EQUIVALENCE_CONFIRMED = "BEHAVIOR_EQUIVALENCE_CONFIRMED" + DELTA_NOT_CONFIRMED = "DELTA_NOT_CONFIRMED" + REGRESSION_DETECTED = "REGRESSION_DETECTED" + FLAKY = "FLAKY" + TIMEOUT = "TIMEOUT" + INVALID = "INVALID" + + +@dataclass(frozen=True, slots=True) +class BehaviorContract: + """One exact command and its expected baseline-to-patch relationship. + + ``BUGFIX`` and ``FEATURE`` default to ``FAIL -> PASS``. ``REFACTOR`` + defaults to ``PASS -> PASS``. The explicit ``baseline`` and ``patched`` + fields allow a caller to serialize the complete contract rather than rely + on those defaults. + + When ``compare_output`` is true, a refactor additionally requires both + executors to return stable output and for its SHA-256 digest to match. + Raw output is never included in serialized evidence. + """ + + id: str + command: tuple[str, ...] + expectation: BehaviorExpectation + baseline: ExpectedBehavior | None = None + patched: ExpectedBehavior | None = None + repetitions: int = 1 + compare_output: bool = False + schema_version: int = 1 + + def __post_init__(self) -> None: + expectation = ( + self.expectation + if isinstance(self.expectation, BehaviorExpectation) + else BehaviorExpectation(str(self.expectation).upper()) + ) + command = tuple(self.command) + if not self.id.strip(): + raise ValueError("behavior contract id must not be empty") + if not command or any(not isinstance(part, str) or not part for part in command): + raise ValueError("behavior contract command must be non-empty exact argv") + if self.repetitions < 1: + raise ValueError("behavior contract repetitions must be at least 1") + default_baseline = ( + ExpectedBehavior.PASS + if expectation is BehaviorExpectation.REFACTOR + else ExpectedBehavior.FAIL + ) + baseline_value = self.baseline if self.baseline is not None else default_baseline + patched_value = self.patched if self.patched is not None else ExpectedBehavior.PASS + baseline = _coerce_expected_behavior(baseline_value) + patched = _coerce_expected_behavior(patched_value) + required_pair = (default_baseline, ExpectedBehavior.PASS) + if (baseline, patched) != required_pair: + raise ValueError( + f"{expectation.value} contracts require " + f"{required_pair[0].value} -> {required_pair[1].value}" + ) + if self.compare_output and expectation is not BehaviorExpectation.REFACTOR: + raise ValueError("output equivalence is only valid for REFACTOR contracts") + object.__setattr__(self, "command", command) + object.__setattr__(self, "expectation", expectation) + object.__setattr__(self, "baseline", baseline) + object.__setattr__(self, "patched", patched) + + @property + def contract_id(self) -> str: + """Compatibility-friendly explicit name for the serialized ``id``.""" + + return self.id + + @property + def argv(self) -> tuple[str, ...]: + """Return the exact argv supplied to both executors.""" + + return self.command + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "id": self.id, + "command": redact_argv(self.command), + "expectation": self.expectation.value, + "baseline": self.baseline.value if self.baseline is not None else None, + "patched": self.patched.value if self.patched is not None else None, + "repetitions": self.repetitions, + "compare_output": self.compare_output, + } + + +@dataclass(frozen=True, slots=True) +class BehaviorExecution: + """Normalized, bounded evidence from one exact-argv execution.""" + + command: tuple[str, ...] + returncode: int | None + duration_seconds: float = 0.0 + timed_out: bool = False + output: str | bytes | None = None + detail: str = "" + invalid: bool = False + + @property + def status(self) -> BehaviorExecutionStatus: + if self.timed_out: + return BehaviorExecutionStatus.TIMEOUT + if self.invalid or self.returncode is None: + return BehaviorExecutionStatus.INVALID + return ( + BehaviorExecutionStatus.PASS if self.returncode == 0 else BehaviorExecutionStatus.FAIL + ) + + @property + def output_sha256(self) -> str | None: + if self.output is None: + return None + payload = self.output if isinstance(self.output, bytes) else self.output.encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + @property + def output_bytes(self) -> int: + if self.output is None: + return 0 + payload = self.output if isinstance(self.output, bytes) else self.output.encode("utf-8") + return len(payload) + + def to_dict(self) -> dict[str, Any]: + return { + "command": redact_argv(self.command), + "status": self.status.value, + "returncode": self.returncode, + "duration_seconds": self.duration_seconds, + "timed_out": self.timed_out, + "output_sha256": self.output_sha256, + "output_bytes": self.output_bytes, + "detail": self.detail, + } + + +@dataclass(frozen=True, slots=True) +class DeltaProofResult: + """Evidence and causal verdict for one behavior contract.""" + + contract: BehaviorContract + verdict: DeltaProofVerdict + baseline_status: BehaviorExecutionStatus + patched_status: BehaviorExecutionStatus + baseline_runs: tuple[BehaviorExecution, ...] + patched_runs: tuple[BehaviorExecution, ...] + reasons: tuple[str, ...] = () + schema_version: int = 1 + + @property + def causally_proven(self) -> bool: + return self.verdict in { + DeltaProofVerdict.CAUSAL_DELTA_CONFIRMED, + DeltaProofVerdict.BEHAVIOR_EQUIVALENCE_CONFIRMED, + } + + @property + def flaky(self) -> bool: + return self.verdict is DeltaProofVerdict.FLAKY + + @property + def timed_out(self) -> bool: + return self.verdict is DeltaProofVerdict.TIMEOUT + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "contract": self.contract.to_dict(), + "verdict": self.verdict.value, + "causally_proven": self.causally_proven, + "baseline_status": self.baseline_status.value, + "patched_status": self.patched_status.value, + "baseline_runs": [run.to_dict() for run in self.baseline_runs], + "patched_runs": [run.to_dict() for run in self.patched_runs], + "reasons": list(self.reasons), + } + + +class DeltaProofEngine: + """Execute one behavior contract against independent baseline and patch states.""" + + def __init__( + self, + baseline_executor: BehaviorExecutor, + patched_executor: BehaviorExecutor, + *, + max_repetitions: int = 10, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if max_repetitions < 1: + raise ValueError("max_repetitions must be at least 1") + self.baseline_executor = baseline_executor + self.patched_executor = patched_executor + self.max_repetitions = max_repetitions + self.clock = clock + + def prove( + self, + contract: BehaviorContract, + *, + timeout_seconds: float = 60.0, + repetitions: int | None = None, + ) -> DeltaProofResult: + """Run exact argv on both states and evaluate only declared expectations.""" + + if timeout_seconds <= 0: + raise ValueError("delta proof timeout must be greater than zero") + run_count = contract.repetitions if repetitions is None else repetitions + if run_count < 1: + raise ValueError("delta proof repetitions must be at least 1") + if run_count > self.max_repetitions: + raise ValueError("delta proof repetitions exceed the configured bound") + + baseline_runs = self._execute_many( + self.baseline_executor, + contract.command, + timeout_seconds, + run_count, + ) + patched_runs = self._execute_many( + self.patched_executor, + contract.command, + timeout_seconds, + run_count, + ) + baseline_status = baseline_runs[0].status + patched_status = patched_runs[0].status + reasons: list[str] = [] + + all_runs = (*baseline_runs, *patched_runs) + if any(run.status is BehaviorExecutionStatus.TIMEOUT for run in all_runs): + reasons.append("at least one behavior-contract execution timed out") + return self._result( + contract, + DeltaProofVerdict.TIMEOUT, + baseline_runs, + patched_runs, + reasons, + ) + if any(run.status is BehaviorExecutionStatus.INVALID for run in all_runs): + reasons.append("at least one executor returned invalid or mismatched evidence") + return self._result( + contract, + DeltaProofVerdict.INVALID, + baseline_runs, + patched_runs, + reasons, + ) + if self._is_flaky(baseline_runs, compare_output=contract.compare_output): + reasons.append("baseline behavior was not stable across repetitions") + if self._is_flaky(patched_runs, compare_output=contract.compare_output): + reasons.append("patched behavior was not stable across repetitions") + if reasons: + return self._result( + contract, + DeltaProofVerdict.FLAKY, + baseline_runs, + patched_runs, + reasons, + ) + + baseline_expectation = contract.baseline + patched_expectation = contract.patched + assert baseline_expectation is not None + assert patched_expectation is not None + expected_baseline = BehaviorExecutionStatus(baseline_expectation.value) + expected_patched = BehaviorExecutionStatus(patched_expectation.value) + if baseline_status is not expected_baseline: + reasons.append( + f"baseline was {baseline_status.value}; expected {expected_baseline.value}" + ) + if patched_status is not expected_patched: + reasons.append( + f"patched state was {patched_status.value}; expected {expected_patched.value}" + ) + if reasons: + verdict = ( + DeltaProofVerdict.REGRESSION_DETECTED + if baseline_status is BehaviorExecutionStatus.PASS + and patched_status is BehaviorExecutionStatus.FAIL + else DeltaProofVerdict.DELTA_NOT_CONFIRMED + ) + return self._result( + contract, + verdict, + baseline_runs, + patched_runs, + reasons, + ) + + if contract.compare_output: + baseline_digest = baseline_runs[0].output_sha256 + patched_digest = patched_runs[0].output_sha256 + if baseline_digest is None or patched_digest is None: + reasons.append("output comparison requested but an executor supplied no output") + return self._result( + contract, + DeltaProofVerdict.INVALID, + baseline_runs, + patched_runs, + reasons, + ) + if baseline_digest != patched_digest: + reasons.append("refactor changed the declared observable output") + return self._result( + contract, + DeltaProofVerdict.REGRESSION_DETECTED, + baseline_runs, + patched_runs, + reasons, + ) + + verdict = ( + DeltaProofVerdict.BEHAVIOR_EQUIVALENCE_CONFIRMED + if contract.expectation is BehaviorExpectation.REFACTOR + else DeltaProofVerdict.CAUSAL_DELTA_CONFIRMED + ) + return self._result( + contract, + verdict, + baseline_runs, + patched_runs, + reasons, + ) + + def run( + self, + contract: BehaviorContract, + *, + timeout_seconds: float = 60.0, + repetitions: int | None = None, + ) -> DeltaProofResult: + """Alias for :meth:`prove` for orchestration code.""" + + return self.prove( + contract, + timeout_seconds=timeout_seconds, + repetitions=repetitions, + ) + + def _execute_many( + self, + executor: BehaviorExecutor, + command: tuple[str, ...], + timeout_seconds: float, + repetitions: int, + ) -> tuple[BehaviorExecution, ...]: + return tuple(self._execute(executor, command, timeout_seconds) for _ in range(repetitions)) + + def _execute( + self, + executor: BehaviorExecutor, + command: tuple[str, ...], + timeout_seconds: float, + ) -> BehaviorExecution: + started = self.clock() + try: + raw = executor(command, timeout_seconds) + except TimeoutError as error: + return BehaviorExecution( + command=command, + returncode=None, + duration_seconds=max(0.0, self.clock() - started), + timed_out=True, + detail=f"executor timeout: {error}", + ) + except (OSError, RuntimeError, TypeError, ValueError) as error: + return BehaviorExecution( + command=command, + returncode=None, + duration_seconds=max(0.0, self.clock() - started), + invalid=True, + detail=f"executor error: {type(error).__name__}: {error}", + ) + elapsed = max(0.0, self.clock() - started) + try: + normalized = self._normalize_execution(raw, command, elapsed) + except (OverflowError, TypeError, ValueError) as error: + return BehaviorExecution( + command=command, + returncode=None, + duration_seconds=elapsed, + invalid=True, + detail=f"invalid executor result: {type(error).__name__}: {error}", + ) + if normalized.command != command: + return BehaviorExecution( + command=normalized.command, + returncode=None, + duration_seconds=elapsed, + invalid=True, + detail="executor result command does not match the behavior contract argv", + ) + effective_duration = max(elapsed, normalized.duration_seconds) + if effective_duration > timeout_seconds and not normalized.timed_out: + return BehaviorExecution( + command=command, + returncode=None, + duration_seconds=effective_duration, + timed_out=True, + output=normalized.output, + detail="executor exceeded the declared timeout", + ) + return normalized + + @staticmethod + def _normalize_execution( + raw: Any, + command: tuple[str, ...], + elapsed: float, + ) -> BehaviorExecution: + if isinstance(raw, BehaviorExecution): + return BehaviorExecution( + command=tuple(raw.command), + returncode=raw.returncode, + duration_seconds=raw.duration_seconds or elapsed, + timed_out=raw.timed_out, + output=raw.output, + detail=raw.detail, + invalid=raw.invalid, + ) + if isinstance(raw, bool): + return BehaviorExecution(command, 0 if raw else 1, elapsed) + if isinstance(raw, int): + return BehaviorExecution(command, raw, elapsed) + if isinstance(raw, Mapping): + advertised = raw.get("command", raw.get("argv", command)) + output = raw.get("output", raw.get("observation", raw.get("stdout"))) + if output is not None and not isinstance(output, (str, bytes)): + return BehaviorExecution( + command=tuple(advertised), + returncode=None, + duration_seconds=elapsed, + invalid=True, + detail="executor output must be text or bytes", + ) + return BehaviorExecution( + command=tuple(advertised), + returncode=_optional_int(raw.get("returncode")), + duration_seconds=float(raw.get("duration_seconds", elapsed)), + timed_out=bool(raw.get("timed_out", False)), + output=output, + detail=str(raw.get("detail", "")), + invalid=bool(raw.get("invalid", False)), + ) + + advertised = getattr(raw, "command", getattr(raw, "argv", command)) + returncode = getattr(raw, "returncode", None) + timed_out = bool(getattr(raw, "timed_out", False)) + output = getattr(raw, "output", getattr(raw, "stdout", None)) + if output is not None and not isinstance(output, (str, bytes)): + output = None + return BehaviorExecution( + command=tuple(advertised), + returncode=_optional_int(returncode), + duration_seconds=float(getattr(raw, "duration_seconds", elapsed)), + timed_out=timed_out, + output=output, + detail=str(getattr(raw, "detail", "")), + invalid=returncode is None and not timed_out, + ) + + @staticmethod + def _is_flaky( + runs: tuple[BehaviorExecution, ...], + *, + compare_output: bool, + ) -> bool: + if len({run.status for run in runs}) > 1: + return True + if not compare_output: + return False + return len({run.output_sha256 for run in runs}) > 1 + + @staticmethod + def _result( + contract: BehaviorContract, + verdict: DeltaProofVerdict, + baseline_runs: tuple[BehaviorExecution, ...], + patched_runs: tuple[BehaviorExecution, ...], + reasons: list[str], + ) -> DeltaProofResult: + return DeltaProofResult( + contract=contract, + verdict=verdict, + baseline_status=baseline_runs[0].status, + patched_status=patched_runs[0].status, + baseline_runs=baseline_runs, + patched_runs=patched_runs, + reasons=tuple(dict.fromkeys(reasons)), + ) + + +def _optional_int(value: Any) -> int | None: + if value is None: + return None + return int(value) + + +def _coerce_expected_behavior(value: ExpectedBehavior | str) -> ExpectedBehavior: + if isinstance(value, ExpectedBehavior): + return value + return ExpectedBehavior(str(value).upper()) + + +# Short alias for callers that prefer result-oriented naming. +DeltaExecutionResult = BehaviorExecution + + +__all__ = [ + "BehaviorContract", + "BehaviorExecution", + "BehaviorExecutionStatus", + "BehaviorExpectation", + "DeltaExecutionResult", + "DeltaProofEngine", + "DeltaProofResult", + "DeltaProofVerdict", + "ExpectedBehavior", +] diff --git a/src/agentdiff/proof/flake.py b/src/agentdiff/proof/flake.py new file mode 100644 index 0000000..d66c076 --- /dev/null +++ b/src/agentdiff/proof/flake.py @@ -0,0 +1,382 @@ +"""Bounded deterministic flake classification beside authoritative proof. + +The firewall samples one exact check repeatedly and records digest-only +evidence. It does not alter :class:`ProofEngine`, start repair, or quarantine +tests; its result exposes narrow eligibility flags that reject noisy evidence. +""" + +from __future__ import annotations + +import time +from typing import Any, Callable, Mapping + +from ._flake_models import ( + FlakeCheck, + FlakeClassification, + FlakeExecution, + FlakeFirewallResult, + FlakeSample, + FlakeSampleStatus, + InfrastructureFailure, + QuarantineCandidate, + _canonical_sha256, +) + +FlakeCheckRunner = Callable[[tuple[str, ...], float], Any] + + +class FlakeFirewall: + """Run exact-argv samples within strict count and duration budgets.""" + + def __init__( + self, + runner: FlakeCheckRunner, + *, + max_samples: int = 10, + max_duration_seconds: float = 300.0, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if isinstance(max_samples, bool) or not isinstance(max_samples, int) or max_samples < 2: + raise ValueError("max_samples must be at least 2") + if max_duration_seconds <= 0: + raise ValueError("max_duration_seconds must be greater than zero") + self.runner = runner + self.max_samples = max_samples + self.max_duration_seconds = max_duration_seconds + self.clock = clock + + def assess( + self, + check: FlakeCheck, + *, + timeout_seconds: float = 60.0, + ) -> FlakeFirewallResult: + """Sample one check repeatedly without granting proof or repair authority.""" + + if timeout_seconds <= 0: + raise ValueError("flake sample timeout must be greater than zero") + started = self.clock() + samples: list[FlakeSample] = [] + reasons: list[str] = [] + planned = min(check.repetitions, self.max_samples) + + if check.repetitions > self.max_samples: + reasons.append("flake sample-count budget reached") + for index in range(1, planned + 1): + elapsed = max(0.0, self.clock() - started) + if elapsed >= self.max_duration_seconds: + reasons.append("flake sampling duration budget reached") + break + samples.append( + self._run_sample( + check, + index, + timeout_seconds=min( + timeout_seconds, + self.max_duration_seconds - elapsed, + ), + ) + ) + + complete = len(samples) == check.repetitions + classification, classification_reasons = _classify( + tuple(samples), + complete=complete, + compare_output=check.compare_output, + ) + reasons.extend(classification_reasons) + candidate = ( + _quarantine_candidate(check, tuple(samples)) + if classification is FlakeClassification.FLAKY + else None + ) + return FlakeFirewallResult( + check=check, + classification=classification, + samples=tuple(samples), + requested_samples=check.repetitions, + complete=complete, + quarantine_candidate=candidate, + reasons=tuple(dict.fromkeys(reasons)), + ) + + def run( + self, + check: FlakeCheck, + *, + timeout_seconds: float = 60.0, + ) -> FlakeFirewallResult: + """Alias for :meth:`assess` for orchestration code.""" + + return self.assess(check, timeout_seconds=timeout_seconds) + + def _run_sample( + self, + check: FlakeCheck, + index: int, + *, + timeout_seconds: float, + ) -> FlakeSample: + started = self.clock() + try: + raw = self.runner(check.command, timeout_seconds) + except TimeoutError as error: + execution = FlakeExecution( + returncode=None, + command=check.command, + timed_out=True, + detail=f"check timed out: {error}", + ) + except (InfrastructureFailure, OSError) as error: + execution = FlakeExecution( + returncode=None, + command=check.command, + infrastructure_failure=True, + detail=f"infrastructure failure: {type(error).__name__}: {error}", + ) + except (RuntimeError, TypeError, ValueError) as error: + execution = FlakeExecution( + returncode=None, + command=check.command, + valid=False, + detail=f"invalid check execution: {type(error).__name__}: {error}", + ) + else: + elapsed = max(0.0, self.clock() - started) + try: + execution = _normalize_execution(raw, check.command, elapsed) + except (OverflowError, TypeError, ValueError) as error: + execution = FlakeExecution( + returncode=None, + command=check.command, + duration_seconds=elapsed, + valid=False, + detail=f"invalid check result: {type(error).__name__}: {error}", + ) + + elapsed = max(0.0, self.clock() - started) + effective_duration = max(elapsed, execution.duration_seconds) + if execution.command != check.command: + execution = FlakeExecution( + returncode=None, + command=check.command, + duration_seconds=effective_duration, + output_sha256=execution.output_sha256, + output_bytes=execution.output_bytes, + valid=False, + detail="check result command does not match configured exact argv", + ) + elif effective_duration > timeout_seconds and not execution.timed_out: + execution = FlakeExecution( + returncode=None, + command=check.command, + duration_seconds=effective_duration, + output_sha256=execution.output_sha256, + output_bytes=execution.output_bytes, + timed_out=True, + detail="check execution exceeded the sample timeout", + ) + + return FlakeSample( + index=index, + command=check.command, + status=execution.status, + returncode=execution.returncode, + duration_seconds=effective_duration, + output_sha256=execution.output_sha256, + output_bytes=execution.output_bytes or 0, + detail=execution.detail, + ) + + +def _normalize_execution( + raw: Any, + command: tuple[str, ...], + elapsed: float, +) -> FlakeExecution: + if isinstance(raw, FlakeExecution): + return FlakeExecution( + returncode=raw.returncode, + command=raw.command or command, + duration_seconds=raw.duration_seconds or elapsed, + output=raw.output, + output_sha256=raw.output_sha256, + output_bytes=raw.output_bytes, + infrastructure_failure=raw.infrastructure_failure, + timed_out=raw.timed_out, + valid=raw.valid, + detail=raw.detail, + ) + if isinstance(raw, bool): + return FlakeExecution(0 if raw else 1, command=command, duration_seconds=elapsed) + if isinstance(raw, int): + return FlakeExecution(raw, command=command, duration_seconds=elapsed) + if isinstance(raw, Mapping): + status = _mapping_status(raw) + returncode = raw.get("returncode") + if returncode is None and status is FlakeSampleStatus.PASS: + returncode = 0 + elif returncode is None and status is FlakeSampleStatus.FAIL: + returncode = 1 + advertised = raw.get("command", raw.get("argv", command)) + return FlakeExecution( + returncode=_optional_int(returncode), + command=tuple(advertised), + duration_seconds=float(raw.get("duration_seconds", elapsed)), + output=raw.get("output", raw.get("stdout")), + output_sha256=raw.get("output_sha256"), + output_bytes=raw.get("output_bytes"), + infrastructure_failure=status is FlakeSampleStatus.INFRASTRUCTURE_FAILURE, + timed_out=status is FlakeSampleStatus.TIMEOUT, + valid=status is not FlakeSampleStatus.INVALID, + detail=str(raw.get("detail", "")), + ) + + advertised = getattr(raw, "command", getattr(raw, "argv", command)) + status = _object_status(raw) + returncode = getattr(raw, "returncode", None) + if returncode is None and status is FlakeSampleStatus.PASS: + returncode = 0 + elif returncode is None and status is FlakeSampleStatus.FAIL: + returncode = 1 + return FlakeExecution( + returncode=_optional_int(returncode), + command=tuple(advertised), + duration_seconds=float(getattr(raw, "duration_seconds", elapsed)), + output=getattr(raw, "output", getattr(raw, "stdout", None)), + output_sha256=getattr(raw, "output_sha256", None), + output_bytes=getattr(raw, "output_bytes", None), + infrastructure_failure=status is FlakeSampleStatus.INFRASTRUCTURE_FAILURE, + timed_out=status is FlakeSampleStatus.TIMEOUT, + valid=status is not FlakeSampleStatus.INVALID, + detail=str(getattr(raw, "detail", "")), + ) + + +def _mapping_status(raw: Mapping[Any, Any]) -> FlakeSampleStatus: + if bool(raw.get("timed_out", False)): + return FlakeSampleStatus.TIMEOUT + if bool(raw.get("infrastructure_failure", raw.get("infrastructure_error", False))): + return FlakeSampleStatus.INFRASTRUCTURE_FAILURE + if not bool(raw.get("valid", True)): + return FlakeSampleStatus.INVALID + if "status" in raw: + return _coerce_sample_status(raw["status"]) + if "passed" in raw: + return FlakeSampleStatus.PASS if bool(raw["passed"]) else FlakeSampleStatus.FAIL + if "returncode" in raw: + return FlakeSampleStatus.PASS if int(raw["returncode"]) == 0 else FlakeSampleStatus.FAIL + raise ValueError("mapping has no explicit check status") + + +def _object_status(raw: Any) -> FlakeSampleStatus: + if bool(getattr(raw, "timed_out", False)): + return FlakeSampleStatus.TIMEOUT + if bool(getattr(raw, "infrastructure_failure", getattr(raw, "infrastructure_error", False))): + return FlakeSampleStatus.INFRASTRUCTURE_FAILURE + if not bool(getattr(raw, "valid", True)): + return FlakeSampleStatus.INVALID + if hasattr(raw, "status"): + return _coerce_sample_status(raw.status) + if hasattr(raw, "passed"): + return FlakeSampleStatus.PASS if bool(raw.passed) else FlakeSampleStatus.FAIL + if hasattr(raw, "returncode"): + return FlakeSampleStatus.PASS if int(raw.returncode) == 0 else FlakeSampleStatus.FAIL + raise TypeError("unsupported check execution result") + + +def _coerce_sample_status(value: Any) -> FlakeSampleStatus: + if isinstance(value, FlakeSampleStatus): + return value + return FlakeSampleStatus(str(value).upper()) + + +def _classify( + samples: tuple[FlakeSample, ...], + *, + complete: bool, + compare_output: bool, +) -> tuple[FlakeClassification, tuple[str, ...]]: + pass_count = sum(sample.status is FlakeSampleStatus.PASS for sample in samples) + fail_count = sum(sample.status is FlakeSampleStatus.FAIL for sample in samples) + inconclusive_count = len(samples) - pass_count - fail_count + + if pass_count and fail_count: + return ( + FlakeClassification.FLAKY, + ("repeated samples produced both passing and failing results",), + ) + if compare_output and samples and not inconclusive_count: + output_digests = {sample.output_sha256 for sample in samples} + if None in output_digests: + return ( + FlakeClassification.INCONCLUSIVE, + ("output comparison requested but at least one sample supplied no digest",), + ) + if len(output_digests) > 1: + return ( + FlakeClassification.FLAKY, + ("repeated samples produced different declared observable output",), + ) + if not complete: + return ( + FlakeClassification.INCONCLUSIVE, + ("requested repeated sampling did not complete",), + ) + if inconclusive_count: + return ( + FlakeClassification.INCONCLUSIVE, + ("infrastructure, timeout, or invalid samples prevent a stability claim",), + ) + if samples and pass_count == len(samples): + return FlakeClassification.STABLE_PASS, () + if samples and fail_count == len(samples): + return FlakeClassification.STABLE_FAIL, () + return FlakeClassification.INCONCLUSIVE, ("no conclusive repeated samples were collected",) + + +def _quarantine_candidate( + check: FlakeCheck, + samples: tuple[FlakeSample, ...], +) -> QuarantineCandidate: + pass_count = sum(sample.status is FlakeSampleStatus.PASS for sample in samples) + fail_count = sum(sample.status is FlakeSampleStatus.FAIL for sample in samples) + sample_digests = tuple(sample.digest for sample in samples) + fingerprint = _canonical_sha256( + { + "check_id": check.id, + "command_digest": check.command_digest, + "sample_digests": list(sample_digests), + } + ) + return QuarantineCandidate( + check_id=check.id, + fingerprint=fingerprint, + pass_count=pass_count, + fail_count=fail_count, + sample_digests=sample_digests, + reason="deterministic repeated sampling observed unstable behavior", + ) + + +def _optional_int(value: Any) -> int | None: + if value is None: + return None + return int(value) + + +FlakeFirewallEngine = FlakeFirewall + + +__all__ = [ + "FlakeCheck", + "FlakeClassification", + "FlakeExecution", + "FlakeFirewall", + "FlakeFirewallEngine", + "FlakeFirewallResult", + "FlakeSample", + "FlakeSampleStatus", + "InfrastructureFailure", + "QuarantineCandidate", +] diff --git a/src/agentdiff/proof/lease.py b/src/agentdiff/proof/lease.py new file mode 100644 index 0000000..144d45d --- /dev/null +++ b/src/agentdiff/proof/lease.py @@ -0,0 +1,427 @@ +"""Fail-closed freshness leases for an already completed proof. + +A lease does not issue, modify, or restamp a certificate. It records the exact +inputs under which an existing proof was established and deterministically +decides whether later repository movement is provably disjoint from that proof. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from agentdiff.pathing import normalize_relative_path + +_GIT_SHA = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_OBSERVATION_FIELDS = frozenset( + { + "base_sha", + "patch_digest", + "proof_plan_digest", + "lockfile_digest", + "runtime_digest", + "changed_paths", + "changed_paths_complete", + "base_lineage_verified", + } +) + + +class ProofLeaseStatus(str, Enum): + """Freshness state of the proof bound by a lease.""" + + VALID = "VALID" + INVALIDATED = "INVALIDATED" + + +class ProofLeaseInvalidation(str, Enum): + """Complete fail-closed invalidation policy for a proof lease.""" + + PATCH_DIGEST_CHANGED = "patch_digest_changed" + PROOF_PLAN_DIGEST_CHANGED = "proof_plan_digest_changed" + LOCKFILE_DIGEST_CHANGED = "lockfile_digest_changed" + RUNTIME_DIGEST_CHANGED = "runtime_digest_changed" + DEPENDENCY_CLOSURE_INCOMPLETE = "dependency_closure_incomplete" + BASE_LINEAGE_UNVERIFIED = "base_lineage_unverified" + CHANGED_PATHS_INCOMPLETE = "changed_paths_incomplete" + DEPENDENCY_CLOSURE_TOUCHED = "dependency_closure_touched" + + +_REQUIRED_INVALIDATIONS = tuple(ProofLeaseInvalidation) + + +def _canonical_sha256(value: object) -> str: + try: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + except (TypeError, ValueError) as error: + raise ValueError("proof lease value is not canonically serializable") from error + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _require_mapping(value: object, *, field: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value): + raise ValueError(f"{field} must be an object with string keys") + return value + + +def _reject_unknown( + value: Mapping[str, Any], + *, + allowed: frozenset[str], + field: str, +) -> None: + unknown = sorted(set(value) - allowed) + if unknown: + raise ValueError(f"{field} contains unknown fields: {', '.join(unknown)}") + + +def _clean_git_sha(value: object, *, field: str) -> str: + if not isinstance(value, str) or not _GIT_SHA.fullmatch(value): + raise ValueError(f"{field} must be a lowercase 40- or 64-character Git SHA") + return value + + +def _clean_digest(value: object, *, field: str) -> str: + if not isinstance(value, str) or not _SHA256.fullmatch(value): + raise ValueError(f"{field} must be a lowercase SHA-256 digest") + return value + + +def _clean_bool(value: object, *, field: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{field} must be a boolean") + return value + + +def _clean_paths(value: object, *, field: str) -> tuple[str, ...]: + if not isinstance(value, (list, tuple)): + raise ValueError(f"{field} must be a list of repository-relative paths") + if len(value) > 100_000: + raise ValueError(f"{field} exceeds the 100000 item limit") + paths: list[str] = [] + for path in value: + if not isinstance(path, str): + raise ValueError(f"{field} must contain only strings") + paths.append(normalize_relative_path(path)) + return tuple(sorted(set(paths))) + + +def _clean_invalidation_rules(value: object) -> tuple[ProofLeaseInvalidation, ...]: + if not isinstance(value, (list, tuple)): + raise ValueError("invalidation_rules must be a list") + try: + rules = tuple(ProofLeaseInvalidation(item) for item in value) + except (TypeError, ValueError) as error: + raise ValueError("invalidation_rules contains an invalid rule") from error + if len(rules) != len(set(rules)): + raise ValueError("invalidation_rules cannot contain duplicates") + if set(rules) != set(_REQUIRED_INVALIDATIONS): + raise ValueError("invalidation_rules must contain the complete fail-closed policy") + return _REQUIRED_INVALIDATIONS + + +@dataclass(frozen=True, slots=True) +class ProofLease: + """Exact proof inputs and the verified dependency closure they cover.""" + + base_sha: str + patch_digest: str + proof_plan_digest: str + lockfile_digest: str + runtime_digest: str + dependency_closure: tuple[str, ...] + closure_complete: bool + invalidation_rules: tuple[ProofLeaseInvalidation, ...] = _REQUIRED_INVALIDATIONS + schema_version: int = 1 + + def __post_init__(self) -> None: + if self.schema_version != 1: + raise ValueError("unsupported ProofLease schema version") + object.__setattr__(self, "base_sha", _clean_git_sha(self.base_sha, field="base_sha")) + for field in ( + "patch_digest", + "proof_plan_digest", + "lockfile_digest", + "runtime_digest", + ): + object.__setattr__(self, field, _clean_digest(getattr(self, field), field=field)) + closure = _clean_paths(self.dependency_closure, field="dependency_closure") + if not closure: + raise ValueError("dependency_closure must contain at least one path") + object.__setattr__(self, "dependency_closure", closure) + object.__setattr__( + self, + "closure_complete", + _clean_bool(self.closure_complete, field="closure_complete"), + ) + object.__setattr__( + self, + "invalidation_rules", + _clean_invalidation_rules(self.invalidation_rules), + ) + + @property + def id(self) -> str: + return f"proof-lease-{_canonical_sha256(self._payload())[:24]}" + + @property + def digest(self) -> str: + return _canonical_sha256(self._payload()) + + def _payload(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "base_sha": self.base_sha, + "patch_digest": self.patch_digest, + "proof_plan_digest": self.proof_plan_digest, + "lockfile_digest": self.lockfile_digest, + "runtime_digest": self.runtime_digest, + "dependency_closure": list(self.dependency_closure), + "closure_complete": self.closure_complete, + "invalidation_rules": [rule.value for rule in self.invalidation_rules], + } + + def to_dict(self) -> dict[str, Any]: + return {**self._payload(), "id": self.id, "digest": self.digest} + + @classmethod + def from_dict(cls, value: object) -> ProofLease: + data = _require_mapping(value, field="ProofLease") + _reject_unknown( + data, + allowed=frozenset( + { + "schema_version", + "id", + "digest", + "base_sha", + "patch_digest", + "proof_plan_digest", + "lockfile_digest", + "runtime_digest", + "dependency_closure", + "closure_complete", + "invalidation_rules", + } + ), + field="ProofLease", + ) + version = data.get("schema_version") + if isinstance(version, bool) or version != 1: + raise ValueError("unsupported ProofLease schema version") + lease = cls( + base_sha=data.get("base_sha", ""), + patch_digest=data.get("patch_digest", ""), + proof_plan_digest=data.get("proof_plan_digest", ""), + lockfile_digest=data.get("lockfile_digest", ""), + runtime_digest=data.get("runtime_digest", ""), + dependency_closure=_clean_paths( + data.get("dependency_closure"), field="dependency_closure" + ), + closure_complete=_clean_bool(data.get("closure_complete"), field="closure_complete"), + invalidation_rules=_clean_invalidation_rules(data.get("invalidation_rules")), + ) + if data.get("id", lease.id) != lease.id: + raise ValueError("ProofLease id does not match its canonical payload") + if data.get("digest", lease.digest) != lease.digest: + raise ValueError("ProofLease digest does not match its canonical payload") + return lease + + +@dataclass(frozen=True, slots=True) +class ProofLeaseObservation: + """Trusted current inputs used to evaluate one lease. + + ``changed_paths_complete`` attests that ``changed_paths`` is the complete + repository-relative change set since ``lease.base_sha``. When the base SHA + moves, ``base_lineage_verified`` must independently attest ancestry. + """ + + base_sha: str + patch_digest: str + proof_plan_digest: str + lockfile_digest: str + runtime_digest: str + changed_paths: tuple[str, ...] = () + changed_paths_complete: bool = False + base_lineage_verified: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "base_sha", _clean_git_sha(self.base_sha, field="base_sha")) + for field in ( + "patch_digest", + "proof_plan_digest", + "lockfile_digest", + "runtime_digest", + ): + object.__setattr__(self, field, _clean_digest(getattr(self, field), field=field)) + object.__setattr__( + self, + "changed_paths", + _clean_paths(self.changed_paths, field="changed_paths"), + ) + object.__setattr__( + self, + "changed_paths_complete", + _clean_bool(self.changed_paths_complete, field="changed_paths_complete"), + ) + object.__setattr__( + self, + "base_lineage_verified", + _clean_bool(self.base_lineage_verified, field="base_lineage_verified"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "base_sha": self.base_sha, + "patch_digest": self.patch_digest, + "proof_plan_digest": self.proof_plan_digest, + "lockfile_digest": self.lockfile_digest, + "runtime_digest": self.runtime_digest, + "changed_paths": list(self.changed_paths), + "changed_paths_complete": self.changed_paths_complete, + "base_lineage_verified": self.base_lineage_verified, + } + + @classmethod + def from_dict(cls, value: object) -> ProofLeaseObservation: + data = _require_mapping(value, field="ProofLeaseObservation") + _reject_unknown(data, allowed=_OBSERVATION_FIELDS, field="ProofLeaseObservation") + return cls( + base_sha=data.get("base_sha", ""), + patch_digest=data.get("patch_digest", ""), + proof_plan_digest=data.get("proof_plan_digest", ""), + lockfile_digest=data.get("lockfile_digest", ""), + runtime_digest=data.get("runtime_digest", ""), + changed_paths=_clean_paths(data.get("changed_paths", []), field="changed_paths"), + changed_paths_complete=_clean_bool( + data.get("changed_paths_complete"), field="changed_paths_complete" + ), + base_lineage_verified=_clean_bool( + data.get("base_lineage_verified"), field="base_lineage_verified" + ), + ) + + +@dataclass(frozen=True, slots=True) +class ProofLeaseEvaluation: + """Deterministic freshness result; never a replacement certificate.""" + + lease_id: str + status: ProofLeaseStatus + current_base_sha: str + invalidations: tuple[ProofLeaseInvalidation, ...] + changed_paths: tuple[str, ...] + intersecting_paths: tuple[str, ...] + disjoint_paths: tuple[str, ...] + + @property + def valid(self) -> bool: + return self.status is ProofLeaseStatus.VALID + + def to_dict(self) -> dict[str, Any]: + return { + "lease_id": self.lease_id, + "status": self.status.value, + "valid": self.valid, + "current_base_sha": self.current_base_sha, + "invalidations": [reason.value for reason in self.invalidations], + "changed_paths": list(self.changed_paths), + "intersecting_paths": list(self.intersecting_paths), + "disjoint_paths": list(self.disjoint_paths), + } + + +class ProofLeaseEvaluator: + """Evaluate freshness using only explicit, deterministic lease bindings.""" + + def evaluate( + self, + lease: ProofLease, + observation: ProofLeaseObservation, + ) -> ProofLeaseEvaluation: + if not isinstance(lease, ProofLease): + raise TypeError("lease must be a ProofLease") + if not isinstance(observation, ProofLeaseObservation): + raise TypeError("observation must be a ProofLeaseObservation") + + invalidations: list[ProofLeaseInvalidation] = [] + comparisons = ( + ( + lease.patch_digest, + observation.patch_digest, + ProofLeaseInvalidation.PATCH_DIGEST_CHANGED, + ), + ( + lease.proof_plan_digest, + observation.proof_plan_digest, + ProofLeaseInvalidation.PROOF_PLAN_DIGEST_CHANGED, + ), + ( + lease.lockfile_digest, + observation.lockfile_digest, + ProofLeaseInvalidation.LOCKFILE_DIGEST_CHANGED, + ), + ( + lease.runtime_digest, + observation.runtime_digest, + ProofLeaseInvalidation.RUNTIME_DIGEST_CHANGED, + ), + ) + invalidations.extend( + reason for expected, current, reason in comparisons if expected != current + ) + + if not lease.closure_complete: + invalidations.append(ProofLeaseInvalidation.DEPENDENCY_CLOSURE_INCOMPLETE) + + base_moved = observation.base_sha != lease.base_sha + workspace_moved = base_moved or bool(observation.changed_paths) + if base_moved and not observation.base_lineage_verified: + invalidations.append(ProofLeaseInvalidation.BASE_LINEAGE_UNVERIFIED) + if workspace_moved and not observation.changed_paths_complete: + invalidations.append(ProofLeaseInvalidation.CHANGED_PATHS_INCOMPLETE) + + intersecting = tuple( + path + for path in observation.changed_paths + if any(_paths_overlap(path, dependency) for dependency in lease.dependency_closure) + ) + if intersecting: + invalidations.append(ProofLeaseInvalidation.DEPENDENCY_CLOSURE_TOUCHED) + + trusted_classification = lease.closure_complete and ( + not workspace_moved or observation.changed_paths_complete + ) + intersecting_set = set(intersecting) + disjoint = ( + tuple(path for path in observation.changed_paths if path not in intersecting_set) + if trusted_classification + else () + ) + status = ProofLeaseStatus.INVALIDATED if invalidations else ProofLeaseStatus.VALID + return ProofLeaseEvaluation( + lease_id=lease.id, + status=status, + current_base_sha=observation.base_sha, + invalidations=tuple(invalidations), + changed_paths=observation.changed_paths, + intersecting_paths=intersecting, + disjoint_paths=disjoint, + ) + + +def _paths_overlap(left: str, right: str) -> bool: + return left == right or left.startswith(f"{right}/") or right.startswith(f"{left}/") diff --git a/src/agentdiff/runtime/__init__.py b/src/agentdiff/runtime/__init__.py index 37bf8c8..08cd872 100644 --- a/src/agentdiff/runtime/__init__.py +++ b/src/agentdiff/runtime/__init__.py @@ -12,7 +12,7 @@ RuntimeControlLevel, RuntimeResult, ) -from .docker import DockerRuntime +from .docker import DockerLifecycleError, DockerRuntime from .local import LocalRuntime from .materialize import ( MaterializationReport, @@ -24,6 +24,7 @@ __all__ = [ "CleanupOutcome", "CleanupReport", + "DockerLifecycleError", "DockerRuntime", "LocalRuntime", "MaterializationReport", diff --git a/src/agentdiff/runtime/docker.py b/src/agentdiff/runtime/docker.py index 3d25873..340fafd 100644 --- a/src/agentdiff/runtime/docker.py +++ b/src/agentdiff/runtime/docker.py @@ -9,6 +9,7 @@ from __future__ import annotations +import errno import os import re import secrets @@ -36,6 +37,14 @@ _ENV_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$") +class DockerLifecycleError(OSError): + """A deterministic timeout from one Docker control-plane lifecycle stage.""" + + def __init__(self, stage: str) -> None: + self.stage = stage + super().__init__(errno.ETIMEDOUT, f"Docker {stage} timed out") + + class DockerRuntime: """Execute exact argv in an ephemeral, resource-bounded Docker container.""" @@ -51,6 +60,7 @@ def __init__( network: str = "none", environment_allowlist: Sequence[str] = (), poll_interval_seconds: float = 0.1, + control_timeout_seconds: float = 120.0, ) -> None: self.root = Path(root).expanduser().resolve(strict=True) if not self.root.is_dir(): @@ -61,14 +71,22 @@ def __init__( if not candidate.is_absolute() or not candidate.is_file(): raise FileNotFoundError(f"Docker executable not found: {executable}") located = str(candidate.resolve(strict=True)) - if not image or any(character in image for character in "\r\n\x00"): - raise ValueError("Docker image must be a non-empty single-line value") + if ( + not image + or image.startswith("-") + or any(character in image for character in "\r\n\x00") + ): + raise ValueError( + "Docker image must be a non-empty single-line value and cannot be an option" + ) if cpus <= 0: raise ValueError("cpus must be greater than zero") if pids_limit <= 0: raise ValueError("pids_limit must be greater than zero") if network not in {"none", "bridge"}: raise ValueError("Docker network must be none or bridge") + if control_timeout_seconds <= 0: + raise ValueError("control_timeout_seconds must be greater than zero") names = tuple(dict.fromkeys(environment_allowlist)) if any(not _ENV_NAME.fullmatch(name) for name in names): raise ValueError("invalid environment allowlist name") @@ -80,6 +98,7 @@ def __init__( self.network = network self.environment_allowlist = names self.poll_interval_seconds = poll_interval_seconds + self.control_timeout_seconds = float(control_timeout_seconds) self._source_dir: Path | None = None self._workspace: Path | None = None self._temporary_root: Path | None = None @@ -128,7 +147,11 @@ def run( create_argv = self._create_argv(container_name, workspace, user, command) container_id: str | None = None try: - created = self._docker_call(create_argv, capture_output=True) + created = self._lifecycle_call( + create_argv, + capture_output=True, + stage="create", + ) if created.returncode != 0: message = (created.stderr or created.stdout or "docker create failed").strip() raise OSError(message) @@ -171,9 +194,10 @@ def run( ) finally: if container_id is not None: - self._docker_call( + self._lifecycle_call( [self.executable, "rm", "--force", "--volumes", container_id], capture_output=True, + stage="cleanup", ) def close(self) -> None: @@ -271,8 +295,21 @@ def _runtime_config(self, user: str) -> dict[str, Any]: "pids_limit": self.pids_limit, "environment_allowlist": list(self.environment_allowlist), "ephemeral_container": True, + "control_timeout_seconds": self.control_timeout_seconds, } + def _lifecycle_call( + self, + argv: Sequence[str], + *, + capture_output: bool, + stage: str, + ) -> subprocess.CompletedProcess[str]: + try: + return self._docker_call(argv, capture_output=capture_output) + except subprocess.TimeoutExpired as error: + raise DockerLifecycleError(stage) from error + def _docker_call( self, argv: Sequence[str], @@ -286,6 +323,7 @@ def _docker_call( capture_output=capture_output, text=True, env=self._docker_environment(), + timeout=self.control_timeout_seconds, ) def _docker_environment(self) -> dict[str, str]: @@ -298,7 +336,7 @@ def _docker_environment(self) -> dict[str, str]: return allowed def _image_digest(self) -> str | None: - inspected = self._docker_call( + inspected = self._lifecycle_call( [ self.executable, "image", @@ -308,6 +346,7 @@ def _image_digest(self) -> str | None: self.image, ], capture_output=True, + stage="inspect", ) if inspected.returncode != 0: return None diff --git a/tests/test_adversarial_proof.py b/tests/test_adversarial_proof.py new file mode 100644 index 0000000..f838530 --- /dev/null +++ b/tests/test_adversarial_proof.py @@ -0,0 +1,102 @@ +"""Focused tests for deterministic adversarial proof.""" + +from __future__ import annotations + +from agentdiff.proof.adversarial import ( + AdversarialProofEngine, + InvalidMutationError, + MutationCase, + MutationVerdict, + MutationVerification, +) +from agentdiff.proof.models import ProofStrengthLabel + + +def test_mutation_score_and_survivor_downgrade_are_truthful() -> None: + cases = tuple( + MutationCase(id=name, payload=name) + for name in ("remove-guard", "invert-condition", "wrong-boundary", "always-success") + ) + + result = AdversarialProofEngine( + lambda case: case.payload, + lambda mutant, _timeout: mutant == "always-success", + ).prove(cases) + + assert [outcome.verdict for outcome in result.outcomes] == [ + MutationVerdict.KILLED, + MutationVerdict.KILLED, + MutationVerdict.KILLED, + MutationVerdict.SURVIVED, + ] + assert result.killed == 3 + assert result.survived == 1 + assert result.score == 0.75 + assert result.score_label == "3 / 4" + assert result.score_percent == 75 + assert result.original_strength is ProofStrengthLabel.STRONG + assert result.strength is ProofStrengthLabel.REVIEW + assert result.downgraded is True + + +def test_all_important_mutants_killed_preserves_strength() -> None: + result = AdversarialProofEngine( + lambda case: case.id, + lambda _mutant, _timeout: False, + ).prove((MutationCase("one"), MutationCase("two"))) + + assert result.score == 1.0 + assert result.strength is ProofStrengthLabel.STRONG + assert result.complete is True + assert result.reasons == () + + +def test_timeout_and_invalid_mutation_are_recorded_not_interpreted() -> None: + def apply(case: MutationCase) -> str: + if case.id == "invalid": + raise InvalidMutationError("hunk no longer applies") + return case.id + + def verify(mutant: str, _timeout: float) -> bool: + if mutant == "timeout": + raise TimeoutError("verification budget") + return False + + result = AdversarialProofEngine(apply, verify).prove( + (MutationCase("invalid"), MutationCase("timeout")) + ) + + assert [outcome.verdict for outcome in result.outcomes] == [ + MutationVerdict.INVALID, + MutationVerdict.TIMEOUT, + ] + assert result.decisive == 0 + assert result.score == 0.0 + assert result.strength is ProofStrengthLabel.REVIEW + + +def test_case_budget_is_bounded_and_reported() -> None: + result = AdversarialProofEngine( + lambda case: case.id, + lambda _mutant, _timeout: False, + max_cases=1, + ).prove((MutationCase("one"), MutationCase("two"))) + + assert len(result.outcomes) == 1 + assert result.complete is False + assert result.total_cases == 2 + assert result.strength is ProofStrengthLabel.REVIEW + assert "mutation case budget reached" in result.reasons + + +def test_reported_verifier_timeout_is_honored() -> None: + result = AdversarialProofEngine( + lambda case: case.id, + lambda _mutant, _timeout: MutationVerification( + passed=False, + duration_seconds=20.0, + ), + ).prove((MutationCase("slow"),), timeout_seconds=5.0) + + assert result.outcomes[0].verdict is MutationVerdict.TIMEOUT + assert result.strength is ProofStrengthLabel.REVIEW diff --git a/tests/test_api_untrusted_generators.py b/tests/test_api_untrusted_generators.py index 8cb3618..60a2f4b 100644 --- a/tests/test_api_untrusted_generators.py +++ b/tests/test_api_untrusted_generators.py @@ -2,17 +2,182 @@ from __future__ import annotations +import errno +import hashlib +import shutil +import subprocess import sys -from typing import TYPE_CHECKING +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest from agentdiff.api import MigrationEngine, get_builtin_manifest -from agentdiff.api.generation_runtime import PrivateGenerationRuntime -from agentdiff.api.generators import CustomCommandGenerator -from agentdiff.runtime import RuntimeControlLevel +from agentdiff.api.generation_runtime import ( + DockerGeneratorRuntime, + PrivateGenerationRuntime, +) +from agentdiff.api.generators import ( + CustomCommandGenerator, + DeterministicASTGenerator, + ExternalCodingAgentGenerator, + GenerationResult, +) +from agentdiff.runtime import ( + CleanupReport, + DockerLifecycleError, + DockerRuntime, + RuntimeResult, +) +from agentdiff.transaction import RunStore from tests.fake_proof import fake_env_factory if TYPE_CHECKING: - from pathlib import Path + from collections.abc import Callable, Iterable, Sequence + + from agentdiff.api.generators import MigrationGenerator + from agentdiff.api.models import MigrationPlan + from agentdiff.runtime import OwnedProcess + + +_STRONG_RUNTIME_CONFIG: dict[str, Any] = { + "image": "generator:test", + "private_workspace": True, + "host_repository_mounted": False, + "docker_socket_mounted": False, + "read_only_rootfs": True, + "no_new_privileges": True, + "network": "none", + "cap_drop": ["ALL"], + "user": "65532:65532", + "cpus": 1.0, + "memory": "512m", + "pids_limit": 64, + "environment_allowlist": [], + "ephemeral_container": True, + "control_timeout_seconds": 120.0, +} + + +class _FakeDockerBackend: + """Test double for DockerRuntime that still uses a private observation copy.""" + + def __init__( + self, + root: str | Path, + *, + mutator: Callable[[Path], None] | None = None, + captured: dict[str, Any] | None = None, + returncode: int = 0, + timed_out: bool = False, + runtime_config: dict[str, Any] | None = None, + stdout_bytes: bytes = b"", + stderr_bytes: bytes = b"", + lifecycle_error_stage: str | None = None, + **kwargs: Any, + ) -> None: + self.root = Path(root) + self.mutator = mutator + self.captured = captured if captured is not None else {} + self.captured["constructor"] = kwargs + self.image = str(kwargs["image"]) + self.returncode = returncode + self.timed_out = timed_out + self.runtime_config = dict(runtime_config or _STRONG_RUNTIME_CONFIG) + self.stdout_bytes = stdout_bytes + self.stderr_bytes = stderr_bytes + self.lifecycle_error_stage = lifecycle_error_stage + self._source: Path | None = None + self._temporary_root: Path | None = None + + def configure_source(self, source_dir: str | Path) -> None: + self._source = Path(source_dir) + + def configure_safety(self, controller: Any) -> None: + del controller + + def run( + self, + argv: Sequence[str], + *, + timeout_seconds: float | None = None, + stdin: Any = None, + stdout: Any = None, + stderr: Any = None, + ) -> RuntimeResult: + del stdin + if self.lifecycle_error_stage is not None: + raise DockerLifecycleError(self.lifecycle_error_stage) + assert self._source is not None + command = tuple(argv) + self.captured["argv"] = command + self.captured["timeout_seconds"] = timeout_seconds + self._temporary_root = Path(tempfile.mkdtemp(prefix="agentdiff-test-docker-")) + workspace = self._temporary_root / "workspace" + shutil.copytree(self._source, workspace, symlinks=False) + if self.mutator is not None: + self.mutator(workspace) + if stdout is not None and self.stdout_bytes: + stdout.write(self.stdout_bytes) + stdout.flush() + if stderr is not None and self.stderr_bytes: + stderr.write(self.stderr_bytes) + stderr.flush() + wrapper_argv = ( + "docker", + "create", + "--user", + "65532:65532", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--network", + "none", + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=64m", + "--cpus", + "1.0", + "--memory", + "512m", + "--pids-limit", + "64", + "--mount", + f"type=bind,src={workspace},dst=/workspace", + "--env", + "HOME=/tmp", + self.image, + *command, + ) + return RuntimeResult( + argv=command, + cwd="/workspace", + returncode=self.returncode, + timed_out=self.timed_out, + duration_seconds=0.01, + backend="docker", + enforcement="isolated_private_workspace", + wrapper_argv=wrapper_argv, + image=self.image, + runtime_config=self.runtime_config, + observation_root=str(workspace), + ) + + def cleanup( + self, + processes: Iterable[OwnedProcess], + *, + grace_period_seconds: float = 1.0, + ) -> CleanupReport: + del processes, grace_period_seconds + return CleanupReport() + + def close(self) -> None: + if self._temporary_root is not None: + shutil.rmtree(self._temporary_root) + self._temporary_root = None def _repository(tmp_path: Path) -> Path: @@ -36,17 +201,79 @@ def _repository(tmp_path: Path) -> Path: return tmp_path +def _runtime_factory( + root: Path, + *, + mutator: Callable[[Path], None] | None = None, + captured: dict[str, Any] | None = None, + returncode: int = 0, + timed_out: bool = False, + runtime_config: dict[str, Any] | None = None, + stdout_bytes: bytes = b"", + stderr_bytes: bytes = b"", + lifecycle_error_stage: str | None = None, + output_limit_bytes_per_stream: int = 1_000_000, + execution_timeout_seconds: float = 900.0, + control_timeout_seconds: float = 120.0, +) -> Callable[[MigrationPlan, MigrationGenerator], DockerGeneratorRuntime]: + def create( + plan: MigrationPlan, + generator: MigrationGenerator, + ) -> DockerGeneratorRuntime: + assert isinstance(generator, CustomCommandGenerator) + + def docker_factory(docker_root: str | Path, **kwargs: Any) -> _FakeDockerBackend: + return _FakeDockerBackend( + docker_root, + mutator=mutator, + captured=captured, + returncode=returncode, + timed_out=timed_out, + runtime_config=runtime_config, + stdout_bytes=stdout_bytes, + stderr_bytes=stderr_bytes, + lifecycle_error_stage=lifecycle_error_stage, + **kwargs, + ) + + return DockerGeneratorRuntime( + root, + plan, + generator, + image="generator:test", + output_limit_bytes_per_stream=output_limit_bytes_per_stream, + execution_timeout_seconds=execution_timeout_seconds, + control_timeout_seconds=control_timeout_seconds, + docker_runtime_factory=docker_factory, + ) + + return create + + +def _plan(root: Path) -> tuple[MigrationEngine, MigrationPlan]: + engine = MigrationEngine( + root, + manifest=get_builtin_manifest("openai", "chat_to_responses"), + ) + usages, impact = engine.scan_and_match() + return engine, engine.create_plan(usages, impact) + + def test_successful_worker_that_changes_nothing_is_not_proven(tmp_path: Path) -> None: root = _repository(tmp_path) + captured: dict[str, Any] = {} + command = ("agent-worker", "--prompt", "leave unchanged; $(touch host)") engine = MigrationEngine( root, manifest=get_builtin_manifest("openai", "chat_to_responses"), - generator=CustomCommandGenerator((sys.executable, "-c", "pass")), + generator=ExternalCodingAgentGenerator(command, name="external-agent"), + generator_runtime_factory=_runtime_factory(root, captured=captured), proof_environment_factory=fake_env_factory(), ) result = engine.run() + assert captured["argv"] == command assert result.proof_verdict == "NOT_PROVEN" assert result.certificate is not None assert result.certificate.verified is False @@ -55,16 +282,17 @@ def test_successful_worker_that_changes_nothing_is_not_proven(tmp_path: Path) -> def test_unexpected_workflow_change_is_rejected(tmp_path: Path) -> None: root = _repository(tmp_path) - command = ( - sys.executable, - "-c", - "from pathlib import Path; p=Path('.github/workflows/deploy.yml'); " - "p.parent.mkdir(parents=True); p.write_text('unsafe: true\\n')", - ) + + def write_workflow(workspace: Path) -> None: + workflow = workspace / ".github" / "workflows" / "deploy.yml" + workflow.parent.mkdir(parents=True) + workflow.write_text("unsafe: true\n", encoding="utf-8") + engine = MigrationEngine( root, manifest=get_builtin_manifest("openai", "chat_to_responses"), - generator=CustomCommandGenerator(command), + generator=CustomCommandGenerator(("agent-worker", "migrate")), + generator_runtime_factory=_runtime_factory(root, mutator=write_workflow), proof_environment_factory=fake_env_factory(), ) @@ -79,24 +307,327 @@ def test_unexpected_workflow_change_is_rejected(tmp_path: Path) -> None: assert not (root / ".github" / "workflows" / "deploy.yml").exists() -def test_private_generation_is_observation_not_an_os_sandbox(tmp_path: Path) -> None: +def test_custom_generator_has_no_host_execution_method(tmp_path: Path) -> None: + root = _repository(tmp_path) + _, plan = _plan(root) + marker = tmp_path / "host-executed" + generator = CustomCommandGenerator( + ( + sys.executable, + "-c", + f"from pathlib import Path; Path({str(marker)!r}).touch()", + ) + ) + + assert not hasattr(generator, "generate") + with pytest.raises(TypeError, match="only DeterministicASTGenerator"): + PrivateGenerationRuntime(plan, generator) # type: ignore[arg-type] + + assert not marker.exists() + + +def test_deterministic_subclass_cannot_gain_in_process_execution(tmp_path: Path) -> None: root = _repository(tmp_path) - generator = CustomCommandGenerator((sys.executable, "-c", "pass")) + marker = tmp_path / "subclass-host-executed" + + class UntrustedSubclass(DeterministicASTGenerator): + def generate(self, plan: MigrationPlan, workspace: Path) -> GenerationResult: + del plan, workspace + marker.touch() + return GenerationResult( + success=True, + generator=self.name, + strategy=self.strategy, + ) + engine = MigrationEngine( root, manifest=get_builtin_manifest("openai", "chat_to_responses"), - generator=generator, + generator=UntrustedSubclass(), ) - usages, impact = engine.scan_and_match() - runtime = PrivateGenerationRuntime(engine.create_plan(usages, impact), generator) + + with pytest.raises(TypeError, match="explicit isolated runtime factory"): + engine.run() + + assert not marker.exists() + + +def test_docker_runtime_preserves_exact_argv_and_requires_security_controls( + tmp_path: Path, +) -> None: + root = _repository(tmp_path) + _, plan = _plan(root) + command = ( + "agent-worker", + "--prompt", + "two words; $(uname)", + "--privileged", + "/var/run/docker.sock", + ) + generator = CustomCommandGenerator(command) + captured: dict[str, Any] = {} + runtime = _runtime_factory(root, captured=captured)(plan, generator) runtime.configure_source(root) - result = runtime.run((generator.command_label,)) + result = runtime.run(command, timeout_seconds=17) runtime.close() - host_capability = next( - capability for capability in result.capabilities if capability.boundary == "host_repository" + assert result.argv == command + assert captured["argv"] == command + assert captured["constructor"] == { + "image": "generator:test", + "network": "none", + "environment_allowlist": (), + "control_timeout_seconds": 120.0, + } + assert captured["timeout_seconds"] == 17 + assert runtime.generation_result == GenerationResult( + success=True, + generator="custom-command", + strategy=generator.strategy, ) - assert result.enforcement == "private_workspace_observation" - assert host_capability.control is RuntimeControlLevel.UNCONTROLLED - assert "not an OS security boundary" in host_capability.mechanism + + +def test_docker_runtime_rejects_weakened_isolation_evidence(tmp_path: Path) -> None: + root = _repository(tmp_path) + _, plan = _plan(root) + generator = CustomCommandGenerator(("agent-worker",)) + weakened = {**_STRONG_RUNTIME_CONFIG, "network": "bridge"} + runtime = _runtime_factory(root, runtime_config=weakened)(plan, generator) + runtime.configure_source(root) + + with pytest.raises(OSError, match="required isolation"): + runtime.run(generator.argv) + runtime.close() + + assert runtime.generation_result is None + + +def test_generator_image_cannot_inject_docker_options(tmp_path: Path) -> None: + root = _repository(tmp_path) + _, plan = _plan(root) + generator = CustomCommandGenerator(("agent-worker",)) + + with pytest.raises(ValueError, match="cannot be an option"): + DockerGeneratorRuntime(root, plan, generator, image="--privileged") + + +def test_generator_output_capture_is_bounded_and_fails_closed(tmp_path: Path) -> None: + root = _repository(tmp_path) + _, plan = _plan(root) + generator = CustomCommandGenerator(("agent-worker",)) + captured: dict[str, Any] = {} + stdout_bytes = b"x" * 20 + stderr_bytes = b"error" + runtime = _runtime_factory( + root, + captured=captured, + stdout_bytes=stdout_bytes, + stderr_bytes=stderr_bytes, + output_limit_bytes_per_stream=8, + )(plan, generator) + runtime.configure_source(root) + + result = runtime.run(generator.argv, timeout_seconds=2_000) + runtime.close() + + assert result.runtime_config is not None + output = result.runtime_config["generator_output"] + assert result.returncode == 125 + assert captured["timeout_seconds"] == 900.0 + assert output == { + "limit_bytes_per_stream": 8, + "limited": True, + "stdout": { + "total_bytes": 20, + "captured_bytes": 8, + "sha256": hashlib.sha256(stdout_bytes).hexdigest(), + "limited": True, + "complete": True, + }, + "stderr": { + "total_bytes": 5, + "captured_bytes": 5, + "sha256": hashlib.sha256(stderr_bytes).hexdigest(), + "limited": False, + "complete": True, + }, + } + assert runtime.generation_result is not None + assert runtime.generation_result.errors == ( + "Docker generator output exceeded the safety limit", + ) + + +def test_output_limit_is_preserved_as_migration_error_evidence(tmp_path: Path) -> None: + root = _repository(tmp_path) + engine = MigrationEngine( + root, + manifest=get_builtin_manifest("openai", "chat_to_responses"), + generator=CustomCommandGenerator(("agent-worker",)), + generator_runtime_factory=_runtime_factory( + root, + stdout_bytes=b"unbounded output", + output_limit_bytes_per_stream=4, + ), + proof_environment_factory=fake_env_factory(), + ) + + result = engine.run() + + assert result.proof_verdict == "NOT_PROVEN" + assert "Docker generator output exceeded the safety limit" in result.errors + runtime_evidence = RunStore.open(root, result.run_id).read_json("runtime.json") + assert runtime_evidence["returncode"] == 125 + assert runtime_evidence["runtime_config"]["generator_output"]["limited"] is True + + +def test_docker_control_calls_have_an_explicit_wall_clock_timeout( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, Any] = {} + + def fake_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + observed["argv"] = argv + observed["timeout"] = kwargs["timeout"] + return subprocess.CompletedProcess(argv, 0, "", "") + + monkeypatch.setattr("agentdiff.runtime.docker.subprocess.run", fake_run) + runtime = DockerRuntime( + tmp_path, + executable=sys.executable, + image="generator:test", + control_timeout_seconds=3.5, + ) + + runtime._docker_call([runtime.executable, "version"], capture_output=True) + + assert observed["timeout"] == 3.5 + + +def test_docker_create_and_inspect_timeouts_have_deterministic_stages( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime = DockerRuntime( + tmp_path, + executable=sys.executable, + image="generator:test", + control_timeout_seconds=0.01, + ) + runtime.configure_source(tmp_path) + + def time_out(argv: Sequence[str], *, capture_output: bool) -> Any: + del capture_output + raise subprocess.TimeoutExpired(argv, 0.01) + + monkeypatch.setattr(runtime, "_docker_call", time_out) + + with pytest.raises(DockerLifecycleError) as create_error: + runtime.run(("agent-worker",), timeout_seconds=1) + assert create_error.value.stage == "create" + runtime.close() + + with pytest.raises(DockerLifecycleError) as inspect_error: + runtime._image_digest() + assert inspect_error.value.stage == "inspect" + + +def test_lifecycle_timeout_is_preserved_as_migration_error_evidence(tmp_path: Path) -> None: + root = _repository(tmp_path) + engine = MigrationEngine( + root, + manifest=get_builtin_manifest("openai", "chat_to_responses"), + generator=CustomCommandGenerator(("agent-worker",)), + generator_runtime_factory=_runtime_factory(root, lifecycle_error_stage="create"), + proof_environment_factory=fake_env_factory(), + ) + + result = engine.run() + + assert result.proof_verdict == "NOT_PROVEN" + assert "Docker generator create timed out" in result.errors + runtime_evidence = RunStore.open(root, result.run_id).read_json("runtime.json") + assert runtime_evidence["execution_error"] == { + "type": "DockerLifecycleError", + "errno": errno.ETIMEDOUT, + } + + +def test_cli_exposes_generator_image_for_api_and_fleet_migrations() -> None: + from agentdiff.cli import build_parser + + parser = build_parser() + api_args = parser.parse_args( + [ + "api", + "migrate", + "--change", + "chat_to_responses", + "--generator", + "command", + "--generator-argv", + "agent-worker", + "--generator-image", + "registry.example/worker@sha256:abc", + ] + ) + fleet_args = parser.parse_args( + [ + "fleet", + "migrate", + "--config", + "fleet.yml", + "--generator-image", + "registry.example/worker:1", + ] + ) + + assert api_args.generator_image == "registry.example/worker@sha256:abc" + assert fleet_args.generator_image == "registry.example/worker:1" + + +def test_docker_timeout_is_a_failed_generation(tmp_path: Path) -> None: + root = _repository(tmp_path) + _, plan = _plan(root) + generator = CustomCommandGenerator(("agent-worker",)) + runtime = _runtime_factory(root, returncode=124, timed_out=True)(plan, generator) + runtime.configure_source(root) + + result = runtime.run(generator.argv, timeout_seconds=0.01) + runtime.close() + + assert result.timed_out is True + assert runtime.generation_result is not None + assert runtime.generation_result.success is False + assert runtime.generation_result.errors == ("Docker generator start/attach timed out",) + + +@pytest.mark.parametrize( + "generator_class", + [CustomCommandGenerator, ExternalCodingAgentGenerator], +) +def test_unavailable_docker_fails_closed_without_host_fallback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + generator_class: type[CustomCommandGenerator], +) -> None: + root = _repository(tmp_path) + marker = tmp_path / "host-fallback" + command = ( + sys.executable, + "-c", + f"from pathlib import Path; Path({str(marker)!r}).touch()", + ) + monkeypatch.setattr("agentdiff.runtime.docker.shutil.which", lambda _executable: None) + engine = MigrationEngine( + root, + manifest=get_builtin_manifest("openai", "chat_to_responses"), + generator=generator_class(command), + ) + + with pytest.raises(FileNotFoundError, match="Docker executable not found"): + engine.run() + + assert not marker.exists() diff --git a/tests/test_delta_proof.py b/tests/test_delta_proof.py new file mode 100644 index 0000000..edac384 --- /dev/null +++ b/tests/test_delta_proof.py @@ -0,0 +1,161 @@ +"""Focused tests for deterministic causal delta proof.""" + +from __future__ import annotations + +from agentdiff.proof.delta import ( + BehaviorContract, + BehaviorExecution, + BehaviorExecutionStatus, + BehaviorExpectation, + DeltaProofEngine, + DeltaProofVerdict, +) + + +def test_bugfix_requires_same_exact_argv_to_fail_then_pass() -> None: + seen: list[tuple[str, tuple[str, ...], float]] = [] + + def baseline(command: tuple[str, ...], timeout: float) -> bool: + seen.append(("baseline", command, timeout)) + return False + + def patched(command: tuple[str, ...], timeout: float) -> bool: + seen.append(("patched", command, timeout)) + return True + + contract = BehaviorContract( + id="expired-token", + command=("pytest", "tests/auth.py::test_expired"), + expectation=BehaviorExpectation.BUGFIX, + ) + result = DeltaProofEngine(baseline, patched).prove(contract, timeout_seconds=12.0) + + assert result.verdict is DeltaProofVerdict.CAUSAL_DELTA_CONFIRMED + assert result.causally_proven is True + assert result.baseline_status is BehaviorExecutionStatus.FAIL + assert result.patched_status is BehaviorExecutionStatus.PASS + assert seen == [ + ("baseline", contract.command, 12.0), + ("patched", contract.command, 12.0), + ] + + +def test_feature_is_not_causal_when_behavior_already_passes_on_base() -> None: + contract = BehaviorContract( + id="new-export", + command=("pytest", "tests/test_export.py::test_json"), + expectation="FEATURE", + ) + result = DeltaProofEngine( + lambda _command, _timeout: True, + lambda _command, _timeout: True, + ).prove(contract) + + assert result.verdict is DeltaProofVerdict.DELTA_NOT_CONFIRMED + assert result.causally_proven is False + assert "baseline was PASS; expected FAIL" in result.reasons + + +def test_bugfix_is_not_causal_when_patch_still_fails() -> None: + contract = BehaviorContract( + id="still-broken", + command=("pytest", "tests/test_bug.py"), + expectation="bugfix", + baseline="fail", + patched="pass", + ) + result = DeltaProofEngine( + lambda _command, _timeout: False, + lambda _command, _timeout: False, + ).prove(contract) + + assert result.verdict is DeltaProofVerdict.DELTA_NOT_CONFIRMED + assert result.causally_proven is False + assert "patched state was FAIL; expected PASS" in result.reasons + + +def test_refactor_can_require_stable_observable_equivalence() -> None: + command = ("python", "contract.py") + contract = BehaviorContract( + id="serialization-refactor", + command=command, + expectation=BehaviorExpectation.REFACTOR, + repetitions=2, + compare_output=True, + ) + + def execution(_command: tuple[str, ...], _timeout: float) -> BehaviorExecution: + return BehaviorExecution(command=command, returncode=0, output='{"ok":true}') + + result = DeltaProofEngine(execution, execution).prove(contract) + + assert result.verdict is DeltaProofVerdict.BEHAVIOR_EQUIVALENCE_CONFIRMED + assert result.causally_proven is True + assert result.to_dict()["baseline_runs"][0]["output_sha256"] + assert "output" not in result.to_dict()["baseline_runs"][0] + + +def test_refactor_reports_changed_observable_output_as_regression() -> None: + command = ("python", "contract.py") + contract = BehaviorContract( + id="same-output", + command=command, + expectation=BehaviorExpectation.REFACTOR, + compare_output=True, + ) + + def baseline(_command: tuple[str, ...], _timeout: float) -> BehaviorExecution: + return BehaviorExecution(command, 0, output="before") + + def patched(_command: tuple[str, ...], _timeout: float) -> BehaviorExecution: + return BehaviorExecution(command, 0, output="after") + + result = DeltaProofEngine(baseline, patched).prove(contract) + + assert result.verdict is DeltaProofVerdict.REGRESSION_DETECTED + assert result.reasons == ("refactor changed the declared observable output",) + + +def test_repetitions_report_flaky_behavior_without_model_judgment() -> None: + baseline_results = iter((False, True)) + contract = BehaviorContract( + id="flaky-bugfix", + command=("pytest", "tests/test_flaky.py"), + expectation=BehaviorExpectation.BUGFIX, + repetitions=2, + ) + result = DeltaProofEngine( + lambda _command, _timeout: next(baseline_results), + lambda _command, _timeout: True, + ).prove(contract) + + assert result.verdict is DeltaProofVerdict.FLAKY + assert result.flaky is True + assert result.reasons == ("baseline behavior was not stable across repetitions",) + + +def test_timeout_and_mismatched_command_are_explicit() -> None: + contract = BehaviorContract( + id="bounded", + command=("pytest", "tests/test_bounded.py"), + expectation=BehaviorExpectation.BUGFIX, + ) + + def timeout(_command: tuple[str, ...], _seconds: float) -> bool: + raise TimeoutError("budget reached") + + timed_out = DeltaProofEngine(timeout, lambda _command, _timeout: True).prove(contract) + assert timed_out.verdict is DeltaProofVerdict.TIMEOUT + assert timed_out.timed_out is True + + mismatched = DeltaProofEngine( + lambda _command, _timeout: BehaviorExecution(("other",), 1), + lambda _command, _timeout: True, + ).prove(contract) + assert mismatched.verdict is DeltaProofVerdict.INVALID + + malformed = DeltaProofEngine( + lambda _command, _timeout: {"command": None, "returncode": "not-an-int"}, + lambda _command, _timeout: True, + ).prove(contract) + assert malformed.verdict is DeltaProofVerdict.INVALID diff --git a/tests/test_flake_firewall.py b/tests/test_flake_firewall.py new file mode 100644 index 0000000..fa697a7 --- /dev/null +++ b/tests/test_flake_firewall.py @@ -0,0 +1,137 @@ +"""Focused tests for deterministic repeated-sample flake classification.""" + +from __future__ import annotations + +from agentdiff.proof import ( + FlakeCheck, + FlakeClassification, + FlakeExecution, + FlakeFirewall, + FlakeSampleStatus, + InfrastructureFailure, +) + + +def test_repeated_exact_argv_passes_are_stable_proof_eligible() -> None: + seen: list[tuple[tuple[str, ...], float]] = [] + + def runner(command: tuple[str, ...], timeout: float) -> FlakeExecution: + seen.append((command, timeout)) + return FlakeExecution(0, command=command, output="3 passed") + + check = FlakeCheck("unit", ("pytest", "tests/test_unit.py"), repetitions=3) + result = FlakeFirewall(runner).assess(check, timeout_seconds=12.0) + + assert result.classification is FlakeClassification.STABLE_PASS + assert result.pass_count == 3 + assert result.fail_count == 0 + assert result.counts_as_proof is True + assert result.repair_eligible is False + assert seen == [(check.command, 12.0)] * 3 + assert len(result.sample_digests) == 3 + assert result.samples[0].output_sha256 + assert "output" not in result.to_dict()["samples"][0] + + +def test_repeated_failures_are_stable_and_only_state_eligible_for_repair() -> None: + result = FlakeFirewall(lambda _command, _timeout: 1).assess( + FlakeCheck("failing", ("pytest", "tests/test_failure.py")) + ) + + assert result.classification is FlakeClassification.STABLE_FAIL + assert result.fail_count == 3 + assert result.counts_as_proof is False + assert result.repair_eligible is True + assert result.may_enter_repair is True + assert result.quarantine_candidate is None + + +def test_mixed_pass_and_fail_is_flaky_and_blocked_from_proof_and_repair() -> None: + outcomes = iter((1, 0, 0)) + result = FlakeFirewall(lambda _command, _timeout: next(outcomes)).assess( + FlakeCheck("intermittent", ("pytest", "tests/test_intermit.py")) + ) + + assert result.classification is FlakeClassification.FLAKY + assert result.pass_count == 2 + assert result.fail_count == 1 + assert result.counts_as_proof is False + assert result.may_enter_repair is False + assert result.quarantine_candidate is not None + assert result.quarantine_candidate.automatic_quarantine is False + assert result.automatic_quarantine is False + assert result.to_dict()["quarantine_candidate"]["fingerprint"] + + +def test_infrastructure_failure_is_distinct_and_inconclusive() -> None: + calls = 0 + + def runner(_command: tuple[str, ...], _timeout: float) -> bool: + nonlocal calls + calls += 1 + if calls == 2: + raise InfrastructureFailure("runner unavailable") + return True + + result = FlakeFirewall(runner).assess(FlakeCheck("infra", ("pytest", "tests/test_infra.py"))) + + assert result.classification is FlakeClassification.INCONCLUSIVE + assert result.infrastructure_failure_count == 1 + assert result.samples[1].status is FlakeSampleStatus.INFRASTRUCTURE_FAILURE + assert result.counts_as_proof is False + assert result.may_enter_repair is False + assert result.quarantine_candidate is None + + +def test_timeout_and_invalid_results_are_inconclusive_not_repair_inputs() -> None: + timed_out = FlakeFirewall( + lambda _command, _timeout: {"timed_out": True, "detail": "capacity"} + ).assess(FlakeCheck("timeout", ("pytest", "tests/test_timeout.py"))) + invalid = FlakeFirewall( + lambda _command, _timeout: {"valid": False, "detail": "malformed runner data"} + ).assess(FlakeCheck("invalid", ("pytest", "tests/test_invalid.py"))) + + assert timed_out.classification is FlakeClassification.INCONCLUSIVE + assert timed_out.timeout_count == 3 + assert timed_out.may_enter_repair is False + assert invalid.classification is FlakeClassification.INCONCLUSIVE + assert invalid.invalid_count == 3 + assert invalid.counts_as_proof is False + + +def test_sample_bound_cannot_be_reported_as_stable() -> None: + result = FlakeFirewall( + lambda _command, _timeout: True, + max_samples=3, + ).assess(FlakeCheck("bounded", ("pytest", "tests/test_bounded.py"), repetitions=5)) + + assert len(result.samples) == 3 + assert result.requested_samples == 5 + assert result.complete is False + assert result.classification is FlakeClassification.INCONCLUSIVE + assert result.counts_as_proof is False + assert "flake sample-count budget reached" in result.reasons + + +def test_declared_output_variance_can_mark_same_exit_status_flaky() -> None: + outputs = iter(("alpha", "beta", "alpha")) + command = ("python", "observable.py") + result = FlakeFirewall( + lambda _command, _timeout: FlakeExecution(0, command=command, output=next(outputs)) + ).assess(FlakeCheck("observable", command, compare_output=True)) + + assert result.classification is FlakeClassification.FLAKY + assert result.pass_count == 3 + assert len({sample.output_sha256 for sample in result.samples}) == 2 + assert result.may_enter_repair is False + assert result.quarantine_candidate is not None + + +def test_mismatched_advertised_command_is_invalid_evidence() -> None: + result = FlakeFirewall( + lambda _command, _timeout: FlakeExecution(0, command=("other",)), + ).assess(FlakeCheck("exact", ("pytest", "tests/test_exact.py"))) + + assert result.classification is FlakeClassification.INCONCLUSIVE + assert result.invalid_count == 3 + assert result.counts_as_proof is False diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py new file mode 100644 index 0000000..4ce1fca --- /dev/null +++ b/tests/test_maintenance.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +import json +import os +import stat +from dataclasses import replace +from typing import TYPE_CHECKING + +import pytest + +import agentdiff.maintenance.store as maintenance_store +from agentdiff.api.manifest import get_builtin_manifest +from agentdiff.maintenance import ( + ChangeDelivery, + ChangeProof, + ChangeSource, + ChangeSpec, + ChangeStrategy, + ChangeTargets, + LocalJSONMaintenanceStore, + MaintenanceEdge, + MaintenanceEntityKind, + MaintenanceGraph, + MaintenanceGraphStore, + MaintenanceNode, + MaintenanceSeverity, + PriorityDecision, + PriorityEngine, + PriorityInputs, + ProofRequirement, + StrategyDisposition, +) + +if TYPE_CHECKING: + from pathlib import Path + + +_NOW = "2026-08-22T12:00:00+00:00" + + +def _spec() -> ChangeSpec: + return ChangeSpec( + kind="upstream_change", + external_id="sdk-4-removal", + title="Remove the retired SDK call", + description="The old call is unavailable in SDK 4.", + source=ChangeSource( + type="sdk_release", + provider="example", + version="4.0.0", + uri="https://example.test/releases/4.0.0", + retrieved_at=_NOW, + content_digest="sha256:release-notes", + provenance=("release-feed", "signed-index"), + ), + targets=ChangeTargets( + symbols=("client.old_call", "Client.old_call"), + packages=("example-sdk",), + ), + severity=MaintenanceSeverity.HIGH, + strategy=ChangeStrategy( + deterministic=StrategyDisposition.PREFERRED, + agent=StrategyDisposition.FALLBACK, + transform_id="example-old-call-v1", + ), + proof=ChangeProof( + required=ProofRequirement.CAUSAL, + behavior_contracts=("old-call-is-replaced",), + ), + delivery=ChangeDelivery(open_pr=True), + deadline="2026-09-30", + confidence=0.9, + ) + + +def _nodes() -> tuple[MaintenanceNode, MaintenanceNode]: + spec = _spec() + change = MaintenanceNode( + kind=MaintenanceEntityKind.CHANGE_SPEC, + key=spec.id, + source=spec.source.uri, + timestamp=_NOW, + status="validated", + provenance=("change-radar", spec.source.uri), + confidence=spec.confidence, + evidence=(spec.digest,), + content_digest=spec.digest, + ) + repository = MaintenanceNode( + kind=MaintenanceEntityKind.REPOSITORY, + key="github.com/example/payments", + source="github-app-installation:42", + timestamp=_NOW, + status="exposed", + provenance=("installation:42", "exposure-scan:123"), + confidence=1.0, + evidence=("blob:payments-client.py",), + content_digest="sha256:repository-tree", + ) + return change, repository + + +def _graph() -> MaintenanceGraph: + change, repository = _nodes() + edge = MaintenanceEdge( + source_id=change.id, + target_id=repository.id, + relationship="affects", + source="exposure-scan:123", + timestamp=_NOW, + status="confirmed", + provenance=("scanner:v1",), + confidence=0.95, + evidence=("blob:payments-client.py#L12",), + ) + return MaintenanceGraph(nodes=(repository, change), edges=(edge,)) + + +def _priority_inputs() -> PriorityInputs: + return PriorityInputs( + security_severity=MaintenanceSeverity.MODERATE, + repository_criticality=50, + usage_count=5, + blast_radius=40, + version_distance=2, + deterministic_transform=True, + test_quality=80, + proof_available=True, + owners=("payments",), + maintenance_window_open=True, + ) + + +def test_change_spec_is_immutable_canonical_and_round_trips() -> None: + spec = _spec() + reordered = replace( + spec, + source=replace( + spec.source, + retrieved_at="2026-08-22T05:00:00-07:00", + provenance=tuple(reversed(spec.source.provenance)), + ), + targets=replace(spec.targets, symbols=tuple(reversed(spec.targets.symbols))), + ) + + assert reordered == spec + assert reordered.id == spec.id + assert reordered.digest == spec.digest + assert ChangeSpec.from_dict(json.loads(json.dumps(spec.to_dict()))) == spec + with pytest.raises(AttributeError): + spec.title = "mutated" # type: ignore[misc] + + revised = replace(spec, description="A more detailed explanation.") + assert revised.id == spec.id + assert revised.digest != spec.digest + assert replace(spec, confidence=-0.0).digest == replace(spec, confidence=0.0).digest + + +def test_change_spec_adapts_existing_api_manifest_without_mutating_it() -> None: + manifest = get_builtin_manifest("openai", "chat_to_responses") + assert manifest is not None + original = manifest.to_dict() + + spec = ChangeSpec.from_api_manifest(manifest) + + assert manifest.to_dict() == original + assert spec.kind == "upstream_change" + assert spec.external_id == manifest.change_id + assert spec.source.provider == manifest.provider + assert spec.targets.symbols == manifest.affected.symbols + assert spec.strategy.deterministic is StrategyDisposition.PREFERRED + assert spec.strategy.agent is StrategyDisposition.FALLBACK + assert spec.proof.required is ProofRequirement.STRONG + assert spec.delivery == ChangeDelivery(open_pr=False, auto_merge=False) + + +def test_change_spec_rejects_malformed_and_tampered_data() -> None: + with pytest.raises(ValueError, match="at least one selector"): + ChangeTargets() + with pytest.raises(ValueError, match="requires"): + ChangeDelivery(auto_merge=True) + with pytest.raises(ValueError, match="object"): + ChangeSpec.from_dict([]) + + unknown = _spec().to_dict() + unknown["surprise"] = True + with pytest.raises(ValueError, match="unknown fields"): + ChangeSpec.from_dict(unknown) + + invalid_severity = _spec().to_dict() + invalid_severity["severity"] = "urgent-ish" + with pytest.raises(ValueError, match="severity"): + ChangeSpec.from_dict(invalid_severity) + + tampered = _spec().to_dict() + tampered["title"] = "Unverified replacement title" + with pytest.raises(ValueError, match="digest"): + ChangeSpec.from_dict(tampered) + + +def test_graph_is_canonical_provenance_rich_and_round_trips() -> None: + graph = _graph() + reordered = MaintenanceGraph( + nodes=tuple(reversed(graph.nodes)), + edges=tuple(reversed(graph.edges)), + ) + + assert reordered == graph + assert reordered.id == graph.id + assert reordered.digest == graph.digest + assert MaintenanceGraph.from_dict(json.loads(json.dumps(graph.to_dict()))) == graph + assert all(node.provenance and node.evidence and node.digest for node in graph.nodes) + assert all(edge.provenance and edge.evidence and edge.digest for edge in graph.edges) + + +def test_graph_rejects_duplicate_dangling_and_tampered_data() -> None: + change, repository = _nodes() + edge = MaintenanceEdge( + source_id=change.id, + target_id=repository.id, + relationship="affects", + source="exposure-scan:123", + timestamp=_NOW, + status="confirmed", + ) + with pytest.raises(ValueError, match="duplicate node"): + MaintenanceGraph(nodes=(change, change)) + with pytest.raises(ValueError, match="duplicate edge"): + MaintenanceGraph(nodes=(change, repository), edges=(edge, edge)) + + dangling = replace(edge, source_id="change-spec-missing") + with pytest.raises(ValueError, match="dangling"): + MaintenanceGraph(nodes=(change, repository), edges=(dangling,)) + + tampered = _graph().to_dict() + tampered["nodes"][0]["status"] = "silently-modified" + with pytest.raises(ValueError, match="digest"): + MaintenanceGraph.from_dict(tampered) + + +def test_local_store_round_trips_deterministically_and_privately(tmp_path: Path) -> None: + path = tmp_path / "state" / "maintenance-graph.json" + store = LocalJSONMaintenanceStore(path) + graph = _graph() + + assert isinstance(store, MaintenanceGraphStore) + assert store.load() is None + assert store.save(graph) == path + first_payload = path.read_bytes() + assert store.load() == graph + store.save(graph) + assert path.read_bytes() == first_payload + if os.name != "nt": + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + +def test_local_store_preserves_old_snapshot_when_replace_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "maintenance-graph.json" + store = LocalJSONMaintenanceStore(path) + graph = _graph() + store.save(graph) + original = path.read_bytes() + outcome = MaintenanceNode( + kind=MaintenanceEntityKind.OUTCOME, + key="deployment:42", + source="post-merge-verifier", + timestamp=_NOW, + status="healthy", + ) + updated = MaintenanceGraph(nodes=(*graph.nodes, outcome), edges=graph.edges) + + def fail_replace(_source: object, _target: object) -> None: + raise OSError("injected replacement failure") + + monkeypatch.setattr(maintenance_store.os, "replace", fail_replace) + with pytest.raises(OSError, match="injected"): + store.save(updated) + + assert path.read_bytes() == original + assert store.load() == graph + assert not tuple(tmp_path.glob(f".{path.name}.*.tmp")) + + +def test_local_store_rejects_symlink_paths(tmp_path: Path) -> None: + outside = tmp_path / "outside.json" + outside.write_text("{}", encoding="utf-8") + leaf_link = tmp_path / "leaf-link.json" + real_parent = tmp_path / "real-parent" + real_parent.mkdir() + parent_link = tmp_path / "parent-link" + try: + leaf_link.symlink_to(outside) + parent_link.symlink_to(real_parent, target_is_directory=True) + except OSError: + pytest.skip("symlinks are unavailable") + + with pytest.raises(ValueError, match="symlink"): + LocalJSONMaintenanceStore(leaf_link).load() + with pytest.raises(ValueError, match="symlink"): + LocalJSONMaintenanceStore(parent_link / "graph.json").save(_graph()) + + +def test_local_store_rejects_invalid_and_oversized_json(tmp_path: Path) -> None: + path = tmp_path / "maintenance-graph.json" + path.write_text("{not-json", encoding="utf-8") + with pytest.raises(ValueError, match="invalid JSON"): + LocalJSONMaintenanceStore(path).load() + + path.write_text("12345", encoding="utf-8") + with pytest.raises(ValueError, match="size limit"): + LocalJSONMaintenanceStore(path, max_bytes=4).load() + + +def test_priority_engine_produces_all_decisions_deterministically() -> None: + engine = PriorityEngine() + base = _priority_inputs() + fix_now = replace( + base, + breaking_deadline="2026-08-27T12:00:00Z", + evaluated_at=_NOW, + ) + urgent = replace( + base, + security_severity=MaintenanceSeverity.HIGH, + repository_criticality=100, + usage_count=15, + blast_radius=100, + version_distance=5, + ) + + cases = { + PriorityDecision.FIX_NOW: fix_now, + PriorityDecision.SCHEDULE: replace(urgent, maintenance_window_open=False), + PriorityDecision.REVIEW: replace(urgent, owners=()), + PriorityDecision.IGNORE: replace(base, usage_count=0), + } + for expected, inputs in cases.items(): + first = engine.decide(inputs) + assert first.decision is expected + assert first == engine.decide(inputs) + assert first.reasons + + +def test_priority_inputs_round_trip_and_reject_ambiguous_data() -> None: + inputs = _priority_inputs() + assert PriorityInputs.from_dict(inputs.to_dict()) == inputs + + malformed = inputs.to_dict() + malformed["maintenance_window_open"] = 1 + with pytest.raises(ValueError, match="boolean"): + PriorityInputs.from_dict(malformed) + with pytest.raises(ValueError, match="between 0 and 100"): + replace(inputs, test_quality=101) + with pytest.raises(ValueError, match="provided together"): + replace(inputs, breaking_deadline="2026-09-01T00:00:00Z") diff --git a/tests/test_maintenance_scheduler.py b/tests/test_maintenance_scheduler.py new file mode 100644 index 0000000..76afd79 --- /dev/null +++ b/tests/test_maintenance_scheduler.py @@ -0,0 +1,395 @@ +"""Focused tests for deterministic side-effect-free maintenance scheduling.""" + +from __future__ import annotations + +import json +from dataclasses import replace + +import pytest + +from agentdiff.maintenance import ( + ActiveMaintenanceRun, + MaintenanceQueueItem, + MaintenanceScheduler, + MaintenanceSchedulerState, + SchedulerAction, + SchedulerLimits, + SchedulerMode, + SchedulerReason, + SchedulerResources, +) + + +def _resources( + *, + cpu_millis: int = 100, + memory_mb: int = 256, + rate_units: int = 1, +) -> SchedulerResources: + return SchedulerResources( + cpu_millis=cpu_millis, + memory_mb=memory_mb, + rate_units=rate_units, + ) + + +def _limits( + *, + global_concurrency: int = 4, + per_repository_concurrency: int = 1, + cpu_millis: int = 4_000, + memory_mb: int = 8_192, + rate_units: int = 100, +) -> SchedulerLimits: + return SchedulerLimits( + global_concurrency=global_concurrency, + per_repository_concurrency=per_repository_concurrency, + cpu_millis=cpu_millis, + memory_mb=memory_mb, + rate_units=rate_units, + ) + + +def _state( + *, + mode: SchedulerMode = SchedulerMode.RUNNING, + limits: SchedulerLimits | None = None, + active_runs: tuple[ActiveMaintenanceRun, ...] = (), + rate_units_used: int = 0, + inputs_complete: bool = True, +) -> MaintenanceSchedulerState: + return MaintenanceSchedulerState( + mode=mode, + limits=limits or _limits(), + active_runs=active_runs, + rate_units_used=rate_units_used, + inputs_complete=inputs_complete, + ) + + +def _item( + stable_id: str, + *, + repository_id: str = "repo/default", + priority: int = 50, + resources: SchedulerResources | None = None, + maintenance_window_open: bool = True, + inputs_complete: bool = True, + deadline: str = "", +) -> MaintenanceQueueItem: + return MaintenanceQueueItem( + stable_id=stable_id, + repository_id=repository_id, + priority=priority, + resources=resources or _resources(), + maintenance_window_open=maintenance_window_open, + inputs_complete=inputs_complete, + deadline=deadline, + ) + + +def _active( + stable_id: str, + *, + repository_id: str = "repo/active", + resources: SchedulerResources | None = None, +) -> ActiveMaintenanceRun: + return ActiveMaintenanceRun( + stable_id=stable_id, + repository_id=repository_id, + resources=resources or _resources(), + ) + + +def test_queue_order_is_priority_then_deadline_then_stable_id() -> None: + items = ( + _item("priority-80", priority=80, deadline="2026-08-01T00:00:00Z"), + _item("no-deadline", priority=90), + _item("same-b", priority=90, deadline="2026-08-02T00:00:00Z"), + _item("later", priority=90, deadline="2026-08-03T00:00:00Z"), + _item("same-a", priority=90, deadline="2026-08-02T00:00:00Z"), + ) + state = _state(limits=_limits(global_concurrency=10, per_repository_concurrency=10)) + scheduler = MaintenanceScheduler() + + forward = scheduler.plan(items, state) + reverse = scheduler.plan(tuple(reversed(items)), state) + + expected = ("same-a", "same-b", "later", "no-deadline", "priority-80") + assert tuple(decision.stable_id for decision in forward.decisions) == expected + assert tuple(decision.queue_position for decision in forward.decisions) == (1, 2, 3, 4, 5) + assert forward.decisions[0].priority == 90 + assert forward.decisions[0].deadline == "2026-08-02T00:00:00+00:00" + assert forward.decisions[0].resources == items[2].resources + assert forward.state_digest == state.digest + assert forward == reverse + assert forward.digest == reverse.digest + + +def test_inputs_are_immutable_canonical_and_round_trip() -> None: + item = _item( + "canonical", + deadline="2026-08-22T05:00:00-07:00", + ) + first = _active("b", repository_id="repo/b") + second = _active("a", repository_id="repo/a") + state = _state( + active_runs=(first, second), + rate_units_used=2, + limits=_limits(per_repository_concurrency=2), + ) + + restored_item = MaintenanceQueueItem.from_dict(json.loads(json.dumps(item.to_dict()))) + restored_state = MaintenanceSchedulerState.from_dict(json.loads(json.dumps(state.to_dict()))) + + assert restored_item == item + assert item.deadline == "2026-08-22T12:00:00+00:00" + assert restored_state == state + assert tuple(run.stable_id for run in state.active_runs) == ("a", "b") + assert replace(item, priority=51).digest != item.digest + with pytest.raises(AttributeError): + item.priority = 10 # type: ignore[misc] + + +def test_pause_and_resume_are_explicit_and_side_effect_free() -> None: + scheduler = MaintenanceScheduler() + running = _state() + paused = running.pause() + item = _item("pause-me") + + paused_plan = scheduler.plan((item,), paused) + + assert running.mode is SchedulerMode.RUNNING + assert paused.mode is SchedulerMode.PAUSED + assert paused_plan.decisions[0].action is SchedulerAction.DEFER + assert paused_plan.decisions[0].reasons == (SchedulerReason.SCHEDULER_PAUSED,) + assert paused.resume() == running + assert scheduler.plan((item,), paused.resume()).decisions[0].action is SchedulerAction.RUN + + +def test_closed_maintenance_window_defers_without_reserving_resources() -> None: + state = _state() + item = _item("closed-window", maintenance_window_open=False) + + plan = MaintenanceScheduler().plan((item,), state) + + assert plan.decisions[0].action is SchedulerAction.DEFER + assert plan.decisions[0].reasons == (SchedulerReason.MAINTENANCE_WINDOW_CLOSED,) + assert plan.planned_active_runs == plan.starting_active_runs + assert plan.planned_cpu_millis == plan.starting_cpu_millis + + +def test_incomplete_scheduler_and_item_inputs_fail_closed() -> None: + state = _state(inputs_complete=False) + item = _item("incomplete", inputs_complete=False) + + decision = MaintenanceScheduler().plan((item,), state).decisions[0] + + assert decision.action is SchedulerAction.DEFER + assert decision.reasons == ( + SchedulerReason.SCHEDULER_INPUTS_INCOMPLETE, + SchedulerReason.ITEM_INPUTS_INCOMPLETE, + ) + + +def test_impossible_resource_request_is_deferred_not_scheduled() -> None: + state = _state(limits=_limits(cpu_millis=100, memory_mb=100, rate_units=1)) + item = _item( + "impossible", + resources=_resources(cpu_millis=101, memory_mb=101, rate_units=2), + ) + + decision = MaintenanceScheduler().plan((item,), state).decisions[0] + + assert decision.action is SchedulerAction.DEFER + assert decision.reasons == ( + SchedulerReason.CPU_REQUEST_EXCEEDS_LIMIT, + SchedulerReason.MEMORY_REQUEST_EXCEEDS_LIMIT, + SchedulerReason.RATE_REQUEST_EXCEEDS_LIMIT, + ) + + +def test_global_concurrency_exhaustion_schedules_for_later() -> None: + active = _active("running") + state = _state( + active_runs=(active,), + limits=_limits(global_concurrency=1, per_repository_concurrency=1), + rate_units_used=active.resources.rate_units, + ) + + decision = ( + MaintenanceScheduler() + .plan((_item("waiting", repository_id="repo/other"),), state) + .decisions[0] + ) + + assert decision.action is SchedulerAction.SCHEDULE + assert decision.reasons == (SchedulerReason.GLOBAL_CONCURRENCY_EXHAUSTED,) + + +def test_per_repository_concurrency_isolated_from_other_repositories() -> None: + active = _active("running", repository_id="repo/a") + state = _state( + active_runs=(active,), + limits=_limits(global_concurrency=3, per_repository_concurrency=1), + rate_units_used=active.resources.rate_units, + ) + same = _item("same", repository_id="repo/a", priority=90) + other = _item("other", repository_id="repo/b", priority=80) + + decisions = MaintenanceScheduler().plan((other, same), state).decisions + + assert decisions[0].stable_id == "same" + assert decisions[0].action is SchedulerAction.SCHEDULE + assert decisions[0].reasons == (SchedulerReason.REPOSITORY_CONCURRENCY_EXHAUSTED,) + assert decisions[1].stable_id == "other" + assert decisions[1].action is SchedulerAction.RUN + + +@pytest.mark.parametrize( + ("limits", "active_resources", "rate_used", "item_resources", "reason"), + ( + ( + _limits(cpu_millis=100), + _resources(cpu_millis=80), + 1, + _resources(cpu_millis=21), + SchedulerReason.CPU_CURRENTLY_EXHAUSTED, + ), + ( + _limits(memory_mb=100), + _resources(memory_mb=80), + 1, + _resources(memory_mb=21), + SchedulerReason.MEMORY_CURRENTLY_EXHAUSTED, + ), + ( + _limits(rate_units=10), + _resources(rate_units=1), + 9, + _resources(rate_units=2), + SchedulerReason.RATE_CURRENTLY_EXHAUSTED, + ), + ), +) +def test_current_resource_pressure_schedules_for_later( + limits: SchedulerLimits, + active_resources: SchedulerResources, + rate_used: int, + item_resources: SchedulerResources, + reason: SchedulerReason, +) -> None: + active = _active("running", resources=active_resources) + state = _state( + limits=limits, + active_runs=(active,), + rate_units_used=rate_used, + ) + item = _item("waiting", repository_id="repo/other", resources=item_resources) + + decision = MaintenanceScheduler().plan((item,), state).decisions[0] + + assert decision.action is SchedulerAction.SCHEDULE + assert decision.reasons == (reason,) + + +def test_batch_planning_reserves_capacity_without_mutating_state() -> None: + state = _state(limits=_limits(global_concurrency=2, per_repository_concurrency=2)) + items = ( + _item("third", repository_id="repo/c", priority=10), + _item("first", repository_id="repo/a", priority=30), + _item("second", repository_id="repo/b", priority=20), + ) + original = state.to_dict() + + plan = MaintenanceScheduler().plan(items, state) + + assert tuple(decision.action for decision in plan.decisions) == ( + SchedulerAction.RUN, + SchedulerAction.RUN, + SchedulerAction.SCHEDULE, + ) + assert plan.decisions[2].reasons == (SchedulerReason.GLOBAL_CONCURRENCY_EXHAUSTED,) + assert plan.planned_active_runs == 2 + assert plan.planned_cpu_millis == 200 + assert plan.planned_memory_mb == 512 + assert plan.planned_rate_units == 2 + assert state.to_dict() == original + + +def test_planned_runs_also_enforce_per_repository_concurrency() -> None: + state = _state(limits=_limits(global_concurrency=3, per_repository_concurrency=1)) + items = ( + _item("repo-a-2", repository_id="repo/a", priority=20), + _item("repo-b", repository_id="repo/b", priority=10), + _item("repo-a-1", repository_id="repo/a", priority=30), + ) + + decisions = MaintenanceScheduler().plan(items, state).decisions + + assert tuple(decision.action for decision in decisions) == ( + SchedulerAction.RUN, + SchedulerAction.SCHEDULE, + SchedulerAction.RUN, + ) + assert decisions[1].reasons == (SchedulerReason.REPOSITORY_CONCURRENCY_EXHAUSTED,) + + +def test_zero_rate_work_can_run_when_rate_budget_is_zero() -> None: + state = _state(limits=_limits(rate_units=0)) + free = _item("no-api", resources=_resources(rate_units=0)) + charged = _item("needs-api", resources=_resources(rate_units=1)) + + decisions = MaintenanceScheduler().plan((charged, free), state).decisions + + assert decisions[0].action is SchedulerAction.DEFER + assert decisions[0].reasons == (SchedulerReason.RATE_REQUEST_EXCEEDS_LIMIT,) + assert decisions[1].action is SchedulerAction.RUN + + +def test_duplicate_or_inconsistent_inputs_are_rejected_fail_closed() -> None: + scheduler = MaintenanceScheduler() + item = _item("duplicate") + with pytest.raises(ValueError, match="duplicate stable IDs"): + scheduler.plan((item, item), _state()) + with pytest.raises(ValueError, match="already active"): + scheduler.plan( + (item,), + _state(active_runs=(_active("duplicate"),), rate_units_used=1), + ) + with pytest.raises(TypeError, match="tuple"): + scheduler.plan([item], _state()) # type: ignore[arg-type] + + over_capacity = ( + _active("one", repository_id="repo/a"), + _active("two", repository_id="repo/b"), + ) + with pytest.raises(ValueError, match="global concurrency"): + _state( + active_runs=over_capacity, + limits=_limits(global_concurrency=1), + rate_units_used=2, + ) + with pytest.raises(ValueError, match="rate budget"): + _state(rate_units_used=101) + with pytest.raises(ValueError, match="active reservations"): + _state(active_runs=(_active("unaccounted"),), rate_units_used=0) + + +def test_malformed_serialized_inputs_are_rejected() -> None: + item = _item("strict") + unknown = item.to_dict() + unknown["surprise"] = True + with pytest.raises(ValueError, match="unknown fields"): + MaintenanceQueueItem.from_dict(unknown) + + missing_window = item.to_dict() + missing_window.pop("maintenance_window_open") + with pytest.raises(ValueError, match="boolean"): + MaintenanceQueueItem.from_dict(missing_window) + with pytest.raises(ValueError, match="timezone"): + replace(item, deadline="2026-08-22T12:00:00") + with pytest.raises(ValueError, match="between 0 and 100"): + replace(item, priority=101) + with pytest.raises(ValueError, match="must be a list"): + MaintenanceSchedulerState.from_dict({**_state().to_dict(), "active_runs": {}}) diff --git a/tests/test_patch_minimizer.py b/tests/test_patch_minimizer.py new file mode 100644 index 0000000..e9e2ff9 --- /dev/null +++ b/tests/test_patch_minimizer.py @@ -0,0 +1,96 @@ +"""Focused tests for budgeted proof-preserving patch minimization.""" + +from __future__ import annotations + +from agentdiff.minimize import ( + MinimizationStatus, + PatchGroup, + PatchMinimizer, + ProofCheckStatus, +) + + +def test_delta_debugging_removes_redundant_groups_and_preserves_order() -> None: + groups = tuple(PatchGroup(name) for name in ("required", "formatting", "comment", "extra")) + original = groups + + result = PatchMinimizer( + lambda candidate: any(group.id == "required" for group in candidate) + ).minimize(groups) + + assert groups == original + assert result.status is MinimizationStatus.MINIMIZED + assert tuple(group.id for group in result.minimized_groups) == ("required",) + assert tuple(group.id for group in result.removed_groups) == ( + "formatting", + "comment", + "extra", + ) + assert result.minimal_within_strategy is True + assert result.changed is True + + +def test_interacting_required_groups_survive_practical_ddmin() -> None: + groups = tuple(PatchGroup(name) for name in ("implementation", "test", "comment", "style")) + + def prove(candidate: tuple[PatchGroup, ...]) -> dict[str, bool]: + ids = {group.id for group in candidate} + return {"proven": {"implementation", "test"} <= ids} + + result = PatchMinimizer(prove).minimize(groups) + + assert tuple(group.id for group in result.kept_groups) == ("implementation", "test") + assert result.status is MinimizationStatus.MINIMIZED + assert all( + attempt.status in {ProofCheckStatus.PROVEN, ProofCheckStatus.NOT_PROVEN} + for attempt in result.attempts + ) + + +def test_original_patch_must_be_proven_before_minimization() -> None: + groups = (PatchGroup("one"), PatchGroup("two")) + result = PatchMinimizer(lambda _candidate: False).minimize(groups) + + assert result.status is MinimizationStatus.INITIAL_NOT_PROVEN + assert result.minimized_groups == groups + assert result.proof_runs == 1 + assert result.combinations == 0 + assert result.changed is False + + +def test_proof_run_budget_stops_without_claiming_minimality() -> None: + groups = (PatchGroup("required"), PatchGroup("extra")) + result = PatchMinimizer( + lambda candidate: any(group.id == "required" for group in candidate), + max_proof_runs=1, + ).minimize(groups) + + assert result.status is MinimizationStatus.BUDGET_EXHAUSTED + assert result.complete is False + assert result.minimal_within_strategy is False + assert result.minimized_groups == groups + assert result.reasons == ("proof-run budget reached",) + + +def test_single_redundant_group_can_reduce_to_empty_patch() -> None: + result = PatchMinimizer(lambda _candidate: True).minimize((PatchGroup("redundant"),)) + + assert result.status is MinimizationStatus.MINIMIZED + assert result.minimized_groups == () + assert tuple(group.id for group in result.removed_groups) == ("redundant",) + + +def test_inconclusive_candidate_does_not_claim_minimality() -> None: + groups = (PatchGroup("required"), PatchGroup("uncertain")) + + def prove(candidate: tuple[PatchGroup, ...]) -> bool: + if len(candidate) < len(groups): + raise TimeoutError("candidate budget") + return True + + result = PatchMinimizer(prove).minimize(groups) + + assert result.status is MinimizationStatus.INCONCLUSIVE + assert result.complete is False + assert result.minimal_within_strategy is False + assert result.minimized_groups == groups diff --git a/tests/test_post_merge_outcome.py b/tests/test_post_merge_outcome.py new file mode 100644 index 0000000..c03de27 --- /dev/null +++ b/tests/test_post_merge_outcome.py @@ -0,0 +1,156 @@ +"""Focused tests for evidence-preserving post-merge outcomes.""" + +from __future__ import annotations + +from agentdiff.delivery import SealedDeliveryIdentity +from agentdiff.outcome import ( + OutcomeCheck, + OutcomeCheckResult, + OutcomeCheckStatus, + OutcomeEvidence, + OutcomeState, + PostMergeContext, + PostMergeOutcomeEvaluator, + PostMergeVerifier, +) + + +def _context() -> PostMergeContext: + return PostMergeContext( + merged_commit_sha="merge-a", + delivery_identity=SealedDeliveryIdentity("base-a", "patch-a", "proof-a"), + pull_request_number=42, + ) + + +def test_all_checks_pass_yields_healthy_and_preserves_evidence() -> None: + evidence = OutcomeEvidence( + source="deployment-api", + summary="deployment completed", + digest="sha256:deployment", + reference="deployments/123", + ) + verifier = PostMergeVerifier( + ( + OutcomeCheck( + "deployment", + lambda _context, _timeout: OutcomeCheckResult( + "deployment", + "deployment", + OutcomeCheckStatus.PASS, + evidence=(evidence,), + ), + kind="deployment", + ), + OutcomeCheck("health", lambda _context, _timeout: True, kind="health-endpoint"), + ) + ) + + result = verifier.verify(_context()) + + assert result.state is OutcomeState.HEALTHY + assert result.maintenance_complete is True + assert result.automatic_production_rollback is False + assert result.checks[0].evidence == (evidence,) + assert result.to_dict()["checks"][0]["evidence"][0]["digest"] == "sha256:deployment" + + +def test_explicit_failure_detects_regression_without_default_rollback() -> None: + result = PostMergeVerifier( + ( + OutcomeCheck("deployment", lambda _context, _timeout: True), + OutcomeCheck( + "error-rate", + lambda _context, _timeout: { + "status": "FAIL", + "evidence": { + "source": "metrics", + "summary": "error rate exceeded threshold", + "digest": "sha256:metrics-window", + }, + }, + kind="error-rate", + ), + ) + ).verify(_context()) + + assert result.state is OutcomeState.REGRESSION_DETECTED + assert result.stop_remaining_campaign is True + assert result.notify_human is True + assert result.prepare_revert is False + assert result.automatic_production_rollback is False + assert "STOP_REMAINING_CAMPAIGN" in result.recommended_actions + assert "PREPARE_REVERT_FOR_REVIEW" not in result.recommended_actions + + +def test_prepare_revert_is_review_only_and_never_automatic_rollback() -> None: + result = PostMergeVerifier((OutcomeCheck("canary", lambda _context, _timeout: False),)).verify( + _context(), prepare_revert_on_regression=True + ) + + assert result.state is OutcomeState.REGRESSION_DETECTED + assert result.prepare_revert is True + assert "PREPARE_REVERT_FOR_REVIEW" in result.recommended_actions + assert result.automatic_production_rollback is False + + +def test_timeout_unavailable_or_invalid_evidence_is_inconclusive() -> None: + def timeout(_context: PostMergeContext, _seconds: float) -> bool: + raise TimeoutError("telemetry budget") + + timed_out = PostMergeVerifier((OutcomeCheck("latency", timeout),)).verify(_context()) + unavailable = PostMergeOutcomeEvaluator().evaluate( + _context(), + ( + OutcomeCheckResult( + "health", + "health-endpoint", + OutcomeCheckStatus.UNAVAILABLE, + evidence=(OutcomeEvidence("health", "endpoint unavailable"),), + ), + ), + ) + + assert timed_out.state is OutcomeState.INCONCLUSIVE + assert timed_out.checks[0].status is OutcomeCheckStatus.TIMEOUT + assert unavailable.state is OutcomeState.INCONCLUSIVE + assert unavailable.checks[0].evidence[0].summary == "endpoint unavailable" + assert unavailable.automatic_production_rollback is False + + +def test_empty_or_budget_truncated_check_set_is_inconclusive() -> None: + empty = PostMergeVerifier(()).verify(_context()) + bounded = PostMergeVerifier( + ( + OutcomeCheck("one", lambda _context, _timeout: True), + OutcomeCheck("two", lambda _context, _timeout: True), + ), + max_checks=1, + ).verify(_context()) + + assert empty.state is OutcomeState.INCONCLUSIVE + assert bounded.state is OutcomeState.INCONCLUSIVE + assert bounded.complete is False + assert len(bounded.checks) == 1 + assert "post-merge check-count budget reached" in bounded.reasons + + +def test_mismatched_plugin_result_id_is_invalid_and_evidence_is_retained() -> None: + evidence = OutcomeEvidence("canary", "wrong subject returned") + result = PostMergeVerifier( + ( + OutcomeCheck( + "expected", + lambda _context, _timeout: OutcomeCheckResult( + "other", + "canary", + OutcomeCheckStatus.PASS, + evidence=(evidence,), + ), + ), + ) + ).verify(_context()) + + assert result.state is OutcomeState.INCONCLUSIVE + assert result.checks[0].status is OutcomeCheckStatus.INVALID + assert result.checks[0].evidence == (evidence,) diff --git a/tests/test_pr_reconciler.py b/tests/test_pr_reconciler.py new file mode 100644 index 0000000..7087299 --- /dev/null +++ b/tests/test_pr_reconciler.py @@ -0,0 +1,104 @@ +"""Focused tests for sealed-identity pull-request reconciliation.""" + +from __future__ import annotations + +from agentdiff.delivery import ( + PRReconciler, + PRReconciliationDecision, + PullRequestSnapshot, + PullRequestState, + SealedDeliveryIdentity, +) + + +def _identity( + *, + base: str = "base-a", + patch: str = "patch-a", + proof: str = "proof-a", +) -> SealedDeliveryIdentity: + return SealedDeliveryIdentity(base, patch, proof) + + +def test_open_pr_with_exact_sealed_identity_is_reused_without_merge_authority() -> None: + desired = _identity() + result = PRReconciler().reconcile( + desired, + PullRequestSnapshot(42, PullRequestState.OPEN, desired, url="https://example/pr/42"), + ) + + assert result.decision is PRReconciliationDecision.REUSE + assert result.exact_identity_match is True + assert result.regeneration_allowed is False + assert result.merge_allowed is False + assert result.to_dict()["desired_identity"]["proof_digest"] == "proof-a" + + +def test_same_base_and_patch_with_new_proof_allows_metadata_update_only() -> None: + result = PRReconciler().reconcile( + _identity(proof="proof-new"), + PullRequestSnapshot(7, PullRequestState.DRAFT, _identity(proof="proof-old")), + ) + + assert result.decision is PRReconciliationDecision.UPDATE + assert result.base_matches is True + assert result.patch_matches is True + assert result.proof_matches is False + assert result.metadata_update_only is True + assert result.regeneration_allowed is False + + +def test_same_patch_on_changed_base_is_stale_and_requires_reproof() -> None: + result = PRReconciler().reconcile( + _identity(base="base-new"), + PullRequestSnapshot(9, PullRequestState.OPEN, _identity(base="base-old")), + ) + + assert result.decision is PRReconciliationDecision.STALE + assert result.requires_reproof is True + assert result.patch_matches is True + assert "reproof" in result.reason + + +def test_different_patch_is_closed_or_superseded_never_rewritten() -> None: + result = PRReconciler().reconcile( + _identity(patch="patch-new"), + PullRequestSnapshot(11, PullRequestState.OPEN, _identity(patch="patch-old")), + ) + + assert result.decision is PRReconciliationDecision.CLOSE_OR_SUPERSEDE + assert result.patch_matches is False + assert result.metadata_update_only is False + assert result.regeneration_allowed is False + + +def test_missing_identity_and_closed_pr_fail_closed() -> None: + desired = _identity() + missing = PRReconciler().reconcile( + desired, + PullRequestSnapshot(13, PullRequestState.OPEN, None), + ) + closed = PRReconciler().reconcile( + desired, + PullRequestSnapshot(14, PullRequestState.CLOSED, desired), + ) + + assert missing.decision is PRReconciliationDecision.STALE + assert missing.exact_identity_match is False + assert closed.decision is PRReconciliationDecision.CLOSE_OR_SUPERSEDE + + +def test_merged_exact_identity_is_terminal_reuse_but_mismatch_is_superseded() -> None: + desired = _identity() + exact = PRReconciler().reconcile( + desired, + PullRequestSnapshot(20, PullRequestState.MERGED, desired), + ) + mismatch = PRReconciler().reconcile( + desired, + PullRequestSnapshot(21, PullRequestState.MERGED, _identity(proof="old-proof")), + ) + + assert exact.decision is PRReconciliationDecision.REUSE + assert exact.merge_allowed is False + assert mismatch.decision is PRReconciliationDecision.CLOSE_OR_SUPERSEDE diff --git a/tests/test_proof_lease.py b/tests/test_proof_lease.py new file mode 100644 index 0000000..85b169a --- /dev/null +++ b/tests/test_proof_lease.py @@ -0,0 +1,280 @@ +"""Focused tests for deterministic, fail-closed proof leases.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace + +import pytest + +from agentdiff.proof import ( + ProofLease, + ProofLeaseEvaluator, + ProofLeaseInvalidation, + ProofLeaseObservation, + ProofLeaseStatus, +) + +_BASE_SHA = "a" * 40 +_MOVED_SHA = "b" * 40 + + +def _digest(label: str) -> str: + return hashlib.sha256(label.encode("utf-8")).hexdigest() + + +def _lease(*, closure_complete: bool = True) -> ProofLease: + return ProofLease( + base_sha=_BASE_SHA, + patch_digest=_digest("patch"), + proof_plan_digest=_digest("proof-plan"), + lockfile_digest=_digest("lockfiles"), + runtime_digest=_digest("runtime"), + dependency_closure=( + "src/auth", + "tests/auth/test_token.py", + "pyproject.toml", + ), + closure_complete=closure_complete, + ) + + +def _observation(lease: ProofLease | None = None) -> ProofLeaseObservation: + bound = lease or _lease() + return ProofLeaseObservation( + base_sha=bound.base_sha, + patch_digest=bound.patch_digest, + proof_plan_digest=bound.proof_plan_digest, + lockfile_digest=bound.lockfile_digest, + runtime_digest=bound.runtime_digest, + ) + + +def test_lease_is_immutable_canonical_and_round_trips() -> None: + lease = _lease() + reordered = ProofLease( + base_sha=lease.base_sha, + patch_digest=lease.patch_digest, + proof_plan_digest=lease.proof_plan_digest, + lockfile_digest=lease.lockfile_digest, + runtime_digest=lease.runtime_digest, + dependency_closure=( + "tests/auth/test_token.py", + "src\\auth", + "pyproject.toml", + "src/auth", + ), + closure_complete=True, + invalidation_rules=tuple(reversed(tuple(ProofLeaseInvalidation))), + ) + + assert reordered == lease + assert reordered.id == lease.id + assert reordered.digest == lease.digest + assert ProofLease.from_dict(json.loads(json.dumps(lease.to_dict()))) == lease + assert tuple(lease.invalidation_rules) == tuple(ProofLeaseInvalidation) + with pytest.raises(AttributeError): + lease.base_sha = _MOVED_SHA # type: ignore[misc] + + +def test_exact_unchanged_inputs_remain_valid_without_reuse_attestations() -> None: + lease = _lease() + result = ProofLeaseEvaluator().evaluate(lease, _observation(lease)) + + assert result.status is ProofLeaseStatus.VALID + assert result.valid is True + assert result.invalidations == () + assert result.changed_paths == () + assert result.lease_id == lease.id + + +def test_complete_disjoint_head_movement_preserves_validity() -> None: + lease = _lease() + observation = replace( + _observation(lease), + base_sha=_MOVED_SHA, + changed_paths=("docs/runbook.md", "README.md", "src/authentication/client.py"), + changed_paths_complete=True, + base_lineage_verified=True, + ) + + result = ProofLeaseEvaluator().evaluate(lease, observation) + + assert result.status is ProofLeaseStatus.VALID + assert result.intersecting_paths == () + assert result.disjoint_paths == ( + "README.md", + "docs/runbook.md", + "src/authentication/client.py", + ) + assert result.current_base_sha == _MOVED_SHA + + +def test_verified_empty_commit_movement_preserves_validity() -> None: + lease = _lease() + observation = replace( + _observation(lease), + base_sha=_MOVED_SHA, + changed_paths_complete=True, + base_lineage_verified=True, + ) + + assert ProofLeaseEvaluator().evaluate(lease, observation).valid is True + + +@pytest.mark.parametrize( + "changed_path", + ( + "src/auth/token.py", + "pyproject.toml", + "tests", + ), +) +def test_dependency_closure_overlap_invalidates_segment_aware(changed_path: str) -> None: + lease = _lease() + observation = replace( + _observation(lease), + base_sha=_MOVED_SHA, + changed_paths=(changed_path,), + changed_paths_complete=True, + base_lineage_verified=True, + ) + + result = ProofLeaseEvaluator().evaluate(lease, observation) + + assert result.status is ProofLeaseStatus.INVALIDATED + assert result.invalidations == (ProofLeaseInvalidation.DEPENDENCY_CLOSURE_TOUCHED,) + assert result.intersecting_paths == (changed_path,) + assert result.disjoint_paths == () + + +def test_incomplete_dependency_closure_always_invalidates() -> None: + lease = _lease(closure_complete=False) + result = ProofLeaseEvaluator().evaluate(lease, _observation(lease)) + + assert result.invalidations == (ProofLeaseInvalidation.DEPENDENCY_CLOSURE_INCOMPLETE,) + assert result.valid is False + assert result.disjoint_paths == () + + +def test_head_movement_requires_complete_paths_and_verified_lineage() -> None: + lease = _lease() + observation = replace( + _observation(lease), + base_sha=_MOVED_SHA, + changed_paths=("README.md",), + ) + + result = ProofLeaseEvaluator().evaluate(lease, observation) + + assert result.invalidations == ( + ProofLeaseInvalidation.BASE_LINEAGE_UNVERIFIED, + ProofLeaseInvalidation.CHANGED_PATHS_INCOMPLETE, + ) + assert result.disjoint_paths == () + + +def test_dirty_workspace_requires_a_complete_changed_path_set() -> None: + lease = _lease() + observation = replace(_observation(lease), changed_paths=("README.md",)) + + result = ProofLeaseEvaluator().evaluate(lease, observation) + + assert result.invalidations == (ProofLeaseInvalidation.CHANGED_PATHS_INCOMPLETE,) + assert result.valid is False + + +@pytest.mark.parametrize( + ("field", "reason"), + ( + ("patch_digest", ProofLeaseInvalidation.PATCH_DIGEST_CHANGED), + ("proof_plan_digest", ProofLeaseInvalidation.PROOF_PLAN_DIGEST_CHANGED), + ("lockfile_digest", ProofLeaseInvalidation.LOCKFILE_DIGEST_CHANGED), + ("runtime_digest", ProofLeaseInvalidation.RUNTIME_DIGEST_CHANGED), + ), +) +def test_every_exact_binding_change_invalidates(field: str, reason: ProofLeaseInvalidation) -> None: + lease = _lease() + observation = replace(_observation(lease), **{field: _digest(f"changed-{field}")}) + + result = ProofLeaseEvaluator().evaluate(lease, observation) + + assert result.status is ProofLeaseStatus.INVALIDATED + assert result.invalidations == (reason,) + + +def test_multiple_invalidations_have_a_stable_auditable_order() -> None: + lease = _lease() + observation = ProofLeaseObservation( + base_sha=_MOVED_SHA, + patch_digest=_digest("changed-patch"), + proof_plan_digest=_digest("changed-plan"), + lockfile_digest=_digest("changed-lock"), + runtime_digest=_digest("changed-runtime"), + changed_paths=("src/auth/token.py",), + ) + + result = ProofLeaseEvaluator().evaluate(lease, observation) + + assert result.invalidations == ( + ProofLeaseInvalidation.PATCH_DIGEST_CHANGED, + ProofLeaseInvalidation.PROOF_PLAN_DIGEST_CHANGED, + ProofLeaseInvalidation.LOCKFILE_DIGEST_CHANGED, + ProofLeaseInvalidation.RUNTIME_DIGEST_CHANGED, + ProofLeaseInvalidation.BASE_LINEAGE_UNVERIFIED, + ProofLeaseInvalidation.CHANGED_PATHS_INCOMPLETE, + ProofLeaseInvalidation.DEPENDENCY_CLOSURE_TOUCHED, + ) + assert result.to_dict()["status"] == "INVALIDATED" + + +def test_observation_round_trip_is_strict_and_canonical() -> None: + observation = replace( + _observation(), + changed_paths=("docs\\guide.md", "README.md", "README.md"), + changed_paths_complete=True, + ) + + restored = ProofLeaseObservation.from_dict(json.loads(json.dumps(observation.to_dict()))) + + assert restored == observation + assert restored.changed_paths == ("README.md", "docs/guide.md") + unknown = observation.to_dict() + unknown["certificate"] = "must-not-be-restamped" + with pytest.raises(ValueError, match="unknown fields"): + ProofLeaseObservation.from_dict(unknown) + + +def test_malformed_or_tampered_lease_data_is_rejected() -> None: + lease = _lease() + with pytest.raises(ValueError, match="Git SHA"): + replace(lease, base_sha="not-a-sha") + with pytest.raises(ValueError, match="SHA-256"): + replace(lease, patch_digest="A" * 64) + with pytest.raises(ValueError, match="safe relative path"): + replace(lease, dependency_closure=("../escape.py",)) + with pytest.raises(ValueError, match="at least one path"): + replace(lease, dependency_closure=()) + with pytest.raises(ValueError, match="boolean"): + replace(lease, closure_complete=1) # type: ignore[arg-type] + with pytest.raises(ValueError, match="object"): + ProofLease.from_dict([]) + + missing_rule = lease.to_dict() + missing_rule["invalidation_rules"] = missing_rule["invalidation_rules"][:-1] + with pytest.raises(ValueError, match="complete fail-closed policy"): + ProofLease.from_dict(missing_rule) + + tampered = lease.to_dict() + tampered["dependency_closure"] = ["src/payments"] + with pytest.raises(ValueError, match="id"): + ProofLease.from_dict(tampered) + + +def test_evaluator_requires_typed_validated_inputs() -> None: + evaluator = ProofLeaseEvaluator() + with pytest.raises(TypeError, match="ProofLease"): + evaluator.evaluate({}, _observation()) # type: ignore[arg-type] + with pytest.raises(TypeError, match="ProofLeaseObservation"): + evaluator.evaluate(_lease(), {}) # type: ignore[arg-type]