From f6a926917e934a970163b2f9fac30f49384d1e5c Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 18 Aug 2026 12:00:16 -0400 Subject: [PATCH] Harden remote display qualification evidence --- benchmark/citrix_ica_hdx/README.md | 9 ++- benchmark/citrix_ica_hdx/fixture.py | 7 +- .../citrix_ica_hdx/run_real_acceptance.py | 52 +++++++++++- claims.yaml | 10 ++- docs/PRODUCT_STATUS.md | 4 +- docs/VERIFICATION.md | 4 +- docs/backends/RDP.md | 25 ++++++ docs/desktop/CITRIX_PIXEL.md | 15 ++++ docs/verification.json | 4 +- openadapt_flow/backends/rdp_backend.py | 79 +++++++++++++----- openadapt_flow/backends/remote_display.py | 59 +++++++++++++- public-artifacts.json | 4 +- tests/test_citrix_real_acceptance.py | 76 ++++++++++++++++- tests/test_rdp_backend.py | 61 ++++++++++---- tests/test_remote_display_backend.py | 81 +++++++++++++++++++ 15 files changed, 436 insertions(+), 54 deletions(-) diff --git a/benchmark/citrix_ica_hdx/README.md b/benchmark/citrix_ica_hdx/README.md index 1ff1f423..c034b534 100644 --- a/benchmark/citrix_ica_hdx/README.md +++ b/benchmark/citrix_ica_hdx/README.md @@ -166,7 +166,10 @@ observation must return both challenges and a signed `observed_at` timestamp. The harness accepts the observation challenge once and rejects a stale, future, or replayed observation. A signed independent collector also binds its current OS observations of the runner, oracle, and collector principals and executable -digests to the configuration, trial, session, and transport. +digests to the configuration, trial, session, transport, and exact display. +The display observation includes the viewport, DPI, scale, window mode, and a +monitor-topology digest. A resize or monitor change outside the configured +trial condition refuses before dispatch. Before each dispatch, a separately authenticated read-only oracle must report a signed `REFUTED` baseline for the exact trial, entity, and effect. The collector @@ -193,6 +196,10 @@ and the fixed condition contract. It emits `VERIFIED` only for a healthy trial whose complete oracle checks pass. A trial with `passed: false` is always `HALTED` or `HALTED_UNCERTAIN`; it is never `VERIFIED`. +The runner receipt must also report zero retries and zero model calls. The +terminal report includes explicit counts for every condition, verified and halt +outcomes, silent incorrect success, over-halt, retries, and model calls. + A commit timeout has the fixed result `HALTED_UNCERTAIN`. The runner reports uncertain delivery, zero retries, and required independent reconciliation. The oracle must independently confirm or refute the effect before the counted trial diff --git a/benchmark/citrix_ica_hdx/fixture.py b/benchmark/citrix_ica_hdx/fixture.py index 95ef898b..c6079081 100644 --- a/benchmark/citrix_ica_hdx/fixture.py +++ b/benchmark/citrix_ica_hdx/fixture.py @@ -35,7 +35,10 @@ from PIL import Image, ImageDraw, ImageFilter, ImageOps -from openadapt_flow.backends.remote_display import RemoteDisplayError, WindowInfo +from openadapt_flow.backends.remote_display import ( + RemoteInputRefused, + WindowInfo, +) # -- fixed synthetic geometry (pixel space of the captured frame) -------------- # Window bounds are screen POINTS; the captured frame is pixels. scale 2.0 (a @@ -335,7 +338,7 @@ def key(self, keycode: int, *, down: bool, flags: list[str]) -> None: if self.paste_blocked and "control" in flags: # Citrix policy disables the clipboard channel: fail LOUD rather # than let a no-op paste look like a completed action. - raise RemoteDisplayError( + raise RemoteInputRefused( "clipboard redirection is disabled by session policy; paste " "was not delivered (refusing to let a no-op look like success)" ) diff --git a/benchmark/citrix_ica_hdx/run_real_acceptance.py b/benchmark/citrix_ica_hdx/run_real_acceptance.py index cca11e71..e2a9925f 100644 --- a/benchmark/citrix_ica_hdx/run_real_acceptance.py +++ b/benchmark/citrix_ica_hdx/run_real_acceptance.py @@ -29,8 +29,9 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey SCHEMA = "openadapt.citrix-real-acceptance.v3" -REPORT_SCHEMA = "openadapt.citrix-real-acceptance-report.v3" +REPORT_SCHEMA = "openadapt.citrix-real-acceptance-report.v4" TRUST_ROOT_SCHEMA = "openadapt.citrix-acceptance-trust-roots.v1" +COLLECTOR_SCHEMA = "openadapt.citrix-independent-collector.v2" SHA256_RE = re.compile(r"[0-9a-f]{64}") NONCE_RE = re.compile(r"[0-9a-f]{32,128}") MAX_COLLECTOR_AGE_S = 300 @@ -664,6 +665,7 @@ def _validate_collector_evidence( "standin", "session_id_sha256", "transport_sha256", + "display", "observed_at", "observed_components", "diagnostic_evidence", @@ -671,7 +673,7 @@ def _validate_collector_evidence( if not isinstance(proof, dict) or set(proof) != required: raise ValueError("collector evidence has incomplete or unknown fields") expected = { - "schema_version": "openadapt.citrix-independent-collector.v1", + "schema_version": COLLECTOR_SCHEMA, "campaign_nonce": config["campaign_nonce"], "config_sha256": config_sha256, "execution_challenge": execution_challenge, @@ -681,6 +683,7 @@ def _validate_collector_evidence( "standin": False, "session_id_sha256": config["fingerprints"]["session"]["session_id_sha256"], "transport_sha256": config["fingerprints"]["ica_hdx"]["transport_sha256"], + "display": config["fingerprints"]["display"], } for key, value in expected.items(): if proof[key] != value: @@ -716,6 +719,7 @@ def _validate_runner_receipt( "condition", "delivery_state", "retry_count", + "model_calls", "collector_evidence_sha256", "reconciliation_required", } @@ -725,8 +729,10 @@ def _validate_runner_receipt( raise ValueError("session runner receipt schema is invalid") if receipt["trial_id"] != trial["id"] or receipt["condition"] != trial["condition"]: raise ValueError("session runner receipt is not bound to the trial") - if receipt["retry_count"] != 0: + if type(receipt["retry_count"]) is not int or receipt["retry_count"] != 0: raise ValueError("a counted acceptance trial must not retry") + if type(receipt["model_calls"]) is not int or receipt["model_calls"] != 0: + raise ValueError("a counted healthy-path acceptance trial must use zero models") if receipt["collector_evidence_sha256"] != collector_evidence_sha256: raise ValueError("session runner receipt is not bound to collector evidence") required_delivery = { @@ -979,6 +985,7 @@ def _write_terminal_report( *, require_primary: bool = False, ) -> None: + _refresh_report_metrics(report) report["journal"] = { "path": str(journal.path), "sha256": _sha256(journal.path), @@ -993,6 +1000,44 @@ def _write_terminal_report( raise +def _refresh_report_metrics(report: dict) -> None: + """Add explicit reliability counts from the retained terminal rows.""" + + rows = [row for row in report.get("trials", []) if isinstance(row, dict)] + condition_counts = { + condition: sum(row.get("condition") == condition for row in rows) + for condition in EXPECTED_OUTCOMES + } + report["metrics"] = { + "trial_count": len(rows), + "condition_counts": condition_counts, + "verified_outcomes": sum(row.get("outcome") == "VERIFIED" for row in rows), + "halted_outcomes": sum(row.get("outcome") == "HALTED" for row in rows), + "halted_uncertain_outcomes": sum( + row.get("outcome") == "HALTED_UNCERTAIN" for row in rows + ), + "silent_incorrect_successes": sum( + row.get("outcome") == "VERIFIED" and row.get("expected") != "VERIFIED" + for row in rows + ), + "over_halts": sum( + row.get("expected") == "VERIFIED" and row.get("outcome") != "VERIFIED" + for row in rows + ), + "delivery_retries": sum( + row.get("retry_count", 0) + for row in rows + if type(row.get("retry_count", 0)) is int + ), + "model_calls": sum( + row["receipt"].get("model_calls", 0) + for row in rows + if isinstance(row.get("receipt"), dict) + and type(row["receipt"].get("model_calls", 0)) is int + ), + } + + def _recover_reserved_campaign(binding: dict) -> dict: output = Path(binding["output"]) fallback = Path(binding["terminal_fallback"]) @@ -1469,6 +1514,7 @@ def main() -> int: print(f"acceptance refused: {exc}", file=sys.stderr) return 2 if not args.execute: + _refresh_report_metrics(report) _atomic_write_json(args.output, report) if not args.execute: print("preflight passed; no trial ran") diff --git a/claims.yaml b/claims.yaml index 349b8f58..f1874dd9 100644 --- a/claims.yaml +++ b/claims.yaml @@ -550,7 +550,10 @@ claims: - path: tests/test_rdp_backend.py proves: >- CI covers the backend/transport contract, framebuffer conversion, - pointer and keyboard delivery, and record-compile-replay conformance. + pointer and keyboard delivery, resize rebaseline between actions, + mid-lease geometry refusal, pre-delivery horizontal-scroll refusal, + typed uncertain transport failures, and record-compile-replay + conformance. - path: tests/test_rdp_multiapp_campaign_contract.py proves: >- Required CI covers the bounded 27-trial FreeRDP campaign contract, @@ -707,8 +710,9 @@ claims: proves: >- Required CI validates the public real-ICA campaign preflight, distinct authority keys, executable and oracle attestations, one-use - nonce journal, crash recovery, uncertain dispatch, and fail-closed - report contract without provisioning infrastructure. + nonce journal, crash recovery, signed display and monitor-topology + observation, explicit reliability metrics, uncertain dispatch, and + fail-closed report contract without provisioning infrastructure. - path: benchmark/citrix_workspace/results.json kind: artifact proves: >- diff --git a/docs/PRODUCT_STATUS.md b/docs/PRODUCT_STATUS.md index ff0b27fb..955f88f3 100644 --- a/docs/PRODUCT_STATUS.md +++ b/docs/PRODUCT_STATUS.md @@ -33,8 +33,8 @@ and its generated view is [`VERIFICATION.md`](VERIFICATION.md). | Native macOS desktop actuation | **Scoped acceptance** | Candidate `b1b61a5` completed 3/3 TextEdit replace-and-save trials with exact file-byte effects and refused a two-window ambiguous selector without changing either file. See the [accepted evidence adjudication](../benchmark/macos_native/textedit_counted_3plus1_b1b61a5_20260717.adjudication.json). | Acceptance covers TextEdit on one macOS 15.7.3 Apple Silicon host and active user session. Customer applications require workflow-specific qualification. | | Native macOS AX structured identity | **Scoped acceptance** | The macOS backend implements the same structured-layer contract as the browser DOM, Windows UIA, and Linux AT-SPI backends: it records a stable AX locator, re-finds the UNIQUE element at replay, refuses ambiguous / truncated / scope-escaping enumeration instead of guessing, and returns structured text under a point. Headless unit CI covers record/locate/refuse; a live-AX TextEdit run produced real evidence ([AX identity adjudication](../benchmark/macos_native/ax_identity_20260720.adjudication.json)); the record→compile→replay conformance test asserts zero model calls on healthy replay. See [`tests/test_macos_structural.py`](../tests/test_macos_structural.py) and the [capability matrix](../tests/test_backend_capability_matrix.py). | The backend uses gated point-bound physical click after structural resolution rather than claiming AXPress everywhere. AX exposure varies by application; controls without durable AX identity use the visual ladder. | | Native Linux desktop actuation | **Scoped acceptance** | The required `linux-atspi-x11` job runs a real GTK3 application against AT-SPI inside an isolated Xvfb/session-D-Bus environment: 3 clean exact-file-effect trials, 3 ambiguous-target refusals, and 3 stale-target refusals. Unit CI covers the remaining window, traversal, capture, physical-input, and portal boundaries. | Acceptance is bounded to the in-tree GTK3 workflow and CI image. Each application and environment retains its own qualification. The built-in driver uses X11; Wayland requires a live operator-approved XDG portal session and refuses without one. | -| RDP | **Scoped acceptance** | Candidate `82a658a` completed 3/3 real-network Aardwolf RDP trials into Windows 11, with a guest-tools file oracle, zero failures, zero silent incorrect successes, zero over-halts, and zero model calls. The public multi-window FreeRDP campaign adds a bounded 27-trial contract with independent SQLite, CSV, and Maildir oracles. See the [accepted batch](../benchmark/rdp/ACCEPTED_BATCH_82A658A.md) and [campaign contract](../benchmark/rdp_multiapp/README.md). | The accepted batch covers the tested 1280×800 transport/input task. The multi-window fixture uses synthetic applications. Target applications, identity/effect rules, session policies, and display conditions are qualified per deployment. | -| Citrix / pixel-only remote display | **Code-qualified** | `--backend citrix` binds an exact Citrix Workspace window, readiness marker, pixel-only ladder, governed run, durable resume, and report; required CI covers those orchestration and refusal contracts. The public real-ICA preflight adds distinct authority keys, executable and oracle attestations, one-use campaign state, crash recovery, and uncertain-dispatch handling. Separately, the retained no-DOM driver qualification passed 3 healthy effect-confirmed trials and 3 drift safe-halts with zero model calls, silent incorrect successes, or false completion, and records `code_readiness_accepted=true`. | The counted stand-in and preflight do not claim live ICA/HDX acceptance. A live result remains bound to the exact Workspace/server/application matrix, customer-approved executable, and independent effect oracle. Deployment-specific recipes, data, and thresholds stay outside the public repository. | +| RDP | **Scoped acceptance** | Candidate `82a658a` completed 3/3 real-network Aardwolf RDP trials into Windows 11, with a guest-tools file oracle, zero failures, zero silent incorrect successes, zero over-halts, and zero model calls. The public multi-window FreeRDP campaign adds a bounded 27-trial contract with independent SQLite, CSV, and Maildir oracles. The backend also rebaselines a changed framebuffer between actions, refuses a change during the exact-frame lease, refuses unsupported horizontal scroll before delivery, and classifies transport failures as uncertain delivery. See the [accepted batch](../benchmark/rdp/ACCEPTED_BATCH_82A658A.md) and [campaign contract](../benchmark/rdp_multiapp/README.md). | The accepted batch covers the tested 1280×800 transport/input task. The multi-window fixture uses synthetic applications. Target applications, identity/effect rules, session policies, and display conditions are qualified per deployment. A composite multi-monitor session remains deployment-qualified evidence, not part of the accepted 1280×800 batch. | +| Citrix / pixel-only remote display | **Code-qualified** | `--backend citrix` binds an exact Citrix Workspace window, readiness marker, pixel-only ladder, governed run, durable resume, and report; required CI covers those orchestration and refusal contracts. The window driver recalculates capture scale after a resize or cross-monitor move and refuses DPI or geometry drift during input. The public real-ICA preflight adds distinct authority keys, executable and oracle attestations, a signed display and monitor-topology observation, explicit reliability metrics, one-use campaign state, crash recovery, and uncertain-dispatch handling. Separately, the retained no-DOM driver qualification passed 3 healthy effect-confirmed trials and 3 drift safe-halts with zero model calls, silent incorrect successes, or false completion, and records `code_readiness_accepted=true`. | The counted stand-in and preflight do not claim live ICA/HDX acceptance. A live result remains bound to the exact Workspace/server/application/display matrix, customer-approved executable, and independent effect oracle. Deployment-specific recipes, data, and thresholds stay outside the public repository. | | Identity verification | **Experimental, armed steps only** | Wrong-entity refusal and adversarial corpora run in CI. | Unarmed clicks have no identity check. Real compiled bundles currently arm only a subset of clicks. | | System-of-record effect verification | **Experimental** | REST, FHIR, SQL, file, and document verifier contracts catch fault classes that screen-only verification misses. A deployment with multiple reviewed read boundaries selects and preflights the strongest evidence tier before input, retains that binding through durable resume, and never downgrades after an action. | Effects are not generally inferred; both authored effects and a configured verifier are required. A selected verifier that becomes unavailable halts or enters reconciliation. | | Lint and certification policies | **Beta** | The CLI reports coverage gaps and refuses bundles that violate a selected policy. | Certification is opt-in; `replay` remains the permissive tutorial path. Use fail-closed `run` for a deployment. | diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index bcf06ab7..8cca61fc 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -281,7 +281,7 @@ | Backing evidence | Kind | Gating / CI stage | Strength | Proves | |---|---|---|---|---| | `tests/e2e/test_parallels_rdp_e2e.py` | test | opt-in (OAFLOW_PARALLELS_RDP_E2E) | validating | Opt-in snapshot-safe real RDP qualification harness with exactly three trials, independent guest-tools oracle, failure taxonomy, and cleanup. | -| `tests/test_rdp_backend.py` | test | ci (required PR gate (test)) | supported | CI covers the backend/transport contract, framebuffer conversion, pointer and keyboard delivery, and record-compile-replay conformance. | +| `tests/test_rdp_backend.py` | test | ci (required PR gate (test)) | supported | CI covers the backend/transport contract, framebuffer conversion, pointer and keyboard delivery, resize rebaseline between actions, mid-lease geometry refusal, pre-delivery horizontal-scroll refusal, typed uncertain transport failures, and record-compile-replay conformance. | | `tests/test_rdp_multiapp_campaign_contract.py` | test | ci (required PR gate (test)) | supported | Required CI covers the bounded 27-trial FreeRDP campaign contract, independent SQLite, CSV, and Maildir oracles, fail-closed visual action preflight, uncertain-delivery handling, and result accounting. | | `benchmark/rdp_multiapp/README.md` | doc | artifact (doc/benchmark) | roadmap | The public synthetic multi-window campaign defines the workflow, faults, independent persisted surfaces, and acceptance denominator. | | `benchmark/rdp/results_82a658a_20260718.sanitized.json` | benchmark | artifact (doc/benchmark) | roadmap | Candidate 82a658a passed 3/3 at 51.845, 10.467, and 7.477 seconds, with zero failures, silent incorrect successes, over-halts, or model calls. | @@ -342,7 +342,7 @@ | `tests/test_cli_new_commands.py` | test | ci (required PR gate (test)) | supported | Required CI restores the recorded Citrix owner, exact title, and readiness binding through durable approve/resume, and refuses before backend construction when resumed configuration is incomplete. | | `tests/test_hosted.py` | test | ci (required PR gate (test)) | supported | Required CI binds a completed local Citrix report to the closed `citrix` execution token without copying target owner, title, or readiness values into the hosted summary. | | `tests/e2e/test_citrix_workspace_standin_e2e.py` | test | opt-in (OAFLOW_CITRIX_STANDIN_E2E) | validating | The dedicated Citrix backend passes three healthy effect-confirmed record->compile->replay trials and three severe-drift safe-halts over a no-DOM surface, with zero model calls, false completion, or silent incorrect success. | -| `tests/test_citrix_real_acceptance.py` | test | ci (required PR gate (test)) | supported | Required CI validates the public real-ICA campaign preflight, distinct authority keys, executable and oracle attestations, one-use nonce journal, crash recovery, uncertain dispatch, and fail-closed report contract without provisioning infrastructure. | +| `tests/test_citrix_real_acceptance.py` | test | ci (required PR gate (test)) | supported | Required CI validates the public real-ICA campaign preflight, distinct authority keys, executable and oracle attestations, one-use nonce journal, crash recovery, signed display and monitor-topology observation, explicit reliability metrics, uncertain dispatch, and fail-closed report contract without provisioning infrastructure. | | `benchmark/citrix_workspace/results.json` | artifact | artifact (doc/benchmark) | roadmap | The retained six-trial record reports code_readiness_accepted=true and ica_hdx_accepted=false, keeping driver readiness distinct from a counted live ICA/HDX qualification. | | `docs/desktop/CITRIX_PIXEL.md` | doc | artifact (doc/benchmark) | roadmap | The Citrix driver model, reusable evidence, exact-deployment acceptance contract, independent-effect boundary, and customer-controlled posture. | diff --git a/docs/backends/RDP.md b/docs/backends/RDP.md index 2fc7a115..6a8d957e 100644 --- a/docs/backends/RDP.md +++ b/docs/backends/RDP.md @@ -60,6 +60,10 @@ down/up (double = the sequence twice); `type_text` sends per-character key down/up; `press` decomposes a key/chord into ordered key down-then-reverse-up events; `scroll` sends a wheel gesture. +The Aardwolf transport supports vertical wheel input only. The backend refuses +any horizontal component before delivery. It never changes a two-axis +demonstrated gesture into a partial vertical action. + ### Coordinate space Everything is in **framebuffer pixels** — the same pixels the resolver emits @@ -69,6 +73,18 @@ MUST report the downsampled `(width, height)`, so screenshot pixels and click pixels stay in one space; no scaling happens in the backend. `AardwolfTransport` runs 1:1 (PIL video-out at the requested width/height). +A resize or display change between actions is supported. The next screenshot +replaces the prior viewport, and the runtime resolves the next target in that +new framebuffer. A dimension change after target resolution and before input +is refused before the first input edge. A qualification-bound remote-frame +contract can deliberately pin one exact geometry; that contract refuses a +resize until the workflow receives a new qualification. + +A transport can expose a multi-monitor remote session as one composite +framebuffer. Its reported dimensions and pointer coordinates must describe +that one bitmap. The accepted RDP batch used one 1280×800 display. It is not +multi-monitor acceptance evidence. + ### Identity model RDP is a pure-pixel substrate, so `FreeRDPBackend` does **not** @@ -92,6 +108,11 @@ disarms the lease. After a successful check, the lease is consumed once, so a multi-character type or double-click remains one gesture and is never retried after an uncertain delivery. +Pointer, keyboard, text, and wheel failures after a transport call starts have +the typed `ActionDeliveryUncertain` result. The runtime does not retry them. It +continues to the configured postcondition and independent effect checks. It +reports `VERIFIED` only when the complete contract confirms the effect. + The first contract intentionally binds the full framebuffer. Dynamic clocks, animations, or other volatile chrome can therefore cause a safe over-halt. Relaxing that behavior requires a qualification-bound mask or protected-region @@ -188,6 +209,10 @@ statistical reliability claim for every Windows application. For a production workflow, record and qualify the customer's exact application under its real account/session policy, DPI and scaling, disconnect/reconnect behavior, latency envelope, identity evidence, and independent effect oracle. +Include every supported resize, full-screen transition, and monitor topology in +that counted matrix. A topology outside the accepted matrix requires a fresh +frame and target resolution. It can require a new qualification when the +deployment pins exact geometry. Citrix ICA/HDX receives a separate counted qualification on its exact Workspace/server/application matrix; the RDP batch is not used as Citrix acceptance evidence. diff --git a/docs/desktop/CITRIX_PIXEL.md b/docs/desktop/CITRIX_PIXEL.md index 24b38b59..009118cc 100644 --- a/docs/desktop/CITRIX_PIXEL.md +++ b/docs/desktop/CITRIX_PIXEL.md @@ -35,6 +35,15 @@ chrome can over-halt. Any mask or protected-region relaxation is part of the specific application/environment qualification, not a permissive global default. +The client window can move, resize, or cross monitors between actions. Each new +capture resolves the exact window again and recalculates the pixel-to-screen +mapping from its current bounds and captured dimensions. The Windows client +requires per-monitor DPI awareness. The macOS client supports negative desktop +coordinates and derives the current Retina scale. An anisotropic mapping, an +unavailable DPI mode, or a geometry change during an actuation lease refuses +input. The current evidence covers this driver contract. It does not claim a +counted real Workspace multi-monitor result. + `openadapt_flow/backends/remote_display.py` implements this contract on macOS (Quartz), and `openadapt_flow/backends/win32_window_client.py` provides the Windows-host client for the same backend (PrintWindow/BitBlt client-area @@ -96,6 +105,12 @@ observable business result. The acceptance record names: - run count, failure taxonomy, silent incorrect success, over-halt, operator intervention, model calls, and time-to-repair. +The public real-acceptance harness requires a signed independent collector to +observe the exact display fingerprint before each trial. That fingerprint +includes width, height, DPI, scale, window mode, and a monitor-topology digest. +The terminal report names trial counts, each condition count, verified and halt +outcomes, silent incorrect success, over-halt, retries, and model calls. + Qualification begins in shadow mode, moves to supervised production writes, and expands only after the fixed workflow meets its acceptance thresholds. Repeated labels or windows, stale foreground bindings, unreadable identity, diff --git a/docs/verification.json b/docs/verification.json index 6c5c8cb3..04e3726c 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -853,7 +853,7 @@ "node_found": null, "ci_job": "test", "junit_status": null, - "proves": "CI covers the backend/transport contract, framebuffer conversion, pointer and keyboard delivery, and record-compile-replay conformance." + "proves": "CI covers the backend/transport contract, framebuffer conversion, pointer and keyboard delivery, resize rebaseline between actions, mid-lease geometry refusal, pre-delivery horizontal-scroll refusal, typed uncertain transport failures, and record-compile-replay conformance." }, { "path": "tests/test_rdp_multiapp_campaign_contract.py", @@ -1114,7 +1114,7 @@ "node_found": null, "ci_job": "test", "junit_status": null, - "proves": "Required CI validates the public real-ICA campaign preflight, distinct authority keys, executable and oracle attestations, one-use nonce journal, crash recovery, uncertain dispatch, and fail-closed report contract without provisioning infrastructure." + "proves": "Required CI validates the public real-ICA campaign preflight, distinct authority keys, executable and oracle attestations, one-use nonce journal, crash recovery, signed display and monitor-topology observation, explicit reliability metrics, uncertain dispatch, and fail-closed report contract without provisioning infrastructure." }, { "path": "benchmark/citrix_workspace/results.json", diff --git a/openadapt_flow/backends/rdp_backend.py b/openadapt_flow/backends/rdp_backend.py index ff784979..858a1350 100644 --- a/openadapt_flow/backends/rdp_backend.py +++ b/openadapt_flow/backends/rdp_backend.py @@ -135,12 +135,13 @@ def wheel(self, dx: int, dy: int) -> None: """Send a wheel gesture by ``(dx, dy)`` framebuffer pixels (Backend convention: positive ``dy`` scrolls content up / view down). - A transport MAY only support vertical scrolling: the real - :class:`AardwolfTransport` drops a non-zero ``dx`` because aardwolf's - wheel API has no horizontal event. A transport that dispatches the - wheel at a cursor position SHOULD use the last pointer location (the - remote OS routes the wheel to the window under the cursor), not a fixed - origin. + A transport MAY only support vertical scrolling. It MUST advertise a + true ``supports_hwheel`` attribute before the backend sends a non-zero + ``dx``. The real :class:`AardwolfTransport` has no horizontal wheel + event and therefore receives no horizontal gesture. A transport that + dispatches the wheel at a cursor position SHOULD use the last pointer + location (the remote OS routes the wheel to the window under the + cursor), not a fixed origin. """ ... @@ -264,8 +265,8 @@ class FreeRDPBackend: Args: transport: The RDP transport to drive (real or fake). - viewport: Optional ``(width, height)`` override; when omitted it is - derived once from the first framebuffer and cached. + viewport: Optional initial ``(width, height)`` value. Every subsequent + screenshot replaces it with the dimensions of that exact frame. connect: When True (default) connect the transport on construction. Pass False if the caller manages the transport lifecycle. max_frame_age_s: Maximum age of the screenshot that established an @@ -817,7 +818,16 @@ def type_text(self, text: str) -> None: with self._input_lock: self._focus_input_surface() self._ensure_input_ready(operation="rdp_type_text") - self._dispatch_text_locked(text) + try: + self._dispatch_text_locked(text, strict_release=True) + except ActionDeliveryUncertain: + raise + except Exception as exc: + raise ActionDeliveryUncertain( + operation="rdp_type_text", + native=False, + cause_type=type(exc).__name__, + ) from exc def press(self, key: str) -> None: """Press a key or chord, e.g. ``'Enter'`` or ``'ControlOrMeta+a'``. @@ -839,7 +849,29 @@ def press(self, key: str) -> None: with self._input_lock: self._focus_input_surface() self._ensure_input_ready(operation="rdp_press") - self._dispatch_key_locked(parts) + physical_key = getattr(self._transport, "physical_key", None) + supports_physical_key = getattr( + self._transport, "supports_physical_key", None + ) + if callable(physical_key) and callable(supports_physical_key): + unsupported = [ + part for part in parts if not supports_physical_key(part) + ] + if unsupported: + raise ValueError( + "RDP transport cannot safely emit physical chord keys: " + f"{unsupported!r}" + ) + try: + self._dispatch_key_locked(parts, strict_release=True) + except ActionDeliveryUncertain: + raise + except Exception as exc: + raise ActionDeliveryUncertain( + operation="rdp_press", + native=False, + cause_type=type(exc).__name__, + ) from exc def select_option(self, text: str, commit_key: str) -> None: """Type and commit one demonstrated option under a single input lock. @@ -1014,19 +1046,28 @@ def _release_keys(self, parts, *, sender=None) -> Optional[Exception]: def scroll(self, dx: int, dy: int) -> None: """Dispatch a wheel gesture by ``(dx, dy)`` pixels. - Limitation — horizontal scroll: the real :class:`AardwolfTransport` - can only emit *vertical* wheel events (aardwolf's ``send_mouse`` - exposes ``WHEEL_UP``/``WHEEL_DOWN`` but no horizontal ``HWHEEL``), so a - non-zero ``dx`` is silently dropped by that transport. This is a - documented capability gap, not a bug in this method; the in-repo - :class:`FakeRDPTransport` models the same drop so a test cannot pass on - a capability the live transport lacks. + The real :class:`AardwolfTransport` supports vertical wheel input only. + The backend refuses a horizontal component before any input unless the + selected transport explicitly advertises ``supports_hwheel``. It never + turns a demonstrated two-axis gesture into a partial action. """ if dx == 0 and dy == 0: return + if dx != 0 and not bool(getattr(self._transport, "supports_hwheel", False)): + raise RuntimeError( + "RDP transport does not support horizontal wheel input; " + "refusing the complete scroll gesture before delivery" + ) with self._input_lock: self._ensure_input_ready(operation="rdp_scroll") - self._transport.wheel(int(dx), int(dy)) + try: + self._transport.wheel(int(dx), int(dy)) + except Exception as exc: + raise ActionDeliveryUncertain( + operation="rdp_scroll", + native=False, + cause_type=type(exc).__name__, + ) from exc # -- lifecycle ----------------------------------------------------------- @@ -1390,6 +1431,8 @@ class AardwolfTransport: keyboard_layout_id: RDP handshake layout id (default 1033 / en-US). """ + supports_hwheel = False + def __init__( self, url: str, diff --git a/openadapt_flow/backends/remote_display.py b/openadapt_flow/backends/remote_display.py index 68f16023..4df320d7 100644 --- a/openadapt_flow/backends/remote_display.py +++ b/openadapt_flow/backends/remote_display.py @@ -87,6 +87,15 @@ class RemoteDisplayError(RuntimeError): """A remote-display capture/inject operation failed (or is not permitted).""" +class RemoteInputRefused(RemoteDisplayError): + """A client proved that it emitted no input edge. + + A ``WindowClient`` can raise this only for a completed pre-delivery check. + Other client exceptions remain delivery-uncertain because the host API can + fail after it starts an input operation. + """ + + class _RemoteDisplayFreshActuationRequired( FreshActuationRequired, RemoteDisplayError, @@ -1251,7 +1260,16 @@ def type_text(self, text: str) -> None: return with self._input_lock: self._ensure_input_ready(operation="remote_type_text") - self._client.type_chars(text) + try: + self._client.type_chars(text) + except RemoteInputRefused: + raise + except Exception as exc: + raise ActionDeliveryUncertain( + operation="remote_type_text", + native=False, + cause_type=type(exc).__name__, + ) from exc def press(self, key: str) -> None: """Press a key or chord, e.g. ``'Enter'`` or ``'ControlOrMeta+a'``. @@ -1266,7 +1284,16 @@ def press(self, key: str) -> None: self._ensure_input_ready(operation="remote_press") # A bare printable key with no modifiers: type it as a character. if len(final) == 1 and not mods: - self._client.type_chars(final) + try: + self._client.type_chars(final) + except RemoteInputRefused: + raise + except Exception as exc: + raise ActionDeliveryUncertain( + operation="remote_press", + native=False, + cause_type=type(exc).__name__, + ) from exc return # Named key, or a modified key: the CLIENT owns the keycode # namespace (macOS virtual key codes vs Windows VKs), so key @@ -1276,10 +1303,25 @@ def press(self, key: str) -> None: raise RemoteDisplayError(f"no key mapping for {final!r} in {key!r}") code, shift = resolved flags = list(mods) + (["shift"] if shift else []) + dispatch_error: Optional[Exception] = None + release_error: Optional[Exception] = None try: self._client.key(code, down=True, flags=flags) - finally: + except RemoteInputRefused: + raise + except Exception as exc: # noqa: BLE001 - host input boundary + dispatch_error = exc + try: self._client.key(code, down=False, flags=flags) + except Exception as exc: # noqa: BLE001 - best-effort release + release_error = exc + failure = dispatch_error or release_error + if failure is not None: + raise ActionDeliveryUncertain( + operation="remote_press", + native=False, + cause_type=type(failure).__name__, + ) from failure def scroll(self, dx: int, dy: int) -> None: """Dispatch a wheel gesture by ``(dx, dy)`` pixels.""" @@ -1287,7 +1329,16 @@ def scroll(self, dx: int, dy: int) -> None: return with self._input_lock: self._ensure_input_ready(operation="remote_scroll") - self._client.scroll(int(dx), int(dy)) + try: + self._client.scroll(int(dx), int(dy)) + except RemoteInputRefused: + raise + except Exception as exc: + raise ActionDeliveryUncertain( + operation="remote_scroll", + native=False, + cause_type=type(exc).__name__, + ) from exc # -- internals ----------------------------------------------------------- diff --git a/public-artifacts.json b/public-artifacts.json index 031d5883..1689e827 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -605,7 +605,7 @@ }, { "path": "claims.yaml", - "sha256": "5423f4f10ef289dc3e232630c8558f12a76b3f11ebbb497feb923a928dc230b7" + "sha256": "bc99b4f9bd568995d480c76aa5430a7826de9bc265f36e1a8f374090c2ba73ba" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1813,7 +1813,7 @@ }, { "path": "docs/verification.json", - "sha256": "538fc0a2dc392a10617da96522191d03aa3d3cbe0288bd4891cd14e1de274f97" + "sha256": "5500a6eda3129b17266364899e970ebb0e102ff3f039a3e7e0eed3d64a432293" }, { "path": "openadapt_flow/console/static/console.css", diff --git a/tests/test_citrix_real_acceptance.py b/tests/test_citrix_real_acceptance.py index 079aa2cb..eed11f0f 100644 --- a/tests/test_citrix_real_acceptance.py +++ b/tests/test_citrix_real_acceptance.py @@ -304,7 +304,7 @@ def _collector( observed_at: datetime | None = None, ) -> dict: payload = { - "schema_version": "openadapt.citrix-independent-collector.v1", + "schema_version": mod.COLLECTOR_SCHEMA, "campaign_nonce": campaign.config["campaign_nonce"], "config_sha256": config_sha256, "execution_challenge": execution_challenge, @@ -314,6 +314,7 @@ def _collector( "standin": False, "session_id_sha256": SHA_A, "transport_sha256": SHA_B, + "display": campaign.config["fingerprints"]["display"], "observed_at": (observed_at or datetime.now(timezone.utc)).isoformat(), "observed_components": campaign.trust_roots["components"], "diagnostic_evidence": _evidence(tmp_path, f"collector-{trial['id']}.json"), @@ -333,6 +334,7 @@ def _receipt(trial: dict, collector_sha256: str) -> dict: "condition": trial["condition"], "delivery_state": delivery, "retry_count": 0, + "model_calls": 0, "reconciliation_required": trial["condition"] == "commit_timeout", "collector_evidence_sha256": collector_sha256, } @@ -560,6 +562,31 @@ def test_collector_is_signed_fresh_and_campaign_bound(tmp_path: Path) -> None: consumed_challenges=set(), ) + wrong_display = _collector( + campaign, + tmp_path, + trial, + digest, + execution_challenge=execution_challenge, + observation_challenge="3" * 64, + ) + wrong_display["payload"]["display"] = { + **wrong_display["payload"]["display"], + "monitor_topology_sha256": SHA_D, + } + wrong_display = _signed(campaign.keys["collector"], wrong_display["payload"]) + with pytest.raises(ValueError, match="display"): + mod._validate_collector_evidence( + _result(wrong_display), + trial=trial, + config=config, + config_sha256=digest, + trust_roots=campaign.trust_roots, + execution_challenge=execution_challenge, + observation_challenge="3" * 64, + consumed_challenges=set(), + ) + def test_runner_receipt_binds_collector_and_never_retries(tmp_path: Path) -> None: campaign = _campaign(tmp_path) @@ -583,6 +610,53 @@ def test_runner_receipt_binds_collector_and_never_retries(tmp_path: Path) -> Non mod._validate_runner_receipt( _result(receipt), trial=trial, collector_evidence_sha256=SHA_C ) + receipt = _receipt(trial, SHA_C) + receipt["model_calls"] = 1 + with pytest.raises(ValueError, match="zero models"): + mod._validate_runner_receipt( + _result(receipt), trial=trial, collector_evidence_sha256=SHA_C + ) + + +def test_report_metrics_name_silent_success_over_halt_retries_and_models() -> None: + report = { + "trials": [ + { + "condition": "healthy", + "expected": "VERIFIED", + "outcome": "HALTED", + "retry_count": 0, + }, + { + "condition": "display_drift", + "expected": "HALTED", + "outcome": "VERIFIED", + "retry_count": 1, + "receipt": {"model_calls": 2}, + }, + ] + } + mod._refresh_report_metrics(report) + assert report["metrics"] == { + "trial_count": 2, + "condition_counts": { + "healthy": 1, + "wrong_session_or_entity": 0, + "ambiguity": 0, + "stale_state": 0, + "display_drift": 1, + "partial_effect": 0, + "reconnect": 0, + "commit_timeout": 0, + }, + "verified_outcomes": 1, + "halted_outcomes": 1, + "halted_uncertain_outcomes": 0, + "silent_incorrect_successes": 1, + "over_halts": 1, + "delivery_retries": 1, + "model_calls": 2, + } def _scripted_run( diff --git a/tests/test_rdp_backend.py b/tests/test_rdp_backend.py index 35704b54..2a7ca366 100644 --- a/tests/test_rdp_backend.py +++ b/tests/test_rdp_backend.py @@ -568,6 +568,24 @@ def test_viewport_override(transport: FakeRDPTransport) -> None: assert b.viewport == (640, 480) +def test_new_screenshot_rebaselines_a_resize_between_actions() -> None: + transport = FakeRDPTransport(app_screens()) + backend = FreeRDPBackend(transport) + backend.screenshot() + backend.click(100, 100) + + resized = transport.screens[0].resize((640, 480)) + transport.screens = [resized] + backend.screenshot() + backend.click(320, 240) + + assert backend.viewport == (640, 480) + assert transport.pointer_events[-2:] == [ + (320, 240, "left", True), + (320, 240, "left", False), + ] + + # -- click --------------------------------------------------------------------- @@ -1257,8 +1275,10 @@ def test_press_releases_all_keys_when_chord_key_raises() -> None: # wrong-action the fix targets (Ctrl+a wipes a field). t = RaisingRDPTransport(app_screens(), raise_on_key_down=frozenset({"a"})) b = FreeRDPBackend(t) - with pytest.raises(TransportError): + with pytest.raises(ActionDeliveryUncertain) as raised: b.press("ControlOrMeta+a") + assert raised.value.operation == "rdp_press" + assert raised.value.cause_type == "TransportError" assert held_keys(t.key_events) == [] # nothing latched down # Specifically: Ctrl went down and came back up. assert ("ctrl", True) in t.key_events @@ -1271,8 +1291,9 @@ def test_press_releases_modifier_when_its_own_down_raises() -> None: # releases every part it attempted to press. t = RaisingRDPTransport(app_screens(), raise_on_key_down=frozenset({"ctrl"})) b = FreeRDPBackend(t) - with pytest.raises(TransportError): + with pytest.raises(ActionDeliveryUncertain) as raised: b.press("ControlOrMeta+a") + assert raised.value.operation == "rdp_press" assert held_keys(t.key_events) == [] assert ("ctrl", False) in t.key_events # released despite its down failing @@ -1282,8 +1303,9 @@ def test_type_text_releases_char_when_key_raises() -> None: # must already be released and no key may be left held. t = RaisingRDPTransport(app_screens(), raise_on_key_down=frozenset({"b"})) b = FreeRDPBackend(t) - with pytest.raises(TransportError): + with pytest.raises(ActionDeliveryUncertain) as raised: b.type_text("ab") + assert raised.value.operation == "rdp_type_text" assert held_keys(t.key_events) == [] assert ("a", True) in t.key_events and ("a", False) in t.key_events @@ -1372,8 +1394,10 @@ def physical_key(self, key: str, down: bool) -> None: transport = _RaisingPhysicalTransport() backend = FreeRDPBackend(transport) - with pytest.raises(TransportError, match="physical r down failed"): + with pytest.raises(ActionDeliveryUncertain) as raised: backend.press("Meta+r") + assert raised.value.operation == "rdp_press" + assert raised.value.cause_type == "TransportError" assert transport.physical_events == [ ("meta", True), ("r", True), @@ -1392,13 +1416,11 @@ def test_scroll_sends_wheel( assert transport.wheel_events == [(0, 400)] -def test_scroll_horizontal_dropped_matching_real_transport( +def test_scroll_horizontal_refuses_before_delivery_matching_real_transport( transport: FakeRDPTransport, backend: FreeRDPBackend ) -> None: - # A horizontal-only scroll must record NOTHING: the real AardwolfTransport - # cannot emit horizontal wheel events (documented limitation), and the fake - # mirrors that so a test can't pass on a capability the live transport lacks. - backend.scroll(120, 0) + with pytest.raises(RuntimeError, match="does not support horizontal"): + backend.scroll(120, 0) assert transport.wheel_events == [] @@ -1412,13 +1434,24 @@ def test_scroll_horizontal_honored_only_when_transport_supports_it() -> None: assert t.wheel_events == [(120, 0)] -def test_scroll_mixed_keeps_vertical_when_horizontal_unsupported( +def test_scroll_mixed_refuses_instead_of_delivering_a_partial_gesture( transport: FakeRDPTransport, backend: FreeRDPBackend ) -> None: - # A diagonal scroll on the pixel-only transport keeps the vertical part and - # drops the horizontal part (rather than dropping the whole gesture). - backend.scroll(120, 400) - assert transport.wheel_events == [(0, 400)] + with pytest.raises(RuntimeError, match="does not support horizontal"): + backend.scroll(120, 400) + assert transport.wheel_events == [] + + +def test_scroll_transport_failure_is_delivery_uncertain() -> None: + class _FailingWheelTransport(FakeRDPTransport): + def wheel(self, dx: int, dy: int) -> None: + raise TransportError("wheel failed after possible delivery") + + backend = FreeRDPBackend(_FailingWheelTransport(app_screens())) + with pytest.raises(ActionDeliveryUncertain) as raised: + backend.scroll(0, 400) + assert raised.value.operation == "rdp_scroll" + assert raised.value.cause_type == "TransportError" def test_scroll_zero_sends_nothing( diff --git a/tests/test_remote_display_backend.py b/tests/test_remote_display_backend.py index ea9b8c17..e9db24ba 100644 --- a/tests/test_remote_display_backend.py +++ b/tests/test_remote_display_backend.py @@ -33,6 +33,7 @@ from openadapt_flow.backends.remote_display import ( RemoteDisplayBackend, RemoteDisplayError, + RemoteInputRefused, WindowInfo, _canonical_rgb_digest, _split_chord, @@ -389,6 +390,34 @@ def test_viewport_and_scale_from_capture() -> None: assert backend._scale == pytest.approx(2.0) # 3024 px / 1512 pt window +def test_new_capture_rebaselines_resize_and_cross_monitor_scale_between_actions() -> ( + None +): + backend, client = _backend(px=(3024, 1888)) + backend.screenshot() + old = client.window + + # Move the client to a left-side, non-Retina monitor and resize it. Negative + # desktop coordinates are valid on a multi-monitor host. A new capture + # establishes the new coordinate space before the next action. + client.window = WindowInfo( + window_id=old.window_id, + owner=old.owner, + title=old.title, + pid=old.pid, + bounds=(-1280.0, 100.0, 1280.0, 800.0), + on_screen=True, + ) + client.windows = [client.window] + client.px = (1280, 800) + backend.screenshot() + backend.click(640, 400) + + assert backend.viewport == (1280, 800) + downs = [call for call in client.calls if call[0] == "mouse" and call[4] is True] + assert downs[-1][1:3] == (-640.0, 500.0) + + def test_screenshot_returns_png() -> None: backend, _ = _backend() png = backend.screenshot() @@ -966,6 +995,18 @@ def test_type_text_routes_to_keycodes() -> None: assert ("type", "Neil-1") in client.calls +def test_type_failure_is_delivery_uncertain() -> None: + class FailingTypeClient(FakeClient): + def type_chars(self, text): + raise RuntimeError("host input failed after possible delivery") + + backend = RemoteDisplayBackend(client=FailingTypeClient(), settle_s=0.0) + with pytest.raises(ActionDeliveryUncertain) as raised: + backend.type_text("Neil-1") + assert raised.value.operation == "remote_type_text" + assert raised.value.cause_type == "RuntimeError" + + def test_press_named_key_enter() -> None: backend, client = _backend() backend.press("Enter") @@ -984,6 +1025,35 @@ def test_press_chord_ctrl_a_uses_control_flag() -> None: assert keys[0][2] is True and keys[-1][2] is False +def test_press_failure_is_uncertain_and_still_attempts_release() -> None: + class FailingKeyClient(FakeClient): + def key(self, keycode, *, down, flags): + super().key(keycode, down=down, flags=flags) + if down: + raise RuntimeError("key failed after possible delivery") + + client = FailingKeyClient() + backend = RemoteDisplayBackend(client=client, settle_s=0.0) + with pytest.raises(ActionDeliveryUncertain) as raised: + backend.press("Enter") + assert raised.value.operation == "remote_press" + assert [call[2] for call in client.calls if call[0] == "key"] == [True, False] + + +def test_typed_pre_delivery_key_refusal_is_preserved() -> None: + class RefusingKeyClient(FakeClient): + def key(self, keycode, *, down, flags): + if down: + raise RemoteInputRefused("session policy refused before input") + super().key(keycode, down=down, flags=flags) + + client = RefusingKeyClient() + backend = RemoteDisplayBackend(client=client, settle_s=0.0) + with pytest.raises(RemoteInputRefused): + backend.press("Enter") + assert [call[2] for call in client.calls if call[0] == "key"] == [] + + def test_press_bare_char_types_it() -> None: backend, client = _backend() backend.press("x") @@ -1002,6 +1072,17 @@ def test_scroll_dispatches() -> None: assert any(c[0] == "scroll" for c in client.calls) +def test_scroll_failure_is_delivery_uncertain() -> None: + class FailingScrollClient(FakeClient): + def scroll(self, dx, dy): + raise RuntimeError("scroll failed after possible delivery") + + backend = RemoteDisplayBackend(client=FailingScrollClient(), settle_s=0.0) + with pytest.raises(ActionDeliveryUncertain) as raised: + backend.scroll(0, 120) + assert raised.value.operation == "remote_scroll" + + def test_fail_loud_when_not_accessibility_trusted() -> None: """A dropped synthetic click must never look like success -> refuse to act.""" backend, _ = _backend(trusted=False)