diff --git a/.gitignore b/.gitignore index e0b754be..baaa4842 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ build/ .DS_Store /runs/ /recordings/ +.openadapt-chrome-profile/ +.openadapt-recording-partial-*/ dist/ benchmark/openemr/finals/ benchmark/openemr/rows.jsonl diff --git a/README.md b/README.md index e7071317..56b3d828 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,10 @@ openadapt-flow record --backend web --url https://your.app --out rec openadapt-flow compile rec --out bundle --name my-task openadapt-flow replay bundle --backend web --url https://your.app +# Browser with an existing local SSO/2FA session: attach one open tab. +openadapt-flow record --backend web --url https://your.app \ + --browser-cdp-endpoint http://127.0.0.1:9222 --out rec + # Native Windows: Capture records the local target window. WAA drives replay. openadapt-flow record --backend windows --window "Target App" \ --task "add a patient note" --out rec @@ -241,6 +245,17 @@ ignoring them; pass them to `replay` or `run`. Drive a real deployment with effects, actuation, durable, and policy sections from one config. Recorded parameter values are the defaults, and `--param` overrides them at replay. +The browser recorder can launch a clean Playwright browser or attach to one +existing local Chromium tab. Attach mode preserves a browser profile that has +already completed sign-in, SSO, or 2FA. It refuses remote CDP endpoints and +ambiguous same-origin tabs. It does not navigate or close the attached browser. +You can resize the tab or move its window between monitors. Flow waits for a +stable CSS-pixel frame and binds the next event to the new viewport. It refuses +an action only if that action overlaps the coordinate-space transition. +See the [browser recording guide](docs/BROWSER_RECORDING.md) for setup, exact +tab selection, secret handling, and the boundary with the Capture Chrome +extension prototype. + **You don't have to name parameters up front.** The recorder passively captures each typed field's label (DOM/accessibility, or nearby OCR on pixel paths), and `compile` proposes a parameter named from it (`"Insurance No."` @@ -295,11 +310,16 @@ Route a production receipt through `sanitize` / `review-sanitized` / `approve-sanitized` before it crosses a trust boundary. -**Secrets never get recorded.** An `input[type=password]` field (or any field -named with `--secret `) is a secret parameter: its value is never written -to the recording, the events log, the compiled bundle, or the saved frames (its -region is redacted). At replay it is injected from the environment and a missing -one fails fast: +**Secret input values stay page-local.** An `input[type=password]` field (or a +field named with `--secret `) becomes a secret parameter. Flow does not +send its literal to Python. It masks the bound field region in saved frames. +For every other piece of page text, **Flow reports it exactly or withholds it +and says why. Flow never rewrites captured text.** Matching uses only the value +a bound element holds at that moment, read live from the DOM; no value is kept +after the field stops holding it. A shadow field whose identity can change must +use a host with the same declared name or ID; Flow masks the complete host. It +refuses an unbound shadow input before it accepts a value. At replay, Flow +injects the secret from the environment and fails fast when it is absent: ```bash openadapt-flow record --backend web --url https://your.app --out rec --secret password @@ -307,6 +327,41 @@ export OPENADAPT_FLOW_SECRET_PASSWORD='…' # supplied at replay openadapt-flow replay bundle --backend web --url https://your.app ``` +Evidence splits in two. **Identity evidence** — the DOM selector, the control +role, the accessible name, the clicked row's identity characters, and the +receiving field's name — is exact or withheld with a stated reason, because +replay compares it against the live page and a rewritten copy would compare +against text the page never showed. **Reflected evidence** — the page URL and +the title — is sampled from Python once the page has settled, never inside the +capture-phase listener, which runs before the page's own handlers and so reads +the previous action's text. + +Within a document, a URL is reduced by **structure**: Flow reports the origin +and the path, keeps every parameter name, and drops the value of any parameter +named after a declared secret field — deterministically, whatever the value is. +A dropped value becomes empty; Flow removes characters from a URL and never +adds characters the page did not show. A single-page application that routes +with `history.pushState` therefore keeps its URL evidence. If the URL Flow is +about to report still holds a value Flow can see, it withholds the whole URL +and warns you that the application put a secret in its own URL — a defect that +exposes it through browser history, logs, proxies and `Referer` headers with or +without Flow. + +That reduction does **not** make a later document safe. A path segment has no +parameter name to identify it, so a server that answers a form submit with a +redirect to `/results/` puts the value where structure cannot reach, and +the new document holds nothing to match it against. Flow therefore withholds +the URL and the title of every document after the one that first held a +declared value. A title has no structure to reduce and follows the same rule +within a document. `meta.json` records everything dropped and everything +withheld, and the CLI prints it. + +This source-time contract does not track an application-defined transform of a +secret or an application copy into an unrelated visible element, and it starts +at the moment a bound field holds the value: text and pixels captured before +then are ordinary recording evidence. Keep every raw recording inside its +approved local boundary. + **Compiled is not the same as certified safe.** `lint` reports a bundle's coverage gaps (clicks that act with no identity check, steps that assert nothing, write steps left mis-classified) with a severity each; `certify` diff --git a/claims.yaml b/claims.yaml index f1874dd9..affb4c90 100644 --- a/claims.yaml +++ b/claims.yaml @@ -65,6 +65,138 @@ claims: proves: >- The deterministic replayer resolves steps, substitutes parameters, enforces postconditions and the risk gate — no model in the loop. + - path: tests/test_browser_attach.py + node: test_live_cdp_attach_records_compiles_and_leaves_browser_running_three_trials + proves: >- + Three real Chromium CDP-attach trials record and compile the same + synthetic workflow, exclude password values before persistence, + preserve CSS-pixel frame/coordinate alignment, and detach without + closing the external browser. The same live campaign covers same-task + replacement, declared open and closed shadow hosts, contenteditable + click metadata, and same-document URL/title reflection through + history.replaceState. A live case records and + compiles actions across viewport and monitor-scale changes. It binds + each event to its exact frame dimensions. A separate live case + refuses an action that overlaps the transition. Unit cases refuse + remote endpoints, cross-origin selectors and navigation, iframe + events, invalid viewport evidence, and ambiguous same-origin tabs. + - path: tests/test_browser_attach.py + node: test_launched_recording_withholds_a_later_document_url_after_a_get_submit + proves: >- + A live Chromium recording submits a same-origin GET form. The + document that submit reaches is a fresh closure that holds no value + to match against, so Flow withholds its URL and its title and says + so, and the recorded surface is stamped before publish. + - path: tests/test_browser_attach.py + node: test_launched_recording_withholds_a_redirect_that_puts_the_value_in_a_path + proves: >- + A server that answers a GET submit with a 302 to `/results/` + puts a declared secret in a path segment, where no parameter name + identifies it. Flow withholds the whole URL for every document after + the one that first held the value, so the literal never reaches + events.jsonl. + - path: tests/test_browser_attach.py + node: test_page_closure_keeps_url_and_identity_evidence_for_a_lowercase_secret + proves: >- + A lowercase secret typed one character at a time into a page whose + URL, title, and button ID share those characters leaves the URL, the + title, the DOM selector, and the accessible name exact. + - path: tests/test_browser_attach.py + node: test_page_closure_keeps_all_evidence_for_a_password_starting_with_a_word + proves: >- + A password that begins with a common English word leaves that word + exact in the page URL, the page title, the clicked row's identity + characters, the accessible name, and an unrelated button ID. Matching + uses only the value the field holds at that moment, so an + intermediate keystroke prefix never becomes a matching value. + - path: tests/test_browser_attach.py + node: test_page_closure_withholds_identity_that_holds_a_declared_value + proves: >- + Identity evidence that holds a declared value is WITHHELD with a + stated reason, never rewritten. Replay compares identity evidence + against the live page, so a rewritten copy would compare against + characters the page never showed, invisibly. No placeholder string + appears anywhere in the recorded evidence. + - path: tests/test_browser_attach.py + node: test_page_closure_withholds_a_stale_reflection_from_a_swapping_input + proves: >- + A page that replaces its input element on every keystroke and writes + the value into its URL and title can end up showing a version the + field no longer holds. Flow withholds that reflected text whole -- + an origin-only URL and an empty title -- instead of matching it + against the current value, and keeps the clicked element's identity + evidence exact. + - path: tests/test_browser_attach.py + node: test_page_closure_still_reports_a_same_document_route_after_the_fix + proves: >- + A single-page application route change through history.pushState does + not build a new document, so the closure that held the declared value + is the one being sampled and its URL is still reported exactly. The + cross-document rule that withholds a later document's URL therefore + does not cost this evidence. + - path: tests/test_browser_attach.py + node: test_page_closure_withholds_a_title_a_consumed_field_produced + proves: >- + A scanner input that writes the badge into the page URL and title and + then clears its own field holds nothing at any moment Flow samples. + Flow arms the document's secret boundary from the input event itself, + so both channels are checked and both are withheld. + - path: tests/test_browser_attach.py + node: test_page_closure_keeps_a_consumed_value_across_a_second_entry + proves: >- + A second scan into the same cleared field does not displace the first + badge while the first badge is still shown in the URL. A value the + next one does not continue was taken by the page, not edited away by + the operator, so it is promoted into the withhold-only set. + - path: tests/test_browser_attach.py + node: test_page_closure_keeps_a_consumed_value_while_another_field_is_live + proves: >- + A second declared field holding a value does not re-expose the first + field's consumed value. The last-value test is per element, so the + same URL is withheld before and after the second field is filled. + - path: tests/test_browser_attach.py + node: test_page_closure_withholds_a_selector_built_from_an_inbound_value + proves: >- + A DOM selector is identity evidence and uses the same value set as + the accessible name and the clicked-row identity. An element id built + from a value carried in an inbound declared parameter is withheld with + a reason, not emitted verbatim. + - path: tests/test_browser_attach.py + node: test_page_closure_drops_only_the_unproven_parameter_value + proves: >- + A query parameter whose value Flow cannot prove predates the moment + the document first held a declared value loses only ITS value. Every + parameter name survives, the path stays exact, and the drop is + recorded with its reason. + - path: tests/test_browser_attach.py + node: test_page_closure_withholds_a_url_that_holds_the_value_in_its_path + proves: >- + A value that no parameter name identifies -- here written into a path + segment -- is caught by detection rather than structure: Flow + withholds the whole URL, marks it, and warns the operator that the + application placed a declared secret into its own URL. + - path: tests/test_browser_attach.py + node: test_page_closure_withholds_identity_after_the_field_is_removed + proves: >- + A single-page wizard that removes its declared field and renders the + value into a summary row cannot leak it into the clicked row's + identity: the value the field held at a commit point is retained for + the single purpose of WITHHOLDING identity text, never to rewrite it + and never for the URL or the title. + - path: tests/test_browser_attach.py + node: test_page_closure_marks_a_withheld_secret_field_name + proves: >- + Flow never reads the visible text of a bound secret field, because + that text is the value. The resulting missing accessible name is + reported as WITHHELD with a reason and counted, never left as a + silent null, while a control field beside it returns its name. + - path: tests/test_browser_attach.py + node: test_page_closure_emits_no_reflected_text_from_the_capture_phase + proves: >- + No browser event carries a URL or a title. The in-page listeners run + in the capture phase, before the page's own handlers, so anything + they read describes the state before the action. Flow samples + reflected evidence from Python at the settled boundary instead. caveats: - >- "Supported" is scoped to the reference headless-browser backend in this @@ -73,6 +205,11 @@ claims: - >- The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix. + - >- + Existing-session attachment is Chromium-only and loopback-only. It + requires a dedicated browser process started with remote debugging. + It does not claim support for the Capture Chrome extension prototype + or direct extension replay. # -------------------------------------------------- deterministic $0 replay - id: deterministic-zero-model-replay diff --git a/docs/BROWSER_RECORDING.md b/docs/BROWSER_RECORDING.md new file mode 100644 index 00000000..6becf391 --- /dev/null +++ b/docs/BROWSER_RECORDING.md @@ -0,0 +1,348 @@ +# Browser recording + +OpenAdapt has one supported browser recording contract. `openadapt-flow` +uses a Playwright page to retain ordered input events, DOM identity, field +geometry, exact before/after frames, and capture-time secret masking. The +result is the same compile-ready recording for both browser entry modes: + +- **Launch mode:** Flow starts a new Chromium browser and opens `--url`. +- **Attach mode:** Flow connects to an existing local Chromium browser and + binds one open tab. This mode keeps a browser session that already completed + sign-in, SSO, or 2FA. + +Both modes are part of the Browser / Playwright Beta surface. Attach mode uses +Chromium DevTools Protocol only as the local connection transport. The +recorder, schema, compiler, secret handling, and governed replay path do not +change. + +## Launch a new recording browser + +```bash +openadapt-flow record --backend web \ + --url https://your.app \ + --out recordings/browser-session +``` + +Flow opens the URL. Perform the workflow. Then press Ctrl-C in the terminal or +close the recording window. + +## Attach an existing signed-in browser + +Start Chromium with a dedicated debugging profile. Reuse this profile for +later recordings if it must retain its signed-in session. Do not enable remote +debugging on a sensitive general-purpose browser profile. + +macOS with Google Chrome: + +```bash +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \ + --remote-debugging-address=127.0.0.1 \ + --remote-debugging-port=9222 \ + --user-data-dir="$HOME/Library/Application Support/OpenAdapt/ChromeRecorderProfile" +``` + +Linux with Google Chrome or Chromium: + +```bash +google-chrome \ + --remote-debugging-address=127.0.0.1 \ + --remote-debugging-port=9222 \ + --user-data-dir="${XDG_DATA_HOME:-$HOME/.local/share}/openadapt/chrome-recorder-profile" +``` + +Windows PowerShell with Google Chrome: + +```powershell +& "$env:ProgramFiles\Google\Chrome\Application\chrome.exe" ` + --remote-debugging-address=127.0.0.1 ` + --remote-debugging-port=9222 ` + --user-data-dir="$env:LOCALAPPDATA\OpenAdapt\ChromeRecorderProfile" +``` + +Open and sign in to the application in that browser. Then attach the recorder: + +```bash +openadapt-flow record --backend web \ + --url https://your.app \ + --browser-cdp-endpoint http://127.0.0.1:9222 \ + --out recordings/browser-session +``` + +Keep the selected tab open during recording. Press `Ctrl-C` in the terminal to +finish. Flow retains the final evidence and then confirms the output path. It +refuses a closed tab and does not publish incomplete metadata. + +Flow selects the sole open HTTP or HTTPS tab on the `--url` origin. It does not +navigate the tab. It also does not close the tab or browser when recording +finishes. + +If two or more open tabs have that origin, Flow refuses to guess. Supply the +exact current URL: + +```bash +openadapt-flow record --backend web \ + --url https://your.app \ + --browser-cdp-endpoint http://127.0.0.1:9222 \ + --browser-page-url 'https://your.app/work/items?view=open' \ + --out recordings/browser-session-selected +``` + +Diagnostic messages omit URL query and fragment values. The exact selector is +used only to bind the requested tab and is not stored as separate attachment +metadata. The CDP endpoint is not stored. The normal recording evidence does +retain the declared app URL and each observed page URL before and after an +action. Those URLs can contain query or fragment values. Flow reduces every +recorded URL by structure: it reports the origin and the path, keeps every +parameter name, and drops the value of any parameter named after a declared +secret field. Treat all other URL data as sensitive recording data. + +## Safety and privacy contract + +- The CDP endpoint must use `localhost` or a loopback IP address and must have + an explicit port. Flow refuses a remote endpoint, URL credentials, query, or + fragment. +- The selected tab must have the same origin as `--url`. Zero matches and + ambiguous matches are refusals. +- The selected tab must stay on that origin for the full recording. A + cross-origin navigation stops the recording and does not produce complete + metadata. +- Do not open a popup or a new tab in the selected tab's browser context while + recording. Flow currently binds one page. It installs a new-page latch on + every candidate context before it reads the accepted page baseline or selects + the recording tab. Listener registration does not replay pre-existing tabs, + so those tabs remain allowed. The selected-context latch records every later + new-page signal, including a tab that acts and closes between recorder polls. + Flow keeps this refusal active through the final Playwright detach so a late + tab cannot produce complete metadata. A refusal leaves the external browser + and its tabs open. +- Attach mode does not combine with `--headless`. The external browser owns + its display mode. +- The `--out` path must not exist. Flow writes to a new temporary sibling and + publishes that directory only after it writes the final metadata. A refusal + removes the temporary output and does not change an existing recording. +- Input event payloads carry a unique recording-session binding and have a + 1 MB limit. Flow removes the current document listeners when it detaches. +- Flow retains one exact frame boundary per logical action. It coalesces only + consecutive input changes from the same bound field session or consecutive + scroll deltas from one observed gesture. If two distinct actions arrive in + one recorder poll, or the second arrives while Flow captures the first + action's after-frame, Flow refuses the recording and discards the temporary + output instead of publishing shared, incorrect evidence. +- Attached screenshots use CSS pixels and the actual live viewport. Thus, DOM + coordinates and retained frame coordinates stay aligned on high-density + displays. +- You can resize the tab or move its window between monitors while no action is + in progress. Flow observes viewport and device-scale changes, waits for a + stable CSS-pixel frame, and then starts a new per-event coordinate baseline. + `meta.json` retains the viewport history. Each frame-backed event retains its + exact `viewport_before` and `viewport_after`. +- Flow refuses only an action that overlaps a resize or monitor-scale change. + In that case, no exact pre-action frame exists in the new coordinate space, + so the recording stops without complete metadata. Stop interacting for a + moment after a resize. Recording then continues automatically. +- `input[type=password]` and fields declared with `--secret FIELD` keep their + literal values inside the page closure. Flow binds a private input-session + identity and a temporary screenshot-mask marker when the field appears in the + document. The + identity remains secret if application code removes the field name or ID + before focus, changes either attribute during input, or replaces the active + input element during the same input session. The listener consumes queued DOM + changes before each focus or input event, including a field that appears, + changes, or receives a replacement inside one JavaScript task. Flow removes + its marker when it detaches, including from a page-owned element that is no + longer in the DOM. +- **Flow reports captured page text exactly, or it withholds that text and + states why. Flow never rewrites captured text.** There is no placeholder + substitution anywhere in the recorder. Removing a value from text that was + already captured is a known-unsolved problem: Englehardt, Acar and Narayanan + measured every major session-replay vendor in 2017 and found that none + redacts displayed content automatically and that all of it leaked, and + PostHog and Sentry still carry open issues for secrets in replay URLs. The + working answer in production tools is capture-time, element-bound, + deny-by-default masking, which is what Flow does. +- Matching uses only the values that bound elements hold at that moment, read + live from the DOM. Flow keeps no value after the field stops holding it, and + a node the page detached is not a source of values: a controlled input that + replaces its element on every keystroke leaves keystroke prefixes on the + nodes it dropped, and those prefixes match ordinary page text by chance. A + replacement inherits the input session of the node it replaced, so a field + with no `name` and no `id` keeps one identity across the swap. +- **Identity evidence** -- the DOM selector (an element ID, `data-testid`, + `data-test`, or `name`), the control role, the accessible name, the clicked + row's identity characters, and the receiving field's name -- is exact or + withheld. Replay resolves and compares these against the live page, so a + rewritten copy would silently compare against characters the page never + showed. Every withheld item states why in the event (`identity_withheld`, + `sid_withheld`), so a DOM identity check can never disarm silently, and the + CLI summary counts the actions. A declared value too short to tell a real + match from an ordinary coincidence is reported separately + (`ambiguous-secret-in-identity`). +- **Reflected evidence** -- the page URL and the document title -- is sampled + from Python at the settled boundary, the same boundary that captures the + after-frame. The in-page listeners run in the capture phase, before the + page's own handlers, so any URL or title they read describes the state before + the action; they therefore emit none. +- **A URL is structure, not one opaque string.** Flow parses it and reports: + - the **origin** and the **path**, always. A path change is the single-page + application case, and a path is app-controlled structure rather than + operator input. Reporting it is what keeps URL evidence usable for a whole + session after a login. + - every parameter **name**, in the query and in a `key=value` fragment. + - a parameter **value**, unless Flow drops it. A value is dropped when the + parameter's NAME is a declared `--secret FIELD` name, or the `name` or `id` + of any field Flow bound (which includes every auto-detected + `input[type=password]`). That drop is deterministic and never looks at the + value, so it holds whatever the operator typed. A same-origin GET submit + carries a field under its own name, so this closes that channel by + structure. Sentry and Datadog redact URLs the same way, by parameter name, + for the same reason: searching for a value inside arbitrary text does not + work. A value is also dropped when Flow cannot prove it predates the moment + this document first held a declared value. Only that ONE value is dropped; + the rest of the URL stays exact. + - **Nothing is invented.** A dropped value becomes empty. Flow removes + characters from a URL; it never adds characters the page did not show. + `meta.json` lists every dropped parameter name and why, and `record` prints + it. +- **The net, which is a detection and never a rewrite.** If the URL Flow is + about to report still holds a value Flow can see, Flow withholds the WHOLE + URL, marks it, and warns the operator that the application put a declared + secret into its own URL. That warning matters on its own: OWASP notes such a + value is already exposed through browser history, server logs, proxies, CDNs + and the `Referer` header, with or without Flow. The net checks a path segment + in both containment directions, so a page that writes the field into its path + as the operator types cannot leave a segment behind. Matching is sound here + and was not sound in earlier revisions of this file: this check now runs only + from Python at the settled boundary, where the page has processed the action, + so it needs no history of previous values. The direction is fail-safe -- a + match withholds, so a false positive costs evidence and can never leak. +- **The title has no structure to exploit.** Flow reports it while it has not + changed since before that document held any declared value, withholds it + otherwise, and applies the same net. +- The page sends `location.origin` beside each event. The origin guard reads + that value and never the reflected text, so withholding a URL never weakens + the origin refusal. +- Flow traverses and observes open shadow roots at every event boundary. If a + shadow field can lose its name, ID, or password type before the first event, + give the shadow host the same `--secret FIELD` name or ID. Flow then masks the + complete host. For a pre-existing closed shadow root, Chromium CDP searches + only for the declared selector and runs the boundary check inside the page; + no node content crosses into Python. Flow refuses before the first frame when + the closed field exists but its host does not bind that declaration. It also + refuses a later unbound shadow input before it accepts a value. +- **The structural reduction applies WITHIN a document. It does not make a + LATER document safe.** Structure closes the QUERY channel, because a form + submits under the field's own name. It does not close the PATH channel: no + parameter name identifies a path segment, so a server that answers a submit + with a redirect to `/results/` puts the value where structure cannot + reach, and the new document's closure holds nothing to match it against. + Once a declared secret field receives input, Flow therefore withholds the + URL **and** the title of every document after the one that first held that + value, exactly as it did before the reduction existed. A document that + receives a declared value of its own is not exempt: holding a value says + nothing about whether it loaded with an earlier document's value in its path. +- A single-page application is unaffected by that rule. `history.pushState` and + `history.replaceState` do NOT build a new document, so the closure that held + the value is the closure being sampled, its URL is reduced by structure, and + a route change is reported exactly. The cross-document rule bites only on a + real navigation. +- A later document does recover the value of any inbound query or fragment + parameter whose NAME is declared and long enough to identify, and uses it to + withhold IDENTITY text there. That is how a results page which prints the + value into a row, an element id, or an accessible name is caught. + `meta.json` records every distinct reason in `structural_text_withheld`, and + `url_dropped_params` records a dropped parameter only for a URL Flow actually + reports; `record` prints one line for each. +- Matching considers four sources, and all four only ever WITHHOLD: + - what a bound field holds now; + - what a bound field held at a COMMIT POINT (`change`, `focusout`, `submit`, + `pagehide`), plus any value the page took from a field and started that + field over from — a value the next one does not CONTINUE was not edited + away by the operator; + - the LAST value each bound field was seen holding, added per element when + that element holds nothing right now. A detached element is added only when + no connected bound field holds anything, so a page that REMOVED its field + is covered while a controlled input that SWAPS its node does not contribute + the trail of one- and two-character values its discarded nodes still + report; + - an inbound declared parameter value. + + Comparison is case-insensitive, because upper-casing an identifier before + showing it is normalisation rather than a transform. Every identity path uses + this same set, including the DOM selector. +- A page that CONSUMES its own field — a scanner that writes the badge into the + URL and clears the input inside the same `input` handler — holds nothing at + any moment Flow samples. Flow arms the document's secret boundary from the + `input` event itself rather than from what the DOM holds at a sample, so such + a document is treated as having received a declared value, its title is + checked, and every later document is withheld. The commit point is what covers a single-page wizard that + removes its form and renders the value into a summary row. A commit is + decided at the microtask checkpoint, so a controlled input that fires + focusout on the node it just replaced commits no keystroke prefix. A spurious + match here can only withhold; it cannot corrupt evidence and it cannot leak, + which is why this is safe where a rewriting rule was not. The URL and the + title never use a committed value for their unchanged-or-withhold proof. +- **What this still costs.** A title is withheld for the rest of a document + once it changes after a declared field has held a value, and for every + document after that one. A parameter value Flow cannot prove predates the + value is dropped even when it is unrelated. Identity text is withheld + whenever it contains a value Flow can see, including by coincidence. Identity + evidence, action coordinates and the recorded before/after frames are + otherwise unaffected, so replay keeps its strongest identity tiers. +- **The stated residual.** The net matches a value Flow can see against the + text the page shows at the settled boundary. An application that updates its + URL on a timer longer than the settle window, or that writes a TRANSFORM of + the value rather than the value, shows text no value Flow can see contains. + The net will not match it, and Flow does NOT keep a previous value to catch + it: that is the rule three reviews broke. Treat every recorded URL as + sensitive recording data. +- Flow protects a declared value from the moment a bound field holds it. Text + and pixels captured BEFORE that moment are ordinary recording evidence. + +- Secret masks cover every frame in the selected page. Before each masked + screenshot, Flow snapshots the frame inventory and its lifecycle generation. + It accepts the in-memory image only when the inventory stays unchanged + through capture. It retries a bounded number of times and refuses persistent + frame churn. A discarded unstable image never reaches disk or metadata. +- The close, new-page, origin, and frame lifecycle guards stay active until + Flow detaches from the external browser. Flow promotes the temporary output + only after detach succeeds and every guard remains clear. + +## Why the Capture Chrome extension is not this path + +The `openadapt-capture` repository contains a custom Chrome extension +prototype. It proved useful DOM event capture, but its current direct +WebSocket and replay design does not implement the supported contract above. +It does not yet bind messages to an authenticated recording session, one tab, +one document, and an ordered acknowledged event stream. It also does not +provide the compiler's exact before/after frame binding and source-time secret +redaction. Its direct DOM replay can dispatch actions without the governed +runtime's identity, policy, fresh-frame, and effect checks. + +The prototype should remain available for development. It can become a +supported acquisition transport after it does all of the following: + +1. Use the shared Flow event and evidence schema. Do not create a second + compiler or replay format. +2. Redact secret values before they cross the extension boundary. +3. Bind and authenticate the browser profile, tab, document, run, session, and + monotonically increasing event sequence. A reconnect must acknowledge or + safely resume events instead of dropping them. +4. Retain exact frame-to-event evidence and bind each event to its current + viewport coordinate system. +5. Send recordings to the existing compiler. Do not perform direct replay. +6. Pass the same three-trial record, compile, secret, ambiguity-refusal, and + browser-lifecycle tests as the Playwright attach mode. + +Until that contract exists, the extension is a prototype component. This label +does not apply to `openadapt-capture` as a whole. Capture is the canonical +native recorder; the browser recorder stays Playwright-native because browser +DOM identity and source-time secret handling are load-bearing. + +## Current boundary + +Attach mode supports local Chromium-family browsers that expose a CDP +endpoint. It requires a browser process started with remote debugging and a +separate user-data directory. It does not claim Firefox, WebKit, arbitrary +Chrome extensions, an ordinary browser process that was not started for local +debugging, cross-origin tab selection, separately qualified cross-frame/iframe +recording, multi-page/popup recording, or direct extension replay. diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 78cd50b8..f6ba95e5 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -18,7 +18,7 @@ For the current evidence behind each maturity claim, see | Capability | Maturity | What the claim means | What it does not mean | | --- | --- | --- | --- | -| Browser record, compile, and replay | **Beta** | The reference browser path runs end to end in automated and clean-environment tests. | It is not evidence for every site, browser extension, authentication flow, or long-running production workload. | +| Browser record, compile, and replay | **Beta** | The launched-browser path runs end to end in automated and clean-environment tests. The attached-browser path reuses the same recorder and passes 3 real Chromium record-and-compile trials with secret exclusion and external-browser survival checks. A live resize case also compiles actions from two viewport and device-scale baselines. | It is not evidence for every site, browser extension, authentication flow, or long-running production workload. Attach mode requires a loopback CDP endpoint and one same-origin Chromium tab. An action that overlaps a resize is refused because its pre-action coordinate evidence is not exact. The custom Capture extension remains a prototype and is not a direct replay path. | | Healthy replay with zero model calls | **Beta** | A run that resolves from retained evidence can execute without a language or vision model; the run report counts model calls. | Zero model calls does not mean zero network traffic. The target application, hosted control plane, remote backend, or effect verifier may still use the network. | | Deterministic re-resolution | **Beta** | Bounded visual or structural drift can be resolved through non-model evidence and recorded as a reviewable change. | It is not general adaptation to a redesigned workflow, changed business rules, or missing evidence. | | AI-assisted repair | **Experimental** | An explicitly enabled model can propose a target or interpret a changed screen. Existing runtime checks still apply. | A model proposal is not authorization, proof of identity, or proof that a business transaction succeeded. | diff --git a/docs/PRODUCT_STATUS.md b/docs/PRODUCT_STATUS.md index 955f88f3..92db2399 100644 --- a/docs/PRODUCT_STATUS.md +++ b/docs/PRODUCT_STATUS.md @@ -23,7 +23,7 @@ and its generated view is [`VERIFICATION.md`](VERIFICATION.md). | Surface | Status | What is proven | Boundary that remains | | --- | --- | --- | --- | | Demonstration compiler and bundle | **Beta** | Browser recording compiles into a parameterized, inspectable bundle in CI. | One demonstration can under-specify intent; production policies and effect bindings still require operator work. | -| Browser / Playwright recording and replay | **Beta** | Record, compile, replay, deterministic drift repair, reports, and refusal all run end to end against MockMed; a bounded OpenEMR result is published separately. | The reference path is not evidence for arbitrary sites, long-term drift, or production reliability. | +| Browser / Playwright recording and replay | **Beta** | Record, compile, replay, deterministic drift repair, reports, and refusal all run end to end against MockMed; a bounded OpenEMR result is published separately. The required browser suite also performs 3 real Chromium CDP-attach record-and-compile trials, checks source-time password exclusion, proves that recorder shutdown leaves the external browser running, and compiles actions across a live viewport and device-scale change. | The reference path is not evidence for arbitrary sites, long-term drift, or production reliability. Attach mode is Chromium-only, loopback-only, and requires a browser started with remote debugging. It refuses an action that overlaps a resize transition. It does not promote the Capture Chrome extension prototype or direct extension replay. | | Healthy zero-model replay | **Beta** | Repeated CI runs use the deterministic ladder with zero model calls. | Optional model grounding is a separate opt-in fallback; a changed app can still halt. | | Deterministic re-resolution | **Beta** | Theme, moved-control, and renamed-control fixtures resolve through non-model rungs and emit reviewable patches. | It covers bounded evidence-preserving drift, not arbitrary workflow or business-logic change. | | AI-assisted repair | **Experimental** | Local/remote VLM contracts, egress gates, refusal behavior, and retention boundaries are tested. | Off by default; model accuracy is not a safety guarantee and real deployment quality is unmeasured. | diff --git a/docs/SURFACES.md b/docs/SURFACES.md index 06de164d..d2f2f4ea 100644 --- a/docs/SURFACES.md +++ b/docs/SURFACES.md @@ -58,6 +58,10 @@ openadapt-flow record --backend web --url https://your.app --out rec openadapt-flow compile rec --out bundle --name my-task openadapt-flow replay bundle --url https://your.app +# Attach the same recorder to one existing signed-in local Chromium tab. +openadapt-flow record --backend web --url https://your.app \ + --browser-cdp-endpoint http://127.0.0.1:9222 --out rec + # Windows: Capture records the local window; the in-guest WAA agent replays it. openadapt-flow record --backend windows --window "Target App" --out rec openadapt-flow compile rec --out bundle --name my-task @@ -100,6 +104,15 @@ session cannot control them, so `record` refuses them instead of accepting an unused flag. Pass them to `replay`/`run`; `run ... --config deploy.yaml --profile standard|regulated` wires the same selection for a real deployment. +The browser attach mode keeps the Playwright-native recording contract. It +binds one same-origin tab and reuses the same event schema, DOM evidence, +before/after frames, secret redaction, compiler, and governed replay path as a +browser that Flow launches. The endpoint is local-loopback only. Flow refuses +ambiguous tabs and does not navigate or close the attached browser. It +rebaselines exact event/frame coordinates after an idle resize or +monitor-scale change. It refuses an action that overlaps that transition. See +[`BROWSER_RECORDING.md`](BROWSER_RECORDING.md). + ## The two remote execution modes Remote systems (a Windows guest, a virtual desktop, a published app) can be diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 8cca61fc..d461bc24 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -30,11 +30,29 @@ | `tests/e2e/test_record_compile_replay.py` | test | ci (required PR gate (e2e-browser)) | supported | Records the MockMed browser demo once, compiles it, and replays it under baseline + theme/move/rename drift and parameter substitution through the headless-browser Backend. | | `tests/test_mockmed.py` | test | ci (required PR gate (test)) | supported | The reference browser demo app and its drift screens render deterministically (no CSS transitions), so replay is repeatable. | | `tests/test_replayer.py` | test | ci (required PR gate (test)) | supported | The deterministic replayer resolves steps, substitutes parameters, enforces postconditions and the risk gate — no model in the loop. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | Three real Chromium CDP-attach trials record and compile the same synthetic workflow, exclude password values before persistence, preserve CSS-pixel frame/coordinate alignment, and detach without closing the external browser. The same live campaign covers same-task replacement, declared open and closed shadow hosts, contenteditable click metadata, and same-document URL/title reflection through history.replaceState. A live case records and compiles actions across viewport and monitor-scale changes. It binds each event to its exact frame dimensions. A separate live case refuses an action that overlaps the transition. Unit cases refuse remote endpoints, cross-origin selectors and navigation, iframe events, invalid viewport evidence, and ambiguous same-origin tabs. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A live Chromium recording submits a same-origin GET form. The document that submit reaches is a fresh closure that holds no value to match against, so Flow withholds its URL and its title and says so, and the recorded surface is stamped before publish. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A server that answers a GET submit with a 302 to `/results/` puts a declared secret in a path segment, where no parameter name identifies it. Flow withholds the whole URL for every document after the one that first held the value, so the literal never reaches events.jsonl. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A lowercase secret typed one character at a time into a page whose URL, title, and button ID share those characters leaves the URL, the title, the DOM selector, and the accessible name exact. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A password that begins with a common English word leaves that word exact in the page URL, the page title, the clicked row's identity characters, the accessible name, and an unrelated button ID. Matching uses only the value the field holds at that moment, so an intermediate keystroke prefix never becomes a matching value. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | Identity evidence that holds a declared value is WITHHELD with a stated reason, never rewritten. Replay compares identity evidence against the live page, so a rewritten copy would compare against characters the page never showed, invisibly. No placeholder string appears anywhere in the recorded evidence. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A page that replaces its input element on every keystroke and writes the value into its URL and title can end up showing a version the field no longer holds. Flow withholds that reflected text whole -- an origin-only URL and an empty title -- instead of matching it against the current value, and keeps the clicked element's identity evidence exact. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A single-page application route change through history.pushState does not build a new document, so the closure that held the declared value is the one being sampled and its URL is still reported exactly. The cross-document rule that withholds a later document's URL therefore does not cost this evidence. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A scanner input that writes the badge into the page URL and title and then clears its own field holds nothing at any moment Flow samples. Flow arms the document's secret boundary from the input event itself, so both channels are checked and both are withheld. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A second scan into the same cleared field does not displace the first badge while the first badge is still shown in the URL. A value the next one does not continue was taken by the page, not edited away by the operator, so it is promoted into the withhold-only set. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A second declared field holding a value does not re-expose the first field's consumed value. The last-value test is per element, so the same URL is withheld before and after the second field is filled. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A DOM selector is identity evidence and uses the same value set as the accessible name and the clicked-row identity. An element id built from a value carried in an inbound declared parameter is withheld with a reason, not emitted verbatim. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A query parameter whose value Flow cannot prove predates the moment the document first held a declared value loses only ITS value. Every parameter name survives, the path stays exact, and the drop is recorded with its reason. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A value that no parameter name identifies -- here written into a path segment -- is caught by detection rather than structure: Flow withholds the whole URL, marks it, and warns the operator that the application placed a declared secret into its own URL. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | A single-page wizard that removes its declared field and renders the value into a summary row cannot leak it into the clicked row's identity: the value the field held at a commit point is retained for the single purpose of WITHHOLDING identity text, never to rewrite it and never for the URL or the title. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | Flow never reads the visible text of a bound secret field, because that text is the value. The resulting missing accessible name is reported as WITHHELD with a reason and counted, never left as a silent null, while a control field beside it returns its name. | +| `tests/test_browser_attach.py` | test | ci (required PR gate (test)) | supported | No browser event carries a URL or a title. The in-page listeners run in the capture phase, before the page's own handlers, so anything they read describes the state before the action. Flow samples reflected evidence from Python at the settled boundary instead. | **Caveats (honest limits):** - "Supported" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below. - The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix. +- Existing-session attachment is Chromium-only and loopback-only. It requires a dedicated browser process started with remote debugging. It does not claim support for the Capture Chrome extension prototype or direct extension replay. ### `deterministic-zero-model-replay` — supported — bound to required CI pass evidence diff --git a/docs/verification.json b/docs/verification.json index 04e3726c..63803a82 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -19,7 +19,8 @@ "strongest_evidence": "supported", "caveats": [ "\"Supported\" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below.", - "The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix." + "The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix.", + "Existing-session attachment is Chromium-only and loopback-only. It requires a dedicated browser process started with remote debugging. It does not claim support for the Capture Chrome extension prototype or direct extension replay." ], "evidence": [ { @@ -57,6 +58,210 @@ "ci_job": "test", "junit_status": null, "proves": "The deterministic replayer resolves steps, substitutes parameters, enforces postconditions and the risk gate \u2014 no model in the loop." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_live_cdp_attach_records_compiles_and_leaves_browser_running_three_trials", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "Three real Chromium CDP-attach trials record and compile the same synthetic workflow, exclude password values before persistence, preserve CSS-pixel frame/coordinate alignment, and detach without closing the external browser. The same live campaign covers same-task replacement, declared open and closed shadow hosts, contenteditable click metadata, and same-document URL/title reflection through history.replaceState. A live case records and compiles actions across viewport and monitor-scale changes. It binds each event to its exact frame dimensions. A separate live case refuses an action that overlaps the transition. Unit cases refuse remote endpoints, cross-origin selectors and navigation, iframe events, invalid viewport evidence, and ambiguous same-origin tabs." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_launched_recording_withholds_a_later_document_url_after_a_get_submit", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A live Chromium recording submits a same-origin GET form. The document that submit reaches is a fresh closure that holds no value to match against, so Flow withholds its URL and its title and says so, and the recorded surface is stamped before publish." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_launched_recording_withholds_a_redirect_that_puts_the_value_in_a_path", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A server that answers a GET submit with a 302 to `/results/` puts a declared secret in a path segment, where no parameter name identifies it. Flow withholds the whole URL for every document after the one that first held the value, so the literal never reaches events.jsonl." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_keeps_url_and_identity_evidence_for_a_lowercase_secret", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A lowercase secret typed one character at a time into a page whose URL, title, and button ID share those characters leaves the URL, the title, the DOM selector, and the accessible name exact." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_keeps_all_evidence_for_a_password_starting_with_a_word", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A password that begins with a common English word leaves that word exact in the page URL, the page title, the clicked row's identity characters, the accessible name, and an unrelated button ID. Matching uses only the value the field holds at that moment, so an intermediate keystroke prefix never becomes a matching value." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_withholds_identity_that_holds_a_declared_value", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "Identity evidence that holds a declared value is WITHHELD with a stated reason, never rewritten. Replay compares identity evidence against the live page, so a rewritten copy would compare against characters the page never showed, invisibly. No placeholder string appears anywhere in the recorded evidence." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_withholds_a_stale_reflection_from_a_swapping_input", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A page that replaces its input element on every keystroke and writes the value into its URL and title can end up showing a version the field no longer holds. Flow withholds that reflected text whole -- an origin-only URL and an empty title -- instead of matching it against the current value, and keeps the clicked element's identity evidence exact." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_still_reports_a_same_document_route_after_the_fix", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A single-page application route change through history.pushState does not build a new document, so the closure that held the declared value is the one being sampled and its URL is still reported exactly. The cross-document rule that withholds a later document's URL therefore does not cost this evidence." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_withholds_a_title_a_consumed_field_produced", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A scanner input that writes the badge into the page URL and title and then clears its own field holds nothing at any moment Flow samples. Flow arms the document's secret boundary from the input event itself, so both channels are checked and both are withheld." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_keeps_a_consumed_value_across_a_second_entry", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A second scan into the same cleared field does not displace the first badge while the first badge is still shown in the URL. A value the next one does not continue was taken by the page, not edited away by the operator, so it is promoted into the withhold-only set." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_keeps_a_consumed_value_while_another_field_is_live", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A second declared field holding a value does not re-expose the first field's consumed value. The last-value test is per element, so the same URL is withheld before and after the second field is filled." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_withholds_a_selector_built_from_an_inbound_value", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A DOM selector is identity evidence and uses the same value set as the accessible name and the clicked-row identity. An element id built from a value carried in an inbound declared parameter is withheld with a reason, not emitted verbatim." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_drops_only_the_unproven_parameter_value", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A query parameter whose value Flow cannot prove predates the moment the document first held a declared value loses only ITS value. Every parameter name survives, the path stays exact, and the drop is recorded with its reason." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_withholds_a_url_that_holds_the_value_in_its_path", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A value that no parameter name identifies -- here written into a path segment -- is caught by detection rather than structure: Flow withholds the whole URL, marks it, and warns the operator that the application placed a declared secret into its own URL." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_withholds_identity_after_the_field_is_removed", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A single-page wizard that removes its declared field and renders the value into a summary row cannot leak it into the clicked row's identity: the value the field held at a commit point is retained for the single purpose of WITHHOLDING identity text, never to rewrite it and never for the URL or the title." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_marks_a_withheld_secret_field_name", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "Flow never reads the visible text of a bound secret field, because that text is the value. The resulting missing accessible name is reported as WITHHELD with a reason and counted, never left as a silent null, while a control field beside it returns its name." + }, + { + "path": "tests/test_browser_attach.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_page_closure_emits_no_reflected_text_from_the_capture_phase", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "No browser event carries a URL or a title. The in-page listeners run in the capture phase, before the page's own handlers, so anything they read describes the state before the action. Flow samples reflected evidence from Python at the settled boundary instead." } ], "errors": [] diff --git a/openadapt_flow/__main__.py b/openadapt_flow/__main__.py index 67a445d8..f81a019a 100644 --- a/openadapt_flow/__main__.py +++ b/openadapt_flow/__main__.py @@ -594,6 +594,10 @@ def _stamp_recording_surface(recording_dir: Path, surface: str) -> None: The compiler binds the compiled bundle to this exact surface. Best-effort by design: a recording written by an older converter without ``meta.json`` simply compiles to a legacy, surface-unbound bundle. + + A recorder that stamps the surface itself before it publishes the + recording leaves nothing to do here. This function then writes nothing, so + a complete recording is never modified after its atomic publish. """ import json @@ -602,6 +606,8 @@ def _stamp_recording_surface(recording_dir: Path, surface: str) -> None: meta = json.loads(meta_path.read_text()) except (OSError, ValueError): return + if meta.get("surface") == surface: + return meta["surface"] = surface meta_path.write_text(json.dumps(meta, indent=2)) @@ -895,7 +901,16 @@ def _cmd_record(args: argparse.Namespace) -> int: print(demo_default_notice(backend, from_last_used=last is not None)) elif profile == "demo": store_last_surface(_report_backend_kind(backend)) + browser_attach_requested = bool( + getattr(args, "browser_cdp_endpoint", None) + or getattr(args, "browser_page_url", None) + ) if backend in ("windows", "macos", "linux", "rdp", "citrix"): + if browser_attach_requested: + raise SystemExit( + "record: --browser-cdp-endpoint and --browser-page-url apply " + "only to --backend web" + ) return _cmd_record_desktop(args, backend) if ( @@ -920,30 +935,143 @@ def _cmd_record(args: argparse.Namespace) -> int: raise SystemExit( "record --backend web requires --url (the app to record against)." ) + if getattr(args, "browser_page_url", None) and not getattr( + args, "browser_cdp_endpoint", None + ): + raise SystemExit("record: --browser-page-url requires --browser-cdp-endpoint") + if getattr(args, "browser_cdp_endpoint", None) and args.headless: + raise SystemExit( + "record: --headless cannot be combined with " + "--browser-cdp-endpoint; the attached browser controls its own " + "display mode" + ) - from openadapt_flow.interactive_recorder import record_interactive - - out = record_interactive( - args.url, - Path(args.out), - secret_fields=tuple(args.secret or ()), - param_fields=tuple(args.param or ()), - identifier_fields=tuple(getattr(args, "identifier", None) or ()), - headless=args.headless, + from openadapt_flow.interactive_recorder import ( + BrowserAttachError, + record_interactive, ) + + try: + out = record_interactive( + args.url, + Path(args.out), + secret_fields=tuple(args.secret or ()), + param_fields=tuple(args.param or ()), + identifier_fields=tuple(getattr(args, "identifier", None) or ()), + headless=args.headless, + cdp_endpoint=getattr(args, "browser_cdp_endpoint", None), + browser_page_url=getattr(args, "browser_page_url", None), + surface="web", + ) + except BrowserAttachError as exc: + raise SystemExit(f"record: browser attachment refused: {exc}") from exc _stamp_recording_surface(out, "web") print(f"Recording written to {out}") secrets = sorted(args.secret or ()) if secrets: print( - "Secret field(s) recorded (values NOT stored): " + "Secret field(s) recorded (no value stored, and each value is " + "redacted from the recorded URL, title, label, and structural " + "text): " + ", ".join(secrets) + ". At replay, export " + ", ".join(f"OPENADAPT_FLOW_SECRET_{name.upper()}" for name in secrets) ) + for notice in _recording_privacy_notices(out): + print(notice) return 0 +def _recording_privacy_notices(recording_dir: Path) -> list[str]: + """Report exactly what the recorder had to withhold, or nothing. + + The claim that a secret value is not stored is the claim the operator uses + to decide whether a recording is safe to keep or to share. Where the + recorder had to drop evidence to keep that claim true, say so here. + """ + import json + + try: + meta = json.loads((Path(recording_dir) / "meta.json").read_text()) + except (OSError, ValueError): + return [] + notices: list[str] = [] + raw_withheld = meta.get("structural_text_withheld") + if raw_withheld: + explained = { + "secret-value-left-its-document": ( + "a declared secret value left its document (for example a form " + "submit that reflects the value into the next URL), and the new " + "document builds a page closure that never saw that value" + ), + "reflected-text-changed-after-a-secret-value": ( + "the page URL or title changed after a declared secret field " + "held a value, so Flow could not prove the new text was not a " + "reflection of that value" + ), + "title-changed-after-a-secret-value": ( + "the page title changed after a declared secret field held a " + "value. A title has no structure to reduce, so Flow withholds " + "it rather than guess whether it reflects the value" + ), + "declared-value-in-url": ( + "the page URL held a declared secret value that no parameter " + "name identified, so structure alone could not remove it" + ), + "declared-value-in-title": ("the page title held a declared secret value"), + "url-cannot-be-parsed": ( + "the page reported a URL that is not an HTTP or HTTPS URL, so " + "Flow could not reduce it by structure" + ), + "opaque-secret-boundary": ( + "a declared secret field sits behind a closed shadow root, " + "which exposes its value to no check at all" + ), + } + for reason in [ + part.strip() for part in str(raw_withheld).split(",") if part.strip() + ]: + notices.append( + "Flow withheld the page URL and title because " + f"{explained.get(reason, reason)}. Those actions carry an " + "origin-only URL and an empty title. Flow reports page text " + "exactly or not at all and never rewrites it, so the recording " + "holds no secret value and no altered text." + ) + dropped = meta.get("url_dropped_params") + if isinstance(dropped, list) and dropped: + names = sorted( + {str(entry.get("name")) for entry in dropped if isinstance(entry, dict)} + ) + notices.append( + "Flow removed the value of these URL parameters and kept their " + f"names: {', '.join(names)}. A parameter named after a declared " + "secret field loses its value in every recorded URL, whatever the " + "value is. The rest of each URL -- origin, path and other " + "parameters -- is exact." + ) + if meta.get("application_placed_secret_in_url") or meta.get( + "application_placed_secret_in_title" + ): + notices.append( + "WARNING: the application put a declared secret value into its own " + "page URL or title. Flow withheld that text, but this is an " + "application defect that exists with or without Flow: OWASP lists " + "browser history, server logs, proxies, CDNs and the Referer " + "header as places such a value is already exposed. Report it to " + "the application owner." + ) + withheld_identity = meta.get("identity_withheld_events") + if isinstance(withheld_identity, int) and withheld_identity > 0: + notices.append( + f"{withheld_identity} action(s) carry no DOM selector: the element " + "identity contained a declared secret value, so Flow refused to " + "record it. Replay uses the remaining identity tiers for those " + "actions." + ) + return notices + + def _cmd_record_desktop(args: argparse.Namespace, backend: str) -> int: """Record a live desktop demonstration for a native/pixel desktop backend. @@ -4491,6 +4619,27 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="URL of the app to record against (required for --backend web)", ) + p.add_argument( + "--browser-cdp-endpoint", + default=None, + metavar="URL", + help=( + "Attach the web recorder to an already-running local Chromium " + "browser through its loopback DevTools endpoint (for example, " + "http://127.0.0.1:9222). The recorder selects a tab on the " + "--url origin and does not launch, navigate, or close the browser." + ), + ) + p.add_argument( + "--browser-page-url", + default=None, + metavar="URL", + help=( + "Exact current URL of the existing tab to record. Use this with " + "--browser-cdp-endpoint when more than one open tab has the " + "--url origin." + ), + ) p.add_argument("--out", required=True, help="Recording output directory") p.add_argument( "--secret", diff --git a/openadapt_flow/backends/playwright_backend.py b/openadapt_flow/backends/playwright_backend.py index 13f1cd9f..478cc1fd 100644 --- a/openadapt_flow/backends/playwright_backend.py +++ b/openadapt_flow/backends/playwright_backend.py @@ -15,7 +15,7 @@ import uuid from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional from urllib.parse import urlsplit if TYPE_CHECKING: # pragma: no cover @@ -33,6 +33,7 @@ ) VIEWPORT: tuple[int, int] = (1280, 800) +_MASKED_SCREENSHOT_ATTEMPTS = 3 _MODIFIER_ALIASES = { "meta": "Meta", @@ -88,6 +89,10 @@ ) +class ScreenshotMaskStabilityError(RuntimeError): + """The browser frame tree changed across every masked screenshot attempt.""" + + @dataclass class _StructuralGuard: """Private one-shot binding retaining only token, scope, and frame selectors.""" @@ -703,13 +708,50 @@ class PlaywrightBackend: such as the demo driver may use locators; replay never does). """ - def __init__(self, page: "Page") -> None: + def __init__( + self, + page: "Page", + *, + screenshot_scale: Literal["css", "device"] = "device", + screenshot_mask_selectors: tuple[str, ...] = (), + structural_state_reader: Optional[Callable[[], dict[str, Any]]] = None, + screenshot_guard: Optional[Callable[[], None]] = None, + ) -> None: """Wrap an existing Playwright page. Args: - page: A page created with viewport 1280x800, deviceScaleFactor=1. + page: A Playwright page. + screenshot_scale: Pixel scale for retained screenshots. The + ordinary launched-browser path uses Playwright's ``device`` + default. A browser attached through CDP uses ``css`` so DOM + event coordinates and retained frame pixels stay in the same + coordinate system even on a high-density display. + screenshot_mask_selectors: CSS selectors whose matching elements + are blacked out by Chromium before screenshot bytes reach + Python. The interactive recorder uses this for password and + declared-secret fields on every retained frame. Locators are + rebuilt from every current document frame for each screenshot + so a child-frame field or a frame added after startup cannot + bypass the mask. + structural_state_reader: Optional source-time sanitized URL/title + reader. Recording paths use it to keep raw reflected secrets + out of Python-side structural evidence. + screenshot_guard: Optional fail-closed check that runs before + Chromium creates screenshot bytes. Recording paths use it to + bind or refuse closed-shadow secret boundaries. """ self.page = page + self._screenshot_scale = screenshot_scale + self._screenshot_mask_selectors = screenshot_mask_selectors + self._structural_state_reader = structural_state_reader + self._screenshot_guard = screenshot_guard + self._screenshot_frame_generation = 0 + self._screenshot_frame_listener = self._handle_screenshot_frame_lifecycle + self._screenshot_frame_tracking = False + if self._screenshot_mask_selectors: + for event in ("frameattached", "framedetached", "framenavigated"): + self.page.on(event, self._screenshot_frame_listener) + self._screenshot_frame_tracking = True # Opaque per-backend key keeps the WeakMap private from ordinary page # code. Python retains only token material keyed by the public # SHA-256 fingerprint; target/row text stays page-local and ephemeral. @@ -726,7 +768,21 @@ def __init__(self, page: "Page") -> None: def viewport(self) -> tuple[int, int]: """(width, height) of the page viewport in pixels.""" size = self.page.viewport_size - if size is None: # pragma: no cover - viewport always set by launch() + if size is None: + # A Chromium page reached through ``connect_over_cdp`` normally + # has no Playwright viewport emulation. Reading the live CSS + # viewport avoids both the old fixed 1280x800 fallback and any + # mutation of the operator's browser window. + try: + live = self.page.evaluate( + "() => ({width: window.innerWidth, height: window.innerHeight})" + ) + width = int(live["width"]) + height = int(live["height"]) + if width > 0 and height > 0: + return (width, height) + except Exception: + pass return VIEWPORT return (size["width"], size["height"]) @@ -762,6 +818,12 @@ def browser_presentation_viewport(self) -> Optional[tuple[int, int, float]]: @property def url(self) -> Optional[str]: """Current page URL, or None if momentarily unobservable.""" + if self._structural_state_reader is not None: + try: + value = self._structural_state_reader().get("url") + return value if isinstance(value, str) else None + except Exception: + return None try: return self.page.url except Exception: @@ -770,6 +832,12 @@ def url(self) -> Optional[str]: @property def page_title(self) -> Optional[str]: """Current page title, or None if momentarily unobservable.""" + if self._structural_state_reader is not None: + try: + value = self._structural_state_reader().get("title") + return value if isinstance(value, str) else None + except Exception: + return None try: return self.page.title() except Exception: @@ -2091,11 +2159,15 @@ def guarded_keyboard_frame(self) -> bytes: """ def capture() -> bytes: + options: dict[str, Any] = {} + if self._screenshot_scale == "css": + options["scale"] = "css" return self.page.screenshot( type="png", full_page=False, caret="initial", style="* { caret-color: transparent !important; }", + **options, ) previous = capture() @@ -2195,9 +2267,71 @@ def type_text_guarded( deliver=lambda locator: locator.press_sequentially(text, timeout=1000), ) + def _handle_screenshot_frame_lifecycle(self, _frame: Any = None) -> None: + """Advance the irreversible frame-tree generation.""" + + self._screenshot_frame_generation += 1 + + def stop_screenshot_mask_tracking(self) -> None: + """Remove recording-only frame listeners from an external page.""" + + if not self._screenshot_frame_tracking: + return + for event in ("frameattached", "framedetached", "framenavigated"): + try: + self.page.remove_listener(event, self._screenshot_frame_listener) + except Exception: + pass + self._screenshot_frame_tracking = False + + @staticmethod + def _same_frames(left: tuple[Any, ...], right: tuple[Any, ...]) -> bool: + return len(left) == len(right) and all( + before is after for before, after in zip(left, right) + ) + def screenshot(self) -> bytes: - """Return the current full-viewport frame as PNG bytes.""" - return self.page.screenshot(type="png", full_page=False) + """Return a stable current full-viewport frame as PNG bytes.""" + if self._screenshot_guard is not None: + self._screenshot_guard() + base_options: dict[str, Any] = {} + if self._screenshot_scale == "css": + base_options["scale"] = "css" + if not self._screenshot_mask_selectors: + return self.page.screenshot(type="png", full_page=False, **base_options) + + for _attempt in range(_MASKED_SCREENSHOT_ATTEMPTS): + generation = self._screenshot_frame_generation + frames = tuple(self.page.frames) + if generation != self._screenshot_frame_generation: + continue + options = dict(base_options) + options["mask"] = [ + frame.locator(selector) + for frame in frames + for selector in self._screenshot_mask_selectors + ] + options["mask_color"] = "#000000" + try: + png = self.page.screenshot(type="png", full_page=False, **options) + # Flush lifecycle events that Chromium sent with or before the + # screenshot response before accepting the in-memory bytes. + self.page.evaluate("() => null") + except Exception: + if generation != self._screenshot_frame_generation: + continue + raise + current_frames = tuple(self.page.frames) + if generation == self._screenshot_frame_generation and self._same_frames( + frames, current_frames + ): + return png + # ``png`` is intentionally discarded here. It never reaches the + # recorder, disk, or a compiled bundle. + raise ScreenshotMaskStabilityError( + "the browser frame tree changed during every secret-masked " + "screenshot attempt; recording was refused" + ) def click(self, x: int, y: int, *, double: bool = False) -> None: """Click (or double-click) at pixel coordinates via the mouse.""" diff --git a/openadapt_flow/compiler/compile.py b/openadapt_flow/compiler/compile.py index 6987ddac..8a6059dd 100644 --- a/openadapt_flow/compiler/compile.py +++ b/openadapt_flow/compiler/compile.py @@ -214,6 +214,78 @@ def _read_png(path: Path) -> Optional[bytes]: return path.read_bytes() if path.exists() else None +def _png_viewport(png: bytes, *, source: str) -> tuple[int, int]: + """Return a PNG's exact ``(width, height)`` or reject invalid evidence.""" + + frame = cv2.imdecode(np.frombuffer(png, dtype=np.uint8), cv2.IMREAD_COLOR) + if frame is None: + raise ValueError(f"could not decode {source} PNG") + return (int(frame.shape[1]), int(frame.shape[0])) + + +def _validated_event_viewport( + event: dict, + *, + key: str, + png: Optional[bytes], + event_index: int, +) -> Optional[tuple[int, int]]: + """Bind an event viewport declaration to its exact retained PNG. + + Older recordings do not carry per-event viewport fields. They remain + valid and use the PNG dimensions as the source of truth. A new recording + that does carry the field must match the retained evidence exactly. + """ + + declared = event.get(key) + actual = ( + _png_viewport(png, source=f"event {event_index} {key}") + if png is not None + else None + ) + if declared is None: + return actual + if ( + not isinstance(declared, (list, tuple)) + or len(declared) != 2 + or any( + isinstance(value, bool) or not isinstance(value, int) for value in declared + ) + or int(declared[0]) <= 0 + or int(declared[1]) <= 0 + ): + raise ValueError( + f"events.jsonl event {event_index} {key} must be two positive integers" + ) + if actual is None: + raise ValueError( + f"events.jsonl event {event_index} declares {key} without its PNG" + ) + normalized = (int(declared[0]), int(declared[1])) + if normalized != actual: + raise ValueError( + f"events.jsonl event {event_index} {key} {normalized} does not match " + f"the retained PNG {actual}" + ) + return actual + + +def _validate_point_in_viewport( + point: Point, + viewport: tuple[int, int], + *, + event_index: int, + label: str, +) -> None: + """Reject pointer evidence outside its declared coordinate space.""" + + if not (0 <= point[0] < viewport[0] and 0 <= point[1] < viewport[1]): + raise ValueError( + f"events.jsonl event {event_index} {label} {point} is outside " + f"viewport {viewport}" + ) + + def _clamped_crop_region( click: Point, frame_w: int, @@ -1715,6 +1787,28 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: ) before_png = _read_png(before_path) after_png = _read_png(after_path) + before_viewport = _validated_event_viewport( + event, + key="viewport_before", + png=before_png, + event_index=i, + ) + after_viewport = _validated_event_viewport( + event, + key="viewport_after", + png=after_png, + event_index=i, + ) + if ( + before_viewport is not None + and after_viewport is not None + and before_viewport != after_viewport + ): + raise ValueError( + f"viewport changed during event {i}: before " + f"{before_viewport[0]}x{before_viewport[1]}, after " + f"{after_viewport[0]}x{after_viewport[1]}" + ) if kind in ("click", "double_click", "right_click", "drag"): if before_png is None: @@ -1722,6 +1816,13 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: f"missing before frame for {kind} event {i} in {recording}" ) click: Point = (int(event["x"]), int(event["y"])) + assert before_viewport is not None + _validate_point_in_viewport( + click, + before_viewport, + event_index=i, + label="pointer", + ) # ``before_png`` is a captured frame we already hold; the decode is # known-valid. cv2's stub types imdecode as Optional, hence the cast. frame = cast( @@ -1874,6 +1975,12 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: drag_end_anchor: Optional[Anchor] = None if kind == "drag": drag_end: Point = (int(event["end_x"]), int(event["end_y"])) + _validate_point_in_viewport( + drag_end, + before_viewport, + event_index=i, + label="drag destination", + ) end_crop_region = _discriminative_crop_region(frame, drag_end) end_template_rel = f"templates/{step_id}_drag_end.png" (bundle / end_template_rel).write_bytes( diff --git a/openadapt_flow/interactive_recorder.py b/openadapt_flow/interactive_recorder.py index 4179f02b..edf72d97 100644 --- a/openadapt_flow/interactive_recorder.py +++ b/openadapt_flow/interactive_recorder.py @@ -1,11 +1,13 @@ """Interactive recorder: capture a demonstration the USER drives live. ``openadapt-flow record --url `` opens a real (headed) Playwright browser -pointed at the user's OWN app and simply watches: it listens to the user's -real clicks, typing, key presses and scrolls via in-page capture-phase DOM -listeners (the same technique ``playwright codegen`` uses) and writes the -EXACT recording format the compiler already consumes (``meta.json`` + -``events.jsonl`` + ``frames/{i:04d}_before.png`` / ``_after.png``). +pointed at the user's OWN app. With ``--browser-cdp-endpoint`` it instead +attaches to one explicitly bound tab in an already-running local Chromium +browser, which preserves an existing SSO or 2FA session. Both modes listen to +the user's real clicks, typing, key presses and scrolls via in-page +capture-phase DOM listeners and write the EXACT recording format the compiler +already consumes (``meta.json`` + ``events.jsonl`` + +``frames/{i:04d}_before.png`` / ``_after.png``). record --url … → compile → replay @@ -28,19 +30,61 @@ mirroring ``PlaywrightBackend.structured_text_at`` exactly, so the compiler's DOM-identity tier arms on interactively-recorded bundles too. -Secrets never touch Python: a field is secret when it is ``input[type= -password]`` or its name/id is passed via ``--secret``. For a secret field the -in-page listener emits NO value at all (only that a secret was typed, plus the -field rectangle for redaction); the literal is never read, never sent over the -pipe, never written to meta/events/frames/bundle. See ``ir.Step.secret`` and -``docs`` for the full contract. +Secret literals never touch Python: a field is secret when it is ``input[type= +password]`` or its name/id is passed via ``--secret``. The page closure binds +the ELEMENT, masks it in every retained frame, and emits no value for it. + +**Flow reports captured page text exactly, or it withholds that text and says +why. Flow never rewrites captured text.** Three earlier revisions of this file +tried instead to remove a remembered value from text that was already captured, +and three independent reviews each found a different defect in the retention +rule that approach needs. That is the expected outcome: Englehardt, Acar and +Narayanan measured every major session-replay vendor in 2017 and found that +none redacts displayed content automatically and that all of it leaked, and +PostHog and Sentry still carry open issues for secrets in replay URLs. The +working answer in production tools is capture-time, element-bound, +deny-by-default masking, which is what this module now implements. + +Three rules follow from it: + +* Matching uses ONLY the values that bound elements hold at match time, read + live from the DOM. No value is kept after the field stops holding it, and a + node the page detached is not a source of values. +* Identity evidence (selector, role, accessible name, clicked-row identity, + and the receiving field's name) is EXACT or WITHHELD with a stated reason. + Replay compares it against the live page, so a rewritten copy would compare + against text the page never held, invisibly. +* Reflected evidence (the page URL and title) is sampled from Python at the + settled boundary, never in the capture-phase listener, which runs before the + page's own handlers and therefore reads the previous action's text. Flow + reports it only while it has not changed since before the document held any + declared value; otherwise it withholds an origin-only URL and an empty title. + +Every withheld item is visible: an ``identity_withheld`` reason on the action, +``meta.json`` keys, and a CLI line. A DOM identity check never disarms +silently. + +See ``ir.Step.secret`` and ``docs`` for the full contract and its boundary. """ from __future__ import annotations +import ctypes +import errno +import io +import ipaddress import json +import os +import re +import shutil +import sys +import tempfile +import uuid from pathlib import Path from typing import Any, Callable, Optional +from urllib.parse import urlsplit + +from PIL import Image from openadapt_flow.backends.playwright_backend import PlaywrightBackend from openadapt_flow.recorder import Recorder @@ -62,16 +106,458 @@ "End", ) + +class BrowserAttachError(RuntimeError): + """A safe browser-attachment precondition was not met.""" + + +_PARTIAL_RECORDING_PREFIX = ".openadapt-recording-partial-" + +# One protocol object group scopes every remote object that the closed-shadow +# privacy scan resolves, so each scan can release its handles in one call. +_PRIVACY_SCAN_OBJECT_GROUP = "openadapt-flow-privacy-scan" + + +def _rename_directory_noreplace(source: Path, destination: Path) -> None: + """Atomically rename a directory without replacing any destination.""" + + if os.name == "nt": + os.rename(source, destination) + return + + libc = ctypes.CDLL(None, use_errno=True) + source_bytes = os.fsencode(source) + destination_bytes = os.fsencode(destination) + result: int + if sys.platform == "linux": + try: + renameat2 = libc.renameat2 + except AttributeError as exc: + raise OSError( + errno.ENOTSUP, + "atomic no-replace directory promotion is unavailable", + destination, + ) from exc + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = int( + renameat2( + -100, # AT_FDCWD + source_bytes, + -100, # AT_FDCWD + destination_bytes, + 1, # RENAME_NOREPLACE + ) + ) + elif sys.platform == "darwin": + try: + renamex_np = libc.renamex_np + except AttributeError as exc: + raise OSError( + errno.ENOTSUP, + "atomic no-replace directory promotion is unavailable", + destination, + ) from exc + renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint] + renamex_np.restype = ctypes.c_int + result = int( + renamex_np( + source_bytes, + destination_bytes, + 0x00000004, # RENAME_EXCL + ) + ) + else: + raise OSError( + errno.ENOTSUP, + "atomic no-replace directory promotion is unavailable", + destination, + ) + + if result != 0: + error_number = ctypes.get_errno() + raise OSError(error_number, os.strerror(error_number), destination) + + +def _secret_screenshot_selectors( + secret_fields: set[str], + *, + marker_attribute: Optional[str] = None, +) -> tuple[str, ...]: + """Return selectors that mask password and declared-secret fields.""" + + selectors = ["input[type='password']"] + for field in sorted(secret_fields): + encoded = _css_string_literal(field) + selectors.append(f"[name={encoded}], [id={encoded}]") + if marker_attribute is not None: + if not re.fullmatch(r"data-oaflow-secret-[0-9a-f]{32}", marker_attribute): + raise BrowserAttachError("the browser secret marker is invalid") + selectors.append(f"[{marker_attribute}]") + return tuple(selectors) + + +def _css_string_literal(value: str) -> str: + """Serialize an exact value as a valid double-quoted CSS string. + + JSON ``\\u`` escapes are not CSS Unicode escapes. Using ``json.dumps`` + therefore made a declared field such as ``päss`` select a different name + and left its later frames unmasked. CSS strings accept Unicode directly; + quotes, backslashes, and control characters need CSS-specific escapes. + """ + + escaped: list[str] = ['"'] + for character in value: + codepoint = ord(character) + if character in {'"', "\\"}: + escaped.append("\\" + character) + elif codepoint == 0: + raise BrowserAttachError( + "a declared secret field name contains a null character and " + "cannot be bound to a safe browser mask" + ) + elif 0xD800 <= codepoint <= 0xDFFF: + raise BrowserAttachError( + "a declared secret field name contains an invalid Unicode " + "surrogate and cannot be bound to a safe browser mask" + ) + elif codepoint < 0x20 or codepoint == 0x7F: + # The trailing space terminates the variable-width CSS hex escape. + escaped.append(f"\\{codepoint:x} ") + else: + escaped.append(character) + escaped.append('"') + return "".join(escaped) + + +def _http_origin(url: str, *, label: str) -> tuple[str, str, int]: + """Return a normalized HTTP origin or refuse an unsafe attach target.""" + + try: + parsed = urlsplit(url) + port = parsed.port + except ValueError as exc: + raise BrowserAttachError(f"{label} is not a valid URL") from exc + scheme = parsed.scheme.lower() + host = (parsed.hostname or "").lower() + if scheme not in {"http", "https"} or not host: + raise BrowserAttachError(f"{label} must be an http:// or https:// URL") + return (scheme, host, port or (443 if scheme == "https" else 80)) + + +def _origin_label(origin: tuple[str, str, int]) -> str: + scheme, host, port = origin + default_port = 443 if scheme == "https" else 80 + rendered_host = f"[{host}]" if ":" in host else host + suffix = "" if port == default_port else f":{port}" + return f"{scheme}://{rendered_host}{suffix}" + + +def _safe_page_label(url: str) -> str: + """Describe a tab without exposing URL credentials, query, or fragment.""" + + try: + parsed = urlsplit(url) + origin = _http_origin(url, label="browser tab URL") + except BrowserAttachError: + return "" + path = parsed.path or "/" + if len(path) > 120: + path = path[:117] + "..." + return _origin_label(origin) + path + + +def validate_browser_cdp_endpoint(endpoint: str) -> str: + """Require an explicit loopback CDP endpoint. + + Browser attachment is local-only. A remote CDP endpoint is effectively a + remote-control credential and can also expose every page in that browser. + The supported recorder does not accept that boundary implicitly. + """ + + try: + parsed = urlsplit(endpoint) + port = parsed.port + except ValueError as exc: + raise BrowserAttachError("the browser CDP endpoint is not a valid URL") from exc + if parsed.scheme.lower() not in {"http", "https", "ws", "wss"}: + raise BrowserAttachError( + "the browser CDP endpoint must use http, https, ws, or wss" + ) + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise BrowserAttachError( + "the browser CDP endpoint must not contain credentials, a query, " + "or a fragment" + ) + host = (parsed.hostname or "").lower() + is_loopback = host == "localhost" + if not is_loopback: + try: + is_loopback = ipaddress.ip_address(host).is_loopback + except ValueError: + is_loopback = False + if not is_loopback: + raise BrowserAttachError( + "the browser CDP endpoint must use localhost or a loopback IP address" + ) + if port is None: + raise BrowserAttachError("the browser CDP endpoint must include a port") + return endpoint + + +def select_attached_page( + browser: Any, + *, + app_url: str, + page_url: Optional[str] = None, +) -> Any: + """Bind one existing same-origin web tab without guessing. + + A sole tab on the declared application origin is unambiguous. If two or + more tabs use that origin, the operator must give the exact current URL. + Query and fragment values are never included in an error message. + """ + + app_origin = _http_origin(app_url, label="the declared app URL") + if ( + page_url is not None + and _http_origin(page_url, label="the selected browser page URL") != app_origin + ): + raise BrowserAttachError( + "the selected browser page URL must have the same origin as " + f"the declared app URL ({_origin_label(app_origin)})" + ) + + matches: list[tuple[Any, str]] = [] + for context in browser.contexts: + for page in context.pages: + try: + current_url = str(page.url) + current_origin = _http_origin(current_url, label="browser tab URL") + except Exception: + continue + if current_origin == app_origin: + matches.append((page, current_url)) + + if page_url is not None: + exact = [page for page, current_url in matches if current_url == page_url] + if len(exact) == 1: + return exact[0] + if not exact: + raise BrowserAttachError( + "no open tab has the exact --browser-page-url on the declared " + f"app origin ({_origin_label(app_origin)})" + ) + raise BrowserAttachError( + "more than one open tab has the exact --browser-page-url; close " + "the duplicate tabs and retry" + ) + + if len(matches) == 1: + return matches[0][0] + if not matches: + raise BrowserAttachError( + "no open tab matches the declared app origin " + f"({_origin_label(app_origin)}); open the app in the attached " + "browser and retry" + ) + labels = sorted({_safe_page_label(current_url) for _, current_url in matches}) + rendered = ", ".join(labels[:5]) + if len(labels) > 5: + rendered += f", and {len(labels) - 5} more" + raise BrowserAttachError( + f"{len(matches)} open tabs match the declared app origin; supply the " + "exact current URL with --browser-page-url. Candidate paths " + f"(query and fragment hidden): {rendered}" + ) + + # In-page recorder script. Installed via add_init_script so it re-arms on every # document (navigations). Emits raw events to the Python side via the # __oaflow_emit binding. __SECRET_NAMES__ / __SPECIAL_KEYS__ are substituted in. _INIT_JS = r""" (() => { - if (window.__oaflowInstalled) return; - window.__oaflowInstalled = true; + const SESSION_ID = __SESSION_ID__; + const BINDING_NAME = __BINDING_NAME__; + const GLOBAL_KEY = '__oaflowRecorder'; + const CLEANUP_KEY = '__oaflowCleanup_' + SESSION_ID; + const previous = window[GLOBAL_KEY]; + if (previous && previous.sessionId === SESSION_ID) return; + if (previous && typeof previous.cleanup === 'function') { + try { previous.cleanup(); } catch (e) {} + } const SECRET_NAMES = __SECRET_NAMES__; + const SECRET_MARKER = __SECRET_MARKER__; const IDENT_NAMES = __IDENT_NAMES__; const SPECIAL = __SPECIAL_KEYS__; + // One identity for this DOCUMENT. The init script builds a fresh closure per + // document, so this closure never saw a value an earlier document received. + // Python compares this id against the document that received a secret and + // withholds every later document's reflected text. + const DOC_ID = SESSION_ID + ':doc:' + String(Date.now()) + ':' + + Math.random().toString(36).slice(2); + // A secret value shorter than this occurs inside ordinary page text by + // chance, so a match cannot be read as a reflection of the secret. Flow + // withholds the text either way, and reports the ambiguous case under its + // own reason so the operator can tell a chance match from a real one. + const MIN_UNAMBIGUOUS_SECRET = 6; + // + // ONE RULE GOVERNS EVERY TEXT THIS CLOSURE PRODUCES: Flow reports captured + // text EXACTLY, or it withholds that text and states why. Flow never + // rewrites captured text. + // + // Post-hoc removal of a secret value from already-captured text is the + // approach the session-replay industry has never made work. Englehardt, + // Acar and Narayanan measured every major replay vendor in 2017 and found + // that no vendor redacts displayed content automatically and that all of it + // leaked; PostHog and Sentry still carry open issues for secrets in replay + // URLs today. The industry answer is capture-time, element-bound, + // deny-by-default masking, which is what this closure does. + // + // A rewrite is also worse than a refusal for identity evidence. Replay + // compares identity text against what the page shows, so a rewritten copy + // is a comparison against text the page never held, and the substitution is + // invisible to that comparison. Withholding disarms the check loudly. + // + const listeners = []; + const secretStates = new WeakMap(); + const closedSecretHosts = new WeakMap(); + const secretBoundaryStates = new WeakMap(); + const inputSessions = new WeakMap(); + const trustedSecretFieldLabels = new WeakMap(); + const stickySecretElements = new Set(); + const observedSecretRoots = new WeakSet(); + const ambiguousSecretReplacements = new WeakSet(); + // Elements that discovery bound during the MutationObserver batch being + // processed right now. A replacement discovery has just bound still needs to + // inherit the state of the node it replaced, including its input session. + let batchDiscoveredSecrets = null; + let nextInputSession = 0; + let activeSecretElement = null; + let activeSecretState = null; + let resizeTimer = null; + let secretObserver = null; + let privacyBoundaryError = null; + let opaqueSecretActive = false; + // Why Flow refused to build identity evidence from free identity text, for + // the identity scope that is being built right now. See identityTextOrNull. + let identityWithheldReason = null; + // Reflected text (the URL and the title) that this document showed while no + // declared secret field held a value. Text that has not changed since then + // CANNOT be a reflection of a value that did not yet exist, so Flow reports + // it exactly. Text that HAS changed may be a reflection of any version of + // the value, including one the field no longer holds, and no rule that + // reads only the current DOM can decide which. Flow withholds it. + // + // These two strings are page text, not secret values. Nothing ever matches + // against them: they answer one question, "did this text exist before the + // secret did", and they are never emitted. + let preSecretUrl = null; + let preSecretTitle = null; + // Sticky: a document that has held a declared value stops refreshing the + // baseline above. It never un-sticks, because a value the field no longer + // holds can still be reflected somewhere in this document. + let documentHeldSecretValue = false; + // Whether the baseline above proves anything. It does not for a document + // that was BORN after some earlier document already received a declared + // value: the URL such a document loads with can already carry that value, so + // its own first sample is not evidence of a time before the value existed. + // Python knows the recording-wide history and reports it on the first read. + let seedTrusted = true; + let seedDecided = false; + // Values a declared field held at a COMMIT POINT -- `change`, `focusout`, + // `submit`, `pagehide`. Used for ONE purpose: deciding whether to WITHHOLD + // identity text after the page removes the field. Never for the URL, never + // for the title, and never to rewrite anything. + // + // Why this is safe where the rule that failed three reviews was not: that + // rule REWROTE text, so a false match corrupted evidence or leaked. Nothing + // here rewrites, so a false match can only withhold. Withholding costs + // evidence; it cannot corrupt and it cannot leak. Identity text is also a + // DIRECT match against the value, while a URL or a title reflects it + // indirectly and can lag, which is why reflected text still uses live values + // only. A commit point is a moment the operator stopped on, not a keystroke, + // and only a CONNECTED element commits, so a controlled input that fires + // focusout on the node it just replaced commits nothing. + const committedSecretValues = new Set(); + // The most recent NON-EMPTY value each bound element held, observed in the + // capture phase before the page's own `input` handler runs. ONE value per + // element, REPLACED on every keystroke -- never a ladder of past values, and + // never used to rewrite anything. + // + // It exists for the page that consumes its own field: a scanner input that + // writes the badge into the URL and then clears the field in the same + // handler, or a wizard that removes the field outright. By the time Python + // samples at the settled boundary, nothing in the DOM holds the value, so + // every other source is empty and the reflected text would be reported. + // + // It is REPLACED, never accumulated, so this is not a ladder of past values + // and a password beginning with an ordinary word never leaves that word + // behind. It is literally the value carried by the last `input` event on + // that element -- NOT "the value the operator stopped on". A page that + // consumes the value mid-stream leaves whatever was typed next, which is + // exactly why a second scan into the same field must not displace the first + // value while the first is still on show. + const lastSecretValues = new WeakMap(); + let eventsStopped = false; + let cleaned = false; + + function listenOn(target, type, handler) { + target.addEventListener(type, handler, true); + listeners.push([target, type, handler]); + } + + function listen(type, handler) { + listenOn(document, type, handler); + } + + function stopEvents() { + if (eventsStopped) return; + eventsStopped = true; + if (resizeTimer !== null) clearTimeout(resizeTimer); + for (const [target, type, handler] of listeners) { + try { target.removeEventListener(type, handler, true); } catch (e) {} + } + } + + function cleanup() { + if (cleaned) return; + cleaned = true; + stopEvents(); + if (secretObserver !== null) secretObserver.disconnect(); + // The Set retains disconnected elements for this exact cleanup. Remove + // the temporary marker before releasing those references so reinserting a + // page-owned node after Flow detaches cannot expose recorder metadata. + for (const el of stickySecretElements) { + try { el.removeAttribute(SECRET_MARKER); } catch (e) {} + } + // No value is filed against these elements, so clearing the Set releases + // everything this closure held about them. + stickySecretElements.clear(); + activeSecretElement = null; + activeSecretState = null; + const current = window[GLOBAL_KEY]; + if (current && current.sessionId === SESSION_ID) { + try { delete window[GLOBAL_KEY]; } catch (e) { window[GLOBAL_KEY] = null; } + } + try { delete window[CLEANUP_KEY]; } catch (e) { window[CLEANUP_KEY] = null; } + } + window[GLOBAL_KEY] = { + sessionId: SESSION_ID, + stopEvents, + cleanup, + privacyStatus: () => ({ok: privacyBoundaryError === null, + error: privacyBoundaryError}), + registerExistingClosedShadowHost, + structuralState: (secretSeenEarlier) => safePageState(secretSeenEarlier), + }; + window[CLEANUP_KEY] = {sessionId: SESSION_ID, stopEvents, cleanup}; function identifierRect() { // Bounding rect of the operator-marked record-identifying field @@ -94,11 +580,506 @@ return null; } - function structuredIdentity(px, py) { + function currentSecretValue(el) { + try { + if (el && el.value != null) return String(el.value); + if (el && (el.isContentEditable + || (el.getAttribute && el.getAttribute('role') === 'textbox'))) { + return String(el.innerText || el.textContent || ''); + } + } catch (e) {} + return ''; + } + + function isConnectedElement(el) { + try { return !!el.isConnected; } catch (e) { return false; } + } + + function liveSecretValues() { + // Every declared value a bound element holds RIGHT NOW, read from the DOM + // at match time. Nothing here survives the call. + // + // Only a CONNECTED element counts. A controlled input that swaps its node + // on every keystroke leaves detached nodes behind, each still holding the + // keystroke prefix it had when the page dropped it. Those prefixes are not + // values the operator entered, and three earlier revisions of this file + // each shipped a different defect while trying to decide which of them to + // keep. This closure keeps none of them: the page tells Flow what the + // field holds, and Flow asks the page every time. + const values = new Set(); + for (const el of stickySecretElements) { + if (!isConnectedElement(el)) continue; + const value = currentSecretValue(el); + if (value) values.add(value); + } + return values; + } + + function declaredSecretParameterNames() { + // Names Flow KNOWS name a secret: the operator's --secret declarations, + // and the name or id of every field Flow bound, which includes every + // auto-detected input[type=password]. This list is structure, not value: + // it is the same whatever the operator typed, so a rule keyed on it is + // deterministic. Sentry and Datadog redact URLs the same way, by parameter + // NAME, because searching for a value inside arbitrary text does not work. + const names = new Set(); + for (const name of SECRET_NAMES) if (name) names.add(String(name)); + for (const el of stickySecretElements) { + const state = secretStates.get(el) || closedSecretHosts.get(el) || null; + if (state && state.field) names.add(String(state.field)); + const attribute = elementName(el); + if (attribute) names.add(String(attribute)); + try { if (el.id) names.add(String(el.id)); } catch (e) {} + } + return names; + } + + function inboundSecretParameterValues() { + // A same-origin GET submit carries the field's value under the field's own + // NAME, because that is how an HTML form works. A document that loads such + // a URL therefore holds a declared value it never saw typed, and its + // closure has no bound element to read it from. Recover it from the URL by + // NAME and use it to WITHHOLD identity text in this document. + // + // Only values long enough to identify. A short one matches ordinary page + // text by chance, and this is a net, not a proof: withholding every + // identity that contains a 3-character query value would cost far more + // evidence than it protects. + const values = new Set(); + const parsed = parseHttpUrl(location.href); + if (parsed === null) return values; + const declared = declaredSecretParameterNames(); + for (const search of [parsed.search, fragmentQuery(parsed.hash)]) { + if (!search) continue; + let params = null; + try { params = new URLSearchParams(search); } catch (e) { params = null; } + if (params === null) continue; + for (const [name, value] of params) { + if (!declared.has(name)) continue; + if (value && value.length >= MIN_UNAMBIGUOUS_SECRET) values.add(value); + } + } + return values; + } + + function identityMatchValues() { + // Every value Flow may use to WITHHOLD text. Never to rewrite it. + // + // The cached value is added PER ELEMENT, whenever THAT element holds + // nothing right now. An earlier revision instead skipped the whole cache + // as soon as ANY declared field held anything, so a second scan into the + // same cleared field, or a second declared field holding a PIN, re-exposed + // the first value that was still sitting in the URL. The document-wide + // guard was the defect; the per-element test is the same idea applied + // where it belongs. + // + // CONNECTED and empty: always add. That is the page that consumed its own + // field -- a scanner writing the badge into the URL and clearing the input + // -- and the element is still there to speak for. + // + // DETACHED: add only when NO connected bound element holds anything. A + // page that REMOVED its field leaves the value nowhere else, so it must be + // added. A controlled input that SWAPS its node leaves a trail of detached + // elements that still report `c`, `ch`, `cha`, and adding those would + // withhold any text containing a one-letter match -- but in that case a + // connected successor does hold the value, so this test excludes them. + const values = liveSecretValues(); + const anythingLive = values.size > 0; + for (const el of stickySecretElements) { + const connected = isConnectedElement(el); + if (connected && currentSecretValue(el)) continue; + if (!connected && anythingLive) continue; + const last = lastSecretValues.get(el) || (connected ? '' : currentSecretValue(el)); + if (last) values.add(last); + } + for (const value of committedSecretValues) values.add(value); + for (const value of inboundSecretParameterValues()) values.add(value); + return values; + } + + function commitConnectedValue(el) { + if (!el || !stickySecretElements.has(el) || !isConnectedElement(el)) return; + const value = currentSecretValue(el); + if (value) committedSecretValues.add(value); + } + + function commitSecretValueFor(node) { + // A commit point that names its element: `change` and `focusout`. + // + // DECIDE AT THE MICROTASK CHECKPOINT, not now. A controlled input that + // replaces its focused node on every keystroke dispatches focusout DURING + // the replacement, while the node can still report itself as connected. + // One checkpoint later the page has certainly dropped it, and the value it + // holds is a keystroke prefix -- not a value the operator stopped on. + // Committing prefixes would withhold identity evidence on every chance + // match, which is the failure this rule exists to avoid. + // + // A real blur is unaffected: the element the operator left is still in the + // document at the checkpoint, and a page handler that removes it runs in a + // LATER task. + const el = secretTextEntryForNode(node) || node; + if (!el || !stickySecretElements.has(el)) return; + try { + Promise.resolve().then(() => commitConnectedValue(el)); + } catch (e) { + commitConnectedValue(el); + } + } + + function commitSecretValues() { + // `submit` and `pagehide`. The document is leaving, so there is no later + // checkpoint to defer to. + for (const el of stickySecretElements) commitConnectedValue(el); + } + + function secretVariants(secret) { + const variants = new Set([secret]); + try { variants.add(encodeURIComponent(secret)); } catch (e) {} + try { + variants.add(new URLSearchParams([['value', secret]]).toString().slice(6)); + } catch (e) {} + return Array.from(variants).filter(Boolean); + } + + function secretValueIn(value, secretValues) { + // DETECT, never rewrite. Returns the reason this text cannot be reported, + // or null when the text holds no declared value and is safe to report + // exactly as the page shows it. + const text = String(value == null ? '' : value); + if (!text) return null; + // Compare case-insensitively. An application that upper-cases or + // lower-cases an identifier before showing it is doing normalisation, not + // an application-defined transform, and it is common enough that an + // exact-case match would miss it. Widening a WITHHOLD test can only + // withhold more; it can never leak and never rewrites anything. + const folded = text.toLowerCase(); + let ambiguousMatch = false; + for (const secret of secretValues) { + if (!secret) continue; + const ambiguous = secret.length < MIN_UNAMBIGUOUS_SECRET; + for (const variant of secretVariants(secret)) { + if (!variant || folded.indexOf(variant.toLowerCase()) < 0) continue; + // A definite match outranks an ambiguous one: the operator reads a + // different meaning into "this text held your value" than into "this + // text could have held your value by chance". A value shorter than + // MIN_UNAMBIGUOUS_SECRET occurs inside ordinary page text often + // enough that its match proves nothing. Either one withholds. + if (!ambiguous) return 'secret-value-in-identity'; + ambiguousMatch = true; + } + } + return ambiguousMatch ? 'ambiguous-secret-in-identity' : null; + } + + function identityTextOrNull(value) { + // IDENTITY / MACHINE EVIDENCE: the accessible name, the control role, the + // clicked row's identity characters, and the receiving field's name. + // Replay re-reads the page and compares it against this text, so the text + // must be what the page held or nothing at all. A rewritten copy would + // make replay compare against characters the page never showed, and + // nothing downstream could see that the substitution happened. + // + // EXACT, or WITHHELD with a reason. Never rewritten. + if (value == null) return value; + const text = String(value); + if (!text) return text; + if (opaqueSecretActive) { + // A closed shadow root does not expose its literal, so Flow cannot rule + // out that this text contains it. + if (identityWithheldReason === null) { + identityWithheldReason = 'opaque-secret-boundary'; + } + return null; + } + const reason = secretValueIn(text, identityMatchValues()); + if (reason === null) return text; + if (identityWithheldReason === null) identityWithheldReason = reason; + return null; + } + + function labelTextOrNull(value) { + // The receiving field's human label. Passive compile-time evidence, not an + // identity tier, so a withheld label does not disarm an identity check and + // is not counted as one. It still follows the one rule: exact or nothing. + if (value == null) return value; + const text = String(value); + if (!text) return text; + if (opaqueSecretActive) return null; + return secretValueIn(text, identityMatchValues()) === null ? text : null; + } + + function originOnlyUrl() { + // The declared origin with an empty path: a URL that carries no value. + try { return location.origin + '/'; } catch (e) { return ''; } + } + + function parseHttpUrl(href) { + try { + const parsed = new URL(String(href)); + if (parsed.protocol === 'http:' || parsed.protocol === 'https:') { + return parsed; + } + } catch (e) {} + return null; + } + + function fragmentQuery(hash) { + // A fragment is either a bare anchor (`#section`) or a second query + // (`#a=1&b=2`). Treat it as a query only when it names things. + const text = String(hash || '').replace(/^#/, ''); + return text.indexOf('=') >= 0 ? text : ''; + } + + function reduceParams(search, context, where, dropped) { + // Return the parameter list with the VALUE of each parameter Flow cannot + // report emptied. NAMES always survive: a name is app structure, and the + // operator needs it to read the evidence. Nothing is invented -- a dropped + // value becomes empty. Flow removes characters; it never adds characters + // the page did not show. + let params = null; + try { params = new URLSearchParams(search); } catch (e) { params = null; } + if (params === null) return null; + const parts = []; + for (const [name, value] of params) { + let reason = null; + if (context.declared.has(name)) { + // By NAME, always, whatever the value is. A same-origin GET submit + // carries a declared field under its own name, so this is the exact + // channel that leaked, closed by structure rather than by matching. + reason = 'declared-secret-parameter'; + } else if (context.requireProof && value) { + const baseline = context.baseline; + const proven = baseline !== null && baseline.get(name) === value; + if (!proven) reason = 'unproven-parameter-value'; + } + if (reason === null) { + parts.push(encodeURIComponent(name) + '=' + encodeURIComponent(value)); + continue; + } + parts.push(encodeURIComponent(name) + '='); + if (value) dropped.push({name: name, where: where, reason: reason}); + } + return parts.join('&'); + } + + function baselineParams(search) { + try { return new URLSearchParams(search); } catch (e) { return null; } + } + + function pathHoldsDeclaredValue(pathname, values) { + // The NET for the PATH, run in BOTH directions. + // + // Direction 1, a segment holds a whole value, is step 5 and the whole-URL + // check already covers it. Direction 2 is the one that matters here: a + // page that writes the field into its path as the operator types leaves a + // segment the value CONTAINS but no longer equals. Matching only the live + // value would miss it, and Flow will not keep a previous value to catch + // it, so it asks the opposite question instead -- does a value Flow can + // see contain this segment? That reads the CURRENT value against the + // CURRENT text. It is not a history and not a prefix ladder, and it only + // ever withholds. + // + // A segment too short to identify is ignored: `/app/edit` inside a long + // passphrase would otherwise withhold the URL of an ordinary page. + const segments = String(pathname || '').split('/').filter(Boolean); + for (const segment of segments) { + if (segment.length < MIN_UNAMBIGUOUS_SECRET) continue; + let decoded = segment; + try { decoded = decodeURIComponent(segment); } catch (e) {} + const folded = segment.toLowerCase(); + const foldedDecoded = decoded.toLowerCase(); + for (const value of values) { + if (!value) continue; + const foldedValue = value.toLowerCase(); + if ( + foldedValue.indexOf(folded) >= 0 + || foldedValue.indexOf(foldedDecoded) >= 0 + ) { + return true; + } + } + } + return false; + } + + function reportedUrl(rawUrl, dropped) { + // A URL is STRUCTURE, not one opaque string. Report the origin and the + // path, which is what a single-page application changes when it routes, + // and which is app-controlled structure rather than operator input. + // Reduce only the parameter values Flow cannot stand behind. + // + // Returns null when Flow cannot parse the URL at all; the caller withholds. + const parsed = parseHttpUrl(rawUrl); + if (parsed === null) return null; + // Once this document has held a declared value, a parameter value Flow + // cannot prove predates that value is dropped. A document whose seed + // baseline Flow does not trust proves nothing at all. + const requireProof = documentHeldSecretValue || !seedTrusted; + const baselineUrl = seedTrusted ? parseHttpUrl(preSecretUrl) : null; + const context = { + declared: declaredSecretParameterNames(), + requireProof: requireProof, + baseline: null, + }; + context.baseline = baselineUrl === null + ? null : baselineParams(baselineUrl.search); + const query = reduceParams(parsed.search, context, 'query', dropped); + const rawFragment = fragmentQuery(parsed.hash); + context.baseline = baselineUrl === null + ? null : baselineParams(fragmentQuery(baselineUrl.hash)); + const fragment = rawFragment + ? reduceParams(rawFragment, context, 'fragment', dropped) : ''; + if (query === null || fragment === null) return null; + // A BARE fragment names nothing (`#section`), so structure cannot reduce + // it. Treat the whole fragment as one unnamed value and apply the same + // proof a named parameter value gets. + let bareFragment = rawFragment ? '' : String(parsed.hash || ''); + if (bareFragment && requireProof) { + const baselineHash = baselineUrl === null + ? '' : String(baselineUrl.hash || ''); + if (baselineHash !== bareFragment) { + dropped.push({ + name: '', where: 'fragment', reason: 'unproven-parameter-value', + }); + bareFragment = ''; + } + } + // Rebuilding normalises percent-encoding. Return exactly what the page + // reports when Flow dropped nothing, so evidence does not drift. User + // information in the authority is never returned: the rebuilt form drops + // it with the rest of the authority. + if (!dropped.length && !parsed.username && !parsed.password) { + return String(rawUrl); + } + const hash = rawFragment ? '#' + fragment : bareFragment; + return parsed.origin + parsed.pathname + + (parsed.search ? '?' + query : '') + hash; + } + + function safePageState(secretSeenEarlier) { + // REFLECTED / CONTEXT EVIDENCE: the page URL and the document title. + // + // STRUCTURE FIRST, DETECTION AS A NET. + // + // The URL is parsed, not treated as one opaque string. The origin and the + // path are reported: a path change is the single-page-application case, + // and the path is app structure, not operator input. Parameter NAMES are + // always reported. A parameter VALUE is dropped when its NAME is a + // declared secret field name -- deterministically, with no reference to + // the value -- or when Flow cannot prove the value predates the moment + // this document first held a declared value. Sentry and Datadog redact + // URLs by parameter name for the same reason: searching for a value inside + // arbitrary text does not work, and OWASP notes that a secret in a URL is + // already exposed through history, logs, proxies and Referer headers. + // + // The net: if the URL Flow is about to report still contains a value a + // bound field holds right now, Flow withholds the whole URL and tells the + // operator that the application put a declared secret into it. That + // matching is sound here and was not sound before, because this function + // now runs only from Python at the settled boundary, where the page has + // processed the action and its reflection matches the value the field + // holds. It needs no history, and the direction is fail-safe: a match + // withholds, so a false positive costs evidence and cannot leak. + // + // The title has no structure to exploit, so it keeps the plain rule: + // report it while it has not changed since before this document held a + // declared value, and withhold it otherwise, plus the same net. + // + // STATED LIMITS, both documented in docs/BROWSER_RECORDING.md: + // * An application that debounces its URL update beyond the settle window + // can still show an earlier value. The net will not match it. Flow does + // NOT keep a previous value to catch that. + // * Text that already carried the value BEFORE any declared field held it + // predates the value and is reported. + if (!seedDecided) { + seedDecided = true; + if (secretSeenEarlier === true) seedTrusted = false; + } + const rawUrl = String(location.href); + const rawTitle = String(document.title == null ? '' : document.title); + const values = liveSecretValues(); + if (values.size > 0) documentHeldSecretValue = true; + if (!documentHeldSecretValue && !opaqueSecretActive && seedTrusted) { + preSecretUrl = rawUrl; + preSecretTitle = rawTitle; + } + const state = { + url: rawUrl, + title: rawTitle, + doc: DOC_ID, + // Whether this document has EVER held a declared value. Python uses it + // to bind the recording-wide secret boundary across documents. + secret: documentHeldSecretValue || opaqueSecretActive, + url_withheld: null, + title_withheld: null, + dropped: [], + secret_in_url: false, + secret_in_title: false, + }; + if (opaqueSecretActive) { + // A closed shadow root exposes its value to no check at all. + state.url_withheld = 'opaque-secret-boundary'; + state.title_withheld = 'opaque-secret-boundary'; + } else { + const dropped = []; + const reduced = reportedUrl(rawUrl, dropped); + // The net runs against every value Flow can see, not only the live one. + // Adding a committed value can only ADD a match, and a match only + // withholds, so this direction cannot leak and cannot corrupt. The + // proof rule above still uses the live value alone. + const netValues = identityMatchValues(); + const parsed = reduced === null ? null : parseHttpUrl(reduced); + if (reduced === null) { + state.url_withheld = 'url-cannot-be-parsed'; + } else if ( + parsed !== null + && pathHoldsDeclaredValue(parsed.pathname, netValues) + ) { + state.url_withheld = 'declared-value-in-url'; + state.secret_in_url = true; + } else if (secretValueIn(reduced, netValues) !== null) { + // The application put a declared secret somewhere structure cannot + // reach -- a path segment, or a parameter it did not name after the + // field. Withhold the whole URL and say so: this is an application + // defect the operator needs to know about. + state.url_withheld = 'declared-value-in-url'; + state.secret_in_url = true; + } else { + state.url = reduced; + state.dropped = dropped; + } + if (documentHeldSecretValue) { + if (secretValueIn(rawTitle, identityMatchValues()) !== null) { + state.title_withheld = 'declared-value-in-title'; + state.secret_in_title = true; + } else if (rawTitle !== preSecretTitle) { + state.title_withheld = 'title-changed-after-a-secret-value'; + } + } + } + if (state.url_withheld !== null) { + state.url = originOnlyUrl(); + state.dropped = []; + } + if (state.title_withheld !== null) state.title = ''; + return state; + } + + function structuredIdentityEvidence(px, py, eventTarget) { + identityWithheldReason = null; + const sid = structuredIdentity(px, py, eventTarget); + return { + sid: sid, + withheld: sid === null ? identityWithheldReason : null, + }; + } + + function structuredIdentity(px, py, eventTarget = null) { // Mirrors PlaywrightBackend.structured_text_at: the REAL characters of the // clicked row (MRN/name/DOB), excluding the clicked target's own cell. try { - const el = document.elementFromPoint(px, py); + refreshSecretBindings(); + const el = eventTarget || document.elementFromPoint(px, py); if (!el) return null; const row = el.closest('tr, [role="row"], li, [role="listitem"]'); if (!row) return null; @@ -107,7 +1088,7 @@ || row.getAttribute('aria-label') || '' ).replace(/\s+/g, ' ').trim(); - if (declared) return declared; + if (declared) return identityTextOrNull(declared); const own = el.closest('td, th, [role="cell"], [role="gridcell"]') || el; own.setAttribute('data-oaflow-own', '1'); let body = ''; @@ -119,14 +1100,14 @@ } finally { own.removeAttribute('data-oaflow-own'); } - const joined = body.replace(/\s+/g, ' ').trim(); + const joined = identityTextOrNull(body.replace(/\s+/g, ' ').trim()); return joined || null; } catch (e) { return null; } } function targetRole(el) { const explicit = el.getAttribute('role'); - if (explicit) return explicit; + if (explicit) return identityTextOrNull(explicit); const tag = el.tagName.toLowerCase(); if (tag === 'button') return 'button'; if (tag === 'a' && el.hasAttribute('href')) return 'link'; @@ -143,74 +1124,461 @@ } function targetName(el) { + // Never inspect visible text from a secret-bound contenteditable or one of + // its descendants. Its innerText is the secret value, not target identity. + // That is a WITHHELD name, not an absent one: the element may well have an + // aria-label, and a control field beside it returns its name normally. Say + // so, or the DOM identity tier disarms silently, exactly as a bare null + // selector once did. + if (secretTextEntryForNode(el)) { + if (identityWithheldReason === null) { + identityWithheldReason = 'secret-field-name-not-read'; + } + return null; + } const aria = (el.getAttribute('aria-label') || '').trim(); - if (aria) return aria; + if (aria) return identityTextOrNull(aria); const labelledBy = (el.getAttribute('aria-labelledby') || '').trim(); if (labelledBy) { const value = labelledBy.split(/\s+/).map((id) => { const node = document.getElementById(id); return node ? (node.textContent || '').trim() : ''; }).filter(Boolean).join(' '); - if (value) return value; + if (value) return identityTextOrNull(value); } if (el.id) { const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); - if (label && (label.textContent || '').trim()) return label.textContent.trim(); + if (label && (label.textContent || '').trim()) { + return identityTextOrNull(label.textContent.trim()); + } } const wrapping = el.closest('label'); if (wrapping && wrapping !== el && (wrapping.textContent || '').trim()) { - return wrapping.textContent.trim(); + return identityTextOrNull(wrapping.textContent.trim()); } for (const attr of ['alt', 'title', 'placeholder']) { const value = (el.getAttribute(attr) || '').trim(); - if (value) return value; + if (value) return identityTextOrNull(value); } const text = (el.innerText || '').replace(/\s+/g, ' ').trim(); - return text ? text.slice(0, 200) : null; + return text ? identityTextOrNull(text.slice(0, 200)) : null; + } + + function identityRefusal(value) { + // Why this identity attribute cannot become evidence, or null when it can. + // A selector is machine evidence in the strictest sense: replay resolves it + // against the live DOM, so a rewritten selector resolves to nothing, or to + // the wrong element. Exact or withheld, never rewritten. + if (opaqueSecretActive) return value ? 'opaque-secret-boundary' : null; + // The SAME value set its siblings use. A selector is the strictest machine + // evidence there is -- replay resolves it against the live DOM -- so it + // must not be the one identity field built from a narrower set. Widening + // this can only withhold more. + return secretValueIn(value, identityMatchValues()); } function uniqueSelector(el) { + // Returns {selector, withheld}. `withheld` names WHY Flow refused to build + // identity evidence from a secret-bearing attribute. The DOM identity tier + // must never disarm silently, so the reason travels with the event instead + // of leaving a bare null selector that looks healthy. + let withheld = null; if (el.id) { - const selector = `#${CSS.escape(el.id)}`; - if (document.querySelectorAll(selector).length === 1) return selector; + const refusal = identityRefusal(el.id); + if (refusal !== null) { + withheld = refusal; + } else { + const selector = `#${CSS.escape(el.id)}`; + if (document.querySelectorAll(selector).length === 1) { + return {selector: selector, withheld: null}; + } + } } for (const attr of ['data-testid', 'data-test', 'name']) { const value = el.getAttribute(attr); if (!value) continue; + const refusal = identityRefusal(value); + if (refusal !== null) { + if (withheld === null) withheld = refusal; + continue; + } const selector = `${el.tagName.toLowerCase()}[${attr}="${CSS.escape(value)}"]`; - if (document.querySelectorAll(selector).length === 1) return selector; + if (document.querySelectorAll(selector).length === 1) { + return {selector: selector, withheld: null}; + } } - return null; + return {selector: null, withheld: withheld}; } - function structuralTarget(px, py) { + function structuralTarget(px, py, eventTarget = null) { try { - const el = document.elementFromPoint(px, py); + refreshSecretBindings(); + const el = eventTarget || document.elementFromPoint(px, py); if (!el) return null; + identityWithheldReason = null; + const identity = uniqueSelector(el); const target = { - selector: uniqueSelector(el), + selector: identity.selector, role: targetRole(el), name: targetName(el), }; - return (target.selector || target.role || target.name) ? target : null; + // A withheld accessible name or role disarms an identity check exactly + // as a withheld selector does, so report either one. + const withheld = identity.withheld || identityWithheldReason; + if (withheld) target.identity_withheld = withheld; + return (target.selector || target.role || target.name || withheld) + ? target : null; } catch (e) { return null; } } - function isSecretEl(el) { - if (!el) return false; - if ((el.type || '').toLowerCase() === 'password') return true; - const n = el.name || '', i = el.id || ''; - return SECRET_NAMES.indexOf(n) >= 0 || SECRET_NAMES.indexOf(i) >= 0; + function isTextEntry(el) { + try { + if (!el || !el.matches) return false; + if (el.matches('textarea, [contenteditable=""], [contenteditable="true"],' + + ' [role="textbox"]')) return true; + if (!el.matches('input')) return false; + const type = (el.getAttribute('type') || 'text').toLowerCase(); + return [ + 'button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', + 'range', 'reset', 'submit', + ].indexOf(type) < 0; + } catch (e) { return false; } + } + + function secretTextEntryForNode(node) { + try { + const el = isTextEntry(node) ? node : ( + node && node.closest && node.closest( + 'input, textarea, [contenteditable=""], [contenteditable="true"],' + + ' [role="textbox"]' + ) + ); + if (!el) return null; + let state = secretStates.get(el) || declaredSecretState(el); + if (!state) state = secretBoundaryStates.get(el.getRootNode()) || null; + if (!state) return null; + if (!secretStates.has(el)) bindSecretState(el, state, false); + return el; + } catch (e) { return null; } + } + + function bindSecretState(el, state, activate = true) { + if (!secretStates.has(el) && !currentSecretValue(el)) { + // A static label observed before typing is field identity, not secret + // text. Cache it before short password prefixes could over-redact it. + const label = fieldLabel(el); + if (label) trustedSecretFieldLabels.set(el, label); + } + secretStates.set(el, state); + // The value stays inside this closure. The element reference is retained + // here, so a pre-filled or still-typed value is read live at match time + // and cannot enter URL, title, or structural metadata. Binding retains the + // ELEMENT, never the value. + stickySecretElements.add(el); + if (activate) { + activeSecretElement = el; + activeSecretState = state; + } + try { + el.setAttribute(SECRET_MARKER, ''); + return el.hasAttribute(SECRET_MARKER); + } catch (e) { return false; } + } + + function inputSessionFor(el) { + let session = inputSessions.get(el) || null; + if (!session) { + nextInputSession += 1; + session = SESSION_ID + ':input:' + String(nextInputSession); + inputSessions.set(el, session); + } + return session; + } + + function elementName(el) { + if (!el) return ''; + try { + return (el.getAttribute && el.getAttribute('name')) || el.name || ''; + } catch (e) { return ''; } + } + + function declaredSecretState(el) { + if (!el) return null; + const n = elementName(el), i = el.id || ''; + if ((el.type || '').toLowerCase() !== 'password' + && SECRET_NAMES.indexOf(n) < 0 && SECRET_NAMES.indexOf(i) < 0) { + return null; + } + return {field: n || i || null, inputSession: inputSessionFor(el)}; + } + + function declaredSecretHostState(host) { + if (!host) return null; + const n = elementName(host); + const i = host.id || ''; + if (SECRET_NAMES.indexOf(n) < 0 && SECRET_NAMES.indexOf(i) < 0) return null; + return {field: n || i, inputSession: inputSessionFor(host)}; + } + + function registerExistingClosedShadowHost(host) { + const state = declaredSecretHostState(host); + if (!state) return false; + // A closed root does not expose its literal. Remove all text metadata for + // the rest of this recording because no check can see its value. + opaqueSecretActive = true; + closedSecretHosts.set(host, state); + return bindSecretState(host, state, false); + } + + function secretStateForInput(el) { + let state = secretStates.get(el) || null; + if (!state) state = closedSecretHosts.get(el) || null; + if (!state) state = declaredSecretState(el); + if (!state && el && el.getRootNode) { + state = secretBoundaryStates.get(el.getRootNode()) || null; + } + if (!state && el && el.getRootNode) { + const root = el.getRootNode(); + const hostState = root && root.host + ? declaredSecretHostState(root.host) : null; + if (hostState) { + closedSecretHosts.set(root.host, hostState); + bindSecretState(root.host, hostState, false); + observeSecretRoot(root, hostState); + state = hostState; + } + } + if (!state && activeSecretState && activeSecretElement + && !activeSecretElement.isConnected && isTextEntry(el)) { + // Some controlled inputs replace their DOM element after each change. + // Programmatic focus transfer is still the same input session. + state = activeSecretState; + } + if (!state) return null; + return {state, maskBound: bindSecretState(el, state)}; + } + + function textEntryCandidates(root) { + const candidates = []; + try { + if (root && root.nodeType === 1 && isTextEntry(root)) candidates.push(root); + if (root && root.querySelectorAll) { + candidates.push(...root.querySelectorAll( + 'input, textarea, [contenteditable=""], [contenteditable="true"],' + + ' [role="textbox"]' + )); + } + } catch (e) {} + return candidates; + } + + function discoverDeclaredSecretHosts(root) { + const candidates = []; + try { + if (root && root.nodeType === 1) candidates.push(root); + if (root && root.querySelectorAll) { + candidates.push(...root.querySelectorAll('[name], [id]')); + } + } catch (e) {} + for (const host of candidates) { + if (isTextEntry(host) || closedSecretHosts.has(host)) continue; + const state = declaredSecretHostState(host); + if (!state) continue; + // A declared non-text element is a complete shadow-host boundary. Its + // internal value can be opaque, so do not retain later text metadata. + opaqueSecretActive = true; + closedSecretHosts.set(host, state); + bindSecretState(host, state, false); + } + } + + function discoverOpenShadowRoots(root) { + if (!root || !root.querySelectorAll) return; + let elements = []; + try { elements = Array.from(root.querySelectorAll('*')); } catch (e) {} + if (root.nodeType === 1) elements.unshift(root); + for (const el of elements) { + try { + if (el.shadowRoot) observeSecretRoot(el.shadowRoot, null); + } catch (e) {} + } + } + + function observeSecretRoot(root, boundaryState) { + if (!root) return; + if (boundaryState) secretBoundaryStates.set(root, boundaryState); + if (!observedSecretRoots.has(root)) { + secretObserver.observe(root, { + attributes: true, attributeOldValue: true, childList: true, subtree: true, + }); + observedSecretRoots.add(root); + } + discoverDeclaredSecrets(root); + discoverOpenShadowRoots(root); + } + + function discoverDeclaredSecrets(root) { + discoverDeclaredSecretHosts(root); + const candidates = textEntryCandidates(root); + for (const el of candidates) { + if (secretStates.has(el)) continue; + const state = declaredSecretState(el) + || secretBoundaryStates.get(el.getRootNode()) || null; + if (!state) continue; + bindSecretState(el, state, false); + if (batchDiscoveredSecrets) batchDiscoveredSecrets.add(el); + } + } + + function stateFromPriorDeclaration(mutation) { + const el = mutation.target; + if (!isTextEntry(el) || typeof mutation.oldValue !== 'string') return null; + const oldValue = mutation.oldValue; + const oldDeclaredName = (mutation.attributeName === 'name' + || mutation.attributeName === 'id') + && SECRET_NAMES.indexOf(oldValue) >= 0; + const oldPasswordType = mutation.attributeName === 'type' + && oldValue.toLowerCase() === 'password'; + if (!oldDeclaredName && !oldPasswordType) return null; + const currentField = elementName(el) || el.id || null; + return { + field: oldDeclaredName ? oldValue : currentField, + inputSession: inputSessionFor(el), + }; + } + + function processSecretMutations(mutations) { + batchDiscoveredSecrets = new Set(); + try { + processSecretMutationBatch(mutations); + } finally { + batchDiscoveredSecrets = null; + } + } + + function processSecretMutationBatch(mutations) { + // Apply every attribute record first. A removed declared field can lose its + // name after removal but before its replacement is appended in the same + // task. The old-value record binds that removed node before rewrite + // matching runs across the complete MutationObserver batch. + for (const mutation of mutations) { + if (mutation.type !== 'attributes') continue; + if (!secretStates.has(mutation.target)) { + const priorState = stateFromPriorDeclaration(mutation); + if (priorState) bindSecretState(mutation.target, priorState, false); + } + discoverDeclaredSecrets(mutation.target); + } + const removedEntries = []; + const addedEntries = []; + for (const mutation of mutations) { + if (mutation.type !== 'childList') continue; + for (const node of mutation.removedNodes) { + removedEntries.push(...textEntryCandidates(node)); + } + for (const node of mutation.addedNodes) { + discoverDeclaredSecrets(node); + addedEntries.push(...textEntryCandidates(node)); + } + } + const detachedRemovedEntries = removedEntries.filter( + (el) => !el.isConnected + ); + const liveAddedEntries = addedEntries.filter((el) => el.isConnected); + const removedSecretEntries = detachedRemovedEntries.filter( + (el) => secretStates.has(el) + ); + const unboundAddedEntries = liveAddedEntries.filter( + // A node discovery bound in THIS batch still counts: discovery derives a + // NEW input session, and a field with no name and no ID has no other + // stable identity, so a controlled input that swaps its node would look + // like a new declared field on every keystroke. + (el) => !secretStates.has(el) || batchDiscoveredSecrets.has(el) + ); + if (removedSecretEntries.length && unboundAddedEntries.length) { + if (detachedRemovedEntries.length === 1 && liveAddedEntries.length === 1 + && removedSecretEntries.length === 1 + && unboundAddedEntries.length === 1) { + bindSecretState( + unboundAddedEntries[0], + secretStates.get(removedSecretEntries[0]), + false + ); + } else { + // A multi-node rewrite has no proven field mapping. Mask every + // possible replacement, but refuse its first input. Assigning one + // removed field/session to all candidates can merge distinct actions + // and replay a secret into the wrong field. + const maskState = secretStates.get(removedSecretEntries[0]); + for (const el of unboundAddedEntries) { + bindSecretState(el, maskState, false); + ambiguousSecretReplacements.add(el); + } + } + } + for (const el of stickySecretElements) { + if (!el.isConnected) continue; + try { + if (!el.hasAttribute(SECRET_MARKER)) el.setAttribute(SECRET_MARKER, ''); + } catch (e) {} + } + } + + function refreshSecretBindings() { + if (secretObserver === null) return; + // A document observer cannot see inside a shadow tree. Traverse every open + // root at the event boundary, then synchronously consume its queued records. + discoverOpenShadowRoots(document); + processSecretMutations(secretObserver.takeRecords()); } + secretObserver = new MutationObserver((mutations) => { + processSecretMutations(mutations); + }); + observeSecretRoot(document.documentElement || document, null); + discoverOpenShadowRoots(document); + // Seed the reflected-text baseline as soon as discovery has bound whatever + // declared fields this document already has. Flow installs this closure at + // document start, so the URL and the title here are what the document showed + // before it could reflect anything the operator typed into it. + // + // A field that ALREADY holds a value is the exception. Flow cannot tell + // whether the text on show already reflects that value, so it takes no + // baseline: every later URL and title from this document is withheld. + if (liveSecretValues().size > 0) { + documentHeldSecretValue = true; + } else { + preSecretUrl = String(location.href); + preSecretTitle = String(document.title == null ? '' : document.title); + } + // NOTE on the title seed. Flow installs this closure at document start, so + // `` has usually not parsed yet and the seeded title is ''. That is + // deliberate and it is not what carries the URL rule: safePageState refreshes + // both baselines at every sample taken while no declared value is held, so + // an ordinary page reaches its real title before the operator types. A + // document that ALREADY holds a value at install keeps the '' baseline and + // therefore withholds its title. That is the fail-closed side. + function fieldLabel(el) { // Best available human label for the receiving field, best-first: // associated <label for=...>, wrapping <label>, aria-label, // aria-labelledby, placeholder, name attribute, title. Mirrors // PlaywrightBackend.focused_field_label exactly. Passive metadata for // the compile-time parameter-proposal pass; NEVER the field's value. + // Exact or withheld, like every other text this closure reports: a label + // rewritten to a placeholder proposes a parameter name the page never + // showed, and the operator confirms that name without being told. try { - const clean = (s) => (s || '').replace(/\s+/g, ' ').trim(); + if (trustedSecretFieldLabels.has(el)) { + // Check the cached label against EVERY declared secret value, not only + // this field's own value. Discovery walks the document in order, so a + // field bound before another declared field caches a label that can + // contain the OTHER field's pre-filled value. + return labelTextOrNull(trustedSecretFieldLabels.get(el)); + } + const clean = (s) => labelTextOrNull( + (s || '').replace(/\s+/g, ' ').trim() + ); if (el.id) { try { const forLabel = document.querySelector( @@ -256,21 +1624,121 @@ return null; } - function emit(o) { try { window.__oaflow_emit(o); } catch (e) {} } + function emit(o) { + try { + // AN EVENT CARRIES NO REFLECTED TEXT. This handler runs in the CAPTURE + // phase, before the page's own listeners, so `location.href` and + // `document.title` here still hold what they held BEFORE this action. + // Sampling them here produced evidence one action out of date, and the + // value history that existed only to repair that staleness is what three + // reviews found separate defects in. + // + // Flow samples the URL and the title from Python instead, through + // structuralState(), at the same settled boundary that captures the + // after-frame. Drop any reflected text a caller attached. + delete o.url; + delete o.title; + // The ORIGIN is structural, not evidence text: Flow already declared it, + // and a secret never becomes part of it. It cannot go stale inside a + // document, because leaving the declared origin stops the recording. + o.__oaflow_origin = location.origin; + o.__oaflow_doc = DOC_ID; + o.__oaflow_doc_holds_secret = ( + opaqueSecretActive || documentHeldSecretValue + || liveSecretValues().size > 0 + ); + o.__oaflow_session = SESSION_ID; + o.__oaflow_top_level = window === window.top; + o.__oaflow_viewport = [Math.round(window.innerWidth), + Math.round(window.innerHeight)]; + o.__oaflow_dpr = Number(window.devicePixelRatio || 1); + const binding = window[BINDING_NAME]; + if (typeof binding === 'function') binding(o); + } catch (e) {} + } + + function inputEventTarget(event) { + try { + const path = event.composedPath ? event.composedPath() : []; + if (path.length && isTextEntry(path[0])) return path[0]; + } catch (e) {} + return event.target; + } + + function deepEventTarget(event) { + try { + const path = event.composedPath ? event.composedPath() : []; + if (path.length && path[0] && path[0].nodeType === 1) return path[0]; + } catch (e) {} + return event.target; + } let pointerDown = null; let suppressClick = false; - document.addEventListener('pointerdown', (e) => { + listenOn(window, 'resize', () => { + if (resizeTimer !== null) clearTimeout(resizeTimer); + resizeTimer = setTimeout(() => { + resizeTimer = null; + emit({kind: 'viewport'}); + }, 100); + }); + listen('focusin', (e) => { + // MutationObserver callbacks run at the microtask checkpoint. Drain queued + // records now so a page cannot add a declared field, remove its identity, + // and focus or type into it in one JavaScript task before classification. + refreshSecretBindings(); + const el = inputEventTarget(e); + if (activeSecretState && activeSecretElement && el !== activeSecretElement + && !activeSecretElement.isConnected && isTextEntry(el)) { + // The page replaced the element the operator was typing into. Continue + // the SAME input session instead of deriving a new one: a field with no + // name and no ID has no other stable identity, so a new session per swap + // would make every keystroke look like a new declared field. + bindSecretState( + el, secretStates.get(activeSecretElement) || activeSecretState + ); + return; + } + const declared = declaredSecretState(el); + if (declared) { + // Bind before the first key event. Application code can remove a + // declared name/id during keydown, beforeinput, or input dispatch. + bindSecretState(el, declared); + return; + } + if (!activeSecretState || el === activeSecretElement) return; + const retained = secretStates.get(el) || null; + if (retained) { + bindSecretState(el, retained); + } else if (activeSecretElement && !activeSecretElement.isConnected + && isTextEntry(el)) { + bindSecretState(el, activeSecretState); + } else { + activeSecretElement = null; + activeSecretState = null; + } + }); + listen('pointerdown', (e) => { if (e.button !== 0) return; + if (activeSecretState && e.target !== activeSecretElement + && !secretStates.has(e.target)) { + // An explicit operator action on another target ends the sticky input + // session. Programmatic replacement without a pointer action does not. + activeSecretElement = null; + activeSecretState = null; + } + const target = deepEventTarget(e); + const rowIdentity = structuredIdentityEvidence(e.clientX, e.clientY, target); pointerDown = { x: Math.round(e.clientX), y: Math.round(e.clientY), - sid: structuredIdentity(e.clientX, e.clientY), - structural: structuralTarget(e.clientX, e.clientY), + sid: rowIdentity.sid, + sid_withheld: rowIdentity.withheld, + structural: structuralTarget(e.clientX, e.clientY, target), idr: identifierRect(), }; - }, true); + }); - document.addEventListener('pointerup', (e) => { + listen('pointerup', (e) => { if (e.button !== 0 || !pointerDown) return; const start = pointerDown; pointerDown = null; @@ -278,58 +1746,175 @@ if (Math.hypot(endX - start.x, endY - start.y) < 5) return; suppressClick = true; setTimeout(() => { suppressClick = false; }, 0); - emit({ + const dragEvent = { kind: 'drag', x: start.x, y: start.y, end_x: endX, end_y: endY, sid: start.sid, structural: start.structural, - end_structural: structuralTarget(endX, endY), idr: start.idr, - url: location.href, title: document.title, - }); - }, true); + end_structural: structuralTarget(endX, endY, deepEventTarget(e)), + idr: start.idr, + }; + if (start.sid_withheld) dragEvent.sid_withheld = start.sid_withheld; + emit(dragEvent); + }); - document.addEventListener('click', (e) => { + listen('click', (e) => { if (e.button !== 0) return; if (suppressClick) { suppressClick = false; return; } - emit({ + const target = deepEventTarget(e); + const rowIdentity = structuredIdentityEvidence(e.clientX, e.clientY, target); + const pointerEvent = { kind: 'click', x: Math.round(e.clientX), y: Math.round(e.clientY), - sid: structuredIdentity(e.clientX, e.clientY), - structural: structuralTarget(e.clientX, e.clientY), + sid: rowIdentity.sid, + structural: structuralTarget(e.clientX, e.clientY, target), idr: identifierRect(), - url: location.href, title: document.title, - }); - }, true); + }; + if (rowIdentity.withheld) pointerEvent.sid_withheld = rowIdentity.withheld; + emit(pointerEvent); + }); - document.addEventListener('contextmenu', (e) => { - emit({ + listen('contextmenu', (e) => { + const target = deepEventTarget(e); + const rowIdentity = structuredIdentityEvidence(e.clientX, e.clientY, target); + const pointerEvent = { kind: 'right_click', x: Math.round(e.clientX), y: Math.round(e.clientY), - sid: structuredIdentity(e.clientX, e.clientY), - structural: structuralTarget(e.clientX, e.clientY), + sid: rowIdentity.sid, + structural: structuralTarget(e.clientX, e.clientY, target), idr: identifierRect(), - url: location.href, title: document.title, - }); - }, true); + }; + if (rowIdentity.withheld) pointerEvent.sid_withheld = rowIdentity.withheld; + emit(pointerEvent); + }); + + // Commit points. After each of these the page can remove the field while + // still showing the value somewhere -- an SPA wizard that replaces its form + // with a summary row is the ordinary case. The value committed here is used + // for ONE purpose, deciding whether to WITHHOLD identity text, and never for + // the URL, the title, or any rewrite. See committedSecretValues. + listen('change', (e) => commitSecretValueFor(inputEventTarget(e))); + listen('focusout', (e) => commitSecretValueFor(inputEventTarget(e))); + listen('submit', commitSecretValues); + listenOn(window, 'pagehide', commitSecretValues); - document.addEventListener('input', (e) => { - const el = e.target; - const secret = isSecretEl(el); + listen('input', (e) => { + refreshSecretBindings(); + const el = inputEventTarget(e); + if (ambiguousSecretReplacements.has(el)) { + privacyBoundaryError = ( + 'a DOM rewrite made a declared secret field identity ambiguous' + ); + emit({kind: 'privacy_refusal'}); + return; + } + const root = el && el.getRootNode ? el.getRootNode() : null; + const isUnboundShadowInput = root && root.host + && !secretStates.has(el) && !secretBoundaryStates.has(root) + && !declaredSecretState(el) && !declaredSecretHostState(root.host); + const isNativeNonTextControl = !!el && !!el.matches && el.matches( + 'select, input, button, option' + ) && !isTextEntry(el); + if (!isTextEntry(el) && !closedSecretHosts.has(el) + && !declaredSecretHostState(el)) { + if (isNativeNonTextControl) return; + privacyBoundaryError = ( + 'a shadow input event did not have a declared secret host boundary' + ); + emit({kind: 'privacy_refusal'}); + return; + } + if (isUnboundShadowInput && SECRET_NAMES.length) { + privacyBoundaryError = ( + 'a shadow input event did not have a declared secret host boundary' + ); + emit({kind: 'privacy_refusal'}); + return; + } + const secretBinding = secretStateForInput(el); + const secret = secretBinding !== null; + if (secret) { + // CAPTURE PHASE, so this runs BEFORE the page's own `input` handler. A + // page that writes the value somewhere and then clears its own field + // does both in that handler; this is the last moment the DOM still holds + // the value. Replace, never accumulate. + const observed = currentSecretValue(el); + if (observed) { + // The cache holds ONE value per element and this is about to replace + // it. A value that the next one does not CONTINUE was not edited away + // by the operator -- the page took it and started the field over. A + // scanner that writes the badge into the URL and clears the field does + // exactly that, and the first badge is still on show while the second + // is being typed. Promote it to the withhold-only committed set, which + // is not per element, so the second scan cannot displace the first. + // + // This cannot promote a keystroke prefix: while the operator types, + // each value continues the one before it. It is checked here, at the + // next input event, rather than at a microtask checkpoint after this + // one -- a checkpoint runs BETWEEN listeners, so it would observe the + // field before the page's own handler had cleared it. + const previous = lastSecretValues.get(el); + if (previous && previous !== observed && observed.indexOf(previous) !== 0) { + committedSecretValues.add(previous); + } + lastSecretValues.set(el, observed); + // ARM THE DOCUMENT BOUNDARY HERE, not from what the DOM holds at the + // settled read. A scanner that clears its own field inside its own + // `input` handler never holds a value at any moment Python samples, so + // a flag derived from the live DOM stayed false for the whole + // recording: the title net never ran, and Python never learned that + // this document had received a declared value at all. This is the + // moment the document PROVABLY held one, and it is what the + // documentation already says -- "once a declared secret field RECEIVES + // INPUT". + documentHeldSecretValue = true; + } + } + if (secret && !isTextEntry(el) && closedSecretHosts.has(el)) { + opaqueSecretActive = true; + } const r = (el.getBoundingClientRect && el.getBoundingClientRect()) || { left: 0, top: 0, width: 0, height: 0 }; + // The receiving field's NAME is machine evidence: it becomes the parameter + // the compiler binds and the replayer fills. A rewritten name would name a + // parameter the page does not have, so it is exact or withheld. + identityWithheldReason = null; + const rawField = elementName(el) || el.id || null; + const field = secret ? secretBinding.state.field : identityTextOrNull(rawField); const o = { kind: 'input', - field: el.name || el.id || null, + field: field, label: fieldLabel(el), secret: secret, + __oaflow_input_session: secret + ? secretBinding.state.inputSession : inputSessionFor(el), rect: [Math.round(r.left), Math.round(r.top), Math.round(r.width), Math.round(r.height)], - url: location.href, title: document.title, }; - // The literal value of a SECRET field is never read or transmitted. - if (!secret) o.value = (el.value != null ? String(el.value) : ''); + if (!secret && rawField !== null && field === null) { + o.identity_withheld = identityWithheldReason; + } + // A secret literal never leaves the page closure. The element reference is + // retained so the value can be READ LIVE at the next match; the value + // itself is not retained anywhere. + if (secret) { + o.__oaflow_secret_mask_bound = secretBinding.maskBound; + } else { + o.value = currentSecretValue(el); + } emit(o); - }, true); + }); - document.addEventListener('keydown', (e) => { + listen('keydown', (e) => { + refreshSecretBindings(); + const el = inputEventTarget(e); + if (ambiguousSecretReplacements.has(el)) { + privacyBoundaryError = ( + 'a DOM rewrite made a declared secret field identity ambiguous' + ); + emit({kind: 'privacy_refusal'}); + return; + } + const keySecretBinding = secretStateForInput(el); + if (keySecretBinding && e.key.length === 1) return; const modifiers = []; if (e.ctrlKey) modifiers.push('ctrl'); if (e.altKey) modifiers.push('alt'); @@ -341,21 +1926,19 @@ if (modifiers.length && !pureModifier && !shiftedText && !e.repeat) { emit({ kind: 'hotkey', key: e.key, modifiers, - url: location.href, title: document.title, - }); + }); return; } if (SPECIAL.indexOf(e.key) < 0) return; - emit({ kind: 'key', key: e.key, url: location.href, title: document.title }); - }, true); + emit({kind: 'key', key: e.key}); + }); - document.addEventListener('wheel', (e) => { + listen('wheel', (e) => { emit({ kind: 'scroll', dx: Math.round(e.deltaX), dy: Math.round(e.deltaY), - url: location.href, title: document.title, }); - }, true); + }); })(); """ @@ -377,6 +1960,8 @@ def __init__( param_fields: tuple[str, ...] = (), identifier_fields: tuple[str, ...] = (), headless: bool = False, + cdp_endpoint: Optional[str] = None, + browser_page_url: Optional[str] = None, poll_ms: int = 60, settle_timeout_s: float = 5.0, settle_stable_frames: int = 2, @@ -386,13 +1971,34 @@ def __init__( Callable[[], Optional[list[dict[str, Any]]]] ] = None, stop_when: Optional[Callable[[], bool]] = None, + surface: str = "web", ) -> None: self._url = url + self._surface = surface self._out_dir = Path(out_dir) self._secret_fields = set(secret_fields) self._param_fields = set(param_fields) self._identifier_fields = set(identifier_fields) self._headless = headless + if browser_page_url and not cdp_endpoint: + raise BrowserAttachError("browser_page_url requires a browser CDP endpoint") + if cdp_endpoint and headless: + raise BrowserAttachError( + "headless mode cannot be combined with an attached browser" + ) + self._cdp_endpoint = ( + validate_browser_cdp_endpoint(cdp_endpoint) if cdp_endpoint else None + ) + self._attached_origin = ( + _http_origin(url, label="the declared app URL") + if self._cdp_endpoint + else None + ) + self._browser_page_url = browser_page_url + self._owns_browser = self._cdp_endpoint is None + self._session_id = uuid.uuid4().hex + self._binding_name = f"__oaflow_emit_{self._session_id}" + self._secret_marker_attribute = f"data-oaflow-secret-{self._session_id}" self._poll_ms = poll_ms self._viewport = viewport # Recording-only, read-only observation. This does not add an effect @@ -409,71 +2015,356 @@ def __init__( self._pyq: list[dict[str, Any]] = [] self._pending_type: Optional[dict[str, Any]] = None self._pending_scroll: Optional[dict[str, Any]] = None + self._listener_error: Optional[BrowserAttachError] = None self.done = False # Set on start(). + self._recording_dir: Optional[Path] = None self._pw = None self._browser = None self.page = None + self._page_close_listener = self._handle_page_close + self._frame_navigation_listener = self._handle_frame_navigation + self._frame_attached_listener = self._handle_frame_tree_change + self._frame_detached_listener = self._handle_frame_tree_change + self._popup_listener = self._handle_popup + self._context_page_listener = self._handle_context_page + self._page_lifecycle_listeners_installed = False + self._context_page_listener_installed = False + self._context_page_latches: list[tuple[Any, Any]] = [] + self._context_page_baselines: list[tuple[Any, tuple[Any, ...]]] = [] + self._context = None + self._privacy_cdp = None + self._context_pages_at_start: tuple[Any, ...] = () + self._finalizing = False self.backend: Optional[PlaywrightBackend] = None self.recorder: Optional[Recorder] = None self._last_frame: bytes = b"" self._last_structural: dict[str, Any] = {} + self._attached_geometry: Optional[tuple[int, int, float]] = None + self._initial_attached_viewport: Optional[tuple[int, int]] = None + self._viewport_dirty = False + self._viewport_history: list[dict[str, Any]] = [] + # Source-time secret boundary state. Each document builds a fresh + # closure, so a later document never saw the value an earlier one + # received. Once a declared secret receives input, reflected text from + # any LATER document is withheld. + self._secret_doc_ids: set[str] = set() + self._first_secret_doc_id: Optional[str] = None + self._structural_text_withheld = False + self._structural_text_withheld_reasons: set[str] = set() + self._identity_withheld_events = 0 + self._dropped_url_parameters: set[tuple[str, str, str]] = set() + self._app_placed_secret_in_url = False + self._app_placed_secret_in_title = False # -- lifecycle ----------------------------------------------------------- def start(self) -> None: - """Launch the browser, install the in-page listeners, capture the - initial settled frame.""" - from openadapt_flow._browser_setup import ensure_chromium_installed + """Launch or attach, install listeners, and capture the first frame.""" - ensure_chromium_installed() - from playwright.sync_api import sync_playwright - - self._pw = sync_playwright().start() + self._prepare_recording_dir() try: - self._browser = self._pw.chromium.launch(headless=self._headless) + if self._owns_browser: + from openadapt_flow._browser_setup import ensure_chromium_installed + + ensure_chromium_installed() + from playwright.sync_api import sync_playwright + + self._pw = sync_playwright().start() + if self._owns_browser: + self._browser = self._pw.chromium.launch(headless=self._headless) + self.page = self._browser.new_page( + viewport={ + "width": self._viewport[0], + "height": self._viewport[1], + }, + device_scale_factor=1, + ) + else: + try: + self._browser = self._pw.chromium.connect_over_cdp( + self._cdp_endpoint + ) + except Exception as exc: + raise BrowserAttachError( + "could not connect to the local Chromium CDP endpoint; " + "confirm that the browser was started with remote " + "debugging and that the endpoint is ready" + ) from exc + self._install_candidate_context_page_latches() + self.page = select_attached_page( + self._browser, + app_url=self._url, + page_url=self._browser_page_url, + ) + + self._context = self.page.context + if self._owns_browser: + self._context.on("page", self._context_page_listener) + self._context_page_latches = [ + (self._context, self._context_page_listener) + ] + self._context_page_listener_installed = True + self._context_pages_at_start = tuple(self._context.pages) + else: + self._bind_selected_context_page_latch() + if self._listener_error is not None: + raise self._listener_error + self.page.on("close", self._page_close_listener) + self.page.on("framenavigated", self._frame_navigation_listener) + self.page.on("frameattached", self._frame_attached_listener) + self.page.on("framedetached", self._frame_detached_listener) + self.page.on("popup", self._popup_listener) + self._page_lifecycle_listeners_installed = True + self.page.expose_binding( + self._binding_name, + lambda source, detail: self._enqueue_browser_event( + detail, + source=source, + ), + ) + init_js = ( + _INIT_JS.replace("__SESSION_ID__", json.dumps(self._session_id)) + .replace("__BINDING_NAME__", json.dumps(self._binding_name)) + .replace("__SECRET_NAMES__", json.dumps(sorted(self._secret_fields))) + .replace( + "__SECRET_MARKER__", + json.dumps(self._secret_marker_attribute), + ) + .replace( + "__IDENT_NAMES__", + json.dumps(sorted(self._identifier_fields)), + ) + .replace("__SPECIAL_KEYS__", json.dumps(list(_SPECIAL_KEYS))) + ) + self.page.add_init_script(init_js) + if self._owns_browser: + self.page.goto(self._url) + try: + self.page.wait_for_load_state("load") + except Exception: + pass + else: + # add_init_script applies after the next navigation. Install + # the same session in every already-open document now. This + # includes child frames, whose local coordinates cannot yet be + # bound to top-level evidence and therefore emit an explicit + # refusal instead of disappearing from the recording. + for frame in list(self.page.frames): + try: + frame.evaluate(init_js) + except Exception as exc: + try: + detached = frame.is_detached() + except Exception: + detached = False + if detached: + continue + raise BrowserAttachError( + "could not install the recording listener in every " + "existing browser frame; recording was refused" + ) from exc + self._guard_screenshot_privacy() + + self.backend = PlaywrightBackend( + self.page, + screenshot_scale="device" if self._owns_browser else "css", + screenshot_mask_selectors=_secret_screenshot_selectors( + self._secret_fields, + marker_attribute=self._secret_marker_attribute, + ), + structural_state_reader=self._read_scrubbed_page_state, + screenshot_guard=self._guard_screenshot_privacy, + ) + assert self._recording_dir is not None + self.recorder = Recorder( + self.backend, + self._recording_dir, + app_url=self._url, + system_of_record_reader=self._system_of_record_reader, + **self._settle, + ) + if self._owns_browser: + self._last_frame = self.recorder._wait_settled() + self._last_structural = self._structural_state() + else: + self._rebaseline_attached_viewport() except Exception: - self._pw.stop() + self.abort() raise - self.page = self._browser.new_page( - viewport={"width": self._viewport[0], "height": self._viewport[1]}, - device_scale_factor=1, - ) - self.page.on("close", lambda _=None: setattr(self, "done", True)) - self.page.expose_binding( - "__oaflow_emit", - lambda source, detail: self._pyq.append(detail), - ) - init_js = ( - _INIT_JS.replace( - "__SECRET_NAMES__", json.dumps(sorted(self._secret_fields)) + + def _register_existing_closed_shadow_boundaries(self) -> None: + """Bind or refuse declared fields inside pre-existing closed roots. + + Page JavaScript cannot traverse a closed shadow root that existed before + attachment. Chromium's DOM search can identify only the declared field + nodes without returning their values. A function executed on each match + performs the root/host check inside that document. No node content or + secret literal crosses the CDP boundary. + """ + + assert self.page is not None + queries = ["input[type='password']"] + for field in sorted(self._secret_fields): + encoded = _css_string_literal(field) + queries.append(f"[name={encoded}], [id={encoded}]") + cdp = self._privacy_cdp + if cdp is None: + assert self._context is not None + try: + cdp = self._context.new_cdp_session(self.page) + cdp.send("DOM.enable") + except Exception as exc: + raise BrowserAttachError( + "could not inspect closed shadow boundaries; recording was " + "refused before retaining a frame" + ) from exc + self._privacy_cdp = cdp + try: + # Populate stable frontend node ids. Depth zero returns only the + # document node; it does not send page attributes or text to Python. + cdp.send("DOM.getDocument", {"depth": 0, "pierce": True}) + for query in queries: + search = cdp.send( + "DOM.performSearch", + {"query": query, "includeUserAgentShadowDOM": False}, + ) + search_id = str(search["searchId"]) + try: + count = int(search.get("resultCount", 0)) + if count <= 0: + continue + results = cdp.send( + "DOM.getSearchResults", + { + "searchId": search_id, + "fromIndex": 0, + "toIndex": count, + }, + ) + for node_id in results.get("nodeIds", []): + resolved = cdp.send( + "DOM.resolveNode", + { + "nodeId": int(node_id), + "objectGroup": _PRIVACY_SCAN_OBJECT_GROUP, + }, + ) + object_id = resolved.get("object", {}).get("objectId") + if not object_id: + raise BrowserAttachError( + "a declared secret node could not be bound before " + "the first frame" + ) + outcome = cdp.send( + "Runtime.callFunctionOn", + { + "objectId": object_id, + "objectGroup": _PRIVACY_SCAN_OBJECT_GROUP, + "functionDeclaration": r"""function(sessionId) { + const root = this.getRootNode && this.getRootNode(); + if (!root || root.mode !== 'closed') { + return {closed: false, registered: true}; + } + const ownerWindow = this.ownerDocument.defaultView; + const recorder = ownerWindow.__oaflowRecorder; + if (!recorder || recorder.sessionId !== sessionId + || typeof recorder.registerExistingClosedShadowHost + !== 'function') { + return {closed: true, registered: false}; + } + return { + closed: true, + registered: Boolean( + recorder.registerExistingClosedShadowHost(root.host) + ), + }; + }""", + "arguments": [{"value": self._session_id}], + "returnByValue": True, + }, + ) + value = outcome.get("result", {}).get("value", {}) + if value.get("closed") and not value.get("registered"): + raise BrowserAttachError( + "a declared secret is inside a pre-existing or " + "newly added closed shadow root whose host is not " + "declared with the same --secret name or id; " + "recording was refused before retaining a frame" + ) + finally: + cdp.send("DOM.discardSearchResults", {"searchId": search_id}) + # This guard runs before every retained screenshot. Release the + # scan's resolved objects so a long recording cannot accumulate + # protocol handles inside the attached browser. + cdp.send( + "Runtime.releaseObjectGroup", + {"objectGroup": _PRIVACY_SCAN_OBJECT_GROUP}, ) - .replace("__IDENT_NAMES__", json.dumps(sorted(self._identifier_fields))) - .replace("__SPECIAL_KEYS__", json.dumps(list(_SPECIAL_KEYS))) - ) - self.page.add_init_script(init_js) - self.page.goto(self._url) + except BrowserAttachError: + raise + except Exception as exc: + raise BrowserAttachError( + "could not prove closed shadow secret boundaries; recording was " + "refused before retaining a frame" + ) from exc + + def _guard_screenshot_privacy(self) -> None: + """Bind or refuse every secret boundary before screenshot bytes exist.""" + + self._register_existing_closed_shadow_boundaries() + self._assert_page_privacy_safe() + + def _assert_page_privacy_safe(self) -> None: + """Refuse a screenshot after an undeclared closed root appears.""" + + if self.page is None: + raise BrowserAttachError("the browser page is unavailable") try: - self.page.wait_for_load_state("load") - except Exception: - pass - self.backend = PlaywrightBackend(self.page) - self.recorder = Recorder( - self.backend, - self._out_dir, - app_url=self._url, - system_of_record_reader=self._system_of_record_reader, - **self._settle, - ) - self._last_frame = self.recorder._wait_settled() - self._last_structural = self._structural_state() + frames = list(self.page.frames) + except Exception as exc: + raise BrowserAttachError( + "could not inventory browser privacy guards" + ) from exc + for frame in frames: + try: + status = frame.evaluate( + """sessionId => { + const recorder = window.__oaflowRecorder; + if (!recorder || recorder.sessionId !== sessionId + || typeof recorder.privacyStatus !== 'function') { + return {ok: false, error: 'the privacy guard is unavailable'}; + } + return recorder.privacyStatus(); + }""", + self._session_id, + ) + except Exception as exc: + try: + detached = frame.is_detached() + except Exception: + detached = False + if detached: + continue + raise BrowserAttachError( + "could not verify every browser secret boundary before " + "retaining a frame" + ) from exc + if not isinstance(status, dict) or status.get("ok") is not True: + raise BrowserAttachError( + str(status.get("error") if isinstance(status, dict) else "") + or "the browser secret boundary is not safe" + ) def run(self) -> Path: """Pump until completion, an operator stop, or a closed window.""" if self._stop_when is None: finish_instruction = ( - "Press Ctrl-C here (or close the browser window) to finish." + "Press Ctrl-C here to finish. Keep the selected browser tab open " + "until Flow confirms the recording." + if not self._owns_browser + else "Press Ctrl-C here (or close the browser window) to finish." ) else: finish_instruction = ( @@ -482,8 +2373,12 @@ def run(self) -> Path: ) print( f"Recording {self._url}\n" - " Perform your workflow in the browser window.\n" - f" {finish_instruction}" + + ( + " Perform your workflow in the selected existing browser tab.\n" + if not self._owns_browser + else " Perform your workflow in the browser window.\n" + ) + + f" {finish_instruction}" ) try: while not self.done: @@ -491,30 +2386,727 @@ def run(self) -> Path: break except KeyboardInterrupt: print("\n[record] stopping…") + except Exception: + self.abort() + raise return self.finish() def run_script(self, script: Callable[[Any, Callable[[], None]], None]) -> Path: """Scripted loop (tests): run ``script(page, pump)`` — which performs synthetic input and calls ``pump()`` to let the recorder drain — then flush and finish.""" - script(self.page, self.pump) + try: + script(self.page, self.pump) + except Exception: + self.abort() + raise return self.finish() def finish(self) -> Path: - """Flush trailing input, write meta.json, tear the browser down.""" + """Flush input, write metadata, and close or detach as appropriate.""" try: + if self._listener_error is not None: + raise self._listener_error + # Bind or refuse every secret boundary first. These page + # round-trips also deliver lifecycle events that Chromium queued + # during the recording, so a stale pre-finalization event is + # judged by recording-time rules instead of aliasing a change + # after the final evidence. + self._guard_screenshot_privacy() + if self._listener_error is not None: + raise self._listener_error + # The operations below retain the final evidence. Arm every + # irreversible lifecycle latch before cleanup and the last + # queue drain. + self._finalizing = True + self._cleanup_page_listeners() + self._drain_event_queue() self._flush_type() self._flush_scroll() - finally: assert self.recorder is not None out = self.recorder.finish() + if self._listener_error is not None: + raise self._listener_error + meta_path = out / "meta.json" + meta = json.loads(meta_path.read_text()) + meta["source"] = ( + "openadapt-flow-playwright" + if self._owns_browser + else "openadapt-flow-playwright-cdp" + ) + # Stamp the surface BEFORE the atomic publish. A recording that the + # publish step still has to modify is not complete when it appears + # at the final path, and a crash in that window publishes a + # surface-unbound recording. + meta["surface"] = self._surface + if self._structural_text_withheld: + # The operator must be able to see that Flow dropped URL and + # title evidence, and why. Silence here would read as evidence + # the page simply did not have. Every distinct reason is named: + # one recording can hit more than one. + meta["structural_text_withheld"] = ",".join( + sorted(self._structural_text_withheld_reasons) + ) + if self._identity_withheld_events: + meta["identity_withheld_events"] = self._identity_withheld_events + if self._dropped_url_parameters: + # Which URL parameter VALUES the recording does not carry, and + # why. The names are app structure and are safe to report. + meta["url_dropped_params"] = [ + {"name": name, "where": where, "reason": reason} + for name, where, reason in sorted(self._dropped_url_parameters) + ] + if self._app_placed_secret_in_url: + meta["application_placed_secret_in_url"] = True + if self._app_placed_secret_in_title: + meta["application_placed_secret_in_title"] = True + if not self._owns_browser: + assert self._initial_attached_viewport is not None + meta["viewport"] = list(self._initial_attached_viewport) + meta["viewport_mode"] = "per-event" + meta["viewport_history"] = list(self._viewport_history) + meta_path.write_text(json.dumps(meta, indent=2)) + if self._listener_error is not None: + raise self._listener_error + self._assert_no_new_pages() + self._stop_browser_connection() + if self._listener_error is not None: + raise self._listener_error + return self._promote_recording() + except Exception: + self.abort() + raise + + def abort(self) -> None: + """Detach and remove only this session's unpublished temporary output.""" + + self.done = True + self._pyq.clear() + try: + self._cleanup_page_listeners() + finally: try: - if self._browser is not None: - self._browser.close() + self._stop_browser_connection() finally: - if self._pw is not None: - self._pw.stop() - return out + self._discard_recording_dir() + + def _prepare_recording_dir(self) -> None: + """Reserve a fresh sibling directory without changing the final path.""" + + if self._recording_dir is not None: + return + if os.path.lexists(self._out_dir): + raise BrowserAttachError( + "the recording output already exists; choose a new --out directory" + ) + try: + self._out_dir.parent.mkdir(parents=True, exist_ok=True) + temporary = tempfile.mkdtemp( + prefix=f"{_PARTIAL_RECORDING_PREFIX}{self._out_dir.name}-", + dir=self._out_dir.parent, + ) + except OSError as exc: + raise BrowserAttachError( + "the temporary recording output could not be created" + ) from exc + self._recording_dir = Path(temporary) + + def _promote_recording(self) -> Path: + """Atomically publish the complete recording at the requested path.""" + + assert self._recording_dir is not None + try: + _rename_directory_noreplace(self._recording_dir, self._out_dir) + except OSError as exc: + if exc.errno in {errno.EEXIST, errno.ENOTEMPTY}: + raise BrowserAttachError( + "the recording output appeared during capture; Flow refused to " + "replace it" + ) from exc + raise BrowserAttachError( + "the complete recording could not be published atomically" + ) from exc + self._recording_dir = None + return self._out_dir + + def _discard_recording_dir(self) -> None: + """Delete only the temporary directory that this session created.""" + + recording_dir, self._recording_dir = self._recording_dir, None + if recording_dir is None or not recording_dir.exists(): + return + shutil.rmtree(recording_dir) + + def _handle_page_close(self, _page: Any = None) -> None: + """Retain a refusal when an attached tab closes before finalization.""" + + self.done = True + if not self._owns_browser and self._listener_error is None: + self._listener_error = BrowserAttachError( + "the selected browser tab closed before Flow could retain the " + "final evidence; recording stopped without complete metadata" + ) + + def _handle_frame_navigation(self, frame: Any) -> None: + """Retain the first selected-main-frame origin violation.""" + + if self._owns_browser or self.page is None or self._listener_error is not None: + return + if self._finalizing: + # The final evidence is already bound. Refuse without another page + # round-trip: an evaluate here would re-enter event dispatch while + # the latch is armed. + self._retain_late_frame_error() + return + try: + if frame is not self.page.main_frame: + return + current_origin = _http_origin( + str(frame.evaluate("() => location.origin")), + label="the selected browser tab URL", + ) + except Exception: + current_origin = None + if current_origin != self._attached_origin: + self._listener_error = BrowserAttachError( + "the selected browser tab left the declared application " + "origin; recording was refused" + ) + self.done = True + + def _retain_late_frame_error(self) -> None: + """Retain one refusal for a post-snapshot frame-tree change.""" + + if self._listener_error is None: + self._listener_error = BrowserAttachError( + "the selected browser tab changed frame state after Flow " + "retained its final evidence; recording was refused" + ) + self.done = True + + def _handle_frame_tree_change(self, _frame: Any = None) -> None: + """Refuse a frame attach/detach after the final evidence snapshot.""" + + if not self._owns_browser and self._finalizing: + self._retain_late_frame_error() + + def _handle_popup(self, _popup: Any = None) -> None: + """Refuse a second page that the selected recording tab opens.""" + + self.done = True + if self._listener_error is None: + self._listener_error = BrowserAttachError( + "the selected browser tab opened a popup or new tab; this " + "recording is bound to one tab, so Flow stopped before " + "publishing incomplete metadata" + ) + + def _handle_context_page(self, _page: Any = None) -> None: + """Irreversibly refuse any page created after context binding.""" + + self.done = True + if self._listener_error is None: + self._listener_error = BrowserAttachError( + "the selected browser context opened a popup or new tab; this " + "recording is bound to its accepted page baseline, so Flow " + "stopped before publishing incomplete metadata" + ) + + def _install_candidate_context_page_latches(self) -> None: + """Latch new pages on every context before attached-page selection.""" + + assert self._browser is not None + try: + contexts = tuple(self._browser.contexts) + except Exception as exc: + raise BrowserAttachError( + "the attached browser context inventory could not be read" + ) from exc + for context in contexts: + try: + context.on("page", self._context_page_listener) + except Exception as exc: + raise BrowserAttachError( + "the attached browser page baseline could not be guarded" + ) from exc + self._context_page_latches.append((context, self._context_page_listener)) + self._context_page_listener_installed = bool(self._context_page_latches) + baselines: list[tuple[Any, tuple[Any, ...]]] = [] + for context in contexts: + try: + baseline = tuple(context.pages) + except Exception as exc: + raise BrowserAttachError( + "the attached browser page baseline could not be read" + ) from exc + baselines.append((context, baseline)) + self._context_page_baselines = baselines + if self._listener_error is not None: + raise self._listener_error + + def _bind_selected_context_page_latch(self) -> None: + """Keep the selected context latch and its pre-listener baseline.""" + + assert self._context is not None + selected_baseline = next( + ( + baseline + for context, baseline in self._context_page_baselines + if context is self._context + ), + None, + ) + if selected_baseline is None: + raise BrowserAttachError( + "the selected browser context was not in the guarded baseline" + ) + retained: list[tuple[Any, Any]] = [] + for context, listener in self._context_page_latches: + if context is self._context: + retained.append((context, listener)) + continue + try: + context.remove_listener("page", listener) + except Exception: + pass + self._context_page_latches = retained + self._context_page_baselines = [(self._context, selected_baseline)] + self._context_pages_at_start = selected_baseline + self._context_page_listener_installed = True + self._assert_no_new_pages() + + def _assert_no_new_pages(self) -> None: + """Retain a refusal if this recording context gained another page.""" + + if self.page is None or not self._context_pages_at_start: + return + try: + current_pages = tuple(self.page.context.pages) + except Exception as exc: + raise BrowserAttachError( + "the selected browser tab page inventory could not be read; " + "recording was refused" + ) from exc + for candidate in current_pages: + if not any( + candidate is existing for existing in self._context_pages_at_start + ): + self._handle_popup(candidate) + break + if self._listener_error is not None: + raise self._listener_error + + def _enqueue_browser_event( + self, + detail: Any, + *, + source: Optional[dict[str, Any]] = None, + ) -> None: + """Accept only a bounded event from this recorder session.""" + + if not isinstance(detail, dict): + return + event = dict(detail) + if event.pop("__oaflow_session", None) != self._session_id: + return + raw_event_origin = event.pop("__oaflow_origin", None) + raw_doc_id = event.pop("__oaflow_doc", None) + doc_holds_secret = event.pop("__oaflow_doc_holds_secret", None) is True + kind = event.get("kind") + if kind == "privacy_refusal": + self._listener_error = BrowserAttachError( + "a shadow input did not have a declared secret host boundary; " + "recording stopped before accepting its value or retaining " + "another frame" + ) + self.done = True + return + secret_mask_bound = event.pop("__oaflow_secret_mask_bound", None) + if ( + kind == "input" + and bool(event.get("secret")) + and secret_mask_bound is not True + ): + self._listener_error = BrowserAttachError( + "a secret input could not retain its screenshot mask identity; " + "recording stopped before accepting the event" + ) + self.done = True + return + raw_input_session = event.pop("__oaflow_input_session", None) + if kind == "input": + expected_prefix = f"{self._session_id}:input:" + if ( + not isinstance(raw_input_session, str) + or not raw_input_session.startswith(expected_prefix) + or not raw_input_session.removeprefix(expected_prefix).isdigit() + ): + self._listener_error = BrowserAttachError( + "the browser emitted an input without a valid bound field " + "session; recording stopped before accepting the event" + ) + self.done = True + return + event["_oaflow_input_session"] = raw_input_session + reported_top_level = bool(event.pop("__oaflow_top_level", True)) + source_is_selected_top_level = reported_top_level + if source is not None: + try: + source_is_selected_top_level = ( + source.get("page") is self.page + and source.get("frame") is self.page.main_frame + ) + except Exception: + source_is_selected_top_level = False + if not source_is_selected_top_level and kind == "viewport": + return + if not source_is_selected_top_level: + self._listener_error = BrowserAttachError( + "an event came from an iframe; cross-frame recording is not " + "qualified, so recording stopped before accepting the event" + ) + self.done = True + return + if kind not in { + "click", + "right_click", + "drag", + "input", + "key", + "hotkey", + "scroll", + "viewport", + }: + return + self._track_secret_document( + kind, event, raw_doc_id, holds_secret=doc_holds_secret + ) + if not self._owns_browser: + if not isinstance(raw_event_origin, str): + self._listener_error = BrowserAttachError( + "a browser event did not report its document origin; " + "recording stopped before accepting the event" + ) + self.done = True + return + try: + # The page sends location.origin beside every event, as its + # own field. The guard must never read reflected evidence + # text: that text can be withheld, and an origin parsed out of + # a withheld URL would refuse a valid recording. + event_origin = _http_origin( + str(raw_event_origin), + label="the browser event origin", + ) + except Exception: + event_origin = None + if event_origin != self._attached_origin: + self._listener_error = BrowserAttachError( + "a browser event came from outside the declared application " + "origin; recording stopped before accepting the event" + ) + self.done = True + return + raw_viewport = event.pop("__oaflow_viewport", None) + raw_dpr = event.pop("__oaflow_dpr", None) + try: + event_geometry = ( + int(raw_viewport[0]), + int(raw_viewport[1]), + round(float(raw_dpr), 6), + ) + except (IndexError, TypeError, ValueError): + event_geometry = (0, 0, 0.0) + if ( + event_geometry[0] <= 0 + or event_geometry[1] <= 0 + or not 0.1 <= event_geometry[2] <= 16.0 + ): + self._listener_error = BrowserAttachError( + "the browser emitted invalid viewport evidence; recording " + "stopped before accepting the event" + ) + self.done = True + return + event["_oaflow_geometry"] = event_geometry + if kind == "viewport": + self._viewport_dirty = True + return + else: + event.pop("__oaflow_viewport", None) + event.pop("__oaflow_dpr", None) + try: + encoded_size = len(json.dumps(event).encode("utf-8")) + except (TypeError, ValueError): + return + if encoded_size > 1_000_000: + self._listener_error = BrowserAttachError( + "the browser emitted an event larger than 1 MB; recording " + "stopped without accepting the event" + ) + self.done = True + return + self._pyq.append(event) + + def _origin_only_url(self) -> str: + """The declared origin with an empty path: a URL that holds no value.""" + + origin = self._attached_origin + if origin is None: + try: + origin = _http_origin(self._url, label="the declared app URL") + except BrowserAttachError: + return "" + scheme, host, port = origin + if port == (443 if scheme == "https" else 80): + return f"{scheme}://{host}/" + return f"{scheme}://{host}:{port}/" + + def _track_secret_document( + self, + kind: str, + event: dict[str, Any], + raw_doc_id: Any, + *, + holds_secret: bool = False, + ) -> None: + """Bind the secret boundary to the document that received the value.""" + + doc_id = raw_doc_id if isinstance(raw_doc_id, str) else None + received_secret_input = kind == "input" and event.get("secret") is True + if doc_id is not None and (holds_secret or received_secret_input): + self._mark_secret_document(doc_id) + # Count the action ONCE, whichever identity evidence Flow withheld: a + # selector, an accessible name or role, the receiving field's name, or + # the clicked row's identity characters. Each one disarms an identity + # check the same way. + withheld_identity = bool( + event.get("sid_withheld") or event.get("identity_withheld") + ) + for key in ("structural", "end_structural"): + target = event.get(key) + if isinstance(target, dict) and target.get("identity_withheld"): + withheld_identity = True + if withheld_identity: + self._identity_withheld_events += 1 + # An event carries no URL or title of its own: Flow samples reflected + # text at the settled boundary instead (see _read_scrubbed_page_state). + # Note the reason here so the operator still learns that an action came + # from a document whose reflected text Flow could no longer prove safe. + if self._secret_document_left(doc_id): + self._note_withheld_structural_text("secret-value-left-its-document") + + def _mark_secret_document(self, doc_id: str) -> None: + """Record that this document received a declared value. + + BOTH markers move together. An earlier revision added to + ``_secret_doc_ids`` from the input-event path but set + ``_first_secret_doc_id`` only from the settled page read, so a document + that never HELD a value at a sampling instant -- a scanner that clears + its own field inside its own ``input`` handler -- reached + ``_secret_doc_ids`` while the marker the cross-document rule keys off + stayed ``None``. Every later document then reported its URL. + """ + + self._secret_doc_ids.add(doc_id) + if self._first_secret_doc_id is None: + self._first_secret_doc_id = doc_id + + def _secret_document_left(self, doc_id: Optional[str]) -> bool: + """True for every document AFTER the one that first held a value. + + Each document builds its own recorder closure, so a later document + never saw the value an earlier one received: no bound element holds it, + nothing was committed in that closure, and a value carried in a PATH + segment has no parameter name to identify it. Such a document cannot + prove the URL it loaded with predates the value, so its reflected text + is withheld. + + A document that receives a declared value of its own is NOT exempt. It + can still have loaded with an earlier document's value in its path, and + holding a value of its own says nothing about that. Only the FIRST + document to hold a declared value reports its own reflected text, and + that document reports it under the in-page rules. + """ + + if self._first_secret_doc_id is None: + return False + return doc_id is None or doc_id != self._first_secret_doc_id + + def _note_withheld_structural_text(self, reason: str) -> None: + """Record WHY Flow withheld reflected text, for the operator notice.""" + + self._structural_text_withheld = True + self._structural_text_withheld_reasons.add(reason) + + def _cleanup_page_listeners(self) -> None: + """Remove this session's current-document listeners before detach.""" + + if self.page is None: + return + try: + frames = list(self.page.frames) + except Exception: + frames = [] + for frame in frames: + try: + frame.evaluate( + """sessionId => { + const current = window.__oaflowRecorder; + const fallback = window['__oaflowCleanup_' + sessionId]; + const owner = current && current.sessionId === sessionId + ? current : fallback; + if (owner && owner.sessionId === sessionId + && typeof owner.stopEvents === 'function') { + owner.stopEvents(); + } + }""", + self._session_id, + ) + except Exception: + continue + + def _cleanup_secret_markers(self) -> None: + """Remove this session's temporary secret-mask attributes.""" + + if self.page is None: + return + try: + frames = list(self.page.frames) + except Exception: + frames = [] + for frame in frames: + try: + frame.evaluate( + """([sessionId, marker]) => { + const current = window.__oaflowRecorder; + const fallback = window['__oaflowCleanup_' + sessionId]; + const owner = current && current.sessionId === sessionId + ? current : fallback; + if (owner && owner.sessionId === sessionId + && typeof owner.cleanup === 'function') { + owner.cleanup(); + } + const roots = [document]; + while (roots.length) { + const root = roots.pop(); + for (const element of root.querySelectorAll('*')) { + if (element.hasAttribute(marker)) { + element.removeAttribute(marker); + } + if (element.shadowRoot) { + roots.push(element.shadowRoot); + } + } + } + }""", + [self._session_id, self._secret_marker_attribute], + ) + except Exception: + continue + + def _stop_browser_connection(self) -> None: + """Close an owned browser or detach without closing an external one.""" + + self._cleanup_page_listeners() + self._cleanup_secret_markers() + privacy_cdp, self._privacy_cdp = self._privacy_cdp, None + browser, self._browser = self._browser, None + playwright, self._pw = self._pw, None + try: + if privacy_cdp is not None: + try: + privacy_cdp.detach() + except Exception: + pass + if self._owns_browser and browser is not None: + browser.close() + finally: + try: + if playwright is not None: + playwright.stop() + finally: + # Keep all local lifecycle latches active until the external + # Playwright connection has detached. They do not remain in + # Chromium after the connection closes. + if self.backend is not None: + self.backend.stop_screenshot_mask_tracking() + self._page_lifecycle_listeners_installed = False + self._context_page_listener_installed = False + self._context_page_latches.clear() + self._context_page_baselines.clear() + self._context = None + + def _drain_event_queue(self) -> bool: + """Process all events already delivered by the page binding.""" + + if self._listener_error is not None: + raise self._listener_error + self._assert_no_new_pages() + batch = self._pyq[:] + del self._pyq[:] + self._validate_event_batch(batch) + rebased = False + if not self._owns_browser: + current_geometry = self._read_attached_geometry() + if self._viewport_dirty or current_geometry != self._attached_geometry: + if batch: + raise BrowserAttachError( + "an action overlapped a browser resize or monitor-scale " + "change; recording stopped because no exact pre-action " + "frame exists in the new coordinate space" + ) + self._rebaseline_attached_viewport() + rebased = True + for event in batch: + if not self._owns_browser: + event_geometry = event.pop("_oaflow_geometry", None) + if event_geometry != self._attached_geometry: + raise BrowserAttachError( + "an action overlapped a browser resize or monitor-scale " + "change; recording stopped because no exact pre-action " + "frame exists in the new coordinate space" + ) + self._process(event) + # A binding callback can arrive while Playwright captures this + # action's after-frame. Revalidate it against the action in flight + # so a later click/key cannot share that frame and then appear as a + # separate, falsely exact step. Same-field input and one scroll + # batch remain safe to coalesce. + self._validate_event_batch([event, *self._pyq]) + if not self._owns_browser and ( + self._viewport_dirty + or self._read_attached_geometry() != self._attached_geometry + ): + raise BrowserAttachError( + "the browser resized or changed monitor scale while an " + "action was being retained; recording stopped without " + "complete metadata" + ) + self._validate_event_batch([event, *self._pyq]) + self._assert_no_new_pages() + if self._listener_error is not None: + raise self._listener_error + return bool(batch) or rebased + + @staticmethod + def _validate_event_batch(batch: list[dict[str, Any]]) -> None: + """Refuse a batch that lacks an exact frame between logical actions.""" + + if len(batch) <= 1: + return + kinds = {event.get("kind") for event in batch} + if kinds == {"scroll"}: + return + if kinds == {"input"}: + sessions = {event.get("_oaflow_input_session") for event in batch} + fields = {event.get("field") for event in batch} + if len(sessions) == 1 and None not in sessions and len(fields) == 1: + return + raise BrowserAttachError( + "more than one logical browser action arrived before Flow could " + "retain an exact intermediate frame; recording was refused" + ) # -- event pump ---------------------------------------------------------- @@ -524,6 +3116,8 @@ def pump(self) -> bool: return self._pump() def _pump(self) -> bool: + if self._listener_error is not None: + raise self._listener_error if self.done: return False try: @@ -531,17 +3125,13 @@ def _pump(self) -> bool: except Exception: self.done = True return False - batch = self._pyq[:] - del self._pyq[:] - if not batch: + if not self._drain_event_queue(): # Distinct scroll gestures are separated by pauses; flush a # completed scroll on idle so each becomes its own step. A type run # is NOT idle-flushed (a mid-word pause must not split it) — it # flushes on the next boundary event or at finish(). self._flush_scroll() return not self._stop_condition_reached() - for ev in batch: - self._process(ev) return not self._stop_condition_reached() def _process(self, ev: dict[str, Any]) -> None: @@ -565,11 +3155,16 @@ def _process(self, ev: dict[str, Any]) -> None: def _accumulate_input(self, ev: dict[str, Any]) -> None: field = ev.get("field") - if self._pending_type is not None and self._pending_type.get("field") != field: + input_session = ev.get("_oaflow_input_session") + if self._pending_type is not None and ( + self._pending_type.get("field") != field + or self._pending_type.get("input_session") != input_session + ): self._flush_type() # focus moved to a different field if self._pending_type is None: self._pending_type = { "field": field, + "input_session": input_session, "label": ev.get("label"), "secret": bool(ev.get("secret")), "value": "", @@ -724,6 +3319,97 @@ def _record_key(self, ev: dict[str, Any]) -> None: # -- internals ----------------------------------------------------------- + def _read_attached_geometry(self) -> tuple[int, int, float]: + """Read the selected tab's origin, CSS viewport, and monitor scale.""" + + assert not self._owns_browser + assert self.page is not None + try: + raw = self.page.evaluate( + """() => ({ + origin: location.origin, + width: window.innerWidth, + height: window.innerHeight, + dpr: window.devicePixelRatio || 1, + })""" + ) + current_origin = _http_origin( + str(raw["origin"]), + label="the selected browser tab URL", + ) + geometry = ( + int(raw["width"]), + int(raw["height"]), + round(float(raw["dpr"]), 6), + ) + except Exception as exc: + raise BrowserAttachError( + "the attached tab geometry could not be read; recording was refused" + ) from exc + if current_origin != self._attached_origin: + raise BrowserAttachError( + "the selected browser tab left the declared application origin; " + "recording was refused" + ) + if geometry[0] <= 0 or geometry[1] <= 0 or not 0.1 <= geometry[2] <= 16.0: + raise BrowserAttachError( + "the attached tab reported invalid viewport or monitor-scale " + "geometry; recording was refused" + ) + return geometry + + def _rebaseline_attached_viewport(self) -> None: + """Resume after an idle resize with a fresh exact CSS-pixel baseline.""" + + assert not self._owns_browser + assert self.recorder is not None + # A deferred input or scroll already has its exact old-space after + # frame. Persist it before the new coordinate space becomes current. + self._flush_type() + self._flush_scroll() + for _attempt in range(3): + before = self._read_attached_geometry() + frame = self.recorder._wait_settled() + after = self._read_attached_geometry() + if self._pyq or self._listener_error is not None: + raise BrowserAttachError( + "an action occurred before the resized browser viewport was " + "rebound to a fresh frame; recording stopped without " + "complete metadata" + ) + with Image.open(io.BytesIO(frame)) as image: + frame_size = image.size + if before == after and frame_size == after[:2]: + self._attached_geometry = after + self._last_frame = frame + self._last_structural = self._structural_state() + self._viewport_dirty = False + viewport = after[:2] + if self._initial_attached_viewport is None: + self._initial_attached_viewport = viewport + entry = { + "before_event": self.recorder.event_count, + "viewport": list(viewport), + "device_scale_factor": after[2], + } + if ( + self._viewport_history + and self._viewport_history[-1]["before_event"] + == entry["before_event"] + ): + self._viewport_history[-1] = entry + elif not self._viewport_history or ( + self._viewport_history[-1]["viewport"] != entry["viewport"] + or self._viewport_history[-1]["device_scale_factor"] + != entry["device_scale_factor"] + ): + self._viewport_history.append(entry) + return + raise BrowserAttachError( + "the attached browser viewport did not settle long enough to bind " + "a new exact frame and coordinate space" + ) + def _advance(self) -> None: """After an IMMEDIATE step (click/key), the current settled frame becomes the next step's BEFORE frame.""" @@ -746,19 +3432,121 @@ def _set_last( if structural_after is not None: self._last_structural = structural_after - def _structural_state(self) -> dict[str, Any]: + def _read_scrubbed_page_state(self) -> dict[str, Any]: + """Sample the page-closure URL and title at a SETTLED boundary. + + This is the only sampling point for reflected evidence. Python calls it + after the page has processed the action, so what the page shows is what + the action produced. The in-page capture-phase listeners deliberately + emit no URL and no title: they run BEFORE the page's own handlers, so + anything they read is one action out of date. + + The page reduces its own URL by STRUCTURE -- origin and path reported, + parameter names kept, the values Flow cannot stand behind emptied -- + and says what it dropped and what it withheld. + + Python supplies the one fact the page cannot know: whether some EARLIER + document already received a declared value. A document born after that + moment can load with the value already in its URL, so its own first + sample proves nothing about a time before the value existed. + """ + + assert self.page is not None + secret_seen_earlier = bool(self._secret_doc_ids) + try: + safe_page_state = self.page.evaluate( + """([sessionId, secretSeenEarlier]) => { + const recorder = window.__oaflowRecorder; + if (!recorder || recorder.sessionId !== sessionId + || typeof recorder.structuralState !== 'function') { + return null; + } + return recorder.structuralState(secretSeenEarlier); + }""", + [self._session_id, secret_seen_earlier], + ) + except Exception as exc: + raise BrowserAttachError( + "could not read scrubbed browser structural state" + ) from exc + if not isinstance(safe_page_state, dict): + raise BrowserAttachError( + "the page-local secret scrubber did not return structural state" + ) state: dict[str, Any] = {} - for attr, key in ( - ("url", "url"), - ("page_title", "title"), - ("page_count", "pages"), - ): - try: - value = getattr(self.backend, attr, None) - except Exception: - value = None - if value is not None: - state[key] = value + for key in ("url", "title"): + value = safe_page_state.get(key) + if not isinstance(value, str): + raise BrowserAttachError( + "the page-local secret scrubber did not return structural state" + ) + state[key] = value + raw_doc_id = safe_page_state.get("doc") + doc_id = raw_doc_id if isinstance(raw_doc_id, str) else None + if doc_id is not None and safe_page_state.get("secret") is True: + self._mark_secret_document(doc_id) + for key in ("url_withheld", "title_withheld"): + reason = safe_page_state.get(key) + if isinstance(reason, str) and reason: + self._note_withheld_structural_text(reason) + # An application that puts a declared secret into its own URL has a + # defect that exists with or without Flow: OWASP lists browser history, + # server logs, proxies, CDNs and the Referer header as places it is + # already exposed. Tell the operator. + if safe_page_state.get("secret_in_url") is True: + self._app_placed_secret_in_url = True + if safe_page_state.get("secret_in_title") is True: + self._app_placed_secret_in_title = True + # A LATER document builds a fresh closure that never saw the value an + # earlier document received, and it cannot prove that the URL it loaded + # with predates that value. A server that answers a form submit with a + # redirect to `/results/<value>` puts the value in the PATH, where no + # parameter name identifies it and no value in the new closure matches + # it. Structure protects the query channel, not this one, so the whole + # URL and the title are withheld. + # + # This does NOT cost the single-page-application evidence: an SPA route + # change is a SAME-document `history.pushState`, so the document that + # held the value is the document being sampled, and its URL is still + # reported exactly. This rule bites only on a real navigation, which is + # exactly where the redirect leak lives. + if self._secret_document_left(doc_id): + self._note_withheld_structural_text("secret-value-left-its-document") + state["url"] = self._origin_only_url() + state["title"] = "" + return state + # Record the drop only for a URL that Flow actually reports. Naming a + # dropped parameter of a URL that never reached disk would tell the + # operator less than nothing. + # + # The drop is a RECORDING-level fact, not a per-action one: a parameter + # named after a declared field loses its value in every URL Flow + # reports, whatever the value is. `meta.json` carries the list. Putting + # it on each event would land it inconsistently, because the Recorder + # builds an action's after-state from the backend's url/title/page-count + # seam alone. + if not safe_page_state.get("url_withheld"): + dropped = safe_page_state.get("dropped") + if isinstance(dropped, list): + for entry in dropped: + if not isinstance(entry, dict): + continue + name = entry.get("name") + if not isinstance(name, str): + continue + self._dropped_url_parameters.add( + (name, str(entry.get("where")), str(entry.get("reason"))) + ) + return state + + def _structural_state(self) -> dict[str, Any]: + state: dict[str, Any] = dict(self._read_scrubbed_page_state()) + try: + page_count = self.backend.page_count + except Exception: + page_count = None + if page_count is not None: + state["pages"] = page_count if self._system_of_record_reader is not None: try: records = self._system_of_record_reader() @@ -788,11 +3576,14 @@ def record_interactive( param_fields: tuple[str, ...] = (), identifier_fields: tuple[str, ...] = (), headless: bool = False, + cdp_endpoint: Optional[str] = None, + browser_page_url: Optional[str] = None, script: Optional[Callable[[Any, Callable[[], None]], None]] = None, system_of_record_reader: Optional[ Callable[[], Optional[list[dict[str, Any]]]] ] = None, stop_when: Optional[Callable[[], bool]] = None, + surface: str = "web", **kwargs: Any, ) -> Path: """Record a live demonstration the user drives against ``url``. @@ -817,6 +3608,13 @@ def record_interactive( remote-display/pixel substrate (Citrix/RDP). headless: Run the browser headless (used by scripted/CI recording; a human recording is headed). + cdp_endpoint: Optional local-loopback Chromium DevTools endpoint. When + set, the recorder attaches to an existing browser and never + launches, navigates, or closes it. + browser_page_url: Exact current tab URL used to disambiguate two or + more open tabs on the declared app origin. Requires + ``cdp_endpoint``. Query and fragment values are not written to + recorder diagnostics. script: Test hook — ``script(page, pump)`` drives synthetic input and pumps the loop; when given, the human wait loop is skipped. system_of_record_reader: Optional read-only observation of the @@ -826,6 +3624,8 @@ def record_interactive( verifier. stop_when: Optional recording completion condition. It is evaluated after queued events are persisted and before the browser closes. + surface: Recorded surface stamped into ``meta.json`` before the + recording is published. The compiler binds the bundle to it. Returns: The recording directory. @@ -837,8 +3637,11 @@ def record_interactive( param_fields=param_fields, identifier_fields=identifier_fields, headless=headless, + cdp_endpoint=cdp_endpoint, + browser_page_url=browser_page_url, system_of_record_reader=system_of_record_reader, stop_when=stop_when, + surface=surface, **kwargs, ) session.start() diff --git a/openadapt_flow/recorder.py b/openadapt_flow/recorder.py index 0c95b707..ca699f69 100644 --- a/openadapt_flow/recorder.py +++ b/openadapt_flow/recorder.py @@ -22,6 +22,8 @@ # structural observations (StructuralBackend), and # sor_before/sor_after (a system-of-record snapshot) # when a recorder-only observer is configured. + # Every frame-backed event also carries + # viewport_before/viewport_after from the actual PNGs. frames/{i:04d}_before.png frames/{i:04d}_after.png # captured after the action settled @@ -208,6 +210,12 @@ def finish(self) -> Path: (self._dir / "meta.json").write_text(json.dumps(meta, indent=2)) return self._dir + @property + def event_count(self) -> int: + """Number of complete events already persisted.""" + + return self._i + # -- internals ----------------------------------------------------------- def _record(self, event: dict[str, Any], act: Callable[[], None]) -> None: @@ -346,7 +354,16 @@ def _commit( after_png = self._redact(after_png, redact_region) (self._frames_dir / f"{i:04d}_before.png").write_bytes(before_png) (self._frames_dir / f"{i:04d}_after.png").write_bytes(after_png) - line: dict[str, Any] = {"i": i, **event} + with Image.open(io.BytesIO(before_png)) as before_image: + before_viewport = [int(before_image.width), int(before_image.height)] + with Image.open(io.BytesIO(after_png)) as after_image: + after_viewport = [int(after_image.width), int(after_image.height)] + line: dict[str, Any] = { + "i": i, + **event, + "viewport_before": before_viewport, + "viewport_after": after_viewport, + } for key, value in structural_before.items(): line[f"{key}_before"] = value if structural_after is None: diff --git a/public-artifacts.json b/public-artifacts.json index 4d9f9736..4e96fc56 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -605,7 +605,7 @@ }, { "path": "claims.yaml", - "sha256": "bc99b4f9bd568995d480c76aa5430a7826de9bc265f36e1a8f374090c2ba73ba" + "sha256": "2243e1fd44a68b40496d4be35303c5cdc3d2416c78d5008d5880898e6340bb82" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1813,7 +1813,7 @@ }, { "path": "docs/verification.json", - "sha256": "5500a6eda3129b17266364899e970ebb0e102ff3f039a3e7e0eed3d64a432293" + "sha256": "3a99ee1287e452279b36cf3529da4fae3c55399dfaa1bd29b056bb4d86819c04" }, { "path": "openadapt_flow/console/static/console.css", diff --git a/pyproject.toml b/pyproject.toml index c4a8cfcc..e66eb613 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,10 @@ packages = ["openadapt_flow"] # list in lockstep with the sdist target and the archive validator below. exclude = [ "/.hypothesis", + "/.openadapt-chrome-profile", + "/.openadapt-recording-partial-*", + "/**/.openadapt-chrome-profile", + "/**/.openadapt-recording-partial-*", "/benchmark/**/api-delta-probe-*", "/benchmark/**/bundle-live*", "/benchmark/**/out", @@ -249,6 +253,10 @@ exclude = [ # fake-patient synthetic fixtures, and bounded aggregate evidence remain. exclude = [ "/.hypothesis", + "/.openadapt-chrome-profile", + "/.openadapt-recording-partial-*", + "/**/.openadapt-chrome-profile", + "/**/.openadapt-recording-partial-*", "/benchmark/**/api-delta-probe-*", "/benchmark/**/bundle-live*", "/benchmark/**/out", diff --git a/scripts/check_release_consistency.py b/scripts/check_release_consistency.py index 219fb5a4..af619ab6 100644 --- a/scripts/check_release_consistency.py +++ b/scripts/check_release_consistency.py @@ -226,6 +226,8 @@ def load_source_policy(path: Path = SOURCE_POLICY_PATH) -> SourcePolicy: } ) RECORDING_METADATA_BASENAMES = frozenset({"events.jsonl", "meta.json"}) +LOCAL_SENSITIVE_ARCHIVE_SEGMENTS = frozenset({".openadapt-chrome-profile"}) +LOCAL_SENSITIVE_ARCHIVE_PREFIXES = (".openadapt-recording-partial-",) RECORDING_AUTHORITY_TOKENS = ( b"access_token", b"api_key", @@ -494,17 +496,27 @@ def _repository_only_evaluation_hits(members: set[str]) -> set[str]: def _generated_or_raw_archive_hits( members: set[str], payloads: dict[str, bytes] ) -> set[str]: - """Return generated sessions, raw media/results, or retained authority. + """Return local secrets, generated sessions, or retained authority. Synthetic fixture source and reviewed aggregate results remain eligible for - publication. What cannot ship is a local run directory, raw video/per-run - result, or recording metadata that retains session authority. + publication. What cannot ship is a browser profile, an unpublished partial + recording, a local run directory, raw video/per-run result, or recording + metadata that retains session authority. """ hits: set[str] = set() for member in members: path = PurePosixPath(member) lower_parts = tuple(part.lower() for part in path.parts) basename = path.name.lower() + if any( + part in LOCAL_SENSITIVE_ARCHIVE_SEGMENTS + or any( + part.startswith(prefix) for prefix in LOCAL_SENSITIVE_ARCHIVE_PREFIXES + ) + for part in lower_parts + ): + hits.add(member) + continue if path.suffix.lower() in RAW_VIDEO_SUFFIXES: hits.add(member) continue @@ -1323,8 +1335,9 @@ def validate_sdist_license_boundary( generated_or_raw = _generated_or_raw_archive_hits(members, payloads) if generated_or_raw: raise ValueError( - "source distribution contains generated/raw benchmark output, " - "recording authority, or per-run evidence: " + "source distribution contains a browser profile, unpublished " + "recording, generated/raw benchmark output, recording authority, " + "or per-run evidence: " f"{sorted(generated_or_raw)}" ) missing = REQUIRED_SDIST_PATHS - members @@ -1437,8 +1450,9 @@ def validate_wheel_license_boundary( generated_or_raw = _generated_or_raw_archive_hits(members, payloads) if generated_or_raw: raise ValueError( - "wheel contains generated/raw benchmark output, recording authority, " - f"or per-run evidence: {sorted(generated_or_raw)}" + "wheel contains a browser profile, unpublished recording, " + "generated/raw benchmark output, recording authority, or per-run " + f"evidence: {sorted(generated_or_raw)}" ) forbidden = { member diff --git a/tests/test_browser_attach.py b/tests/test_browser_attach.py new file mode 100644 index 00000000..2b0e9ad6 --- /dev/null +++ b/tests/test_browser_attach.py @@ -0,0 +1,4910 @@ +"""Safe attachment of the Playwright recorder to an existing Chromium tab.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from io import BytesIO +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from urllib.request import urlopen + +import pytest +from PIL import Image + +import openadapt_flow.interactive_recorder as interactive_recorder_module +from openadapt_flow.__main__ import main +from openadapt_flow.backends.playwright_backend import ( + PlaywrightBackend, + ScreenshotMaskStabilityError, +) +from openadapt_flow.compiler import compile_recording +from openadapt_flow.interactive_recorder import ( + BrowserAttachError, + InteractiveRecorder, + _secret_screenshot_selectors, + record_interactive, + select_attached_page, + validate_browser_cdp_endpoint, +) + + +class _Page: + def __init__(self, url: str) -> None: + self.url = url + + +class _FakePrivacyCdp: + """Empty-page CDP stand-in: the closed-shadow scan finds no secret node.""" + + def send(self, method: str, params: dict | None = None) -> dict: + if method == "DOM.performSearch": + return {"searchId": "fake-search", "resultCount": 0} + return {} + + def detach(self) -> None: + return None + + +def _browser(*urls: str): + return SimpleNamespace( + contexts=[SimpleNamespace(pages=[_Page(url) for url in urls])] + ) + + +@pytest.mark.parametrize( + "endpoint", + [ + "http://localhost:9222", + "http://127.0.0.1:9222", + "http://127.9.8.7:9222", + "ws://[::1]:9222/devtools/browser/abc", + ], +) +def test_cdp_endpoint_accepts_only_explicit_loopback(endpoint: str) -> None: + assert validate_browser_cdp_endpoint(endpoint) == endpoint + + +@pytest.mark.parametrize( + ("endpoint", "message"), + [ + ("http://example.com:9222", "loopback"), + ("http://localhost", "include a port"), + ("ftp://localhost:9222", "must use http"), + ("http://user:pass@localhost:9222", "must not contain credentials"), + ("http://localhost:9222?token=secret", "must not contain credentials"), + ], +) +def test_cdp_endpoint_refuses_unsafe_boundaries(endpoint: str, message: str) -> None: + with pytest.raises(BrowserAttachError, match=message): + validate_browser_cdp_endpoint(endpoint) + + +def test_attach_selects_the_only_tab_on_the_declared_origin() -> None: + wanted = _Page("https://app.example.test/work/12?record=private") + browser = SimpleNamespace( + contexts=[ + SimpleNamespace( + pages=[ + _Page("chrome://settings/"), + _Page("https://unrelated.example.test/"), + wanted, + ] + ) + ] + ) + assert select_attached_page(browser, app_url="https://app.example.test/") is wanted + + +def test_attach_requires_exact_url_when_origin_has_multiple_tabs() -> None: + browser = _browser( + "https://app.example.test/one?patient=SECRET_ONE#private", + "https://app.example.test/two?patient=SECRET_TWO", + ) + with pytest.raises(BrowserAttachError) as caught: + select_attached_page(browser, app_url="https://app.example.test/") + message = str(caught.value) + assert "--browser-page-url" in message + assert "/one" in message and "/two" in message + assert "SECRET_ONE" not in message and "SECRET_TWO" not in message + + +def test_attach_exact_url_selects_one_tab_without_navigation() -> None: + selected = "https://app.example.test/two?record=42" + browser = _browser("https://app.example.test/one", selected) + page = select_attached_page( + browser, + app_url="https://app.example.test/", + page_url=selected, + ) + assert page.url == selected + + +def test_attach_refuses_cross_origin_page_selector_without_echoing_it() -> None: + private_url = "https://other.example.test/path?token=DO_NOT_PRINT" + with pytest.raises(BrowserAttachError) as caught: + select_attached_page( + _browser("https://app.example.test/"), + app_url="https://app.example.test/", + page_url=private_url, + ) + assert "same origin" in str(caught.value) + assert private_url not in str(caught.value) + assert "DO_NOT_PRINT" not in str(caught.value) + + +def test_attached_backend_uses_live_css_viewport_and_css_screenshot() -> None: + class Page: + viewport_size = None + + def __init__(self) -> None: + self.screenshot_options = None + + def evaluate(self, _script): + return {"width": 1440, "height": 900} + + def screenshot(self, **kwargs): + self.screenshot_options = kwargs + return b"png" + + page = Page() + backend = PlaywrightBackend(page, screenshot_scale="css") # type: ignore[arg-type] + assert backend.viewport == (1440, 900) + assert backend.screenshot() == b"png" + assert page.screenshot_options["scale"] == "css" + + +def test_attached_backend_uses_source_sanitized_structural_state() -> None: + page = SimpleNamespace( + url="https://app.example.test/?token=RAW-SECRET", + title=lambda: "RAW-SECRET", + ) + backend = PlaywrightBackend( # type: ignore[arg-type] + page, + structural_state_reader=lambda: { + "url": "https://app.example.test/?token=[secret]", + "title": "[secret]", + }, + ) + + assert backend.url == "https://app.example.test/?token=[secret]" + assert backend.page_title == "[secret]" + + +def test_backend_runs_privacy_guard_before_screenshot_bytes_exist() -> None: + calls: list[str] = [] + + class Page: + def screenshot(self, **_kwargs): + calls.append("screenshot") + return b"png" + + backend = PlaywrightBackend( # type: ignore[arg-type] + Page(), + screenshot_guard=lambda: calls.append("guard"), + ) + assert backend.screenshot() == b"png" + assert calls == ["guard", "screenshot"] + + def refuse() -> None: + calls.append("refuse") + raise BrowserAttachError("unsafe closed shadow boundary") + + refusing = PlaywrightBackend(Page(), screenshot_guard=refuse) # type: ignore[arg-type] + with pytest.raises(BrowserAttachError, match="closed shadow"): + refusing.screenshot() + assert calls == ["guard", "screenshot", "refuse"] + + +def test_backend_masks_password_and_declared_secret_fields_on_every_frame() -> None: + class Frame: + def __init__(self, name: str) -> None: + self.name = name + + def locator(self, selector): + return f"locator:{self.name}:{selector}" + + class Page: + viewport_size = {"width": 1280, "height": 800} + + def __init__(self) -> None: + self.screenshot_options: list[dict] = [] + self.frames = [Frame("main"), Frame("child")] + self.listeners: dict[str, list] = {} + self.attach_on_next_screenshot = True + + def on(self, event, listener): + self.listeners.setdefault(event, []).append(listener) + + def remove_listener(self, event, listener): + self.listeners[event].remove(listener) + + def evaluate(self, _script): + return None + + def screenshot(self, **kwargs): + self.screenshot_options.append(kwargs) + if self.attach_on_next_screenshot: + self.attach_on_next_screenshot = False + frame = Frame("late-child") + self.frames.append(frame) + for listener in self.listeners.get("frameattached", []): + listener(frame) + return b"png" + + selectors = ( + "input[type='password']", + '[name="token"], [id="token"]', + ) + page = Page() + backend = PlaywrightBackend( # type: ignore[arg-type] + page, + screenshot_mask_selectors=selectors, + ) + + assert backend.screenshot() == b"png" + assert backend.screenshot() == b"png" + assert len(page.screenshot_options) == 3 + assert page.screenshot_options[0]["mask"] == [ + f"locator:{frame}:{selector}" + for frame in ("main", "child") + for selector in selectors + ] + assert page.screenshot_options[1]["mask"] == [ + f"locator:{frame}:{selector}" + for frame in ("main", "child", "late-child") + for selector in selectors + ] + assert page.screenshot_options[2]["mask"] == page.screenshot_options[1]["mask"] + for options in page.screenshot_options: + assert options["mask_color"] == "#000000" + backend.stop_screenshot_mask_tracking() + assert not any(page.listeners.values()) + + +def test_declared_secret_selectors_use_css_string_escaping() -> None: + selectors = _secret_screenshot_selectors({"päss", 'quote"\\line\nend'}) + + assert '[name="päss"], [id="päss"]' in selectors + assert ( + '[name="quote\\"\\\\line\\a end"], [id="quote\\"\\\\line\\a end"]' in selectors + ) + with pytest.raises(BrowserAttachError, match="null character"): + _secret_screenshot_selectors({"unsafe\x00field"}) + with pytest.raises(BrowserAttachError, match="Unicode surrogate"): + _secret_screenshot_selectors({"unsafe\ud800field"}) + + +def test_attached_recorder_api_refuses_incompatible_options(tmp_path: Path) -> None: + with pytest.raises(BrowserAttachError, match="requires a browser CDP"): + InteractiveRecorder( + "https://app.example.test/", + tmp_path / "recording", + browser_page_url="https://app.example.test/work", + ) + with pytest.raises(BrowserAttachError, match="headless"): + InteractiveRecorder( + "https://app.example.test/", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + headless=True, + ) + + +def test_attached_recorder_reads_geometry_and_refuses_origin_drift( + tmp_path: Path, +) -> None: + session = InteractiveRecorder( + "https://app.example.test/", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + origin = {"value": "https://app.example.test"} + session.page = SimpleNamespace( + evaluate=lambda _script: { + "origin": origin["value"], + "width": 1280, + "height": 800, + "dpr": 2, + }, + ) + assert session._read_attached_geometry() == (1280, 800, 2.0) + + origin["value"] = "https://other.example.test" + with pytest.raises(BrowserAttachError) as caught: + session._read_attached_geometry() + assert "left the declared application origin" in str(caught.value) + assert "DO_NOT_PRINT" not in str(caught.value) + + +def test_attached_recorder_retains_main_frame_origin_violation( + tmp_path: Path, +) -> None: + out = tmp_path / "recording" + session = InteractiveRecorder( + "https://app.example.test/", + out, + cdp_endpoint="http://127.0.0.1:9222", + ) + session._prepare_recording_dir() + origin = {"value": "https://other.example.test"} + frame = SimpleNamespace(evaluate=lambda _script: origin["value"]) + session.page = SimpleNamespace(main_frame=frame) + + session._handle_frame_navigation(frame) + origin["value"] = "https://app.example.test" + session._handle_frame_navigation(frame) + + assert session.done is True + assert session._listener_error is not None + assert "left the declared application origin" in str(session._listener_error) + with pytest.raises(BrowserAttachError, match="left the declared"): + session.finish() + assert not out.exists() + assert not list(tmp_path.glob(".openadapt-recording-partial-*")) + + +def test_attached_recorder_refuses_existing_output_without_changing_it( + tmp_path: Path, +) -> None: + out = tmp_path / "recording" + frames = out / "frames" + frames.mkdir(parents=True) + (out / "meta.json").write_text('{"id":"complete-existing"}\n') + (out / "events.jsonl").write_text('{"i":0,"kind":"key"}\n') + (frames / "0000_before.png").write_bytes(b"SENSITIVE-FRAME") + before = { + path.relative_to(out): path.read_bytes() + for path in out.rglob("*") + if path.is_file() + } + session = InteractiveRecorder( + "https://app.example.test/", + out, + cdp_endpoint="http://127.0.0.1:9222", + ) + + with pytest.raises(BrowserAttachError, match="output already exists"): + session._prepare_recording_dir() + session.abort() + + after = { + path.relative_to(out): path.read_bytes() + for path in out.rglob("*") + if path.is_file() + } + assert after == before + assert not list(tmp_path.glob(".openadapt-recording-partial-*")) + + +def test_attached_recorder_refuses_output_created_during_promotion( + tmp_path: Path, +) -> None: + out = tmp_path / "recording" + session = InteractiveRecorder( + "https://app.example.test/", + out, + cdp_endpoint="http://127.0.0.1:9222", + ) + session._prepare_recording_dir() + assert session._recording_dir is not None + (session._recording_dir / "complete.txt").write_text("new recording") + + out.mkdir() + original_identity = out.stat() + with pytest.raises(BrowserAttachError, match="appeared during capture"): + session._promote_recording() + + current_identity = out.stat() + assert (current_identity.st_dev, current_identity.st_ino) == ( + original_identity.st_dev, + original_identity.st_ino, + ) + session.abort() + assert out.is_dir() + assert not list(tmp_path.glob(".openadapt-recording-partial-*")) + + +def test_attached_tab_close_discards_partial_output(tmp_path: Path) -> None: + out = tmp_path / "recording" + session = InteractiveRecorder( + "https://app.example.test/", + out, + cdp_endpoint="http://127.0.0.1:9222", + ) + session._prepare_recording_dir() + assert session._recording_dir is not None + (session._recording_dir / "meta.json").write_text('{"id":"not-final"}\n') + + session._handle_page_close() + + with pytest.raises(BrowserAttachError, match="selected browser tab closed"): + session.finish() + assert not out.exists() + assert not list(tmp_path.glob(".openadapt-recording-partial-*")) + + +def test_attached_tab_close_during_finalization_discards_metadata( + tmp_path: Path, +) -> None: + out = tmp_path / "recording" + session = InteractiveRecorder( + "https://app.example.test/", + out, + cdp_endpoint="http://127.0.0.1:9222", + ) + session._prepare_recording_dir() + + class ClosingPage: + frames: list = [] + + def __init__(self) -> None: + self.origin = "https://app.example.test" + self.main_frame = SimpleNamespace(evaluate=lambda _script: self.origin) + + def evaluate(self, _script): + return { + "origin": self.origin, + "width": 1280, + "height": 800, + "dpr": 1, + } + + session.page = ClosingPage() + session._page_lifecycle_listeners_installed = True + session._privacy_cdp = _FakePrivacyCdp() + session._attached_geometry = (1280, 800, 1.0) + session._initial_attached_viewport = (1280, 800) + + class ClosingRecorder: + def finish(self): + assert session._recording_dir is not None + (session._recording_dir / "meta.json").write_text( + json.dumps({"viewport": [1280, 800]}) + ) + return session._recording_dir + + session.recorder = ClosingRecorder() # type: ignore[assignment] + session._pw = SimpleNamespace(stop=lambda: session._handle_page_close()) + + with pytest.raises(BrowserAttachError, match="selected browser tab closed"): + session.finish() + assert not out.exists() + assert not list(tmp_path.glob(".openadapt-recording-partial-*")) + + +def test_attached_popup_during_finalization_discards_metadata( + tmp_path: Path, +) -> None: + out = tmp_path / "recording" + session = InteractiveRecorder( + "https://app.example.test/", + out, + cdp_endpoint="http://127.0.0.1:9222", + ) + session._prepare_recording_dir() + + class PopupPage: + frames: list = [] + + def __init__(self) -> None: + self.origin = "https://app.example.test" + self.main_frame = SimpleNamespace(evaluate=lambda _script: self.origin) + + def evaluate(self, _script): + return { + "origin": self.origin, + "width": 1280, + "height": 800, + "dpr": 1, + } + + session.page = PopupPage() + session._page_lifecycle_listeners_installed = True + session._privacy_cdp = _FakePrivacyCdp() + session._attached_geometry = (1280, 800, 1.0) + session._initial_attached_viewport = (1280, 800) + + class FinalizingRecorder: + def finish(self): + assert session._recording_dir is not None + (session._recording_dir / "meta.json").write_text( + json.dumps({"viewport": [1280, 800]}) + ) + return session._recording_dir + + session.recorder = FinalizingRecorder() # type: ignore[assignment] + session._pw = SimpleNamespace( + stop=lambda: session._handle_popup(SimpleNamespace(url="about:blank")) + ) + + with pytest.raises(BrowserAttachError, match="popup or new tab"): + session.finish() + assert not out.exists() + assert not list(tmp_path.glob(".openadapt-recording-partial-*")) + + +@pytest.mark.parametrize("late_event", ["context_page", "origin", "frame"]) +def test_attached_late_lifecycle_event_discards_metadata( + tmp_path: Path, + late_event: str, +) -> None: + out = tmp_path / f"recording-{late_event}" + session = InteractiveRecorder( + "https://app.example.test/", + out, + cdp_endpoint="http://127.0.0.1:9222", + ) + session._prepare_recording_dir() + + class LifecyclePage: + frames: list = [] + + def __init__(self) -> None: + self.origin = "https://app.example.test" + self.main_frame = SimpleNamespace(evaluate=lambda _script: self.origin) + + def evaluate(self, _script): + return { + "origin": self.origin, + "width": 1280, + "height": 800, + "dpr": 1, + } + + session.page = LifecyclePage() + session._page_lifecycle_listeners_installed = True + session._privacy_cdp = _FakePrivacyCdp() + session._attached_geometry = (1280, 800, 1.0) + session._initial_attached_viewport = (1280, 800) + + class FinalizingRecorder: + def finish(self): + assert session._recording_dir is not None + (session._recording_dir / "meta.json").write_text( + json.dumps({"viewport": [1280, 800]}) + ) + return session._recording_dir + + session.recorder = FinalizingRecorder() # type: ignore[assignment] + + def emit_late_event() -> None: + if late_event == "context_page": + session._handle_context_page(SimpleNamespace(url="about:blank")) + elif late_event == "origin": + session.page.origin = "https://other.example.test" + session._handle_frame_navigation(session.page.main_frame) + else: + session._handle_frame_tree_change(SimpleNamespace()) + + session._pw = SimpleNamespace(stop=emit_late_event) + + with pytest.raises(BrowserAttachError): + session.finish() + assert not out.exists() + assert not list(tmp_path.glob(".openadapt-recording-partial-*")) + + +def test_attached_recorder_refuses_a_new_context_page(tmp_path: Path) -> None: + session = InteractiveRecorder( + "https://app.example.test/", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + selected = SimpleNamespace() + context = SimpleNamespace(pages=[selected]) + selected.context = context + session.page = selected + session._context_pages_at_start = (selected,) + + context.pages.append(SimpleNamespace(context=context)) + + with pytest.raises(BrowserAttachError, match="popup or new tab"): + session._assert_no_new_pages() + assert session.done is True + + +def test_attached_recorder_refuses_iframe_events(tmp_path: Path) -> None: + session = InteractiveRecorder( + "https://app.example.test/", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": False, + "kind": "click", + "x": 10, + "y": 20, + } + ) + assert session.done is True + assert session._pyq == [] + assert session._listener_error is not None + assert "iframe" in str(session._listener_error) + + session.done = False + session._listener_error = None + selected_frame = object() + session.page = SimpleNamespace(main_frame=selected_frame) + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": True, + "kind": "click", + "x": 10, + "y": 20, + }, + source={"page": session.page, "frame": object()}, + ) + assert session.done is True + assert session._listener_error is not None + assert "iframe" in str(session._listener_error) + + +def test_attached_recorder_refuses_invalid_viewport_evidence(tmp_path: Path) -> None: + session = InteractiveRecorder( + "https://app.example.test/", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + selected_frame = object() + session.page = SimpleNamespace(main_frame=selected_frame) + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": True, + "__oaflow_viewport": [0, 800], + "__oaflow_dpr": 2, + "__oaflow_origin": "https://app.example.test", + "kind": "click", + "url": "https://app.example.test/work", + "x": 10, + "y": 20, + }, + source={"page": session.page, "frame": selected_frame}, + ) + assert session.done is True + assert session._pyq == [] + assert session._listener_error is not None + assert "invalid viewport evidence" in str(session._listener_error) + + +@pytest.mark.parametrize( + "batch", + [ + [{"kind": "click"}, {"kind": "click"}], + [ + {"kind": "input", "field": "note", "_oaflow_input_session": "a"}, + {"kind": "click"}, + ], + [ + {"kind": "input", "field": "note", "_oaflow_input_session": "a"}, + {"kind": "key", "key": "Enter"}, + ], + [{"kind": "scroll", "dy": 10}, {"kind": "click"}], + [ + {"kind": "input", "field": "note", "_oaflow_input_session": "a"}, + {"kind": "input", "field": "note", "_oaflow_input_session": "b"}, + ], + ], +) +def test_browser_event_batch_refuses_multiple_logical_actions( + batch: list[dict], +) -> None: + with pytest.raises(BrowserAttachError, match="more than one logical"): + InteractiveRecorder._validate_event_batch(batch) + + +@pytest.mark.parametrize( + "batch", + [ + [ + {"kind": "input", "field": "note", "_oaflow_input_session": "a"}, + {"kind": "input", "field": "note", "_oaflow_input_session": "a"}, + ], + [{"kind": "scroll", "dy": 10}, {"kind": "scroll", "dy": 20}], + ], +) +def test_browser_event_batch_preserves_one_coalescible_action( + batch: list[dict], +) -> None: + InteractiveRecorder._validate_event_batch(batch) + + +def test_cli_threads_attach_contract_to_the_recorder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + captured: dict = {} + + def fake_record(url, out_dir, **kwargs): + captured["url"] = url + captured.update(kwargs) + out_dir.mkdir(parents=True) + (out_dir / "meta.json").write_text(json.dumps({"source": "fake"})) + return out_dir + + monkeypatch.setattr( + "openadapt_flow.interactive_recorder.record_interactive", fake_record + ) + selected = "https://app.example.test/work?record=42" + rc = main( + [ + "record", + "--backend", + "web", + "--url", + "https://app.example.test/", + "--browser-cdp-endpoint", + "http://127.0.0.1:9222", + "--browser-page-url", + selected, + "--out", + str(tmp_path / "recording"), + ] + ) + assert rc == 0 + assert captured["url"] == "https://app.example.test/" + assert captured["cdp_endpoint"] == "http://127.0.0.1:9222" + assert captured["browser_page_url"] == selected + + +def test_cli_refuses_attach_flags_on_the_wrong_surface(tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="apply only to --backend web"): + main( + [ + "record", + "--backend", + "windows", + "--browser-cdp-endpoint", + "http://127.0.0.1:9222", + "--out", + str(tmp_path / "recording"), + ] + ) + + +def test_cli_refuses_page_selector_without_endpoint(tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="requires --browser-cdp-endpoint"): + main( + [ + "record", + "--backend", + "web", + "--url", + "https://app.example.test/", + "--browser-page-url", + "https://app.example.test/work", + "--out", + str(tmp_path / "recording"), + ] + ) + + +def test_cli_refuses_headless_attachment(tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="--headless cannot be combined"): + main( + [ + "record", + "--backend", + "web", + "--url", + "https://app.example.test/", + "--browser-cdp-endpoint", + "http://127.0.0.1:9222", + "--headless", + "--out", + str(tmp_path / "recording"), + ] + ) + + +_ATTACH_HTML = b"""<!doctype html> +<html><head><title>Attach recorder test + + + + + + + + + + + + + + + +""" + +_CLOSED_SHADOW_HTML = b""" +Closed shadow test + + +""" + + +_GET_FORM_HTML = b""" +Token form +
+ + + +
+""" + + +@pytest.fixture(scope="module") +def attach_app_url() -> str: + class Handler(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - stdlib callback name + if self.path.startswith("/closed-shadow"): + payload = _CLOSED_SHADOW_HTML + elif self.path.startswith("/get-form"): + payload = _GET_FORM_HTML + else: + payload = _ATTACH_HTML + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}/" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _chromium_executable() -> Path | None: + configured = os.environ.get("OPENADAPT_TEST_CHROMIUM_EXECUTABLE") + candidates = [Path(configured)] if configured else [] + try: + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + candidates.append(Path(playwright.chromium.executable_path)) + except Exception: + pass + for command in ("google-chrome", "google-chrome-stable", "chromium"): + found = shutil.which(command) + if found: + candidates.append(Path(found)) + candidates.append( + Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome") + ) + return next((candidate for candidate in candidates if candidate.is_file()), None) + + +def _activate_app_tab(endpoint: str, app_url: str) -> None: + """Bring the app tab back to the front through the DevTools HTTP API. + + A refusal trial can leave the popup or new tab it created as the active + tab. Chromium throttles rendering for the now-background app tab, so the + next attach trial's first evidence screenshot can stall on a slow runner. + A real operator records in a visible tab; restore that precondition. + """ + + with urlopen(f"{endpoint}/json/list", timeout=5) as response: + targets = json.load(response) + for target in targets: + if target.get("type") == "page" and str(target.get("url", "")).startswith( + app_url + ): + with urlopen( + f"{endpoint}/json/activate/{target['id']}", timeout=5 + ) as response: + assert response.status == 200 + return + raise AssertionError("the app tab was not found for activation") + + +@pytest.mark.timeout(60) +def test_launched_browser_refuses_static_unbound_closed_shadow_password( + attach_app_url: str, + tmp_path: Path, +) -> None: + """Owned launch inventories closed roots before its first screenshot.""" + + if _chromium_executable() is None: + pytest.skip("no Chromium executable is installed") + output = tmp_path / "launched-static-closed-shadow" + with pytest.raises(BrowserAttachError, match="closed shadow root"): + record_interactive( + f"{attach_app_url}closed-shadow", + output, + headless=True, + script=lambda _page, _pump: None, + ) + assert not output.exists() + + +@pytest.mark.timeout(30) +def test_page_closure_scrubs_replaced_prefilled_and_reflected_secrets() -> None: + """Real Chromium proves the page-local guard before screenshot handling.""" + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-privacy-test" + binding_name = "__oaflow_emit_page_closure_test" + secret_fields = ( + "prefilled-secret", + "reordered-secret", + "ambiguous-secret", + "reflected-secret", + "contenteditable-secret", + "label-equals-secret", + "altgr-secret", + ) + init_js = ( + interactive_recorder_module._INIT_JS.replace( + "__SESSION_ID__", json.dumps(session_id) + ) + .replace("__BINDING_NAME__", json.dumps(binding_name)) + .replace("__SECRET_NAMES__", json.dumps(secret_fields)) + .replace("__SECRET_MARKER__", json.dumps("data-oaflow-secret-test")) + .replace("__IDENT_NAMES__", "[]") + .replace("__SPECIAL_KEYS__", "[]") + ) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + prefilled = "PREFILLED SECRET MUST NOT CROSS" + page.route( + "http://privacy.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "" + "
" + "
" + ), + ), + ) + page.goto("http://privacy.test/") + page.locator("[name='prefilled-secret']").evaluate( + "(element, secret) => { element.value = secret; document.title = secret; }", + prefilled, + ) + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + state = page.evaluate("() => window.__oaflowRecorder.structuralState()") + assert prefilled not in json.dumps(state) + + reordered = "REORDERED SECRET MUST NOT CROSS" + page.evaluate( + """secret => { + const parent = document.querySelector('#rewrite'); + const declared = document.createElement('input'); + declared.name = 'reordered-secret'; + parent.appendChild(declared); + parent.removeChild(declared); + declared.removeAttribute('name'); + const replacement = document.createElement('input'); + parent.appendChild(replacement); + replacement.value = secret; + replacement.dispatchEvent(new Event('input', {bubbles: true})); + }""", + reordered, + ) + page.evaluate( + """() => { + const parent = document.querySelector('#ambiguous-rewrite'); + const declared = document.createElement('input'); + declared.name = 'ambiguous-secret'; + parent.append(declared, document.createElement('input')); + const possible = document.createElement('input'); + parent.replaceChildren(document.createElement('input'), possible); + possible.value = 'AMBIGUOUS VALUE MUST NOT CROSS'; + possible.dispatchEvent(new Event('input', {bubbles: true})); + }""" + ) + + reflected = "REFLECTED SECRET MUST NOT CROSS" + page.evaluate( + """secret => { + const field = document.createElement('input'); + field.name = 'reflected-secret'; + document.body.appendChild(field); + field.value = secret; + field.dispatchEvent(new Event('input', {bubbles: true})); + history.replaceState({}, '', '/?token=' + encodeURIComponent(secret)); + document.title = secret; + const button = document.querySelector('button'); + button.id = secret; + button.setAttribute('role', secret); + button.setAttribute('aria-label', secret); + button.click(); + }""", + reflected, + ) + contenteditable = "CONTENTEDITABLE SECRET MUST NOT CROSS" + page.evaluate( + """secret => { + const field = document.createElement('div'); + field.contentEditable = 'true'; + field.setAttribute('name', 'contenteditable-secret'); + field.innerText = secret; + document.body.appendChild(field); + field.dispatchEvent(new Event('input', {bubbles: true})); + field.click(); + }""", + contenteditable, + ) + page.evaluate( + """async () => { + const label = document.createElement('label'); + label.htmlFor = 'label-equals-secret'; + label.textContent = 'Password'; + const field = document.createElement('input'); + field.id = 'label-equals-secret'; + field.name = 'label-equals-secret'; + document.body.append(label, field); + await Promise.resolve(); + field.value = 'Password'; + field.dispatchEvent(new Event('input', {bubbles: true})); + + const altGr = document.createElement('input'); + altGr.name = 'altgr-secret'; + document.body.appendChild(altGr); + altGr.dispatchEvent(new KeyboardEvent('keydown', { + key: '@', ctrlKey: true, altKey: true, bubbles: true, + })); + + const editable = document.createElement('div'); + editable.contentEditable = 'true'; + editable.innerText = 'VISIBLE CONTENTEDITABLE'; + document.body.appendChild(editable); + editable.dispatchEvent(new Event('input', {bubbles: true})); + + const aria = document.createElement('div'); + aria.setAttribute('role', 'textbox'); + aria.textContent = 'VISIBLE ARIA TEXTBOX'; + document.body.appendChild(aria); + aria.dispatchEvent(new Event('input', {bubbles: true})); + + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + document.body.appendChild(checkbox); + checkbox.dispatchEvent(new Event('input', {bubbles: true})); + const select = document.createElement('select'); + select.innerHTML = ''; + document.body.appendChild(select); + select.dispatchEvent(new Event('input', {bubbles: true})); + }""" + ) + page.wait_for_timeout(50) + finally: + browser.close() + + payload = json.dumps(events) + assert prefilled not in payload + assert reordered not in payload + assert "AMBIGUOUS VALUE MUST NOT CROSS" not in payload + assert reflected not in payload + assert contenteditable not in payload + input_events = [event for event in events if event.get("kind") == "input"] + assert len(input_events) == 6 + assert sum(event.get("secret") is True for event in input_events) == 4 + label_event = next( + event for event in input_events if event.get("field") == "label-equals-secret" + ) + # Flow does not rewrite captured text. A label that holds another declared + # field's value is WITHHELD, not replaced by a placeholder: a placeholder + # would propose a parameter name the page never showed. + assert label_event["label"] is None + assert { + event.get("value") for event in input_events if not event.get("secret") + } == { + "VISIBLE CONTENTEDITABLE", + "VISIBLE ARIA TEXTBOX", + } + assert sum(event.get("kind") == "privacy_refusal" for event in events) == 1 + assert not any( + event.get("kind") == "hotkey" and event.get("key") == "@" for event in events + ) + click = next(event for event in events if event.get("kind") == "click") + assert click["structural"]["selector"] is None + # Identity evidence is EXACT or WITHHELD. Replay compares the role and the + # accessible name against the live page, so a placeholder here would make + # replay compare against characters the page never held. + assert click["structural"]["role"] is None + assert click["structural"]["name"] is None + # A refused DOM identity says WHY. It is never a bare null selector that + # reads as "this element simply had no stable identity". + assert click["structural"]["identity_withheld"] == "secret-value-in-identity" + + +@pytest.mark.timeout(300) +def test_live_cdp_attach_records_compiles_and_leaves_browser_running_three_trials( + attach_app_url: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Three real Chromium trials cover attach, secrets, frames, and detach.""" + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + from playwright.sync_api import sync_playwright + + profile = tmp_path / "chrome-profile" + profile.mkdir() + process = subprocess.Popen( + [ + str(executable), + "--headless=new", + "--no-sandbox", + "--no-first-run", + "--no-default-browser-check", + "--disable-background-networking", + "--remote-debugging-address=127.0.0.1", + "--remote-debugging-port=0", + f"--user-data-dir={profile}", + "--window-size=1280,800", + attach_app_url, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + port_file = profile / "DevToolsActivePort" + deadline = time.monotonic() + 20 + while not port_file.is_file() and time.monotonic() < deadline: + if process.poll() is not None: + pytest.fail("Chromium exited before its CDP endpoint was ready") + time.sleep(0.05) + assert port_file.is_file(), "Chromium CDP endpoint did not become ready" + port = int(port_file.read_text().splitlines()[0]) + endpoint = f"http://127.0.0.1:{port}" + + for trial in range(3): + # Trial 1 types a LOWERCASE secret whose characters occur in the + # attach URL http://127.0.0.1:/. An uppercase phrase shares + # no character with that URL, and that blind spot hid a redaction + # defect that rewrote the URL of every event and refused the whole + # recording on the first keystroke. + secret = ( + "hunter2-attach-secret" + if trial == 1 + else f"ATTACH-SECRET-{trial}-NEVER-PERSIST" + ) + secret_rect: dict[str, int] = {} + + def drive(page, pump, *, secret_value=secret, trial_number=trial): + page.evaluate( + """() => { + document.querySelector('#note').value = ''; + document.querySelector('#password').value = ''; + delete document.body.dataset.saved; + }""" + ) + page.click("#note") + pump() + page.keyboard.type(f"trial-{trial_number}") + pump() + pump() + page.click("#password") + box = page.locator("#password").bounding_box() + assert box is not None + secret_rect.update( + x=round(box["x"]), + y=round(box["y"]), + width=round(box["width"]), + height=round(box["height"]), + ) + pump() + page.keyboard.type(secret_value) + pump() + pump() + page.click("#save") + pump() + pump() + assert page.get_attribute("body", "data-saved") == "yes" + + recording = record_interactive( + attach_app_url, + tmp_path / f"recording-{trial}", + secret_fields=("password",), + param_fields=("note",), + cdp_endpoint=endpoint, + script=drive, + ) + meta = json.loads((recording / "meta.json").read_text()) + events_text = (recording / "events.jsonl").read_text() + assert meta["source"] == "openadapt-flow-playwright-cdp" + assert meta["secret_params"] == ["password"] + assert secret not in json.dumps(meta) + assert secret not in events_text + assert meta["viewport"][0] > 0 and meta["viewport"][1] > 0 + events = [json.loads(line) for line in events_text.splitlines()] + secret_event = next(event for event in events if event.get("secret")) + next_before = Image.open( + recording / "frames" / f"{int(secret_event['i']) + 1:04d}_before.png" + ).convert("RGB") + crop = next_before.crop( + ( + secret_rect["x"], + secret_rect["y"], + secret_rect["x"] + secret_rect["width"], + secret_rect["y"] + secret_rect["height"], + ) + ) + assert all(extrema == (0, 0) for extrema in crop.getextrema()) + + bundle = tmp_path / f"bundle-{trial}" + workflow = compile_recording( + recording, + bundle, + name=f"attached-browser-{trial}", + ) + assert workflow.steps + assert workflow.secret_params == ["password"] + + # Finishing a recording detaches Playwright. It does not close the + # operator's external browser or its selected tab. + assert process.poll() is None + with urlopen(f"{endpoint}/json/version", timeout=2) as response: + assert response.status == 200 + + unicode_secret = "INTERNATIONAL-SECRET-NEVER-PERSIST" + unicode_secret_rect: dict[str, int] = {} + + def record_unicode_secret(page, pump): + field = page.locator('[name="päss"]') + field.fill("") + box = field.bounding_box() + assert box is not None + unicode_secret_rect.update( + x=round(box["x"]), + y=round(box["y"]), + width=round(box["width"]), + height=round(box["height"]), + ) + field.click() + pump() + page.keyboard.type(unicode_secret) + pump() + pump() + page.click("#save") + pump() + pump() + + unicode_recording = record_interactive( + attach_app_url, + tmp_path / "recording-unicode-secret", + secret_fields=("päss",), + cdp_endpoint=endpoint, + script=record_unicode_secret, + ) + unicode_events_text = (unicode_recording / "events.jsonl").read_text() + unicode_meta_text = (unicode_recording / "meta.json").read_text() + assert unicode_secret not in unicode_events_text + assert unicode_secret not in unicode_meta_text + unicode_events = [json.loads(line) for line in unicode_events_text.splitlines()] + unicode_event = next(event for event in unicode_events if event.get("secret")) + unicode_next_before = Image.open( + unicode_recording + / "frames" + / f"{int(unicode_event['i']) + 1:04d}_before.png" + ).convert("RGB") + unicode_crop = unicode_next_before.crop( + ( + unicode_secret_rect["x"], + unicode_secret_rect["y"], + unicode_secret_rect["x"] + unicode_secret_rect["width"], + unicode_secret_rect["y"] + unicode_secret_rect["height"], + ) + ) + assert all(extrema == (0, 0) for extrema in unicode_crop.getextrema()) + assert process.poll() is None + + def assert_mutating_secret_stays_private( + *, + field_selector: str, + declared_field: str, + secret: str, + output_name: str, + remove_identity_before_focus: bool = False, + ) -> None: + secret_rect: dict[str, int] = {} + + def type_through_mutation(page, pump): + field = page.query_selector(field_selector) + assert field is not None + if remove_identity_before_focus: + field.evaluate( + """element => { + element.removeAttribute('name'); + element.removeAttribute('id'); + }""" + ) + box = field.bounding_box() + assert box is not None + secret_rect.update( + x=round(box["x"]), + y=round(box["y"]), + width=round(box["width"]), + height=round(box["height"]), + ) + field.click() + pump() + page.keyboard.type(secret) + pump() + pump() + page.click("#save") + pump() + pump() + + recording = record_interactive( + attach_app_url, + tmp_path / output_name, + secret_fields=(declared_field,), + cdp_endpoint=endpoint, + script=type_through_mutation, + ) + for artifact in recording.rglob("*"): + if artifact.is_file(): + assert secret.encode() not in artifact.read_bytes() + events = [ + json.loads(line) + for line in (recording / "events.jsonl").read_text().splitlines() + ] + secret_events = [event for event in events if event.get("secret")] + assert secret_events + assert all(event.get("text") is None for event in secret_events) + for frame_path in (recording / "frames").glob("*.png"): + frame = Image.open(frame_path).convert("RGB") + crop = frame.crop( + ( + secret_rect["x"], + secret_rect["y"], + secret_rect["x"] + secret_rect["width"], + secret_rect["y"] + secret_rect["height"], + ) + ) + assert all(extrema == (0, 0) for extrema in crop.getextrema()) + + pre_focus_secret = "PREINPUT-SECRET-NEVER-PERSIST" + assert_mutating_secret_stays_private( + field_selector="#pre-focus-secret", + declared_field="pre-focus-secret", + secret=pre_focus_secret, + output_name="recording-pre-focus-secret", + remove_identity_before_focus=True, + ) + attribute_secret = "ATTRIBUTE-MUTATION-SECRET-NEVER-PERSIST" + assert_mutating_secret_stays_private( + field_selector="#sticky-secret", + declared_field="sticky-secret", + secret=attribute_secret, + output_name="recording-sticky-secret", + ) + replacement_secret = "REPLACEMENT-SECRET-NEVER-PERSIST" + assert_mutating_secret_stays_private( + field_selector="#replacement-secret", + declared_field="replacement-secret", + secret=replacement_secret, + output_name="recording-replacement-secret", + ) + captured_output = capsys.readouterr() + for secret in (pre_focus_secret, attribute_secret, replacement_secret): + assert secret not in captured_output.out + assert secret not in captured_output.err + assert process.poll() is None + + dynamic_secret = "DYNAMIC-SECRET-LITERAL-NEVER-PERSIST" + dynamic_secret_rect: dict[str, int] = {} + + def add_remove_and_type_dynamic_secret(page, pump): + rect = page.evaluate( + """secret => { + const field = document.createElement('input'); + field.id = 'dynamic-secret'; + field.name = 'dynamic-secret'; + field.style.cssText = 'width:220px;height:40px;border:0'; + document.body.appendChild(field); + field.removeAttribute('name'); + field.removeAttribute('id'); + field.value = secret; + field.dispatchEvent(new Event('input', {bubbles: true})); + const box = field.getBoundingClientRect(); + return { + x: Math.round(box.left), y: Math.round(box.top), + width: Math.round(box.width), height: Math.round(box.height), + }; + }""", + dynamic_secret, + ) + dynamic_secret_rect.update(rect) + pump() + pump() + + dynamic_recording = record_interactive( + attach_app_url, + tmp_path / "recording-dynamic-secret", + secret_fields=("dynamic-secret",), + cdp_endpoint=endpoint, + script=add_remove_and_type_dynamic_secret, + ) + for artifact in dynamic_recording.rglob("*"): + if artifact.is_file(): + assert dynamic_secret.encode() not in artifact.read_bytes() + dynamic_events = [ + json.loads(line) + for line in (dynamic_recording / "events.jsonl").read_text().splitlines() + ] + assert len(dynamic_events) == 1 + assert dynamic_events[0].get("secret") is True + dynamic_after = Image.open( + dynamic_recording / "frames" / "0000_after.png" + ).convert("RGB") + dynamic_crop = dynamic_after.crop( + ( + dynamic_secret_rect["x"], + dynamic_secret_rect["y"], + dynamic_secret_rect["x"] + dynamic_secret_rect["width"], + dynamic_secret_rect["y"] + dynamic_secret_rect["height"], + ) + ) + assert all(extrema == (0, 0) for extrema in dynamic_crop.getextrema()) + assert process.poll() is None + + pre_input_replacement_secret = ( + "PRE-INPUT-REPLACEMENT-SECRET-LITERAL-NEVER-PERSIST" + ) + pre_input_replacement_rect: dict[str, int] = {} + + def replace_dynamic_secret_before_first_input(page, pump): + rect = page.evaluate( + """secret => { + const declared = document.createElement('input'); + declared.id = 'pre-input-replacement-secret'; + declared.name = 'pre-input-replacement-secret'; + declared.style.cssText = 'width:220px;height:40px;border:0'; + document.body.appendChild(declared); + const replacement = declared.cloneNode(true); + replacement.removeAttribute('name'); + replacement.removeAttribute('id'); + declared.replaceWith(replacement); + replacement.value = secret; + replacement.dispatchEvent(new Event('input', {bubbles: true})); + const box = replacement.getBoundingClientRect(); + return { + x: Math.round(box.left), y: Math.round(box.top), + width: Math.round(box.width), height: Math.round(box.height), + }; + }""", + pre_input_replacement_secret, + ) + pre_input_replacement_rect.update(rect) + pump() + pump() + + pre_input_replacement_recording = record_interactive( + attach_app_url, + tmp_path / "recording-pre-input-replacement-secret", + secret_fields=("pre-input-replacement-secret",), + cdp_endpoint=endpoint, + script=replace_dynamic_secret_before_first_input, + ) + for artifact in pre_input_replacement_recording.rglob("*"): + if artifact.is_file(): + assert ( + pre_input_replacement_secret.encode() not in artifact.read_bytes() + ) + pre_input_replacement_events = [ + json.loads(line) + for line in (pre_input_replacement_recording / "events.jsonl") + .read_text() + .splitlines() + ] + assert len(pre_input_replacement_events) == 1 + assert pre_input_replacement_events[0].get("secret") is True + pre_input_replacement_after = Image.open( + pre_input_replacement_recording / "frames" / "0000_after.png" + ).convert("RGB") + pre_input_replacement_crop = pre_input_replacement_after.crop( + ( + pre_input_replacement_rect["x"], + pre_input_replacement_rect["y"], + pre_input_replacement_rect["x"] + pre_input_replacement_rect["width"], + pre_input_replacement_rect["y"] + pre_input_replacement_rect["height"], + ) + ) + assert all( + extrema == (0, 0) for extrema in pre_input_replacement_crop.getextrema() + ) + assert process.poll() is None + + open_shadow_secret = "OPEN-SHADOW-SECRET-LITERAL-NEVER-PERSIST" + + def type_open_shadow_secret_after_identity_removal(page, pump): + page.evaluate( + """secret => { + const host = document.createElement('x-open-secret'); + host.id = 'open-shadow-secret'; + document.body.appendChild(host); + const root = host.attachShadow({mode: 'open'}); + const field = document.createElement('input'); + field.name = 'open-shadow-secret'; + field.style.cssText = 'width:220px;height:40px;border:0'; + root.appendChild(field); + field.removeAttribute('name'); + field.value = secret; + field.dispatchEvent(new Event('input', { + bubbles: true, composed: true, + })); + }""", + open_shadow_secret, + ) + pump() + pump() + page.evaluate("() => document.querySelector('x-open-secret').remove()") + + open_shadow_recording = record_interactive( + attach_app_url, + tmp_path / "recording-open-shadow-secret", + secret_fields=("open-shadow-secret",), + cdp_endpoint=endpoint, + script=type_open_shadow_secret_after_identity_removal, + ) + for artifact in open_shadow_recording.rglob("*"): + if artifact.is_file(): + assert open_shadow_secret.encode() not in artifact.read_bytes() + open_shadow_events = [ + json.loads(line) + for line in (open_shadow_recording / "events.jsonl") + .read_text() + .splitlines() + ] + assert len(open_shadow_events) == 1 + assert open_shadow_events[0].get("secret") is True + + future_closed_secret = "FUTURE-CLOSED-SECRET-LITERAL-NEVER-PERSIST" + future_closed_rect: dict[str, int] = {} + + def type_future_closed_shadow_secret(page, pump): + rect = page.evaluate( + """secret => { + const host = document.createElement('x-future-closed-secret'); + host.id = 'future-closed-secret'; + host.style.cssText = 'display:block;width:240px;height:50px'; + document.body.appendChild(host); + const root = host.attachShadow({mode: 'closed'}); + const field = document.createElement('input'); + field.style.cssText = 'width:220px;height:40px;border:0'; + root.appendChild(field); + field.value = secret; + field.dispatchEvent(new Event('input', { + bubbles: true, composed: true, + })); + const box = host.getBoundingClientRect(); + return { + x: Math.round(box.left), y: Math.round(box.top), + width: Math.round(box.width), height: Math.round(box.height), + }; + }""", + future_closed_secret, + ) + future_closed_rect.update(rect) + pump() + pump() + page.evaluate( + "() => document.querySelector('x-future-closed-secret').remove()" + ) + + future_closed_recording = record_interactive( + attach_app_url, + tmp_path / "recording-future-closed-secret", + secret_fields=("future-closed-secret",), + cdp_endpoint=endpoint, + script=type_future_closed_shadow_secret, + ) + for artifact in future_closed_recording.rglob("*"): + if artifact.is_file(): + assert future_closed_secret.encode() not in artifact.read_bytes() + future_closed_events = [ + json.loads(line) + for line in (future_closed_recording / "events.jsonl") + .read_text() + .splitlines() + ] + assert len(future_closed_events) == 1 + assert future_closed_events[0].get("secret") is True + future_closed_after = Image.open( + future_closed_recording / "frames" / "0000_after.png" + ).convert("RGB") + future_closed_crop = future_closed_after.crop( + ( + future_closed_rect["x"], + future_closed_rect["y"], + future_closed_rect["x"] + future_closed_rect["width"], + future_closed_rect["y"] + future_closed_rect["height"], + ) + ) + assert all(extrema == (0, 0) for extrema in future_closed_crop.getextrema()) + + late_closed_secret = "LATE-CLOSED-SECRET-LITERAL-NEVER-PERSIST" + late_closed_output = tmp_path / "recording-late-unbound-closed-secret" + + def expose_late_unbound_closed_secret(page, pump): + page.evaluate( + """secret => { + const host = document.createElement('x-late-closed-secret'); + host.id = 'different-late-closed-host'; + document.body.appendChild(host); + const root = host.attachShadow({mode: 'closed'}); + const field = document.createElement('input'); + field.name = 'late-closed-secret'; + field.value = secret; + root.appendChild(field); + document.title = secret; + }""", + late_closed_secret, + ) + page.click("#save") + pump() + + with pytest.raises(BrowserAttachError, match="closed shadow root"): + record_interactive( + attach_app_url, + late_closed_output, + secret_fields=("late-closed-secret",), + cdp_endpoint=endpoint, + script=expose_late_unbound_closed_secret, + ) + assert not late_closed_output.exists() + with sync_playwright() as late_cleanup_playwright: + late_cleanup_browser = late_cleanup_playwright.chromium.connect_over_cdp( + endpoint + ) + late_cleanup_page = select_attached_page( + late_cleanup_browser, + app_url=attach_app_url, + ) + late_cleanup_page.evaluate( + """() => { + document.querySelector('#different-late-closed-host').remove(); + document.title = 'Attach recorder test'; + }""" + ) + + contenteditable_secret = "CONTENTEDITABLE-SECRET-LITERAL-NEVER-PERSIST" + + def type_and_click_secret_contenteditable(page, pump): + page.evaluate( + """secret => { + const field = document.createElement('div'); + field.contentEditable = 'true'; + field.setAttribute('name', 'contenteditable-secret'); + field.setAttribute('role', 'textbox'); + field.style.cssText = 'width:260px;height:40px;border:0'; + document.body.appendChild(field); + field.innerText = secret; + field.dispatchEvent(new Event('input', { + bubbles: true, composed: true, + })); + }""", + contenteditable_secret, + ) + pump() + pump() + page.locator('[name="contenteditable-secret"]').click() + pump() + pump() + page.locator('[name="contenteditable-secret"]').evaluate( + "element => element.remove()" + ) + + contenteditable_recording = record_interactive( + attach_app_url, + tmp_path / "recording-contenteditable-secret", + secret_fields=("contenteditable-secret",), + cdp_endpoint=endpoint, + script=type_and_click_secret_contenteditable, + ) + for artifact in contenteditable_recording.rglob("*"): + if artifact.is_file(): + assert contenteditable_secret.encode() not in artifact.read_bytes() + contenteditable_events = [ + json.loads(line) + for line in (contenteditable_recording / "events.jsonl") + .read_text() + .splitlines() + ] + assert any(event.get("secret") is True for event in contenteditable_events) + click_event = next( + event for event in contenteditable_events if event.get("kind") == "click" + ) + assert contenteditable_secret not in json.dumps(click_event) + + reflected_secret = "URL TITLE SECRET LITERAL NEVER PERSIST" + + def reflect_secret_into_url_title_and_target(page, pump): + page.evaluate( + """secret => { + const field = document.createElement('input'); + field.name = 'reflected-secret'; + document.body.appendChild(field); + field.addEventListener('input', () => { + history.replaceState({}, '', '/?token=' + encodeURIComponent(secret)); + document.title = 'Result ' + secret; + document.querySelector('#save').setAttribute( + 'aria-label', 'Save ' + secret + ); + }); + field.value = secret; + field.dispatchEvent(new Event('input', { + bubbles: true, composed: true, + })); + }""", + reflected_secret, + ) + pump() + pump() + page.click("#save") + pump() + pump() + page.evaluate( + """() => { + history.replaceState({}, '', '/'); + document.title = 'Attach recorder'; + document.querySelector('#save').removeAttribute('aria-label'); + document.querySelector('[name="reflected-secret"]').remove(); + }""" + ) + + reflected_recording = record_interactive( + attach_app_url, + tmp_path / "recording-reflected-secret", + secret_fields=("reflected-secret",), + cdp_endpoint=endpoint, + script=reflect_secret_into_url_title_and_target, + ) + encoded_reflected_secret = reflected_secret.replace(" ", "%20") + for artifact in reflected_recording.rglob("*"): + if artifact.is_file(): + payload = artifact.read_bytes() + assert reflected_secret.encode() not in payload + assert encoded_reflected_secret.encode() not in payload + + replaced_guard_output = tmp_path / "recording-replaced-privacy-guard" + + def replace_page_privacy_guard(page, pump): + page.evaluate( + """() => { + const host = document.createElement('x-guard-secret'); + document.body.appendChild(host); + const root = host.attachShadow({mode: 'open'}); + const field = document.createElement('input'); + field.name = 'guard-secret'; + field.value = 'GUARD-SECRET-NEVER-PERSIST'; + root.appendChild(field); + field.dispatchEvent(new Event('input', { + bubbles: true, composed: true, + })); + }""" + ) + pump() + page.evaluate( + "() => { window.__oaflowRecorder = {sessionId: 'replaced'}; }" + ) + + with pytest.raises(BrowserAttachError, match="privacy guard is unavailable"): + record_interactive( + attach_app_url, + replaced_guard_output, + secret_fields=("guard-secret",), + cdp_endpoint=endpoint, + script=replace_page_privacy_guard, + ) + assert not replaced_guard_output.exists() + with sync_playwright() as guard_cleanup_playwright: + guard_cleanup_browser = guard_cleanup_playwright.chromium.connect_over_cdp( + endpoint + ) + guard_cleanup_page = select_attached_page( + guard_cleanup_browser, + app_url=attach_app_url, + ) + remaining_markers = guard_cleanup_page.evaluate( + """() => { + const host = document.querySelector('x-guard-secret'); + const field = host.shadowRoot.querySelector('input'); + const markers = Array.from(field.attributes).filter( + (attribute) => attribute.name.startsWith('data-oaflow-secret-') + ); + host.remove(); + delete window.__oaflowRecorder; + return markers.length; + }""" + ) + assert remaining_markers == 0 + + existing_closed_secret = "EXISTING-CLOSED-SECRET-LITERAL-NEVER-PERSIST" + with sync_playwright() as setup_playwright: + setup_browser = setup_playwright.chromium.connect_over_cdp(endpoint) + setup_page = select_attached_page(setup_browser, app_url=attach_app_url) + setup_page.evaluate( + """secret => { + const host = document.createElement('x-existing-closed-secret'); + host.id = 'existing-closed-secret'; + host.style.cssText = 'display:block;width:240px;height:50px'; + document.body.appendChild(host); + const root = host.attachShadow({mode: 'closed'}); + const field = document.createElement('input'); + field.name = 'existing-closed-secret'; + field.value = secret; + field.style.cssText = 'width:220px;height:40px;border:0'; + root.appendChild(field); + document.title = secret; + host.writeSecret = (secret) => { + field.value = secret; + field.dispatchEvent(new Event('input', { + bubbles: true, composed: true, + })); + }; + }""", + existing_closed_secret, + ) + + def type_existing_closed_shadow_secret(page, pump): + page.evaluate( + """secret => document.querySelector( + '#existing-closed-secret' + ).writeSecret(secret)""", + existing_closed_secret, + ) + pump() + pump() + + existing_closed_recording = record_interactive( + attach_app_url, + tmp_path / "recording-existing-closed-secret", + secret_fields=("existing-closed-secret",), + cdp_endpoint=endpoint, + script=type_existing_closed_shadow_secret, + ) + for artifact in existing_closed_recording.rglob("*"): + if artifact.is_file(): + assert existing_closed_secret.encode() not in artifact.read_bytes() + + with sync_playwright() as refusal_playwright: + refusal_browser = refusal_playwright.chromium.connect_over_cdp(endpoint) + refusal_page = select_attached_page(refusal_browser, app_url=attach_app_url) + refusal_page.evaluate( + """() => { + document.querySelector('#existing-closed-secret').remove(); + document.title = 'Attach recorder test'; + const host = document.createElement('x-undeclared-closed-secret'); + host.id = 'different-closed-host'; + document.body.appendChild(host); + const root = host.attachShadow({mode: 'closed'}); + const field = document.createElement('input'); + field.name = 'undeclared-closed-secret'; + root.appendChild(field); + }""" + ) + + refused_closed_output = tmp_path / "recording-refused-existing-closed-secret" + with pytest.raises( + BrowserAttachError, + match="pre-existing or newly added closed shadow", + ): + record_interactive( + attach_app_url, + refused_closed_output, + secret_fields=("undeclared-closed-secret",), + cdp_endpoint=endpoint, + script=lambda _page, _pump: None, + ) + assert not refused_closed_output.exists() + with sync_playwright() as refusal_cleanup_playwright: + refusal_cleanup_browser = ( + refusal_cleanup_playwright.chromium.connect_over_cdp(endpoint) + ) + refusal_cleanup_page = select_attached_page( + refusal_cleanup_browser, + app_url=attach_app_url, + ) + refusal_cleanup_page.evaluate( + """() => document.querySelector( + '#different-closed-host' + ).remove()""" + ) + assert process.poll() is None + + moved_secret = "MOVED-FINAL-SECRET-NEVER-PERSIST" + moved_secret_rect: dict[str, int] = {} + + def move_secret_and_finish_without_pump(page, _pump): + rect = page.evaluate( + """secret => { + const field = document.createElement('input'); + field.id = 'moved-final-secret'; + field.name = 'moved-final-secret'; + field.dataset.oaMovedFinalSecret = 'yes'; + field.style.cssText = [ + 'position:fixed', 'left:20px', 'top:180px', + 'width:220px', 'height:40px', 'border:0', 'z-index:1000', + ].join(';'); + document.body.appendChild(field); + field.removeAttribute('name'); + field.removeAttribute('id'); + field.value = secret; + field.dispatchEvent(new Event('input', {bubbles: true})); + field.style.left = '700px'; + field.style.top = '300px'; + const box = field.getBoundingClientRect(); + return { + x: Math.round(box.left), y: Math.round(box.top), + width: Math.round(box.width), height: Math.round(box.height), + }; + }""", + moved_secret, + ) + moved_secret_rect.update(rect) + + moved_recording = record_interactive( + attach_app_url, + tmp_path / "recording-moved-final-secret", + secret_fields=("moved-final-secret",), + cdp_endpoint=endpoint, + script=move_secret_and_finish_without_pump, + ) + for artifact in moved_recording.rglob("*"): + if artifact.is_file(): + assert moved_secret.encode() not in artifact.read_bytes() + moved_events = [ + json.loads(line) + for line in (moved_recording / "events.jsonl").read_text().splitlines() + ] + assert len(moved_events) == 1 + assert moved_events[0].get("secret") is True + moved_after = Image.open(moved_recording / "frames" / "0000_after.png").convert( + "RGB" + ) + moved_crop = moved_after.crop( + ( + moved_secret_rect["x"], + moved_secret_rect["y"], + moved_secret_rect["x"] + moved_secret_rect["width"], + moved_secret_rect["y"] + moved_secret_rect["height"], + ) + ) + assert all(extrema == (0, 0) for extrema in moved_crop.getextrema()) + from playwright.sync_api import sync_playwright + + with sync_playwright() as cleanup_playwright: + cleanup_browser = cleanup_playwright.chromium.connect_over_cdp(endpoint) + cleanup_page = select_attached_page( + cleanup_browser, + app_url=attach_app_url, + ) + cleanup_page.evaluate( + """() => document.querySelector( + '[data-oa-moved-final-secret="yes"]' + ).remove()""" + ) + assert process.poll() is None + + detached_marker_session = InteractiveRecorder( + attach_app_url, + tmp_path / "recording-detached-secret-marker", + secret_fields=("detached-cleanup-secret",), + cdp_endpoint=endpoint, + ) + detached_marker_session.start() + assert detached_marker_session.page is not None + detached_marker_session.page.evaluate( + """() => { + const field = document.createElement('input'); + field.name = 'detached-cleanup-secret'; + document.body.appendChild(field); + window.__detachedOaSecret = field; + }""" + ) + detached_marker_session.page.wait_for_timeout(0) + assert detached_marker_session.page.evaluate( + """() => Array.from(window.__detachedOaSecret.attributes) + .some((attribute) => attribute.name.startsWith('data-oaflow-secret-'))""" + ) + detached_marker_session.page.evaluate( + "() => window.__detachedOaSecret.remove()" + ) + detached_marker_session.finish() + + detached_marker_probe = InteractiveRecorder( + attach_app_url, + tmp_path / "recording-detached-secret-marker-probe", + cdp_endpoint=endpoint, + ) + detached_marker_probe.start() + assert detached_marker_probe.page is not None + assert not detached_marker_probe.page.evaluate( + """() => Array.from(window.__detachedOaSecret.attributes) + .some((attribute) => attribute.name.startsWith('data-oaflow-secret-'))""" + ) + detached_marker_probe.page.evaluate("() => delete window.__detachedOaSecret") + detached_marker_probe.abort() + assert process.poll() is None + + child_secret_rect: dict[str, int] = {} + + def retain_top_level_action_with_child_secret(page, pump): + child_secret = page.frame_locator("#child").locator("#frame-password") + box = child_secret.bounding_box() + assert box is not None + child_secret_rect.update( + x=round(box["x"]), + y=round(box["y"]), + width=round(box["width"]), + height=round(box["height"]), + ) + page.click("#note") + pump() + pump() + + child_secret_recording = record_interactive( + attach_app_url, + tmp_path / "recording-child-frame-secret", + cdp_endpoint=endpoint, + script=retain_top_level_action_with_child_secret, + ) + child_before = Image.open( + child_secret_recording / "frames" / "0000_before.png" + ).convert("RGB") + child_crop = child_before.crop( + ( + child_secret_rect["x"], + child_secret_rect["y"], + child_secret_rect["x"] + child_secret_rect["width"], + child_secret_rect["y"] + child_secret_rect["height"], + ) + ) + assert all(extrema == (0, 0) for extrema in child_crop.getextrema()) + assert process.poll() is None + + frame_race_session = InteractiveRecorder( + attach_app_url, + tmp_path / "recording-frame-race-probe", + cdp_endpoint=endpoint, + ) + frame_race_session.start() + try: + race_page = frame_race_session.page + race_backend = frame_race_session.backend + assert race_page is not None and race_backend is not None + original_screenshot = race_page.screenshot + masked_races = 0 + for trial in range(30): + race_page.evaluate( + """() => { + const previous = document.querySelector('#race-frame'); + if (previous) previous.remove(); + }""" + ) + race_page.wait_for_timeout(0) + state = {"attach": True} + + def attach_after_frame_snapshot(**kwargs): + if state["attach"]: + state["attach"] = False + race_page.evaluate( + """trial => { + const frame = document.createElement('iframe'); + frame.id = 'race-frame'; + frame.style.cssText = [ + 'position:fixed', 'left:20px', 'top:160px', + 'width:220px', 'height:80px', 'border:0', + ].join(';'); + frame.srcdoc = ``; + document.body.appendChild(frame); + }""", + trial, + ) + return original_screenshot(**kwargs) + + with monkeypatch.context() as patch_context: + patch_context.setattr( + race_page, + "screenshot", + attach_after_frame_snapshot, + ) + png = race_backend.screenshot() + password = race_page.frame_locator("#race-frame").locator( + "#race-password" + ) + box = password.bounding_box() + assert box is not None + image = Image.open(BytesIO(png)).convert("RGB") + crop = image.crop( + ( + round(box["x"]), + round(box["y"]), + round(box["x"] + box["width"]), + round(box["y"] + box["height"]), + ) + ) + assert all(extrema == (0, 0) for extrema in crop.getextrema()) + masked_races += 1 + assert masked_races == 30 + + churn_attempts = 0 + + def attach_and_detach_during_every_capture(**kwargs): + nonlocal churn_attempts + churn_attempts += 1 + race_page.evaluate( + """attempt => { + const frame = document.createElement('iframe'); + frame.id = `churn-${attempt}`; + frame.srcdoc = ''; + document.body.appendChild(frame); + frame.remove(); + }""", + churn_attempts, + ) + return original_screenshot(**kwargs) + + with monkeypatch.context() as patch_context: + patch_context.setattr( + race_page, + "screenshot", + attach_and_detach_during_every_capture, + ) + with pytest.raises(ScreenshotMaskStabilityError, match="frame tree"): + race_backend.screenshot() + assert churn_attempts == 3 + finally: + if frame_race_session.page is not None: + frame_race_session.page.evaluate( + """() => { + const frame = document.querySelector('#race-frame'); + if (frame) frame.remove(); + }""" + ) + frame_race_session.abort() + assert not (tmp_path / "recording-frame-race-probe").exists() + assert process.poll() is None + + interleaved_recording = tmp_path / "recording-interleaved-action-refusal" + interleaved_session = InteractiveRecorder( + attach_app_url, + interleaved_recording, + cdp_endpoint=endpoint, + ) + interleaved_session.start() + assert interleaved_session.page is not None + assert interleaved_session.backend is not None + interleaved_session.page.click("#note") + original_backend_screenshot = interleaved_session.backend.screenshot + interleaved_action_injected = False + + def screenshot_after_second_action() -> bytes: + nonlocal interleaved_action_injected + if not interleaved_action_injected: + interleaved_action_injected = True + interleaved_session.page.click("#save") + interleaved_session.page.wait_for_timeout(0) + return original_backend_screenshot() + + monkeypatch.setattr( + interleaved_session.backend, + "screenshot", + screenshot_after_second_action, + ) + try: + with pytest.raises(BrowserAttachError, match="more than one logical"): + interleaved_session.pump() + finally: + interleaved_session.abort() + assert interleaved_action_injected + assert not (interleaved_recording / "meta.json").exists() + assert process.poll() is None + + def rapid_pointer_pointer(page, pump): + page.click("#note") + page.click("#save") + pump() + + def rapid_input_submit(page, pump): + page.evaluate("document.querySelector('#note').value = ''") + page.click("#note") + pump() + page.keyboard.type("rapid-input-submit") + page.click("#save") + pump() + + def rapid_input_enter(page, pump): + page.evaluate("document.querySelector('#note').value = ''") + page.click("#note") + pump() + page.keyboard.type("rapid-input-enter") + page.keyboard.press("Enter") + pump() + + def rapid_scroll_click(page, pump): + page.mouse.wheel(0, 40) + page.click("#note") + pump() + + for case_name, drive_rapid_actions in ( + ("pointer-pointer", rapid_pointer_pointer), + ("input-submit", rapid_input_submit), + ("input-enter", rapid_input_enter), + ("scroll-click", rapid_scroll_click), + ): + rapid_recording = tmp_path / f"recording-rapid-{case_name}" + with pytest.raises(BrowserAttachError, match="more than one logical"): + record_interactive( + attach_app_url, + rapid_recording, + cdp_endpoint=endpoint, + script=drive_rapid_actions, + ) + assert not (rapid_recording / "meta.json").exists() + assert process.poll() is None + + def coalesce_one_field(page, pump): + page.evaluate("document.querySelector('#note').value = ''") + page.click("#note") + pump() + page.keyboard.type("same-field-input-coalesces") + pump() + pump() + + coalesced_recording = record_interactive( + attach_app_url, + tmp_path / "recording-coalesced-input", + cdp_endpoint=endpoint, + script=coalesce_one_field, + ) + coalesced_events = [ + json.loads(line) + for line in (coalesced_recording / "events.jsonl").read_text().splitlines() + ] + assert ( + len([event for event in coalesced_events if event["kind"] == "type"]) == 1 + ) + assert process.poll() is None + + popup_recording = tmp_path / "recording-popup-refusal" + + def open_popup(page, pump): + with page.expect_popup() as popup_info: + page.click("#open-popup") + popup_info.value.wait_for_load_state() + pump() + + with pytest.raises(BrowserAttachError, match="popup or new tab"): + record_interactive( + attach_app_url, + popup_recording, + cdp_endpoint=endpoint, + script=open_popup, + ) + assert not (popup_recording / "meta.json").exists() + assert process.poll() is None + with urlopen(f"{endpoint}/json/list", timeout=2) as response: + popup_targets = json.load(response) + assert any(target.get("url") == "about:blank" for target in popup_targets) + + _activate_app_tab(endpoint, attach_app_url) + popup_activity_recording = tmp_path / "recording-popup-activity-refusal" + + def act_inside_popup(page, pump): + with page.expect_popup() as popup_info: + page.click("#open-popup") + popup = popup_info.value + popup.set_content( + "" + ) + popup.fill("#popup-note", "activity-that-must-not-disappear") + popup.click("#popup-save") + pump() + + with pytest.raises(BrowserAttachError, match="popup or new tab"): + record_interactive( + attach_app_url, + popup_activity_recording, + cdp_endpoint=endpoint, + script=act_inside_popup, + ) + assert not (popup_activity_recording / "meta.json").exists() + assert process.poll() is None + with urlopen(f"{endpoint}/json/version", timeout=2) as response: + assert response.status == 200 + + _activate_app_tab(endpoint, attach_app_url) + short_page_recording = tmp_path / "recording-short-page-refusal" + + def act_in_short_lived_context_page(page, pump): + temporary = page.context.new_page() + temporary.set_content( + "" + ) + temporary.fill("#short-note", "activity-that-must-not-disappear") + temporary.click("#short-save") + temporary.close() + assert len(page.context.pages) >= 1 + pump() + + with pytest.raises(BrowserAttachError, match="popup or new tab"): + record_interactive( + attach_app_url, + short_page_recording, + cdp_endpoint=endpoint, + script=act_in_short_lived_context_page, + ) + assert not (short_page_recording / "meta.json").exists() + assert process.poll() is None + with urlopen(f"{endpoint}/json/version", timeout=2) as response: + assert response.status == 200 + + _activate_app_tab(endpoint, attach_app_url) + prebaseline_recording = tmp_path / "recording-prebaseline-page-refusal" + original_select_attached_page = interactive_recorder_module.select_attached_page + prebaseline_page_acted = False + + def select_after_short_lived_page(browser, *, app_url, page_url=None): + nonlocal prebaseline_page_acted + selected = original_select_attached_page( + browser, + app_url=app_url, + page_url=page_url, + ) + temporary = selected.context.new_page() + temporary.set_content( + "" + ) + temporary.fill("#pre-note", "prebaseline-activity-must-not-disappear") + temporary.click("#pre-save") + prebaseline_page_acted = True + temporary.close() + return selected + + with monkeypatch.context() as patch_context: + patch_context.setattr( + interactive_recorder_module, + "select_attached_page", + select_after_short_lived_page, + ) + with pytest.raises(BrowserAttachError, match="popup or new tab"): + record_interactive( + attach_app_url, + prebaseline_recording, + cdp_endpoint=endpoint, + script=lambda _page, _pump: None, + ) + assert prebaseline_page_acted + assert not (prebaseline_recording / "meta.json").exists() + assert process.poll() is None + + _activate_app_tab(endpoint, attach_app_url) + baseline_getter_recording = tmp_path / "recording-baseline-getter-refusal" + from playwright.sync_api import BrowserContext + + original_pages_getter = BrowserContext.pages.fget + assert original_pages_getter is not None + baseline_getter_page_acted = False + + def pages_with_short_lived_action(context): + nonlocal baseline_getter_page_acted + existing = original_pages_getter(context) + if not baseline_getter_page_acted: + baseline_getter_page_acted = True + temporary = context.new_page() + temporary.set_content( + "" + ) + temporary.fill("#gap-note", "baseline-gap-action-must-not-disappear") + temporary.click("#gap-save") + temporary.close() + return existing + + with monkeypatch.context() as patch_context: + patch_context.setattr( + BrowserContext, + "pages", + property(pages_with_short_lived_action), + ) + with pytest.raises(BrowserAttachError, match="popup or new tab"): + record_interactive( + attach_app_url, + baseline_getter_recording, + cdp_endpoint=endpoint, + script=lambda _page, _pump: None, + ) + assert baseline_getter_page_acted + assert not (baseline_getter_recording / "meta.json").exists() + assert process.poll() is None + + _activate_app_tab(endpoint, attach_app_url) + late_page_recording = tmp_path / "recording-late-page-refusal" + late_page_session = InteractiveRecorder( + attach_app_url, + late_page_recording, + cdp_endpoint=endpoint, + ) + late_page_session.start() + assert late_page_session.page is not None + assert late_page_session._pw is not None + original_playwright_stop = late_page_session._pw.stop + + def stop_after_short_lived_page() -> None: + temporary = late_page_session.page.context.new_page() + temporary.set_content( + "" + ) + temporary.fill("#late-note", "late-activity-must-not-disappear") + temporary.click("#late-save") + temporary.close() + original_playwright_stop() + + monkeypatch.setattr(late_page_session._pw, "stop", stop_after_short_lived_page) + with pytest.raises(BrowserAttachError, match="popup or new tab"): + late_page_session.finish() + assert not (late_page_recording / "meta.json").exists() + assert process.poll() is None + with urlopen(f"{endpoint}/json/version", timeout=2) as response: + assert response.status == 200 + + _activate_app_tab(endpoint, attach_app_url) + cleanup_frame_recording = tmp_path / "recording-cleanup-frame-refusal" + cleanup_frame_session = InteractiveRecorder( + attach_app_url, + cleanup_frame_recording, + cdp_endpoint=endpoint, + ) + cleanup_frame_session.start() + assert cleanup_frame_session.page is not None + original_cleanup = cleanup_frame_session._cleanup_page_listeners + cleanup_race_injected = False + + def cleanup_after_frame_race() -> None: + nonlocal cleanup_race_injected + if not cleanup_race_injected: + cleanup_race_injected = True + cleanup_frame_session.page.evaluate( + """() => { + const frame = document.createElement('iframe'); + frame.srcdoc = ''; + document.body.appendChild(frame); + frame.remove(); + }""" + ) + cleanup_frame_session.page.wait_for_timeout(0) + original_cleanup() + + monkeypatch.setattr( + cleanup_frame_session, + "_cleanup_page_listeners", + cleanup_after_frame_race, + ) + with pytest.raises(BrowserAttachError, match="changed frame state"): + cleanup_frame_session.finish() + assert cleanup_race_injected + assert not (cleanup_frame_recording / "meta.json").exists() + assert process.poll() is None + + iframe_recording = tmp_path / "recording-iframe-refusal" + + def click_inside_existing_iframe(page, pump): + page.frame_locator("#child").locator("#inside").click() + pump() + + with pytest.raises(BrowserAttachError, match="iframe"): + record_interactive( + attach_app_url, + iframe_recording, + cdp_endpoint=endpoint, + script=click_inside_existing_iframe, + ) + assert not (iframe_recording / "meta.json").exists() + assert process.poll() is None + + origin_bounce_recording = tmp_path / "recording-origin-bounce-refusal" + other_origin = attach_app_url.replace("127.0.0.1", "localhost") + + def leave_origin_and_return(page, pump): + page.goto(other_origin) + page.goto(attach_app_url) + pump() + + with pytest.raises(BrowserAttachError, match="left the declared"): + record_interactive( + attach_app_url, + origin_bounce_recording, + cdp_endpoint=endpoint, + script=leave_origin_and_return, + ) + assert not origin_bounce_recording.exists() + assert process.poll() is None + + overlap_recording = tmp_path / "recording-resize-overlap-refusal" + + def resize_during_action(page, pump): + cdp = page.context.new_cdp_session(page) + cdp.send( + "Emulation.setDeviceMetricsOverride", + { + "width": 1000, + "height": 650, + "deviceScaleFactor": 2, + "mobile": False, + }, + ) + page.click("#note") + pump() + + with pytest.raises(BrowserAttachError, match="overlapped"): + record_interactive( + attach_app_url, + overlap_recording, + cdp_endpoint=endpoint, + script=resize_during_action, + ) + assert not (overlap_recording / "meta.json").exists() + assert process.poll() is None + + resized_recording = tmp_path / "recording-resized" + + def resize_then_record(page, pump): + page.evaluate("document.querySelector('#note').value = ''") + page.click("#note") + pump() + page.keyboard.type("before-resize") + pump() + pump() + cdp = page.context.new_cdp_session(page) + cdp.send( + "Emulation.setDeviceMetricsOverride", + { + "width": 900, + "height": 600, + "deviceScaleFactor": 1, + "mobile": False, + }, + ) + pump() + page.click("#note") + pump() + page.keyboard.type("after-resize") + pump() + pump() + + recording = record_interactive( + attach_app_url, + resized_recording, + cdp_endpoint=endpoint, + script=resize_then_record, + ) + meta = json.loads((recording / "meta.json").read_text()) + events = [ + json.loads(line) + for line in (recording / "events.jsonl").read_text().splitlines() + ] + assert meta["viewport_mode"] == "per-event" + assert meta["viewport_history"][-1]["viewport"] == [900, 600] + assert len(meta["viewport_history"]) >= 2 + assert events + assert {tuple(event["viewport_before"]) for event in events} == { + (1000, 650), + (900, 600), + } + assert all( + event["viewport_before"] == event["viewport_after"] for event in events + ) + resized_bundle = tmp_path / "bundle-resized" + resized_workflow = compile_recording( + recording, + resized_bundle, + name="attached-browser-resized", + ) + assert resized_workflow.steps + assert process.poll() is None + with urlopen(f"{endpoint}/json/version", timeout=2) as response: + assert response.status == 200 + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +# --------------------------------------------------------------------------- +# Source-time secret boundary: a short keystroke prefix must not corrupt or +# abort anything. Every OTHER live secret in this file is an uppercase phrase +# that shares no character with an http://127.0.0.1:/ URL, which hid +# three defects. The cases below use lowercase secrets whose first characters +# occur in the page URL, the page title, and the element identity. +# --------------------------------------------------------------------------- + + +def _sample_reflected_state(page: Any, session_id: str) -> dict: + """Sample reflected evidence exactly the way the recorder does. + + ``InteractiveRecorder._read_scrubbed_page_state`` calls this same entry + point, from Python, at a settled boundary after the page has processed the + action. The in-page capture-phase listeners emit no URL and no title at + all, so this is the ONLY path by which reflected text reaches a recording. + Tests assert here for that reason: an assertion on an event field would + test a channel that no longer reaches disk. + """ + + return page.evaluate( + """sessionId => { + const recorder = window.__oaflowRecorder; + if (!recorder || recorder.sessionId !== sessionId) return null; + return recorder.structuralState(); + }""", + session_id, + ) + + +def _page_closure_init_js(session_id: str, binding_name: str, secrets: tuple) -> str: + return ( + interactive_recorder_module._INIT_JS.replace( + "__SESSION_ID__", json.dumps(session_id) + ) + .replace("__BINDING_NAME__", json.dumps(binding_name)) + .replace("__SECRET_NAMES__", json.dumps(list(secrets))) + .replace("__SECRET_MARKER__", json.dumps("data-oaflow-secret-test")) + .replace("__IDENT_NAMES__", "[]") + .replace("__SPECIAL_KEYS__", "[]") + ) + + +@pytest.mark.timeout(60) +def test_page_closure_keeps_url_and_identity_evidence_for_a_lowercase_secret() -> None: + """A typed prefix must never rewrite the URL, the title, or the identity. + + Real Chromium types ``charlie1`` one character at a time into a declared + secret field on ``http://host.test/hospital/charts``. The prefixes ``c``, + ``ch``, ``cha`` and ``char`` occur in that URL, in the page title, and in + the id of an unrelated button. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-lowercase-test" + binding_name = "__oaflow_emit_lowercase_test" + init_js = _page_closure_init_js(session_id, binding_name, ("password",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "charlie1" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "Charts home" + "" + "" + ), + ), + ) + page.goto("http://host.test/hospital/charts") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#password") + page.keyboard.type(secret) + page.click("#chart-save") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + payload = json.dumps(events) + assert secret not in payload + assert events + # The origin travels beside the event, so the origin guard reads a value + # no redaction rule can touch. Every event stays on the real origin. + assert {event["__oaflow_origin"] for event in events} == {"http://host.test"} + # An event carries NO reflected text: the capture-phase listener runs + # before the page's own handlers, so anything it read would be one action + # out of date. + assert not any("url" in event or "title" in event for event in events) + # This page never changes its URL or its title, so both predate the secret + # value and both stay EXACT -- including the path segment ``charts``, which + # contains the typed prefix ``char``. + assert reflected_state["url"] == "http://host.test/hospital/charts" + assert reflected_state["title"] == "Charts home" + assert reflected_state["url_withheld"] is None + secret_inputs = [ + event + for event in events + if event.get("kind") == "input" and event.get("secret") is True + ] + assert len(secret_inputs) == len(secret) + clicks = [event for event in events if event.get("kind") == "click"] + click = clicks[-1] + # The DOM identity tier stays armed: an unrelated button keeps its exact + # id and name, and nothing is withheld. + assert click["structural"]["selector"] == "#chart-save" + assert click["structural"]["name"] == "Save chart" + assert "identity_withheld" not in click["structural"] + + +@pytest.mark.timeout(60) +def test_page_closure_scrubs_a_cached_label_holding_another_declared_secret() -> None: + """A cached field label must be scrubbed against EVERY declared secret. + + Discovery walks the document in order. A declared field that appears + BEFORE another declared field caches its label while the other field + already holds a pre-filled value. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-cached-label-test" + binding_name = "__oaflow_emit_cached_label_test" + init_js = _page_closure_init_js( + session_id, binding_name, ("confirm-secret", "primary-secret") + ) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + primary = "hunter2-primary-value" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "Sign in" + f"" + "" + "" + ), + ), + ) + page.goto("http://host.test/sign-in") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#confirm-secret") + page.keyboard.type("second-value") + page.wait_for_timeout(50) + finally: + browser.close() + + payload = json.dumps(events) + assert primary not in payload + labels = { + event.get("label") + for event in events + if event.get("kind") == "input" and event.get("field") == "confirm-secret" + } + # WITHHELD, not rewritten. The cached label holds the OTHER declared + # field's pre-filled value, so Flow reports no label for this field rather + # than a placeholder the page never showed. + assert labels == {None} + + +def test_attached_recorder_reads_the_origin_the_page_reports(tmp_path: Path) -> None: + """The origin guard must not parse the scrubbed URL text. + + A declared secret that shares one character with the tab URL used to + rewrite the URL of every event, which refused the whole recording with a + false diagnosis on the first keystroke. + """ + + session = InteractiveRecorder( + "http://host.test/app", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + selected_frame = object() + session.page = SimpleNamespace(main_frame=selected_frame) + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": True, + "__oaflow_viewport": [1280, 800], + "__oaflow_dpr": 1.0, + "__oaflow_origin": "http://host.test", + "__oaflow_doc": "doc-1", + "kind": "click", + "url": "http://host.test/[secret]", + "x": 10, + "y": 20, + }, + source={"page": session.page, "frame": selected_frame}, + ) + assert session._listener_error is None + assert session.done is False + assert len(session._pyq) == 1 + + # An event without a reported origin is refused, and says exactly that. + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": True, + "__oaflow_viewport": [1280, 800], + "__oaflow_dpr": 1.0, + "kind": "click", + "url": "http://host.test/app", + "x": 10, + "y": 20, + }, + source={"page": session.page, "frame": selected_frame}, + ) + assert session.done is True + assert "did not report its document origin" in str(session._listener_error) + + +def test_structural_text_is_withheld_after_a_secret_leaves_its_document( + tmp_path: Path, +) -> None: + """A later document cannot scrub a value the previous document received.""" + + session = InteractiveRecorder( + "http://host.test/app", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + selected_frame = object() + session.page = SimpleNamespace(main_frame=selected_frame) + + def send(doc_id: str, event: dict) -> None: + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": True, + "__oaflow_viewport": [1280, 800], + "__oaflow_dpr": 1.0, + "__oaflow_origin": "http://host.test", + "__oaflow_doc": doc_id, + **event, + }, + source={"page": session.page, "frame": selected_frame}, + ) + + page_state = { + "url": "http://host.test/app", + "title": "App", + "doc": "doc-1", + "secret": True, + "url_withheld": None, + "title_withheld": None, + "dropped": [], + "secret_in_url": False, + "secret_in_title": False, + } + session.page.evaluate = lambda _js, _args: dict(page_state) + + send( + "doc-1", + { + "kind": "input", + "field": "token", + "secret": True, + "__oaflow_secret_mask_bound": True, + "__oaflow_input_session": f"{session._session_id}:input:1", + }, + ) + assert session._listener_error is None + # An event carries no reflected text of its own. The recorder samples it at + # the settled boundary, and the document that received the value can still + # report text that predates the value. + assert "url" not in session._pyq[-1] + assert session._read_scrubbed_page_state() == { + "url": "http://host.test/app", + "title": "App", + } + assert session._structural_text_withheld is False + + # A same-origin GET form submit builds a NEW document. Its closure never + # saw the value, and it cannot prove the URL it loaded with predates that + # value: a server that answers the submit with a redirect to + # `/results/` puts the value in the PATH, which no parameter name + # identifies. Structure protects the query channel, not that one, so the + # whole URL and the title are withheld. + page_state.update( + { + "url": "http://host.test/results?token=", + "title": "Done", + "doc": "doc-2", + "secret": False, + "dropped": [ + { + "name": "token", + "where": "query", + "reason": "declared-secret-parameter", + } + ], + } + ) + send("doc-2", {"kind": "click"}) + assert session._listener_error is None + assert session._read_scrubbed_page_state() == { + "url": "http://host.test/", + "title": "", + } + assert session._structural_text_withheld is True + assert session._structural_text_withheld_reasons == { + "secret-value-left-its-document" + } + # A drop is recorded only for a URL Flow reports. This one was withheld + # whole, so nothing is claimed about its parameters. + assert session._dropped_url_parameters == set() + + +def test_recording_privacy_notices_report_what_flow_withheld(tmp_path: Path) -> None: + """The operator decides on the recording from exactly these lines.""" + + from openadapt_flow.__main__ import _recording_privacy_notices + + recording = tmp_path / "recording" + recording.mkdir() + assert _recording_privacy_notices(recording) == [] + (recording / "meta.json").write_text( + json.dumps({"surface": "web", "identity_withheld_events": 2}) + ) + (notice,) = _recording_privacy_notices(recording) + assert "no DOM selector" in notice + (recording / "meta.json").write_text( + json.dumps( + { + "surface": "web", + "structural_text_withheld": "secret-value-left-its-document", + } + ) + ) + (notice,) = _recording_privacy_notices(recording) + assert "withheld the page URL and title" in notice + + +def test_stamping_a_recorded_surface_does_not_rewrite_a_published_recording( + tmp_path: Path, +) -> None: + """The recorder stamps the surface, so the publish step writes nothing.""" + + from openadapt_flow.__main__ import _stamp_recording_surface + + recording = tmp_path / "recording" + recording.mkdir() + meta_path = recording / "meta.json" + meta_path.write_text(json.dumps({"surface": "web", "source": "test"})) + before = meta_path.read_bytes() + _stamp_recording_surface(recording, "web") + assert meta_path.read_bytes() == before + _stamp_recording_surface(recording, "windows") + assert json.loads(meta_path.read_text())["surface"] == "windows" + + +@pytest.mark.timeout(120) +def test_launched_recording_withholds_a_later_document_url_after_a_get_submit( + attach_app_url: str, + tmp_path: Path, +) -> None: + """A same-origin GET submit builds a NEW document, and it is withheld. + + Structure closes the QUERY channel: the parameter keeps its name and loses + its value. It does not close the PATH channel, because no parameter name + identifies a path segment, and a server that answers the submit with a + redirect to `/results/` uses exactly that. A fresh closure holds no + value to match it against either. So a document that comes after the one + that first held a declared value reports an origin-only URL and an empty + title. Flow stamps the surface before it publishes and says what it + withheld. + """ + + if _chromium_executable() is None: + pytest.skip("no Chromium executable is installed") + secret = "hunter2-token-value" + + def drive(page, pump): + page.click("#token") + pump() + page.keyboard.type(secret) + pump() + pump() + page.click("#submit-token") + pump() + pump() + + recording = record_interactive( + f"{attach_app_url}get-form", + tmp_path / "recording-get-form", + secret_fields=("token",), + headless=True, + script=drive, + ) + body = "\n".join( + path.read_text(errors="replace") for path in sorted(recording.glob("*.json*")) + ) + assert secret not in body + assert f"token={secret}" not in body + events = [ + json.loads(line) + for line in (recording / "events.jsonl").read_text().splitlines() + ] + submit = events[-1] + # The document reached by the submit reports an origin-only URL and an + # empty title. Nothing about the results document survives. + assert submit["url_after"] == attach_app_url + assert submit["title_after"] == "" + meta = json.loads((recording / "meta.json").read_text()) + # Stamped before the atomic publish, not mutated afterwards. + assert meta["surface"] == "web" + assert meta["structural_text_withheld"] == "secret-value-left-its-document" + # A drop is recorded only for a URL Flow actually reports. This URL was + # withheld whole, so naming a dropped parameter would say less than + # nothing. + assert "url_dropped_params" not in meta + from openadapt_flow.__main__ import _recording_privacy_notices + + notices = _recording_privacy_notices(recording) + assert any("withheld the page URL and title" in n for n in notices) + + +@pytest.mark.timeout(60) +def test_page_closure_keeps_evidence_when_a_secret_input_swaps_its_node() -> None: + """A controlled input that swaps its node per keystroke leaves prefixes. + + Each removed node keeps the value it held. Those are keystroke prefixes of + the value the page still holds, and treating them as declared values would + withhold unrelated evidence on a chance match. The complete value stays + redacted. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-swap-test" + binding_name = "__oaflow_emit_swap_test" + init_js = _page_closure_init_js(session_id, binding_name, ("swap-secret",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "charlie1" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "Charts home" + "' + "" + ), + ), + ) + page.goto("http://host.test/hospital/charts") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("[name='swap-secret']") + page.keyboard.type(secret) + page.click("#chart-save") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + payload = json.dumps(events) + assert secret not in payload + clicks = [event for event in events if event.get("kind") == "click"] + click = clicks[-1] + assert click["structural"]["selector"] == "#chart-save" + assert click["structural"]["name"] == "Save chart" + assert "identity_withheld" not in click["structural"] + # Only a CONNECTED node's value is ever matched, so the prefixes the + # detached nodes still hold cannot withhold unrelated evidence. This page + # never changes its URL, so the URL predates the value and stays exact. + assert reflected_state["url"] == "http://host.test/hospital/charts" + assert reflected_state["url_withheld"] is None + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_reflected_text_the_field_no_longer_matches() -> None: + """The shorter value left in the field must never reach reflected text. + + The operator types a value, leaves the field, returns and deletes one + character. The field now holds a SHORTER value, and the page still shows + the longer one in its URL and title until the next keystroke reflects + again. No rule that reads only the current DOM can scrub that, so Flow + withholds the reflected text and reports why. Reproduction (a). + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-backspace-test" + binding_name = "__oaflow_emit_backspace_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "session" + "' + "" + ), + ), + ) + page.goto("http://host.test/charts") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type("hunter2") + page.click("#chart-save") + page.click("#token") + page.keyboard.press("Backspace") + page.click("#chart-save") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + raw = page.evaluate("() => [location.href, document.title]") + finally: + browser.close() + + payload = json.dumps(events) + # `hunter` is what the field holds now; `hunter2` is what it held before. + # Neither may reach an event or the sampled reflected state, in whole or in + # part. + assert "hunter" not in payload + assert "hunter" not in json.dumps(reflected_state) + # The page really does still show the value: this is a live leak the + # withholding rule is closing, not a page that never reflected. + assert "hunter" in raw[0] and "hunter" in raw[1] + assert reflected_state["url"] == "http://host.test/" + assert reflected_state["title"] == "" + assert reflected_state["url_withheld"] == "declared-value-in-url" + # Identity evidence is unaffected: the button holds no declared value, so + # it stays exact. + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["structural"]["selector"] == "#chart-save" + + +@pytest.mark.timeout(60) +def test_page_closure_keeps_evidence_when_an_unnamed_password_swaps_its_node() -> None: + """A password field with no name and no ID still needs a stable key. + + Discovery derives a NEW input session for each replacement, so without an + inherited session every keystroke looks like a new declared field, no + prefix is ever recognised, and every later scrub becomes ambiguous. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-unnamed-swap-test" + binding_name = "__oaflow_emit_unnamed_swap_test" + init_js = _page_closure_init_js(session_id, binding_name, ()) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "charlie1" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "Charts home" + "' + "" + ), + ), + ) + page.goto("http://host.test/hospital/charts") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("input[type=password]") + page.keyboard.type(secret) + page.click("#chart-save") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + payload = json.dumps(events) + assert secret not in payload + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["structural"]["selector"] == "#chart-save" + assert click["structural"]["name"] == "Save chart" + assert "identity_withheld" not in click["structural"] + assert reflected_state["url"] == "http://host.test/hospital/charts" + assert reflected_state["url_withheld"] is None + + +@pytest.mark.timeout(60) +def test_page_closure_marks_a_withheld_name_and_row_identity() -> None: + """A withheld accessible name or row identity must be visible too. + + A withheld selector already carries its reason. A name or a row identity + that Flow withholds disarms the same identity check, so it must carry one + as well, and the recorder must count the action. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-withheld-name-test" + binding_name = "__oaflow_emit_withheld_name_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "Charts" + "" + "
  • Alice Example row" + "
" + ), + ), + ) + page.goto("http://host.test/list") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#token") + # One character. It is too short to tell a reflection from a + # coincidence, so every text that contains it is withheld. + page.keyboard.type("a") + page.click("button") + page.wait_for_timeout(50) + finally: + browser.close() + + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["structural"]["selector"] is None + assert click["structural"]["name"] is None + assert click["structural"]["identity_withheld"] == "ambiguous-secret-in-identity" + assert click["sid"] is None + assert click["sid_withheld"] == "ambiguous-secret-in-identity" + + +def test_withheld_row_identity_is_counted_for_the_operator(tmp_path: Path) -> None: + """A withheld row identity counts as a withheld identity, like a selector.""" + + session = InteractiveRecorder( + "http://host.test/app", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + selected_frame = object() + session.page = SimpleNamespace(main_frame=selected_frame) + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": True, + "__oaflow_viewport": [1280, 800], + "__oaflow_dpr": 1.0, + "__oaflow_origin": "http://host.test", + "__oaflow_doc": "doc-1", + "kind": "click", + "url": "http://host.test/app", + "sid": None, + "sid_withheld": "ambiguous-secret-in-identity", + "structural": {"selector": None, "role": "button", "name": None}, + "x": 10, + "y": 20, + }, + source={"page": session.page, "frame": selected_frame}, + ) + assert session._listener_error is None + assert session._identity_withheld_events == 1 + + +# --------------------------------------------------------------------------- +# Regression tests for the redaction redesign (fourth review round). +# +# Three earlier revisions tried to remove a REMEMBERED value from text that was +# already captured. Three independent reviews each found a different defect in +# the retention rule that approach needs. The tests below pin the rules that +# replaced it: match only what a bound element holds right now, report identity +# evidence exactly or withhold it, and sample reflected text at the settled +# boundary. Each test states the defect it closes and fails on commit 7acd716. +# --------------------------------------------------------------------------- + + +@pytest.mark.timeout(60) +def test_page_closure_keeps_all_evidence_for_a_password_starting_with_a_word() -> None: + """A password that begins with a common word rewrites nothing. Finding 1. + + The operator types ``invoice-2026-quarterly-passphrase`` on a page whose + URL, title, clicked-row identity and button id all contain the word + ``invoice``. While the field held exactly ``invoice`` that word was a live + declared value; the retention rule then kept it for the rest of the + recording, and every one of those five pieces of evidence was rewritten or + nulled. Nothing is retained now, so at click time the only declared value + is the complete passphrase, which none of that evidence contains. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-common-word-test" + binding_name = "__oaflow_emit_common_word_test" + init_js = _page_closure_init_js(session_id, binding_name, ("password",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "invoice-2026-quarterly-passphrase" + row_identity = "MRN 44120 invoice Alice Example" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "invoice queue 2026" + "" + f"
  • {row_identity}" + "" + "
" + ), + ), + ) + page.goto("http://host.test/invoices/2026-queue") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#password") + page.keyboard.type(secret) + page.click("#invoice-submit") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + payload = json.dumps(events) + assert secret not in payload + click = [event for event in events if event.get("kind") == "click"][-1] + # IDENTITY / MACHINE EVIDENCE, exact. Replay re-reads the page and compares + # against these, so each one must be what the page held. + assert click["sid"] == row_identity + assert "sid_withheld" not in click + assert click["structural"]["selector"] == "#invoice-submit" + assert click["structural"]["name"] == "Post invoice" + assert "identity_withheld" not in click["structural"] + # REFLECTED / CONTEXT EVIDENCE, exact: neither changed after the field held + # a value, so neither can be a reflection of it. + assert reflected_state["url"] == "http://host.test/invoices/2026-queue" + assert reflected_state["title"] == "invoice queue 2026" + assert reflected_state["url_withheld"] is None + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_a_stale_reflection_from_a_swapping_input() -> None: + """A node swap combined with an as-you-type reflection. The second P1. + + The page replaces its input element on every keystroke AND writes the value + into the URL and the title -- but stops reflecting past eight characters, + so what it shows at the end is a PREFIX the field no longer holds. The + retention rule treated that prefix as a droppable keystroke artifact of the + replaced node, so nothing redacted it and it reached the recording. Flow + now proves reflected text safe only while it has not changed since before + the field held a value, so this text is withheld whole. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-stale-swap-test" + binding_name = "__oaflow_emit_stale_swap_test" + init_js = _page_closure_init_js(session_id, binding_name, ("swap-secret",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "charlie-alpha" + stale_prefix = "charlie-" + reflect = ( + "if (this.value.length <= 8) {" + "document.title = 'session for ' + this.value;" + "history.replaceState({}, '', '/charts/' + this.value);" + "}" + "const next = document.createElement('input');" + "next.name = 'swap-secret';" + "next.type = 'password';" + "next.setAttribute('oninput', this.getAttribute('oninput'));" + "next.value = this.value;" + "this.replaceWith(next);" + "next.focus();" + ) + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "session" + "" + "" + ), + ), + ) + page.goto("http://host.test/charts") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("[name='swap-secret']") + page.keyboard.type(secret) + page.click("#chart-save") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + raw = page.evaluate("() => [location.href, document.title]") + finally: + browser.close() + + # The page really is showing a value the field no longer holds. Without + # this the test would pass on a page that simply never reflected. + assert raw == [ + f"http://host.test/charts/{stale_prefix}", + f"session for {stale_prefix}", + ] + payload = json.dumps(events) + assert secret not in payload + assert stale_prefix not in payload + assert stale_prefix not in json.dumps(reflected_state) + assert reflected_state["url"] == "http://host.test/" + assert reflected_state["title"] == "" + assert reflected_state["url_withheld"] == "declared-value-in-url" + # Identity evidence is untouched by the withholding: the button holds no + # declared value, so replay keeps its strongest identity tier. + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["structural"]["selector"] == "#chart-save" + assert click["structural"]["name"] == "Save chart" + assert "identity_withheld" not in click["structural"] + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_identity_that_holds_a_declared_value() -> None: + """Identity evidence is EXACT or WITHHELD-AND-MARKED. Never rewritten. + + The page copies the typed value into a row. The old rule rewrote the row + identity to ``Ticket [secret] owner`` and reported nothing, so replay would + have compared against characters the page never showed and no downstream + check could see the substitution. Flow withholds the field and marks it. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-identity-rewrite-test" + binding_name = "__oaflow_emit_identity_rewrite_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "alpha-charlie-9" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "Tickets" + "' + "
  • Ticket owner" + "
" + ), + ), + ) + page.goto("http://host.test/tickets") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type(secret) + page.click("#save") + page.wait_for_timeout(50) + finally: + browser.close() + + payload = json.dumps(events) + assert secret not in payload + # No rewritten copy of anything exists: Flow writes no placeholder into + # captured text, so the placeholder string appears nowhere at all. + assert "[secret]" not in payload + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["sid"] is None + assert click["sid_withheld"] == "secret-value-in-identity" + # The button itself holds no declared value, so its identity is untouched. + assert click["structural"]["selector"] == "#save" + + +@pytest.mark.timeout(60) +def test_page_closure_emits_no_reflected_text_from_the_capture_phase() -> None: + """An event carries no URL and no title, because it cannot carry a true one. + + The in-page listeners run in the CAPTURE phase, before the page's own + handlers. Any URL or title read there belongs to the state BEFORE the + action, so an event that carried one described the previous screen. That + staleness was the only reason the closure kept a value history at all. + Flow samples reflected text from Python at the settled boundary instead. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-no-event-text-test" + binding_name = "__oaflow_emit_no_event_text_test" + init_js = _page_closure_init_js(session_id, binding_name, ()) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "step one" + "' + ), + ), + ) + page.goto("http://host.test/one") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#next") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + assert events + for event in events: + assert "url" not in event + assert "title" not in event + # Sampled after the page processed the click, so it is the screen the click + # produced -- not the one it left. A capture-phase read would say "/one" + # and "step one" here. + assert reflected_state["url"] == "http://host.test/two" + assert reflected_state["title"] == "step two" + assert reflected_state["url_withheld"] is None + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_after_a_shorter_value_replaces_a_reflection() -> None: + """Reproduction (b): blur, clear, retype something shorter. + + The page reflects the first value, the operator leaves the field, the page + clears it, and the operator types a shorter value that the page does not + reflect. The URL still shows the first value, and nothing in the DOM can + identify it any more. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-cleared-field-test" + binding_name = "__oaflow_emit_cleared_field_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + first = "hunter2-primary" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "session" + "' + "" + ), + ), + ) + page.goto("http://host.test/charts") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type(first) + page.click("#chart-save") + # The page clears the field and stops reflecting. + page.evaluate( + """() => { + const el = document.getElementById('token'); + el.dataset.done = '1'; + el.value = ''; + }""" + ) + page.click("#token") + page.keyboard.type("abc") + page.click("#chart-save") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + raw_url = page.evaluate("() => location.href") + finally: + browser.close() + + assert raw_url == f"http://host.test/charts/{first}" + payload = json.dumps(events) + assert first not in payload + assert first not in json.dumps(reflected_state) + assert reflected_state["url"] == "http://host.test/" + assert reflected_state["url_withheld"] == "declared-value-in-url" + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_a_reflection_the_field_extended_past() -> None: + """Reproduction (c): ``alpha-one`` reflected, then extended to + ``alpha-one-two``. + + The URL keeps the first value. Matching the current value against it finds + nothing, because the current value only CONTAINS the reflected one -- it is + not equal to it and does not appear in the URL. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-extended-value-test" + binding_name = "__oaflow_emit_extended_value_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "session" + "' + "" + ), + ), + ) + page.goto("http://host.test/charts") + page.expose_binding( + binding_name, + lambda _source, detail: events.append(detail), + ) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type("alpha-one-two") + page.click("#chart-save") + page.wait_for_timeout(50) + reflected_state = _sample_reflected_state(page, session_id) + raw_url = page.evaluate("() => location.href") + finally: + browser.close() + + assert raw_url == "http://host.test/charts/alpha-one" + payload = json.dumps(events) + assert "alpha-one" not in payload + assert "alpha-one" not in json.dumps(reflected_state) + assert reflected_state["url"] == "http://host.test/" + assert reflected_state["url_withheld"] == "declared-value-in-url" + + +def test_withheld_reflected_text_names_every_reason_for_the_operator( + tmp_path: Path, +) -> None: + """One recording can withhold reflected text for more than one reason.""" + + from openadapt_flow.__main__ import _recording_privacy_notices + + recording = tmp_path / "recording" + recording.mkdir() + (recording / "meta.json").write_text( + json.dumps( + { + "surface": "web", + "structural_text_withheld": ( + "reflected-text-changed-after-a-secret-value," + "secret-value-left-its-document" + ), + } + ) + ) + notices = _recording_privacy_notices(recording) + assert len(notices) == 2 + assert any( + "changed after a declared secret field held a value" in n for n in notices + ) + assert any("left its document" in n for n in notices) + # Flow never rewrites captured text, and the operator is told so. + assert all("never rewrites it" in notice for notice in notices) + + +def test_reflected_text_withheld_reasons_reach_the_recording_metadata( + tmp_path: Path, +) -> None: + """Every distinct reason Flow withheld reflected text reaches meta.json.""" + + session = InteractiveRecorder( + "http://host.test/app", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + page_state = { + "url": "http://host.test/", + "title": "", + "doc": "doc-1", + "secret": True, + "url_withheld": "declared-value-in-url", + "title_withheld": "declared-value-in-title", + "dropped": [], + "secret_in_url": True, + "secret_in_title": True, + } + session.page = SimpleNamespace( + main_frame=object(), + evaluate=lambda _js, _args: dict(page_state), + ) + # The page withheld both: the application put the value into its own URL + # and its own title, where structure could not remove it. + assert session._read_scrubbed_page_state() == { + "url": "http://host.test/", + "title": "", + } + assert session._app_placed_secret_in_url is True + assert session._app_placed_secret_in_title is True + # A later document adds the cross-document reason. + page_state.update( + { + "doc": "doc-2", + "secret": False, + "url_withheld": None, + "title_withheld": None, + "secret_in_url": False, + "secret_in_title": False, + "url": "http://host.test/next", + } + ) + assert session._read_scrubbed_page_state() == { + "url": "http://host.test/", + "title": "", + } + assert session._structural_text_withheld_reasons == { + "declared-value-in-url", + "declared-value-in-title", + "secret-value-left-its-document", + } + + +@pytest.mark.timeout(60) +def test_page_closure_warns_when_the_application_puts_a_secret_in_its_url() -> None: + """A value structure cannot name is caught by the net, and reported. + + The operator opens a URL that already carries the value and then types it + into a declared field. Nothing about the URL changed, so the baseline rule + reports it. The net catches it anyway: the URL Flow is about to report + holds a value a bound field is holding right now. Flow withholds the whole + URL and tells the operator, because an application that puts a secret in a + URL has a defect that exists with or without Flow -- OWASP lists browser + history, server logs, proxies and the Referer header as places it is + already exposed. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-secret-in-url-test" + binding_name = "__oaflow_emit_secret_in_url_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + from playwright.sync_api import sync_playwright + + secret = "quarterly-passphrase-9" + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "session" + "" + "" + ), + ), + ) + # NOT a declared parameter name, so structure alone cannot find it. + page.goto(f"http://host.test/charts?ref={secret}") + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + before = _sample_reflected_state(page, session_id) + page.click("#token") + page.keyboard.type(secret) + page.click("#chart-save") + page.wait_for_timeout(50) + after = _sample_reflected_state(page, session_id) + finally: + browser.close() + + # Before anything is typed, nothing declared holds a value, so this URL is + # ordinary page context and is reported. + assert before["url"] == f"http://host.test/charts?ref={secret}" + assert before["url_withheld"] is None + # Once a bound field holds it, the net fires and the operator is warned. + assert secret not in json.dumps(after) + assert after["url"] == "http://host.test/" + assert after["url_withheld"] == "declared-value-in-url" + assert after["secret_in_url"] is True + + +@pytest.mark.timeout(60) +def test_page_closure_states_its_debounce_limit() -> None: + """The stated residual, pinned so a change to it has to be deliberate. + + The net matches the value a bound field holds NOW against the text the page + shows NOW. An application that updates its URL on a timer longer than the + settle window still shows an earlier value at the moment Flow samples, and + the net will not match it. Flow does NOT keep a previous value to catch + that: the rule that kept previous values is the one three reviews broke. + + Here the page reflects only the first eight characters and then stops, and + the field goes on to hold more. The PATH net catches this particular shape + -- the value Flow can see contains that path segment -- so the URL is + withheld. What is NOT caught, and is written down in + docs/BROWSER_RECORDING.md, is a reflection that no value Flow can see + contains: an application-defined transform of the value. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-transform-limit-test" + binding_name = "__oaflow_emit_transform_limit_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=str(executable), + headless=True, + args=["--no-sandbox"], + ) + try: + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill( + content_type="text/html", + body=( + "session" + "' + "" + ), + ), + ) + page.goto("http://host.test/charts") + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type("quarterly-passphrase-9") + page.click("#chart-save") + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + raw_url = page.evaluate("() => location.href") + finally: + browser.close() + + # The page really is showing a transform of the value. + assert raw_url == "http://host.test/charts/9-esarhpssap-ylretrauq" + # Flow reports it: no value it can see contains that text, in either + # direction. This is the declared limit, not an oversight. + assert state["url"] == raw_url + assert state["url_withheld"] is None + + +# --------------------------------------------------------------------------- +# Round-4 blockers and the structured-URL redesign. +# +# Every test below FAILS at ff5d71e and passes here. +# --------------------------------------------------------------------------- + + +def _closure_page(playwright, executable, body: str, url: str, session_id: str): + browser = playwright.chromium.launch( + executable_path=str(executable), headless=True, args=["--no-sandbox"] + ) + page = browser.new_page() + page.route( + "http://host.test/**", + lambda route: route.fulfill(content_type="text/html", body=body), + ) + page.goto(url) + return browser, page + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_identity_after_the_field_is_removed() -> None: + """Blocker P1-B. An SPA wizard removes the field and shows the value. + + The operator types into a declared field, the page replaces the form with a + summary row holding that value, and the operator clicks the row. Matching + only what a CONNECTED element holds finds nothing, because the field is + gone. Flow retains the value the field held at a COMMIT POINT and uses it + for exactly one thing: withholding identity text. It is never used to + rewrite anything, and never for the URL or the title. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-wizard-test" + binding_name = "__oaflow_emit_wizard_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "hunter2-token-value" + body = ( + "Wizard" + "
" + "
" + "" + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, executable, body, "http://host.test/wizard", session_id + ) + try: + page.expose_binding( + binding_name, lambda _source, detail: events.append(detail) + ) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type(secret) + page.click("#next") + page.click("#confirm") + page.wait_for_timeout(50) + finally: + browser.close() + + assert secret not in json.dumps(events) + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["sid"] is None + assert click["sid_withheld"] == "secret-value-in-identity" + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_identity_from_an_inbound_declared_parameter() -> None: + """Blocker P1-B across documents. The value arrives under its own name. + + A same-origin GET submit carries the field's value under the field's NAME, + because that is how an HTML form works. The results document builds a fresh + closure with no bound element holding that value, so it recovers the value + from its own URL by NAME and uses it to withhold identity text. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-inbound-param-test" + binding_name = "__oaflow_emit_inbound_param_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "hunter2-token-value" + body = ( + "Results" + "" + "
Token" + secret + "
" + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, + executable, + body, + f"http://host.test/results?token={secret}", + session_id, + ) + try: + page.expose_binding( + binding_name, lambda _source, detail: events.append(detail) + ) + page.evaluate(init_js) + page.click("#confirm") + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + assert secret not in json.dumps(events) + assert secret not in json.dumps(state) + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["sid"] is None + assert click["sid_withheld"] == "secret-value-in-identity" + # The URL keeps the parameter NAME and loses its VALUE, by name. + assert state["url"] == "http://host.test/results?token=" + assert state["dropped"] == [ + {"name": "token", "where": "query", "reason": "declared-secret-parameter"} + ] + + +@pytest.mark.timeout(60) +def test_page_closure_marks_a_withheld_secret_field_name() -> None: + """Blocker P2-A. The last silent null. + + Flow never reads the visible text of a bound secret field, because that + text IS the value. A declared field with an aria-label therefore reports no + accessible name. That is a WITHHELD name, not an absent one: a control + field beside it returns its name normally. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-secret-name-test" + binding_name = "__oaflow_emit_secret_name_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + body = ( + "Login" + "" + "" + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, executable, body, "http://host.test/login", session_id + ) + try: + page.expose_binding( + binding_name, lambda _source, detail: events.append(detail) + ) + page.evaluate(init_js) + page.click("#note") + page.click("#token") + page.wait_for_timeout(50) + finally: + browser.close() + + clicks = [event for event in events if event.get("kind") == "click"] + note = next(c for c in clicks if c["structural"]["selector"] == "#note") + token = next(c for c in clicks if c["structural"]["selector"] == "#token") + assert note["structural"]["name"] == "Note field" + assert "identity_withheld" not in note["structural"] + assert token["structural"]["name"] is None + assert token["structural"]["identity_withheld"] == "secret-field-name-not-read" + + +@pytest.mark.timeout(60) +def test_page_closure_reports_a_single_page_app_route_change() -> None: + """The evidence a whole-URL refusal used to destroy. + + A single-page application routes after a login. The path is app structure, + not operator input, so Flow reports the origin and the path exactly. The + old rule withheld an origin-only URL for the rest of the document, which + cost the URL evidence of an entire session on every well-behaved app. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-spa-route-test" + binding_name = "__oaflow_emit_spa_route_test" + init_js = _page_closure_init_js(session_id, binding_name, ()) + from playwright.sync_api import sync_playwright + + body = ( + "Sign in" + "" + "' + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, executable, body, "http://host.test/login", session_id + ) + try: + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#password") + page.keyboard.type("hunter2-token-value") + page.click("#sign-in") + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + assert state["url"] == "http://host.test/dashboard/overview" + assert state["url_withheld"] is None + assert state["dropped"] == [] + + +@pytest.mark.timeout(60) +def test_page_closure_drops_only_the_unproven_parameter_value() -> None: + """One parameter value is dropped; the path and the others stay exact. + + A parameter Flow cannot prove predates the value loses only its own value. + The whole URL is not withheld for it, and every parameter NAME survives. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-one-param-test" + binding_name = "__oaflow_emit_one_param_test" + init_js = _page_closure_init_js(session_id, binding_name, ()) + from playwright.sync_api import sync_playwright + + body = ( + "Charts" + "" + "' + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, + executable, + body, + "http://host.test/charts?view=open&filter=start", + session_id, + ) + try: + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#password") + page.keyboard.type("hunter2-token-value") + page.click("#go") + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + finally: + browser.close() + + # `view` is unchanged since before the field held a value, so it is proven + # and reported. `filter` changed, so only ITS value is dropped. + assert state["url"] == "http://host.test/charts?view=open&filter=" + assert state["url_withheld"] is None + assert state["dropped"] == [ + {"name": "filter", "where": "query", "reason": "unproven-parameter-value"} + ] + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_a_url_that_holds_the_value_in_its_path() -> None: + """The net: a value structure cannot name, in a path segment.""" + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-path-secret-test" + binding_name = "__oaflow_emit_path_secret_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + from playwright.sync_api import sync_playwright + + secret = "hunter2-token-value" + body = ( + "Charts" + "" + "" + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, executable, body, "http://host.test/charts", session_id + ) + try: + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type(secret) + page.click("#save") + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + raw_url = page.evaluate("() => location.href") + finally: + browser.close() + + assert raw_url == f"http://host.test/charts/{secret}" + assert secret not in json.dumps(state) + assert state["url"] == "http://host.test/" + assert state["url_withheld"] == "declared-value-in-url" + assert state["secret_in_url"] is True + + +# --------------------------------------------------------------------------- +# Round-5: the URL PATH channel, and the evidence the fix must NOT cost. +# +# Every test below FAILS at 07372d5 and passes here. +# --------------------------------------------------------------------------- + + +@pytest.mark.timeout(180) +def test_launched_recording_withholds_a_redirect_that_puts_the_value_in_a_path( + tmp_path: Path, +) -> None: + """A REST redirect answers a GET submit with `/results/`. + + Nothing in the new document can find that value: no bound element holds it, + the closure committed nothing, and no parameter NAME identifies a path + segment. Structure closes the query channel and not this one, so the whole + URL is withheld for every document after the one that first held a declared + value. + """ + + if _chromium_executable() is None: + pytest.skip("no Chromium executable is installed") + import threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + from urllib.parse import parse_qs, urlparse + + secret = "hunter2-primary" + form = ( + b"Token form" + b'
' + b'' + b'' + b"
" + ) + results = ( + b"Results" + b'
  • open
' + b"" + ) + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + parsed = urlparse(self.path) + if parsed.path == "/lookup": + value = parse_qs(parsed.query).get("token", [""])[0] + self.send_response(302) + self.send_header("Location", f"/results/{value}") + self.send_header("Content-Length", "0") + self.end_headers() + return + payload = results if parsed.path.startswith("/results") else form + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + app_url = f"http://127.0.0.1:{server.server_address[1]}/" + try: + + def drive(page, pump): + page.click("#token") + pump() + page.keyboard.type(secret) + pump() + pump() + page.click("#submit-token") + pump() + pump() + page.click("#cell") + pump() + pump() + + recording = record_interactive( + f"{app_url}form", + tmp_path / "recording-redirect", + secret_fields=("token",), + headless=True, + script=drive, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + body = "\n".join( + path.read_text(errors="replace") for path in sorted(recording.glob("*.json*")) + ) + assert secret not in body + events = [ + json.loads(line) + for line in (recording / "events.jsonl").read_text().splitlines() + ] + assert events[-1]["url_after"] == app_url + assert events[-1]["title_after"] == "" + meta = json.loads((recording / "meta.json").read_text()) + assert meta["structural_text_withheld"] == "secret-value-left-its-document" + + +@pytest.mark.timeout(60) +def test_page_closure_still_reports_a_same_document_route_after_the_fix() -> None: + """The cross-document rule must not cost the SPA evidence again. + + A single-page application routes with `history.pushState`, which does NOT + build a new document. The closure that held the value is the closure being + sampled, so its URL is still reported exactly. The cross-document rule + bites only on a real navigation, which is where the redirect leak lives. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-pushstate-test" + binding_name = "__oaflow_emit_pushstate_test" + init_js = _page_closure_init_js(session_id, binding_name, ()) + from playwright.sync_api import sync_playwright + + body = ( + "Sign in" + "" + "' + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, executable, body, "http://host.test/login", session_id + ) + try: + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#password") + page.keyboard.type("hunter2-primary") + page.click("#sign-in") + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + doc_id = state["doc"] + # Same document: the recorder closure was not rebuilt, so its + # document id is unchanged and Python's cross-document rule cannot + # apply to it. + after = _sample_reflected_state(page, session_id) + finally: + browser.close() + + assert state["url"] == "http://host.test/dashboard/overview" + assert state["url_withheld"] is None + assert after["doc"] == doc_id + + +def test_a_same_document_route_is_never_treated_as_a_later_document( + tmp_path: Path, +) -> None: + """The Python half of the same claim, without a browser. + + Only the FIRST document to hold a declared value reports its reflected + text. A same-document route change keeps that document id, so it keeps its + URL; a real navigation produces a new id and is withheld. + """ + + session = InteractiveRecorder( + "http://host.test/app", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + page_state = { + "url": "http://host.test/login", + "title": "Sign in", + "doc": "doc-1", + "secret": True, + "url_withheld": None, + "title_withheld": None, + "dropped": [], + "secret_in_url": False, + "secret_in_title": False, + } + session.page = SimpleNamespace( + main_frame=object(), + evaluate=lambda _js, _args: dict(page_state), + ) + assert session._read_scrubbed_page_state()["url"] == "http://host.test/login" + # Same document, new route: reported exactly. + page_state["url"] = "http://host.test/dashboard/overview" + assert ( + session._read_scrubbed_page_state()["url"] + == "http://host.test/dashboard/overview" + ) + assert session._structural_text_withheld is False + # A real navigation builds a new document. Even one that holds a declared + # value of its own is withheld: holding a value says nothing about whether + # it loaded with an EARLIER document's value in its path. + page_state.update({"doc": "doc-2", "url": "http://host.test/results/x"}) + assert session._read_scrubbed_page_state() == { + "url": "http://host.test/", + "title": "", + } + assert session._structural_text_withheld_reasons == { + "secret-value-left-its-document" + } + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_a_selector_built_from_an_inbound_value() -> None: + """A selector is identity too, and it used the narrower value set. + + In a document reached by a GET submit the accessible name and the row + identity were correctly refused, while the element id went out verbatim as + `#row-`. All three identity paths now use the same value set. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-inbound-selector-test" + binding_name = "__oaflow_emit_inbound_selector_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + events: list[dict] = [] + from playwright.sync_api import sync_playwright + + secret = "hunter2-primary" + body = ( + "Results" + f"
  • open" + "
" + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, + executable, + body, + f"http://host.test/results?token={secret}", + session_id, + ) + try: + page.expose_binding( + binding_name, lambda _source, detail: events.append(detail) + ) + page.evaluate(init_js) + page.click("#cell-" + secret) + page.wait_for_timeout(50) + finally: + browser.close() + + assert secret not in json.dumps(events) + click = [event for event in events if event.get("kind") == "click"][-1] + assert click["structural"]["selector"] is None + assert click["structural"]["identity_withheld"] == "secret-value-in-identity" + + +# --------------------------------------------------------------------------- +# Round-6: a page that CONSUMES its own field. +# +# A scanner input writes the badge into the URL and clears the field inside its +# own `input` handler. Nothing in the DOM holds the value at any moment Python +# samples. Each test below FAILS at d1a762e and passes here. +# --------------------------------------------------------------------------- + + +_SCANNER_BODY = ( + "Scan station" + "" + "" + "" +) + + +@pytest.mark.timeout(60) +def test_page_closure_withholds_a_title_a_consumed_field_produced() -> None: + """F1. The title branch was gated on a flag the live DOM never set. + + `documentHeldSecretValue` was derived from what a bound field HOLDS at a + sample. A field the page clears inside its own `input` handler holds + nothing at every sample, so the flag stayed false for the whole recording + and the title check was never reached, while the URL check ran regardless. + The flag is now armed in the capture-phase `input` handler, which is the + moment the document provably held a value -- and is what the documentation + already said: "once a declared secret field RECEIVES INPUT". + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-consumed-title-test" + binding_name = "__oaflow_emit_consumed_title_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + from playwright.sync_api import sync_playwright + + secret = "hunter2-primary" + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, + executable, + _SCANNER_BODY, + "http://host.test/station", + session_id, + ) + try: + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type(secret) + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + raw = page.evaluate("() => [location.href, document.title]") + finally: + browser.close() + + # The page really is showing the value in both channels. + assert raw == [f"http://host.test/scan/{secret}", f"Scan {secret}"] + assert secret not in json.dumps(state) + assert state["secret"] is True + assert state["title"] == "" + assert state["title_withheld"] == "declared-value-in-title" + assert state["url_withheld"] == "declared-value-in-url" + + +def test_an_input_event_alone_arms_the_cross_document_boundary( + tmp_path: Path, +) -> None: + """F2. Both document markers must move together. + + `_track_secret_document` added to `_secret_doc_ids` from the input event + but set `_first_secret_doc_id` only from the settled page read. A document + that never HELD a value at a sampling instant therefore reached one marker + and not the other, and the cross-document rule -- which keys off the second + -- never engaged, so every later document reported its URL. + """ + + session = InteractiveRecorder( + "http://host.test/app", + tmp_path / "recording", + cdp_endpoint="http://127.0.0.1:9222", + ) + selected_frame = object() + session.page = SimpleNamespace(main_frame=selected_frame) + session._enqueue_browser_event( + { + "__oaflow_session": session._session_id, + "__oaflow_top_level": True, + "__oaflow_viewport": [1280, 800], + "__oaflow_dpr": 1.0, + "__oaflow_origin": "http://host.test", + "__oaflow_doc": "doc-1", + "__oaflow_doc_holds_secret": False, + "kind": "input", + "field": "token", + "secret": True, + "__oaflow_secret_mask_bound": True, + "__oaflow_input_session": f"{session._session_id}:input:1", + }, + source={"page": session.page, "frame": selected_frame}, + ) + assert session._listener_error is None + assert session._secret_doc_ids == {"doc-1"} + assert session._first_secret_doc_id == "doc-1" + # A later document is therefore withheld, which is the whole point. + assert session._secret_document_left("doc-2") is True + assert session._secret_document_left("doc-1") is False + + +@pytest.mark.timeout(60) +def test_page_closure_keeps_a_consumed_value_across_a_second_entry() -> None: + """F3. One field, two scans. + + Badge one is consumed into the URL and the field is cleared; the operator + starts badge two. The per-element cache holds ONE value and badge two was + about to displace badge one while badge one was still on show. A value the + next one does not CONTINUE was not edited away by the operator -- the page + took it -- so it is promoted into the withhold-only committed set, which is + not per element. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-two-scans-test" + binding_name = "__oaflow_emit_two_scans_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token",)) + from playwright.sync_api import sync_playwright + + first = "hunter2-primary" + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, + executable, + _SCANNER_BODY, + "http://host.test/station", + session_id, + ) + try: + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type(first) + page.wait_for_timeout(30) + page.keyboard.type("beta9t") # badge two, still live + page.wait_for_timeout(50) + state = _sample_reflected_state(page, session_id) + live = page.evaluate("() => document.getElementById('token').value") + finally: + browser.close() + + assert live == "beta9t", "badge two must still be live for this to be the case" + assert first not in json.dumps(state) + assert state["url_withheld"] == "declared-value-in-url" + + +@pytest.mark.timeout(60) +def test_page_closure_keeps_a_consumed_value_while_another_field_is_live() -> None: + """F4. Two declared fields. + + The first is consumed into the URL and cleared; the second still holds a + PIN. The cache used to be skipped for the WHOLE document as soon as + anything held a value, so the same URL was withheld before the PIN was + typed and reported afterwards. The test is per element now. + """ + + executable = _chromium_executable() + if executable is None: + pytest.skip("no Chromium executable is installed") + session_id = "page-closure-two-fields-test" + binding_name = "__oaflow_emit_two_fields_test" + init_js = _page_closure_init_js(session_id, binding_name, ("token", "pin")) + from playwright.sync_api import sync_playwright + + first = "hunter2-primary" + body = _SCANNER_BODY.replace( + "", + "", + ) + with sync_playwright() as playwright: + browser, page = _closure_page( + playwright, executable, body, "http://host.test/station", session_id + ) + try: + page.expose_binding(binding_name, lambda _source, _detail: None) + page.evaluate(init_js) + page.click("#token") + page.keyboard.type(first) + page.wait_for_timeout(30) + before_pin = _sample_reflected_state(page, session_id) + page.click("#pin") + page.keyboard.type("4821") + page.wait_for_timeout(50) + after_pin = _sample_reflected_state(page, session_id) + finally: + browser.close() + + # The SAME URL, withheld before the PIN and withheld after it. A second + # declared field holding a value says nothing about the first field's. + assert before_pin["url_withheld"] == "declared-value-in-url" + assert first not in json.dumps(after_pin) + assert after_pin["url_withheld"] == "declared-value-in-url" diff --git a/tests/test_compiler.py b/tests/test_compiler.py index 4e8d219c..bacc88a5 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -142,6 +142,173 @@ def compiled(tmp_path_factory: pytest.TempPathFactory): class TestCompileRecording: + def test_per_event_viewports_can_change_between_steps(self, tmp_path: Path) -> None: + recording = tmp_path / "recording" + (recording / "frames").mkdir(parents=True) + first = blank() + second = np.full((600, 900, 3), 245, dtype=np.uint8) + frames = {0: (first, first), 1: (second, second)} + for i, (before, after) in frames.items(): + write_frame(recording, i, "before", before) + write_frame(recording, i, "after", after) + events = [ + { + "i": 0, + "kind": "key", + "key": "Tab", + "t": 1.0, + "viewport_before": [1280, 800], + "viewport_after": [1280, 800], + }, + { + "i": 1, + "kind": "key", + "key": "Enter", + "t": 2.0, + "viewport_before": [900, 600], + "viewport_after": [900, 600], + }, + ] + (recording / "events.jsonl").write_text( + "\n".join(json.dumps(event) for event in events) + "\n" + ) + (recording / "meta.json").write_text( + json.dumps( + { + "id": "resized-recording", + "created_at": "2026-08-18T00:00:00+00:00", + "viewport": [1280, 800], + "viewport_mode": "per-event", + "viewport_history": [ + { + "before_event": 0, + "viewport": [1280, 800], + "device_scale_factor": 2, + }, + { + "before_event": 1, + "viewport": [900, 600], + "device_scale_factor": 1, + }, + ], + "params": {}, + } + ) + ) + + workflow = compile_recording( + recording, + tmp_path / "bundle", + name="resized-recording", + ) + assert len(workflow.steps) == 2 + assert workflow.viewport == VIEWPORT + + def test_per_event_viewport_cannot_change_within_one_step( + self, tmp_path: Path + ) -> None: + recording = tmp_path / "recording" + (recording / "frames").mkdir(parents=True) + write_frame(recording, 0, "before", blank()) + write_frame( + recording, + 0, + "after", + np.full((600, 900, 3), 245, dtype=np.uint8), + ) + (recording / "events.jsonl").write_text( + json.dumps( + { + "i": 0, + "kind": "key", + "key": "Enter", + "t": 1.0, + "viewport_before": [1280, 800], + "viewport_after": [900, 600], + } + ) + + "\n" + ) + (recording / "meta.json").write_text( + json.dumps( + { + "id": "mid-event-resize", + "created_at": "2026-08-18T00:00:00+00:00", + "viewport": list(VIEWPORT), + "viewport_mode": "per-event", + "params": {}, + } + ) + ) + + with pytest.raises(ValueError, match="viewport changed during event 0"): + compile_recording(recording, tmp_path / "bundle", name="mid-resize") + + def test_per_event_viewport_must_match_retained_png(self, tmp_path: Path) -> None: + recording = tmp_path / "recording" + (recording / "frames").mkdir(parents=True) + write_frame(recording, 0, "before", blank()) + write_frame(recording, 0, "after", blank()) + (recording / "events.jsonl").write_text( + json.dumps( + { + "i": 0, + "kind": "key", + "key": "Enter", + "t": 1.0, + "viewport_before": [900, 600], + "viewport_after": [1280, 800], + } + ) + + "\n" + ) + (recording / "meta.json").write_text( + json.dumps( + { + "id": "mismatched-viewport", + "created_at": "2026-08-18T00:00:00+00:00", + "viewport": list(VIEWPORT), + "params": {}, + } + ) + ) + + with pytest.raises(ValueError, match="does not match the retained PNG"): + compile_recording(recording, tmp_path / "bundle", name="mismatch") + + def test_pointer_must_be_inside_event_viewport(self, tmp_path: Path) -> None: + recording = tmp_path / "recording" + (recording / "frames").mkdir(parents=True) + write_frame(recording, 0, "before", blank()) + write_frame(recording, 0, "after", blank()) + (recording / "events.jsonl").write_text( + json.dumps( + { + "i": 0, + "kind": "click", + "x": 1280, + "y": 50, + "t": 1.0, + "viewport_before": [1280, 800], + "viewport_after": [1280, 800], + } + ) + + "\n" + ) + (recording / "meta.json").write_text( + json.dumps( + { + "id": "outside-viewport", + "created_at": "2026-08-18T00:00:00+00:00", + "viewport": list(VIEWPORT), + "params": {}, + } + ) + ) + + with pytest.raises(ValueError, match="is outside viewport"): + compile_recording(recording, tmp_path / "bundle", name="outside") + def test_incomplete_frame_pair_fails_loud(self, tmp_path: Path) -> None: recording = tmp_path / "recording" (recording / "frames").mkdir(parents=True) diff --git a/tests/test_recorder.py b/tests/test_recorder.py index 89fbf993..2f0e0426 100644 --- a/tests/test_recorder.py +++ b/tests/test_recorder.py @@ -95,6 +95,8 @@ def test_recorder_writes_recording_format(tmp_path: Path) -> None: assert events[3]["text"] == "world" assert events[3]["param"] == "note" assert events[4]["key"] == "Enter" + assert all(e["viewport_before"] == [1280, 800] for e in events) + assert all(e["viewport_after"] == [1280, 800] for e in events) times = [e["t"] for e in events] assert times == sorted(times) assert all(t >= 0 for t in times) diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index bc1d855c..39547299 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -70,6 +70,12 @@ "/tests/test_reliability.py", } EPHEMERAL_BUILD_EXCLUDES = {"/.hypothesis"} +LOCAL_SENSITIVE_EXCLUDES = { + "/.openadapt-chrome-profile", + "/.openadapt-recording-partial-*", + "/**/.openadapt-chrome-profile", + "/**/.openadapt-recording-partial-*", +} GENERATED_BENCHMARK_EXCLUDES = { "/benchmark/**/api-delta-probe-*", "/benchmark/**/bundle-live*", @@ -139,11 +145,18 @@ def test_required_wheel_job_builds_and_validates_actual_sdist() -> None: def test_wheel_and_sdist_exclude_repository_only_evidence() -> None: pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text()) targets = pyproject["tool"]["hatch"]["build"]["targets"] + gitignore = set((ROOT / ".gitignore").read_text().splitlines()) assert SOURCE_BOUNDARY_EXCLUDES <= set(targets["wheel"]["exclude"]) assert SOURCE_BOUNDARY_EXCLUDES <= set(targets["sdist"]["exclude"]) assert EPHEMERAL_BUILD_EXCLUDES <= set(targets["wheel"]["exclude"]) assert EPHEMERAL_BUILD_EXCLUDES <= set(targets["sdist"]["exclude"]) + assert LOCAL_SENSITIVE_EXCLUDES <= set(targets["wheel"]["exclude"]) + assert LOCAL_SENSITIVE_EXCLUDES <= set(targets["sdist"]["exclude"]) + assert { + ".openadapt-chrome-profile/", + ".openadapt-recording-partial-*/", + } <= gitignore assert GENERATED_BENCHMARK_EXCLUDES <= set(targets["wheel"]["exclude"]) assert GENERATED_BENCHMARK_EXCLUDES <= set(targets["sdist"]["exclude"]) @@ -165,6 +178,21 @@ def test_wheel_and_sdist_exclude_repository_only_evidence() -> None: assert "tests/test_reliability.py" in REPOSITORY_ONLY_EVALUATION_EXACT_PATHS +def test_local_sensitive_exclusions_apply_at_nested_paths() -> None: + pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text()) + targets = pyproject["tool"]["hatch"]["build"]["targets"] + gitignore = set((ROOT / ".gitignore").read_text().splitlines()) + recursive = { + "/**/.openadapt-chrome-profile", + "/**/.openadapt-recording-partial-*", + } + + assert recursive <= set(targets["wheel"]["exclude"]) + assert recursive <= set(targets["sdist"]["exclude"]) + assert ".openadapt-chrome-profile/" in gitignore + assert ".openadapt-recording-partial-*/" in gitignore + + def test_public_source_tree_excludes_private_data_and_recipes(tmp_path: Path) -> None: allowed = tmp_path / "benchmark/reliability" allowed.mkdir(parents=True) @@ -573,6 +601,30 @@ def test_sdist_requires_mit_license_and_excludes_openimis_surface( validate_sdist_license_boundary(mixed) +def test_release_artifacts_refuse_local_browser_and_partial_recordings( + tmp_path: Path, +) -> None: + sdist_base = {*REQUIRED_SDIST_PATHS, "PKG-INFO"} + wheel_base = { + "openadapt_flow-1.0.dist-info/licenses/LICENSE", + "openadapt_flow-1.0.dist-info/METADATA", + } + sensitive_paths = ( + ".openadapt-chrome-profile/Cookies", + ".openadapt-recording-partial-session/events.jsonl", + ) + for index, sensitive in enumerate(sensitive_paths): + sdist = tmp_path / f"sensitive-{index}.tar.gz" + _write_sdist(sdist, {*sdist_base, sensitive}) + with pytest.raises(ValueError, match="browser profile, unpublished"): + validate_sdist_license_boundary(sdist) + + wheel = tmp_path / f"sensitive-{index}.whl" + _write_wheel(wheel, {*wheel_base, sensitive}) + with pytest.raises(ValueError, match="browser profile, unpublished"): + validate_wheel_license_boundary(wheel) + + def test_wheel_refuses_private_corpus_material(tmp_path: Path) -> None: """The wheel must reject private source-policy material.