From 62eb122ac2bf006799a2d2dd4deae62af52e6543 Mon Sep 17 00:00:00 2001 From: Adam Rowe Date: Fri, 16 Oct 2026 12:43:06 -0400 Subject: [PATCH 1/4] fix(scroll): refuse an unreadable capture before scroll top/bottom decides `scroll bottom` coerced a failed or sparse capture to an empty tree, which the edge analyzer reads as "no hidden content below", so it reported "Already at bottom" about a screen it never saw. Both scroll loops now run on one `runScrollPasses` loop in capture-kit: observe, refuse an unreadable observation, ask the caller's stop condition, then spend a pass. The daemon defines the refusal once (`scrollCaptureRefusal`, built on `sparseCaptureQuality`) and both `scroll top`/`scroll bottom` and `scroll --until` pass it in, so an edge scroll now fails with `scroll_edge_capture_unreadable` instead of an edge verdict. Fixes #2468 --- .../scroll-edge-state-passes.test.ts | 120 ++++++++++++++ .../src/snapshot/scroll-edge-state.ts | 128 +++++++++++---- .../__tests__/scroll-runtime.test.ts | 70 ++++++++ .../src/execution/scroll-capture-refusal.ts | 32 ++++ .../daemon/src/execution/scroll-runtime.ts | 36 ++-- packages/daemon/src/execution/scroll-until.ts | 154 +++++------------- 6 files changed, 382 insertions(+), 158 deletions(-) create mode 100644 packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-passes.test.ts create mode 100644 packages/daemon/src/execution/scroll-capture-refusal.ts diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-passes.test.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-passes.test.ts new file mode 100644 index 0000000000..5e2a0e355e --- /dev/null +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-passes.test.ts @@ -0,0 +1,120 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { runScrollEdgePasses, runScrollPasses } from '../scroll-edge-state.ts'; + +type Frame = { readable: boolean; done: boolean; fingerprint?: string }; + +const REFUSED = new AppError('COMMAND_FAILED', 'refused', { reason: 'refused' }); +const PASS_LIMIT = new AppError('COMMAND_FAILED', 'limit', { reason: 'limit' }); +const NO_PROGRESS = new AppError('COMMAND_FAILED', 'stuck', { reason: 'stuck' }); + +/** Drives the shared loop over scripted frames, counting what it asked for. */ +async function run(frames: Frame[], passLimit = 10) { + const calls = { observe: 0, shouldStop: 0, scroll: 0, settle: 0 }; + const outcome = runScrollPasses({ + observe: async () => frames[Math.min(calls.observe++, frames.length - 1)]!, + refuse: (frame) => (frame.readable ? undefined : REFUSED), + shouldStop: async (frame) => { + calls.shouldStop += 1; + return frame.done; + }, + passLimit, + passLimitError: () => PASS_LIMIT, + fingerprint: async (frame) => frame.fingerprint, + noProgressError: () => NO_PROGRESS, + scroll: async () => ({ pass: ++calls.scroll }), + settleAfterPass: async () => { + calls.settle += 1; + }, + }); + return { outcome, calls }; +} + +test('runScrollPasses: a stop on the first observation costs no gesture', async () => { + const { outcome, calls } = await run([{ readable: true, done: true }]); + assert.deepEqual(await outcome, { passes: 0, result: undefined }); + assert.equal(calls.scroll, 0); +}); + +test('runScrollPasses: scrolls and settles once per pass and reports the last gesture', async () => { + const { outcome, calls } = await run([ + { readable: true, done: false, fingerprint: 'a' }, + { readable: true, done: false, fingerprint: 'b' }, + { readable: true, done: true }, + ]); + assert.deepEqual(await outcome, { passes: 2, result: { pass: 2 } }); + assert.equal(calls.settle, 2); +}); + +test('runScrollPasses: refuses an unreadable observation before asking the stop condition', async () => { + const { outcome, calls } = await run([{ readable: false, done: true }]); + await assert.rejects(outcome, (error) => error === REFUSED); + assert.equal(calls.shouldStop, 0); + assert.equal(calls.scroll, 0); +}); + +test('runScrollPasses: refuses an observation that goes unreadable after a pass', async () => { + const { outcome, calls } = await run([ + { readable: true, done: false, fingerprint: 'a' }, + { readable: false, done: true }, + ]); + await assert.rejects(outcome, (error) => error === REFUSED); + assert.equal(calls.scroll, 1); +}); + +test('runScrollPasses: spends exactly its pass budget, then raises the caller error', async () => { + const budget = 3; + const frames = Array.from({ length: budget + 1 }, (_, index) => ({ + readable: true, + done: false, + fingerprint: `f${index}`, + })); + const { outcome, calls } = await run(frames, budget); + await assert.rejects(outcome, (error) => error === PASS_LIMIT); + assert.equal(calls.scroll, budget); +}); + +/** + * The budget is checked before the stuck window, so an explicit small budget reports the budget + * rather than no-progress (the order `scroll --until` documented before the loops were shared). + */ +test('runScrollPasses: a surface that never shifts stops as no-progress, the budget winning ties', async () => { + const stuck = { readable: true, done: false, fingerprint: 'same' }; + const unbounded = await run([stuck], 10); + await assert.rejects(unbounded.outcome, (error) => error === NO_PROGRESS); + assert.equal(unbounded.calls.scroll, 3); + + const bounded = await run([stuck], 3); + await assert.rejects(bounded.outcome, (error) => error === PASS_LIMIT); +}); + +test('runScrollPasses: an observation without a fingerprint never trips no-progress', async () => { + const { outcome, calls } = await run([{ readable: true, done: false }], 5); + await assert.rejects(outcome, (error) => error === PASS_LIMIT); + assert.equal(calls.scroll, 5); +}); + +test('runScrollEdgePasses: a refused capture raises the typed edge refusal before any gesture', async () => { + let scrolls = 0; + await assert.rejects( + runScrollEdgePasses({ + edge: 'bottom', + captureState: async () => ({ + refused: { reason: 'sparse-tree', detail: 'AX bridge unavailable' }, + }), + scroll: async () => { + scrolls += 1; + }, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_edge_capture_unreadable'); + assert.equal(error.details?.edge, 'bottom'); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + assert.match(error.message, /AX bridge unavailable/); + return true; + }, + ); + assert.equal(scrolls, 0); +}); diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index 240c1237f6..698c422e12 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -35,6 +35,16 @@ export type ScrollEdgeState = { fingerprint?: string; }; +/** + * Why a capture cannot answer a scroll loop's question at all. Never an outcome about the content: + * a failed or sparse read coerced to an empty tree is what the edge analyzer reads as "no hidden + * content", so it has to be refused before anything decides on it. + */ +export type ScrollCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string }; + +/** What one edge capture yields: an analyzed state, or the refusal that kept it from being read. */ +export type ScrollEdgeObservation = ScrollEdgeState | { refused: ScrollCaptureRefusal }; + export type ScrollEdgeTarget = { point?: Point; nodeIndex?: number; @@ -143,9 +153,24 @@ export function pushScrollSurfaceSignature( if (recentSignatures.length > window) recentSignatures.shift(); } -export async function runScrollEdgePasses(params: { - edge: ScrollEdge; - captureState: (scope?: string) => Promise; +/** + * The one scroll pass loop: observe, refuse an unreadable observation, ask the caller's stop + * condition, then spend a pass. `scroll top`/`scroll bottom` and `scroll --until` differ only in + * what stops them and what they call a failure, so every other rule lives here once: the refusal + * lands before the stop condition reads anything, the budget is checked before the stuck window so + * an explicit small budget still reports itself, and the first observation precedes the first + * gesture so an already-satisfied stop costs no scroll. + */ +export async function runScrollPasses(params: { + observe: () => Promise; + refuse: (observation: TObservation) => AppError | undefined; + /** True ends the loop with this pass count; a terminal outcome of its own throws instead. */ + shouldStop: (observation: TObservation, passes: number) => Promise; + passLimit: number; + passLimitError: (passes: number) => AppError; + /** The surface signature the stuck window tracks; `undefined` keeps a pass out of it. */ + fingerprint: (observation: TObservation) => Promise; + noProgressError: (passes: number) => AppError; scroll: () => Promise; /** * Waits for the previous fling to come to rest before the next capture. A rubber-banded scroll @@ -155,19 +180,58 @@ export async function runScrollEdgePasses(params: { */ settleAfterPass?: () => Promise; }): Promise<{ passes: number; result?: TResult }> { - const { edge, captureState, scroll, settleAfterPass = async () => {} } = params; - let state = await captureState(); - if (state.scope) { - state = await captureState(state.scope); - } - + const { observe, refuse, shouldStop, passLimit, scroll, settleAfterPass } = params; let passes = 0; let result: TResult | undefined; const recentSignatures: string[] = []; - pushScrollSurfaceSignature(recentSignatures, state.fingerprint, SCROLL_EDGE_STUCK_WINDOW); - while (state.canScroll) { - if (passes >= SCROLL_EDGE_PASS_LIMIT) { - throw new AppError( + while (true) { + const observation = await observe(); + const refusal = refuse(observation); + if (refusal) throw refusal; + if (await shouldStop(observation, passes)) return { passes, result }; + if (passes >= passLimit) throw params.passLimitError(passes); + pushScrollSurfaceSignature( + recentSignatures, + await params.fingerprint(observation), + SCROLL_EDGE_STUCK_WINDOW, + ); + if (scrollSurfaceIsStuck(recentSignatures)) throw params.noProgressError(passes); + result = await scroll(); + passes += 1; + await settleAfterPass?.(); + } +} + +export async function runScrollEdgePasses(params: { + edge: ScrollEdge; + captureState: (scope?: string) => Promise; + scroll: () => Promise; + settleAfterPass?: () => Promise; +}): Promise<{ passes: number; result?: TResult }> { + const { edge, captureState, scroll, settleAfterPass } = params; + // The first capture discovers the container; every later one reads the scope the previous state + // named, so the loop verifies against the scroller it decided on rather than the whole tree. + let scope: string | undefined; + let discovered = false; + const observe = async (): Promise => { + let observation = await captureState(scope); + if (!discovered) { + discovered = true; + if (!('refused' in observation) && observation.scope) { + observation = await captureState(observation.scope); + } + } + if (!('refused' in observation)) scope = observation.scope; + return observation; + }; + return await runScrollPasses({ + observe, + refuse: (observation) => + 'refused' in observation ? buildScrollEdgeCaptureError(edge, observation.refused) : undefined, + shouldStop: async (observation) => !('refused' in observation) && !observation.canScroll, + passLimit: SCROLL_EDGE_PASS_LIMIT, + passLimitError: (passes) => + new AppError( 'COMMAND_FAILED', `scroll ${edge} reached the safety limit before the snapshot showed the edge`, { @@ -176,21 +240,13 @@ export async function runScrollEdgePasses(params: { passes, hint: 'The scoped scroll container still reports hidden content. Run scroll --until to stop on the element you are after, or snapshot -i to inspect the current state.', }, - ); - } - - result = await scroll(); - passes += 1; - await settleAfterPass(); - state = await captureState(state.scope); - - pushScrollSurfaceSignature(recentSignatures, state.fingerprint, SCROLL_EDGE_STUCK_WINDOW); - if (state.canScroll && scrollSurfaceIsStuck(recentSignatures)) { - throw buildScrollEdgeNoProgressError(edge, passes); - } - } - - return { passes, result }; + ), + fingerprint: async (observation) => + 'refused' in observation ? undefined : observation.fingerprint, + noProgressError: (passes) => buildScrollEdgeNoProgressError(edge, passes), + scroll, + ...(settleAfterPass ? { settleAfterPass } : {}), + }); } /** @@ -278,6 +334,22 @@ export function scrollNoProgressHint( ); } +function buildScrollEdgeCaptureError(edge: ScrollEdge, refusal: ScrollCaptureRefusal): AppError { + return new AppError( + 'COMMAND_FAILED', + `scroll ${edge} could not read the screen: ${refusal.detail}`, + { + reason: 'scroll_edge_capture_unreadable', + edge, + captureRefusal: refusal.reason, + hint: + refusal.reason === 'no-capture' + ? 'Run snapshot -i to see whether the app is producing an accessibility tree at all, and retry once it does.' + : 'The accessibility tree came back sparse, so it cannot show whether content remains. Run screenshot, inspect the image, and navigate by coordinates until snapshot -i reports a full tree.', + }, + ); +} + function buildScrollEdgeVerificationError( edge: ScrollEdge, scope: string | undefined, diff --git a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts index b967ad239b..d0cec49d98 100644 --- a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts @@ -344,6 +344,76 @@ test('bound scroll bottom keeps scoped capture failures scoped', async () => { assert.equal(snapshotCount, 2); }); +/** + * A failed or sparse read used to coerce to an empty tree, which the edge analyzer reads as "no + * hidden content below", so `scroll bottom` answered "Already at bottom" about a screen it never + * saw (#2468). Each case counts gestures, so the refusal is proven to land before any scroll. + */ +test.each([ + { name: 'a capture with no tree', frame: { backend: 'xctest' }, refusal: 'no-capture' }, + { name: 'an empty tree', frame: { backend: 'xctest', nodes: [] }, refusal: 'no-capture' }, + { + name: 'a tree the backend calls sparse', + frame: { + ...makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }, + refusal: 'sparse-tree', + }, +])( + 'bound scroll bottom refuses $name instead of reporting the edge', + async ({ frame, refusal }) => { + let scrolls = 0; + await assert.rejects( + () => + runScroll( + ['bottom'], + {}, + { + scroll: async () => { + scrolls += 1; + return {}; + }, + captureSnapshot: async () => frame, + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'scroll_edge_capture_unreadable'); + assert.equal(error.details?.edge, 'bottom'); + assert.equal(error.details?.captureRefusal, refusal); + return true; + }, + ); + assert.equal(scrolls, 0); + }, +); + +test('bound scroll bottom refuses a capture that goes unreadable mid-loop', async () => { + let scrolls = 0; + await assert.rejects( + () => + runScroll( + ['bottom'], + {}, + { + scroll: async () => { + scrolls += 1; + return {}; + }, + captureSnapshot: async () => + scrolls === 0 + ? makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }) + : { backend: 'xctest', nodes: [] }, + }, + ), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_edge_capture_unreadable', + ); + assert.equal(scrolls, 1); +}); + function makeScrollSnapshot(options: { hiddenBelow: boolean; message: string }) { return { backend: 'xctest' as const, diff --git a/packages/daemon/src/execution/scroll-capture-refusal.ts b/packages/daemon/src/execution/scroll-capture-refusal.ts new file mode 100644 index 0000000000..9e5f59130b --- /dev/null +++ b/packages/daemon/src/execution/scroll-capture-refusal.ts @@ -0,0 +1,32 @@ +import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; +import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { sparseCaptureQuality } from '@agent-device/selectors/absence-observation'; +import type { ScrollCaptureRefusal } from '@agent-device/capture-kit/scroll-edge-state'; + +/** + * Whether a scroll capture can answer anything at all, for every scroll loop. Sparseness is the + * question absence assertions ask, answered by the same helper rather than a second definition of + * readable. Truncation is deliberately NOT refused: a truncated tree is real and readable with its + * tail missing, and refusing it would fail large screens where the answer is plainly in view. + */ +export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefusal | undefined { + const nodes = result.nodes; + if (nodes === undefined) { + return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; + } + if (nodes.length === 0) { + return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; + } + const sparse = sparseCaptureQuality({ + backend: result.backend as SnapshotState['backend'], + nodes: nodes as SnapshotNode[], + ...(result.quality ? { snapshotQuality: result.quality } : {}), + }); + if (sparse) { + return { + reason: 'sparse-tree', + detail: sparse.reason ?? 'the capture backend reported a sparse tree', + }; + } + return undefined; +} diff --git a/packages/daemon/src/execution/scroll-runtime.ts b/packages/daemon/src/execution/scroll-runtime.ts index 2f7fd959ac..b4247f28f0 100644 --- a/packages/daemon/src/execution/scroll-runtime.ts +++ b/packages/daemon/src/execution/scroll-runtime.ts @@ -26,9 +26,11 @@ import { runScrollEdgePasses, scrollSurfaceFingerprint, type ScrollEdge, - type ScrollEdgeState, + type ScrollEdgeObservation, } from '@agent-device/capture-kit/scroll-edge-state'; +import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; import { formatScrollUntilMessage, runScrollUntilVisible } from './scroll-until.ts'; +import { scrollCaptureRefusal } from './scroll-capture-refusal.ts'; import { publicPlatformString } from '@agent-device/kernel/device'; import type { CommandFlags } from '@agent-device/contracts/command'; import { withSuccessText } from '@agent-device/kernel/success-text'; @@ -295,7 +297,7 @@ function swipeEvidence(result: Record): ScrollSwipeEvidence { }; } -/** Repeats the pass while the verified state still moves; the capture needs no guard here. */ +/** Repeats the pass while the verified state still moves; the loop refuses an unreadable capture. */ async function executeEdgeScroll( runtime: BoundScrollEdge, edge: ScrollEdge, @@ -309,9 +311,9 @@ async function executeEdgeScroll( const edgeResult = await runScrollEdgePasses({ edge, captureState: async (stateScope) => { - const state = await captureEdgeState(runtime, edge, stateScope, context); - scope = state.scope ?? scope; - return state; + const observation = await captureEdgeState(runtime, edge, stateScope, context); + if (!('refused' in observation)) scope = observation.scope ?? scope; + return observation; }, scroll: async () => await scrollOnce(runtime, target, options, context), settleAfterPass: async () => { @@ -355,22 +357,30 @@ async function executeUntilScroll( ); } +/** + * One edge read, refused when the capture cannot show the screen. Refusal is decided on the capture + * the analyzer last read — after an empty scoped tree, that is the unscoped retry — so neither a + * failed read nor a sparse one reaches the edge verdict as "no hidden content" (#2468). + */ async function captureEdgeState( runtime: BoundScrollEdge, edge: ScrollEdge, scope: string | undefined, context: DaemonCommandContext, -): Promise { - return await captureScrollEdgeState({ +): Promise { + let lastCapture: SnapshotResult | undefined; + const state = await captureScrollEdgeState({ edge, scope, - captureNodes: async (snapshotScope) => - ( - await runtime.operations.captureSnapshot( - scrollCaptureInput(context, { scope: snapshotScope }), - ) - ).nodes ?? [], + captureNodes: async (snapshotScope) => { + lastCapture = await runtime.operations.captureSnapshot( + scrollCaptureInput(context, { scope: snapshotScope }), + ); + return lastCapture.nodes ?? []; + }, }); + const refused = lastCapture && scrollCaptureRefusal(lastCapture); + return refused ? { refused } : state; } /** The single lexical owner of the bound call (R53); the edge binding satisfies this shape too. */ diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index b33d6a1e27..98b8152499 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -2,21 +2,20 @@ import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import { evaluateIsPredicate } from '@agent-device/selectors'; -import { sparseCaptureQuality } from '@agent-device/selectors/absence-observation'; import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import { canScrollFurtherAtEdge, - pushScrollSurfaceSignature, + runScrollPasses, scrollNoProgressHint, scrollSurfaceFingerprint, - scrollSurfaceIsStuck, verticalEdgeFor, - type ScrollEdge, + type ScrollCaptureRefusal, } from '@agent-device/capture-kit/scroll-edge-state'; +import { scrollCaptureRefusal } from './scroll-capture-refusal.ts'; /** * Everything `scroll --until ` needs beyond the ordinary scroll: when a pass has arrived, @@ -35,15 +34,6 @@ import { */ export const SCROLL_UNTIL_PASS_LIMIT = 12; -/** - * Cap for the vertical stuck-signature window fed to `scrollSurfaceIsStuck`. Matches the capture-kit - * edge loop so both loops notice a stuck container after the same handful of non-advancing passes. - */ -const SCROLL_UNTIL_STUCK_WINDOW = 6; - -/** Why a capture cannot answer the `--until` question at all. Never an outcome about the content. */ -type ScrollUntilCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string }; - /** * Scrolls until the selector matches a node that is on screen. * @@ -65,87 +55,45 @@ export async function runScrollUntilVisible(params: { scroll: () => Promise; }): Promise<{ passes: number; result?: TResult }> { const { selector, direction, platform, capture, scroll } = params; - const passLimit = params.passLimit ?? SCROLL_UNTIL_PASS_LIMIT; const edge = verticalEdgeFor(direction); - let passes = 0; - let result: TResult | undefined; - const recentSignatures: string[] = []; - - let scrolling = false; + let scrolled = false; try { - while (true) { - const decision = await decideUntilPass({ - captured: await capture(), - selector, - direction, - platform, - edge, - passes, - passLimit, - recentSignatures, - }); - if (decision.visible) { - return { passes, ...(result === undefined ? {} : { result }) }; - } - scrolling = true; - result = await scroll(); - scrolling = false; - passes += 1; - } + const { passes, result } = await runScrollPasses({ + observe: capture, + refuse: (captured) => { + const refusal = scrollCaptureRefusal(captured); + return refusal ? scrollUntilCaptureError(direction, selector, refusal) : undefined; + }, + shouldStop: async (captured, passes) => { + const nodes = (captured.nodes ?? []) as SnapshotNode[]; + if (await isSelectorVisible(nodes, selector, platform)) return true; + if (edge && !(await canScrollFurtherAtEdge(nodes, edge))) { + throw scrollUntilNotFoundError(direction, selector, 'edge-reached', passes); + } + return false; + }, + passLimit: params.passLimit ?? SCROLL_UNTIL_PASS_LIMIT, + passLimitError: (passes) => + scrollUntilNotFoundError(direction, selector, 'pass-limit', passes), + // Vertical only: a stuck container revisits the same one or two signatures. A horizontal + // scroll has no analyzer to read a surface off, so it is bounded by the pass budget alone. + fingerprint: async (captured) => + edge + ? await scrollSurfaceFingerprint((captured.nodes ?? []) as SnapshotNode[], edge) + : undefined, + noProgressError: (passes) => scrollUntilNoProgressError(direction, selector, passes), + scroll: async () => { + scrolled = true; + return await scroll(); + }, + }); + return { passes, ...(result === undefined ? {} : { result }) }; } catch (error) { - if (passes === 0 && !scrolling && error instanceof AppError) { - throw discloseDispatch(error, 'no'); - } + if (!scrolled && error instanceof AppError) throw discloseDispatch(error, 'no'); throw error; } } -/** - * What one capture tells us about the `--until` loop, answered in the order that matters: refuse an - * unusable read, report an arrived target, then the two ways a pass cannot usefully continue. Every - * terminal condition raises its own typed error; a non-terminal pass returns `{ visible: false }` so - * the caller flings once more and asks again. The stuck window mutates in place so the loop keeps one - * running history across passes. - */ -async function decideUntilPass(params: { - captured: SnapshotResult; - selector: string; - direction: ScrollDirection; - platform: Platform | PublicPlatform; - edge: ScrollEdge | undefined; - passes: number; - passLimit: number; - recentSignatures: string[]; -}): Promise<{ visible: boolean }> { - const { captured, selector, direction, platform, edge, passes, passLimit, recentSignatures } = - params; - const refusal = captureRefusal(captured); - if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); - const nodes = (captured.nodes ?? []) as SnapshotNode[]; - if (await isSelectorVisible(nodes, selector, platform)) { - return { visible: true }; - } - if (edge && !(await canScrollFurtherAtEdge(nodes, edge))) { - throw scrollUntilNotFoundError(direction, selector, 'edge-reached', passes); - } - if (passes >= passLimit) { - throw scrollUntilNotFoundError(direction, selector, 'pass-limit', passes); - } - if (edge) { - // Vertical only: a stuck container revisits the same one or two signatures. Checked after the - // pass-limit branch so an explicit small budget still reports `scroll_until_pass_limit`. - pushScrollSurfaceSignature( - recentSignatures, - await scrollSurfaceFingerprint(nodes, edge), - SCROLL_UNTIL_STUCK_WINDOW, - ); - if (scrollSurfaceIsStuck(recentSignatures)) { - throw scrollUntilNoProgressError(direction, selector, passes); - } - } - return { visible: false }; -} - export function formatScrollUntilMessage( direction: ScrollDirection, selector: string, @@ -187,34 +135,6 @@ async function isSelectorVisible( ); } -/** - * Sparseness is the same question absence assertions ask, answered by the same helper rather than a - * second definition of readable. Truncation is deliberately NOT refused: a truncated tree is real - * and readable with its tail missing, and refusing it would fail large screens where the target is - * plainly in view. - */ -function captureRefusal(result: SnapshotResult): ScrollUntilCaptureRefusal | undefined { - const nodes = result.nodes; - if (nodes === undefined) { - return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; - } - if (nodes.length === 0) { - return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; - } - const sparse = sparseCaptureQuality({ - backend: result.backend as SnapshotState['backend'], - nodes: nodes as SnapshotNode[], - ...(result.quality ? { snapshotQuality: result.quality } : {}), - }); - if (sparse) { - return { - reason: 'sparse-tree', - detail: sparse.reason ?? 'the capture backend reported a sparse tree', - }; - } - return undefined; -} - /** The content ran out, or the budget did. Separate messages: the corrective action differs. */ function scrollUntilNotFoundError( direction: ScrollDirection, @@ -272,7 +192,7 @@ function scrollUntilNoProgressError( function scrollUntilCaptureError( direction: ScrollDirection, selector: string, - refusal: ScrollUntilCaptureRefusal, + refusal: ScrollCaptureRefusal, ): AppError { return new AppError( 'COMMAND_FAILED', From d6eb2a936d95a8362d15728c433e48a54a7e09b0 Mon Sep 17 00:00:00 2001 From: Adam Rowe Date: Fri, 16 Oct 2026 13:57:38 -0400 Subject: [PATCH 2/4] refactor(scroll): keep the backend's sparse reason and one scope owner Check sparse quality before the empty-tree refusal, so a backend that ships no nodes with a sparse verdict (the Android helper's presentation failure) reports its own reason rather than a generic no-capture. Read refused observations through one `isScrollEdgeRefused` guard, and hand the edge loop's scope to `settleAfterPass` so the daemon no longer tracks a second copy. --- .../src/snapshot/scroll-edge-state.ts | 29 ++++++++++++------- .../__tests__/scroll-runtime.test.ts | 11 +++++++ .../src/execution/scroll-capture-refusal.ts | 8 +++-- .../daemon/src/execution/scroll-runtime.ts | 11 ++----- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index 698c422e12..a6b7c529b9 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -45,6 +45,12 @@ export type ScrollCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detai /** What one edge capture yields: an analyzed state, or the refusal that kept it from being read. */ export type ScrollEdgeObservation = ScrollEdgeState | { refused: ScrollCaptureRefusal }; +function isScrollEdgeRefused( + observation: ScrollEdgeObservation, +): observation is { refused: ScrollCaptureRefusal } { + return 'refused' in observation; +} + export type ScrollEdgeTarget = { point?: Point; nodeIndex?: number; @@ -140,10 +146,10 @@ export function scrollSurfaceIsStuck(recentSignatures: readonly string[]): boole /** * Records one pass's surface signature, capped to the trailing window the stuck detector reads. An * empty signature is skipped so a container with no positioned descendants stays on its edge/pass - * signals instead of stacking blanks. Both scroll loops push through here so "one window" has one - * definition rather than a copy that can drift. + * signals instead of stacking blanks. `runScrollPasses` is its one caller, so every scroll loop + * reads the same window. */ -export function pushScrollSurfaceSignature( +function pushScrollSurfaceSignature( recentSignatures: string[], fingerprint: string | undefined, window: number, @@ -206,7 +212,8 @@ export async function runScrollEdgePasses(params: { edge: ScrollEdge; captureState: (scope?: string) => Promise; scroll: () => Promise; - settleAfterPass?: () => Promise; + /** The rest-wait, handed the scope the loop is deciding on so it watches the same container. */ + settleAfterPass?: (scope: string | undefined) => Promise; }): Promise<{ passes: number; result?: TResult }> { const { edge, captureState, scroll, settleAfterPass } = params; // The first capture discovers the container; every later one reads the scope the previous state @@ -217,18 +224,20 @@ export async function runScrollEdgePasses(params: { let observation = await captureState(scope); if (!discovered) { discovered = true; - if (!('refused' in observation) && observation.scope) { + if (!isScrollEdgeRefused(observation) && observation.scope) { observation = await captureState(observation.scope); } } - if (!('refused' in observation)) scope = observation.scope; + if (!isScrollEdgeRefused(observation)) scope = observation.scope; return observation; }; return await runScrollPasses({ observe, refuse: (observation) => - 'refused' in observation ? buildScrollEdgeCaptureError(edge, observation.refused) : undefined, - shouldStop: async (observation) => !('refused' in observation) && !observation.canScroll, + isScrollEdgeRefused(observation) + ? buildScrollEdgeCaptureError(edge, observation.refused) + : undefined, + shouldStop: async (observation) => !isScrollEdgeRefused(observation) && !observation.canScroll, passLimit: SCROLL_EDGE_PASS_LIMIT, passLimitError: (passes) => new AppError( @@ -242,10 +251,10 @@ export async function runScrollEdgePasses(params: { }, ), fingerprint: async (observation) => - 'refused' in observation ? undefined : observation.fingerprint, + isScrollEdgeRefused(observation) ? undefined : observation.fingerprint, noProgressError: (passes) => buildScrollEdgeNoProgressError(edge, passes), scroll, - ...(settleAfterPass ? { settleAfterPass } : {}), + ...(settleAfterPass ? { settleAfterPass: async () => await settleAfterPass(scope) } : {}), }); } diff --git a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts index d0cec49d98..b62d7702a9 100644 --- a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts @@ -360,6 +360,17 @@ test.each([ }, refusal: 'sparse-tree', }, + { + // The Android helper's presentation failure: no nodes, and a sparse verdict naming why. The + // backend's reason is the more useful answer, so it outranks the generic empty-tree refusal. + name: 'an empty tree the backend calls sparse', + frame: { + backend: 'xctest', + nodes: [], + quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }, + refusal: 'sparse-tree', + }, ])( 'bound scroll bottom refuses $name instead of reporting the edge', async ({ frame, refusal }) => { diff --git a/packages/daemon/src/execution/scroll-capture-refusal.ts b/packages/daemon/src/execution/scroll-capture-refusal.ts index 9e5f59130b..b7fa26276a 100644 --- a/packages/daemon/src/execution/scroll-capture-refusal.ts +++ b/packages/daemon/src/execution/scroll-capture-refusal.ts @@ -14,9 +14,8 @@ export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefus if (nodes === undefined) { return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; } - if (nodes.length === 0) { - return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; - } + // Sparseness first: a backend that names why its tree is unusable (the Android helper's + // presentation failure ships no nodes and a sparse verdict) gives the better answer. const sparse = sparseCaptureQuality({ backend: result.backend as SnapshotState['backend'], nodes: nodes as SnapshotNode[], @@ -28,5 +27,8 @@ export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefus detail: sparse.reason ?? 'the capture backend reported a sparse tree', }; } + if (nodes.length === 0) { + return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; + } return undefined; } diff --git a/packages/daemon/src/execution/scroll-runtime.ts b/packages/daemon/src/execution/scroll-runtime.ts index b4247f28f0..6a60fd79be 100644 --- a/packages/daemon/src/execution/scroll-runtime.ts +++ b/packages/daemon/src/execution/scroll-runtime.ts @@ -305,18 +305,11 @@ async function executeEdgeScroll( options: ResolvedScrollExecutionOptions, context: DaemonCommandContext, ): Promise> { - // The loop discovers its scope from the first capture; the rest-wait has to watch the same - // scoped container the loop decides on, so it reads the scope this closure records. - let scope: string | undefined; const edgeResult = await runScrollEdgePasses({ edge, - captureState: async (stateScope) => { - const observation = await captureEdgeState(runtime, edge, stateScope, context); - if (!('refused' in observation)) scope = observation.scope ?? scope; - return observation; - }, + captureState: async (scope) => await captureEdgeState(runtime, edge, scope, context), scroll: async () => await scrollOnce(runtime, target, options, context), - settleAfterPass: async () => { + settleAfterPass: async (scope) => { await pollForScrollRest( async () => (await runtime.operations.captureSnapshot(scrollCaptureInput(context, { scope }))) From c4f56112edf62a10326a224cb14512fae1fe10c3 Mon Sep 17 00:00:00 2001 From: Adam Rowe Date: Fri, 16 Oct 2026 14:18:37 -0400 Subject: [PATCH 3/4] fix(scroll): keep the capture refusal inside scroll-until The separate scroll-capture-refusal module added one eagerly evaluated module to every daemon entry, which the eager-closure budget rejects. Define the refusal in scroll-until.ts, which scroll-runtime already imports, so both loops share it at no extra load cost. --- .../src/execution/scroll-capture-refusal.ts | 34 ----------------- .../daemon/src/execution/scroll-runtime.ts | 7 +++- packages/daemon/src/execution/scroll-until.ts | 37 +++++++++++++++++-- 3 files changed, 39 insertions(+), 39 deletions(-) delete mode 100644 packages/daemon/src/execution/scroll-capture-refusal.ts diff --git a/packages/daemon/src/execution/scroll-capture-refusal.ts b/packages/daemon/src/execution/scroll-capture-refusal.ts deleted file mode 100644 index b7fa26276a..0000000000 --- a/packages/daemon/src/execution/scroll-capture-refusal.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; -import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; -import { sparseCaptureQuality } from '@agent-device/selectors/absence-observation'; -import type { ScrollCaptureRefusal } from '@agent-device/capture-kit/scroll-edge-state'; - -/** - * Whether a scroll capture can answer anything at all, for every scroll loop. Sparseness is the - * question absence assertions ask, answered by the same helper rather than a second definition of - * readable. Truncation is deliberately NOT refused: a truncated tree is real and readable with its - * tail missing, and refusing it would fail large screens where the answer is plainly in view. - */ -export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefusal | undefined { - const nodes = result.nodes; - if (nodes === undefined) { - return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; - } - // Sparseness first: a backend that names why its tree is unusable (the Android helper's - // presentation failure ships no nodes and a sparse verdict) gives the better answer. - const sparse = sparseCaptureQuality({ - backend: result.backend as SnapshotState['backend'], - nodes: nodes as SnapshotNode[], - ...(result.quality ? { snapshotQuality: result.quality } : {}), - }); - if (sparse) { - return { - reason: 'sparse-tree', - detail: sparse.reason ?? 'the capture backend reported a sparse tree', - }; - } - if (nodes.length === 0) { - return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; - } - return undefined; -} diff --git a/packages/daemon/src/execution/scroll-runtime.ts b/packages/daemon/src/execution/scroll-runtime.ts index 6a60fd79be..0716eadc0f 100644 --- a/packages/daemon/src/execution/scroll-runtime.ts +++ b/packages/daemon/src/execution/scroll-runtime.ts @@ -29,8 +29,11 @@ import { type ScrollEdgeObservation, } from '@agent-device/capture-kit/scroll-edge-state'; import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; -import { formatScrollUntilMessage, runScrollUntilVisible } from './scroll-until.ts'; -import { scrollCaptureRefusal } from './scroll-capture-refusal.ts'; +import { + formatScrollUntilMessage, + runScrollUntilVisible, + scrollCaptureRefusal, +} from './scroll-until.ts'; import { publicPlatformString } from '@agent-device/kernel/device'; import type { CommandFlags } from '@agent-device/contracts/command'; import { withSuccessText } from '@agent-device/kernel/success-text'; diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index 98b8152499..48a87c5fb4 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -2,7 +2,8 @@ import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { sparseCaptureQuality } from '@agent-device/selectors/absence-observation'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import { evaluateIsPredicate } from '@agent-device/selectors'; import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; @@ -15,7 +16,6 @@ import { verticalEdgeFor, type ScrollCaptureRefusal, } from '@agent-device/capture-kit/scroll-edge-state'; -import { scrollCaptureRefusal } from './scroll-capture-refusal.ts'; /** * Everything `scroll --until ` needs beyond the ordinary scroll: when a pass has arrived, @@ -23,7 +23,8 @@ import { scrollCaptureRefusal } from './scroll-capture-refusal.ts'; * * One module beside the route that runs it. `scroll` reaches a device in exactly one place (ADR * 0019, `scroll-runtime.ts`), so there is no second caller to keep in agreement and no reason for - * this to be a package surface. + * this to be a package surface. `scroll top`/`scroll bottom` borrow `scrollCaptureRefusal` from + * here so both loops share one definition of readable without loading another module. */ /** @@ -135,6 +136,36 @@ async function isSelectorVisible( ); } +/** + * Whether a scroll capture can answer anything at all, for every scroll loop. Sparseness is the + * question absence assertions ask, answered by the same helper rather than a second definition of + * readable. Truncation is deliberately NOT refused: a truncated tree is real and readable with its + * tail missing, and refusing it would fail large screens where the answer is plainly in view. + */ +export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefusal | undefined { + const nodes = result.nodes; + if (nodes === undefined) { + return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; + } + // Sparseness first: a backend that names why its tree is unusable (the Android helper's + // presentation failure ships no nodes and a sparse verdict) gives the better answer. + const sparse = sparseCaptureQuality({ + backend: result.backend as SnapshotState['backend'], + nodes: nodes as SnapshotNode[], + ...(result.quality ? { snapshotQuality: result.quality } : {}), + }); + if (sparse) { + return { + reason: 'sparse-tree', + detail: sparse.reason ?? 'the capture backend reported a sparse tree', + }; + } + if (nodes.length === 0) { + return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; + } + return undefined; +} + /** The content ran out, or the budget did. Separate messages: the corrective action differs. */ function scrollUntilNotFoundError( direction: ScrollDirection, From 640fd43d0e32c451fb92f1d983e54f0bdbe79e7d Mon Sep 17 00:00:00 2001 From: Adam Rowe Date: Fri, 16 Oct 2026 14:29:08 -0400 Subject: [PATCH 4/4] fix(scroll): keep the sparse reason when a capture omits its tree A capture with no nodes field and a sparse verdict reported a generic no-capture. Check sparse quality against an empty tree first, so the backend's reason survives whether the tree is empty or missing. --- .../daemon/src/execution/__tests__/scroll-runtime.test.ts | 8 ++++++++ packages/daemon/src/execution/scroll-until.ts | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts index b62d7702a9..0dd7118984 100644 --- a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts @@ -363,6 +363,14 @@ test.each([ { // The Android helper's presentation failure: no nodes, and a sparse verdict naming why. The // backend's reason is the more useful answer, so it outranks the generic empty-tree refusal. + name: 'a missing tree the backend calls sparse', + frame: { + backend: 'xctest', + quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }, + refusal: 'sparse-tree', + }, + { name: 'an empty tree the backend calls sparse', frame: { backend: 'xctest', diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index 48a87c5fb4..d309f8fe97 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -144,14 +144,11 @@ async function isSelectorVisible( */ export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefusal | undefined { const nodes = result.nodes; - if (nodes === undefined) { - return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; - } // Sparseness first: a backend that names why its tree is unusable (the Android helper's // presentation failure ships no nodes and a sparse verdict) gives the better answer. const sparse = sparseCaptureQuality({ backend: result.backend as SnapshotState['backend'], - nodes: nodes as SnapshotNode[], + nodes: (nodes ?? []) as SnapshotNode[], ...(result.quality ? { snapshotQuality: result.quality } : {}), }); if (sparse) { @@ -160,6 +157,9 @@ export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefus detail: sparse.reason ?? 'the capture backend reported a sparse tree', }; } + if (nodes === undefined) { + return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; + } if (nodes.length === 0) { return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; }