From 1029621c891dc55622a019904a8b7a42a6d48329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:29:49 +0200 Subject: [PATCH 01/10] feat(scroll): support explicit budgets for long target searches --- .../src/flag-definitions-action.ts | 12 +++ packages/contracts/src/cli-flags.ts | 1 + packages/contracts/src/client-gesture.ts | 2 + packages/contracts/src/request-envelope.ts | 1 + packages/contracts/src/scroll-command.ts | 23 +++++- packages/daemon/src/core/dispatch-context.ts | 1 + .../__tests__/scroll-runtime.test.ts | 76 +++++++++++++++++++ .../daemon/src/execution/scroll-runtime.ts | 2 + src/commands/command-flags.ts | 1 + src/commands/interaction/index.ts | 10 ++- src/commands/interaction/interactions.ts | 1 + src/commands/interaction/metadata.ts | 12 ++- src/commands/schema/cli-help.ts | 2 +- website/docs/docs/commands.md | 2 +- 14 files changed, 140 insertions(+), 6 deletions(-) diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 5634505506..4271b3acc4 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -1,5 +1,6 @@ import { RESPONSE_LEVELS } from '@agent-device/kernel/contracts'; import { RECORDING_SCOPE_VALUES } from '@agent-device/contracts/recording'; +import { SCROLL_UNTIL_MAX_PASSES } from '@agent-device/contracts/scroll-command'; import type { FlagDefinition } from './flag-types.ts'; export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ @@ -175,6 +176,17 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ // and fails on another. recorded: true, }, + { + key: 'maxPasses', + names: ['--max-passes'], + type: 'int', + min: 1, + max: SCROLL_UNTIL_MAX_PASSES, + usageLabel: '--max-passes ', + usageDescription: 'Scroll --until: maximum gestures (default 12, up to 200)', + projectConfig: false, + recorded: true, + }, { key: 'doubleTap', names: ['--double-tap'], diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index b5d5402c30..30cb73740f 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -114,6 +114,7 @@ export type CliFlags = CloudProviderProfileFields & pixels?: number; /** Scroll: repeat passes until this selector is visible on screen. */ until?: string; + maxPasses?: number; doubleTap?: boolean; verify?: boolean; settle?: boolean; diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index cc21f0a6d1..89c8d15d78 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -151,4 +151,6 @@ export type ScrollOptions = DeviceCommandBaseOptions & durationMs?: number; /** Repeat scroll passes until this selector is visible on screen, then stop. */ until?: string; + /** Maximum scroll gestures for an until search (1–200, default 12). */ + maxPasses?: number; }; diff --git a/packages/contracts/src/request-envelope.ts b/packages/contracts/src/request-envelope.ts index b111723465..2f11845678 100644 --- a/packages/contracts/src/request-envelope.ts +++ b/packages/contracts/src/request-envelope.ts @@ -41,6 +41,7 @@ export type CommandExecutionOptions = Partial & pixels?: number; /** Scroll: repeat passes until this selector is visible on screen. */ until?: string; + maxPasses?: number; doubleTap?: boolean; verify?: boolean; settle?: boolean; diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index 52e6f89fcb..dd13019a22 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -4,6 +4,7 @@ import type { ScrollDirection } from './scroll-gesture.ts'; import type { Rect } from '@agent-device/kernel/snapshot'; export const SCROLL_DURATION_MAX_MS = 10_000; +export const SCROLL_UNTIL_MAX_PASSES = 200; export const DEFAULT_MOBILE_SCROLL_DURATION_MS = 300; export const DEFAULT_IOS_SCROLL_DURATION_MS = 400; export const DEFAULT_IOS_SCROLL_AMOUNT = 0.65; @@ -83,8 +84,28 @@ export function assertExclusiveScrollDistanceInputs( * than resolved by precedence, so neither stop condition can silently win. */ export function assertScrollUntilCompatible( - input: Readonly<{ edge?: 'top' | 'bottom'; until?: string }>, + input: Readonly<{ edge?: 'top' | 'bottom'; until?: string; maxPasses?: number }>, ): void { + if (input.maxPasses !== undefined) { + if (input.until === undefined) { + throw new AppError('INVALID_ARGS', 'scroll --max-passes requires --until', { + reason: 'scroll_max_passes_requires_until', + }); + } + if ( + !Number.isInteger(input.maxPasses) || + input.maxPasses < 1 || + input.maxPasses > SCROLL_UNTIL_MAX_PASSES + ) { + throw new AppError( + 'INVALID_ARGS', + `scroll maxPasses must be an integer from 1 to ${SCROLL_UNTIL_MAX_PASSES}`, + { + reason: 'scroll_max_passes_invalid', + }, + ); + } + } if (input.until === undefined || input.edge === undefined) return; throw new AppError( 'INVALID_ARGS', diff --git a/packages/daemon/src/core/dispatch-context.ts b/packages/daemon/src/core/dispatch-context.ts index 0c143e4ccd..3625426184 100644 --- a/packages/daemon/src/core/dispatch-context.ts +++ b/packages/daemon/src/core/dispatch-context.ts @@ -50,6 +50,7 @@ export const DISPATCH_CONTEXT_FLAG_KEYS = [ 'jitterPx', 'pixels', 'until', + 'maxPasses', 'doubleTap', 'backMode', 'pauseMs', diff --git a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts index 4219f10ba7..cdf38ed4ac 100644 --- a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts @@ -570,6 +570,82 @@ test('bound scroll rejects --until on an edge direction before any device work', ); }); +test('bound scroll --until honors a larger explicit pass budget', async () => { + let scrolls = 0; + const result = await runScroll( + ['down'], + { until: 'label=Email', maxPasses: 14 }, + { + captureSnapshot: async () => ({ + nodes: [ + ...untilNodes(scrolls >= 13 ? 300 : 2400, true), + { + index: 3, + depth: 1, + parentIndex: 1, + type: 'Text', + label: `Page ${scrolls}`, + rect: { x: 0, y: 100, width: 200, height: 40 }, + }, + ], + }), + scroll: async () => { + scrolls += 1; + return { pixels: 480 }; + }, + }, + ); + assert.equal(result.passes, 13); + assert.equal(scrolls, 13); +}); + +test('bound scroll --until stops at a smaller explicit pass budget', async () => { + let scrolls = 0; + await assert.rejects( + () => + runScroll( + ['down'], + { until: 'label=Email', maxPasses: 2 }, + { + captureSnapshot: async () => ({ nodes: [...untilNodes(2400 - scrolls * 100, true)] }), + scroll: async () => { + scrolls += 1; + return {}; + }, + }, + ), + (error: unknown) => + error instanceof AppError && + error.details?.reason === 'scroll_until_pass_limit' && + error.details?.passes === 2, + ); + assert.equal(scrolls, 2); +}); + +test.each([ + { until: undefined, maxPasses: 2 }, + { until: 'label=Email', maxPasses: 0 }, + { until: 'label=Email', maxPasses: 201 }, + { until: 'label=Email', maxPasses: 1.5 }, +])('bound scroll rejects invalid pass budgets before device work: %j', async (flags) => { + let calls = 0; + await assert.rejects( + () => + runScroll(['down'], flags, { + captureSnapshot: async () => { + calls += 1; + return { nodes: untilNodes(300, true) }; + }, + scroll: async () => { + calls += 1; + return {}; + }, + }), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); + assert.equal(calls, 0); +}); + test('bound scroll --until is refused at admission when the owner declares no capture', async () => { const resolved = await resolveBoundScrollRuntime({ platformServices: daemonPlatformServicesFixture(), diff --git a/packages/daemon/src/execution/scroll-runtime.ts b/packages/daemon/src/execution/scroll-runtime.ts index e58671749b..b721f43c2e 100644 --- a/packages/daemon/src/execution/scroll-runtime.ts +++ b/packages/daemon/src/execution/scroll-runtime.ts @@ -125,6 +125,7 @@ export async function resolveBoundScrollRuntime( const stopCondition = { ...(target.edge === undefined ? {} : { edge: target.edge }), ...(until === undefined ? {} : { until }), + ...(params.context.maxPasses === undefined ? {} : { maxPasses: params.context.maxPasses }), }; assertScrollUntilCompatible(stopCondition); const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); @@ -351,6 +352,7 @@ async function executeUntilScroll( selector, direction: target.direction, platform: publicPlatformString(device), + passLimit: context.maxPasses, capture: async () => await runtime.operations.captureSnapshot(scrollCaptureInput(context)), scroll: async () => await scrollOnce(runtime, target, options, context), }); diff --git a/src/commands/command-flags.ts b/src/commands/command-flags.ts index e2c103d857..909b6fdbd0 100644 --- a/src/commands/command-flags.ts +++ b/src/commands/command-flags.ts @@ -90,6 +90,7 @@ function buildFlags(options: InternalRequestOptions): CommandFlags { jitterPx: options.jitterPx, pixels: options.pixels, until: options.until, + maxPasses: options.maxPasses, doubleTap: options.doubleTap, verify: options.verify, settle: options.settle, diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 21c635cb6e..5d281d198c 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -442,9 +442,15 @@ const interactionCliSchemas = { }, scroll: { usageOverride: 'scroll [amount]', - usageFlags: ['until', 'pixels', 'durationMs', 'settle'], + usageFlags: ['until', 'maxPasses', 'pixels', 'durationMs', 'settle'], positionalArgs: ['directionOrEdge', 'amount?'], - allowedFlags: ['pixels', 'durationMs', 'until', ...postActionObservationCliFlags('scroll')], + allowedFlags: [ + 'pixels', + 'durationMs', + 'until', + 'maxPasses', + ...postActionObservationCliFlags('scroll'), + ], }, } as const satisfies Record; diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 79154b99d1..2cade9f31a 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -105,6 +105,7 @@ export const interactionCliReaders = { pixels: flags.pixels, durationMs: flags.durationMs, until: flags.until, + maxPasses: flags.maxPasses, }), // The one observation-only reader in this file: `get` can be excluded from a // repair-armed heal by default, so it also takes the `--record` opt-in diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 0a2dcfedcd..0eb33fa38e 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -10,7 +10,10 @@ import { type TransformGesturePayload, readGesturePayload, } from '@agent-device/contracts/gesture-input'; -import { SCROLL_DURATION_MAX_MS } from '@agent-device/contracts/scroll-command'; +import { + SCROLL_DURATION_MAX_MS, + SCROLL_UNTIL_MAX_PASSES, +} from '@agent-device/contracts/scroll-command'; import { IS_PREDICATES } from '@agent-device/contracts/is-predicate'; import { SCROLL_DIRECTIONS, @@ -156,6 +159,13 @@ const scrollFields = { until: stringField( 'Repeat scroll passes until this selector is visible on screen, then stop. Not valid with the top/bottom edge directions, which carry their own stop condition.', ), + maxPasses: integerField( + 'Maximum scroll gestures for an until search (default 12). Requires until.', + { + min: 1, + max: SCROLL_UNTIL_MAX_PASSES, + }, + ), durationMs: integerField('Scroll duration in milliseconds when the backend supports pacing.', { min: 0, max: SCROLL_DURATION_MAX_MS, diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 84f2fbd017..f0cfa08a93 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -208,7 +208,7 @@ Shapes: agent-device gesture pinch 0.5 200 400 agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is 12 gestures. For a longer list, add --max-passes (1–200); it requires --until and does not extend request timeouts. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 462316cfda..bf6bf0dab8 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -495,7 +495,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' ### Scroll - `scroll [amount]` takes the finger path as a fraction of the viewport (`0.5` is half), or `--pixels ` for a fixed distance. Large distances are clamped. The app's scroll physics decide the final offset, so a scroll does not land on an exact position. -- `scroll --until ` repeats passes until the selector is on screen and fails if the content runs out first. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay & E2E](/docs/replay-e2e). +- `scroll --until ` repeats passes until the selector is on screen and fails if the content runs out first. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend request timeouts. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay & E2E](/docs/replay-e2e). - Default snapshot text summarizes off-screen interactive content instead of listing refs. `scroll --settle` (and `back --settle`) waits for the UI to go quiet and returns the diff with fresh refs, so you don't need another `snapshot -i`. Settle is best-effort and never fails the action. - On Android, settle ignores an allowlist of system-bar containers and shade/keyguard status headers; other SystemUI overlays remain content. Appium settle also applies package rules for keyboard and system chrome. - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. From 24d75b7ac57cd1daeebde20797bd840d72ec8dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:36:05 +0200 Subject: [PATCH 02/10] fix(scroll): preserve long searches in replay and honor cancellation --- .../src/internal/__tests__/script.test.ts | 16 +++++++++ .../ad-script/src/internal/script-utils.ts | 9 ++--- .../src/flag-definitions-action.ts | 7 ++-- packages/contracts/src/scroll-command.ts | 1 + .../__tests__/scroll-runtime.test.ts | 34 +++++++++++++++++++ .../daemon/src/execution/scroll-runtime.ts | 10 ++++-- .../daemon/src/execution/scroll-until.test.ts | 9 ++--- packages/daemon/src/execution/scroll-until.ts | 11 ++---- src/commands/interaction/interactions.test.ts | 11 ++++++ src/commands/interaction/metadata.ts | 3 +- src/commands/schema/cli-help.ts | 6 +++- 11 files changed, 92 insertions(+), 25 deletions(-) diff --git a/packages/ad-script/src/internal/__tests__/script.test.ts b/packages/ad-script/src/internal/__tests__/script.test.ts index b5220ea54b..40f44cbb61 100644 --- a/packages/ad-script/src/internal/__tests__/script.test.ts +++ b/packages/ad-script/src/internal/__tests__/script.test.ts @@ -224,6 +224,22 @@ test('scroll replay script keeps an amount positional beside the stop condition' assert.equal(parsed[0]?.flags.until, 'label=Email'); }); +test('scroll replay script preserves an explicit long-search budget', () => { + const actions: SessionAction[] = [ + { + ts: Date.now(), + command: 'scroll', + positionals: ['down'], + flags: { until: 'id="row-512"', maxPasses: 90 }, + }, + ]; + const script = formatReplayScriptForTest(actions); + assert.match(script, /--max-passes 90/); + const reparsed = parseReplayScriptDetailed(script).actions[0]; + assert.deepEqual(reparsed?.positionals, ['down']); + assert.deepEqual(reparsed?.flags, { until: 'id="row-512"', maxPasses: 90 }); +}); + test('a scroll --until value with spaces survives as one selector when quoted', () => { const parsed = parseReplayScriptDetailed( String.raw`scroll down --until "label=\"Sign in\" || label=\"Log in\""`, diff --git a/packages/ad-script/src/internal/script-utils.ts b/packages/ad-script/src/internal/script-utils.ts index dd45aa73b5..ebb9a74f3d 100644 --- a/packages/ad-script/src/internal/script-utils.ts +++ b/packages/ad-script/src/internal/script-utils.ts @@ -68,6 +68,7 @@ const TYPING_NUMERIC_FLAG_MAP = new Map([['--delay-ms', 'dela */ const SCROLL_SCRIPT_FLAG_MAP = new Map([ ['--until', { key: 'until', kind: 'string' }], + ['--max-passes', { key: 'maxPasses', kind: 'int' }], ]); /** @@ -89,7 +90,7 @@ const WAIT_SCRIPT_FLAG_MAP = new Map([ /** How one script flag token carries its value. */ type ScriptFlagEntry = { - key: 'until' | 'snapshotRaw' | 'snapshotDepth' | 'snapshotScope'; + key: 'until' | 'maxPasses' | 'snapshotRaw' | 'snapshotDepth' | 'snapshotScope'; kind: 'boolean' | 'int' | 'string'; }; @@ -384,12 +385,12 @@ export function appendGenericActionScriptArgs(parts: string[], action: SessionAc if (action.command === 'fold' && action.flags?.keyframes !== undefined) { parts.push('--keyframes', formatScriptArg(action.flags.keyframes)); } - // #3197: `scroll`'s stop condition is part of the step's meaning, so the writer - // emits it beside the parser that reads it back. Only `--until` is declared - // recorded, so only `--until` can arrive here on a recorded action. if (action.command === 'scroll' && typeof action.flags?.until === 'string') { parts.push('--until', formatScriptArg(action.flags.until)); } + if (action.command === 'scroll' && typeof action.flags?.maxPasses === 'number') { + parts.push('--max-passes', String(action.flags.maxPasses)); + } if (action.command === 'wait') { appendWaitSnapshotScriptFlags(parts, action.flags); } diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 4271b3acc4..07f5ceb5d1 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -1,6 +1,9 @@ import { RESPONSE_LEVELS } from '@agent-device/kernel/contracts'; import { RECORDING_SCOPE_VALUES } from '@agent-device/contracts/recording'; -import { SCROLL_UNTIL_MAX_PASSES } from '@agent-device/contracts/scroll-command'; +import { + DEFAULT_SCROLL_UNTIL_PASSES, + SCROLL_UNTIL_MAX_PASSES, +} from '@agent-device/contracts/scroll-command'; import type { FlagDefinition } from './flag-types.ts'; export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ @@ -183,7 +186,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 1, max: SCROLL_UNTIL_MAX_PASSES, usageLabel: '--max-passes ', - usageDescription: 'Scroll --until: maximum gestures (default 12, up to 200)', + usageDescription: `Scroll --until: maximum gestures (default ${DEFAULT_SCROLL_UNTIL_PASSES}, up to ${SCROLL_UNTIL_MAX_PASSES})`, projectConfig: false, recorded: true, }, diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index dd13019a22..08454bc30e 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -5,6 +5,7 @@ import type { Rect } from '@agent-device/kernel/snapshot'; export const SCROLL_DURATION_MAX_MS = 10_000; export const SCROLL_UNTIL_MAX_PASSES = 200; +export const DEFAULT_SCROLL_UNTIL_PASSES = 12; export const DEFAULT_MOBILE_SCROLL_DURATION_MS = 300; export const DEFAULT_IOS_SCROLL_DURATION_MS = 400; export const DEFAULT_IOS_SCROLL_AMOUNT = 0.65; diff --git a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts index cdf38ed4ac..ef4c4818cc 100644 --- a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts @@ -622,6 +622,40 @@ test('bound scroll --until stops at a smaller explicit pass budget', async () => assert.equal(scrolls, 2); }); +test.each(['capture', 'scroll'] as const)( + 'bound scroll --until stops after cancellation during %s', + async (phase) => { + const controller = new AbortController(); + const reason = new AppError('COMMAND_FAILED', 'request canceled', { + reason: 'request_canceled', + }); + let captures = 0; + let scrolls = 0; + await assert.rejects( + () => + runScroll( + ['down'], + { until: 'label=Email', maxPasses: 200, signal: controller.signal }, + { + captureSnapshot: async () => { + captures += 1; + if (phase === 'capture') controller.abort(reason); + return { nodes: untilNodes(2400 - scrolls * 100, true) }; + }, + scroll: async () => { + scrolls += 1; + controller.abort(reason); + return {}; + }, + }, + ), + (error: unknown) => error instanceof AppError && error.details?.reason === 'request_canceled', + ); + assert.equal(captures, 1); + assert.equal(scrolls, phase === 'capture' ? 0 : 1); + }, +); + test.each([ { until: undefined, maxPasses: 2 }, { until: 'label=Email', maxPasses: 0 }, diff --git a/packages/daemon/src/execution/scroll-runtime.ts b/packages/daemon/src/execution/scroll-runtime.ts index b721f43c2e..9298364fe0 100644 --- a/packages/daemon/src/execution/scroll-runtime.ts +++ b/packages/daemon/src/execution/scroll-runtime.ts @@ -353,8 +353,14 @@ async function executeUntilScroll( direction: target.direction, platform: publicPlatformString(device), passLimit: context.maxPasses, - capture: async () => await runtime.operations.captureSnapshot(scrollCaptureInput(context)), - scroll: async () => await scrollOnce(runtime, target, options, context), + capture: async () => { + context.signal?.throwIfAborted(); + return await runtime.operations.captureSnapshot(scrollCaptureInput(context)); + }, + scroll: async () => { + context.signal?.throwIfAborted(); + return await scrollOnce(runtime, target, options, context); + }, }); return withSuccessText( { diff --git a/packages/daemon/src/execution/scroll-until.test.ts b/packages/daemon/src/execution/scroll-until.test.ts index 411ae2caf0..4a3ae41311 100644 --- a/packages/daemon/src/execution/scroll-until.test.ts +++ b/packages/daemon/src/execution/scroll-until.test.ts @@ -3,11 +3,8 @@ import { test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; -import { - SCROLL_UNTIL_PASS_LIMIT, - formatScrollUntilMessage, - runScrollUntilVisible, -} from './scroll-until.ts'; +import { DEFAULT_SCROLL_UNTIL_PASSES } from '@agent-device/contracts/scroll-command'; +import { formatScrollUntilMessage, runScrollUntilVisible } from './scroll-until.ts'; /** The provenance every `SnapshotResult` carries; the fields under test are the rest. */ function capture(fields: Partial): SnapshotResult { @@ -246,7 +243,7 @@ test('the default budget is the shared constant', async () => { scroll: async () => ({}), }), (error: unknown) => - error instanceof AppError && error.details?.passes === SCROLL_UNTIL_PASS_LIMIT, + error instanceof AppError && error.details?.passes === DEFAULT_SCROLL_UNTIL_PASSES, ); }); diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index d309f8fe97..4286ea8347 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -1,5 +1,6 @@ import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; +import { DEFAULT_SCROLL_UNTIL_PASSES } from '@agent-device/contracts/scroll-command'; import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; @@ -27,14 +28,6 @@ import { * here so both loops share one definition of readable without loading another module. */ -/** - * How many gestures one `scroll --until` may spend before it gives up. A pass costs a capture plus - * a gesture, so this is the request's whole cost ceiling, not a retry budget: 12 passes at the - * honored 0.8-viewport maximum cover roughly ten screens, which is past the point where a list is - * better reached by `scroll bottom` or a search field. - */ -export const SCROLL_UNTIL_PASS_LIMIT = 12; - /** * Scrolls until the selector matches a node that is on screen. * @@ -73,7 +66,7 @@ export async function runScrollUntilVisible(params: { } return false; }, - passLimit: params.passLimit ?? SCROLL_UNTIL_PASS_LIMIT, + passLimit: params.passLimit ?? DEFAULT_SCROLL_UNTIL_PASSES, passLimitError: (passes) => scrollUntilNotFoundError(direction, selector, 'pass-limit', passes), // Vertical only: a stuck container revisits the same one or two signatures. A horizontal diff --git a/src/commands/interaction/interactions.test.ts b/src/commands/interaction/interactions.test.ts index 0f64f3dd85..6a4e107c29 100644 --- a/src/commands/interaction/interactions.test.ts +++ b/src/commands/interaction/interactions.test.ts @@ -3,9 +3,20 @@ import { interactionCliReaders, interactionDaemonWriters } from './interactions. import { selectorCliReaders } from './selectors.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { CliFlags } from '@agent-device/contracts/command'; +import { parseArgs } from '../../cli/parser/args.ts'; const BASE_FLAGS: CliFlags = { json: false, help: false, version: false }; +test('scroll carries the CLI long-search budget to the daemon', () => { + const parsed = parseArgs(['scroll', 'down', '--until', 'id="row-512"', '--max-passes', '90'], { + strictFlags: true, + }); + const input = interactionCliReaders.scroll(parsed.positionals, parsed.flags); + const request = interactionDaemonWriters.scroll(input); + expect(request.options.until).toBe('id="row-512"'); + expect(request.options.maxPasses).toBe(90); +}); + test('swipe writes only typed daemon input', () => { const request = interactionDaemonWriters.swipe({ from: { x: 10, y: 20 }, diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 0eb33fa38e..102e3c95de 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -11,6 +11,7 @@ import { readGesturePayload, } from '@agent-device/contracts/gesture-input'; import { + DEFAULT_SCROLL_UNTIL_PASSES, SCROLL_DURATION_MAX_MS, SCROLL_UNTIL_MAX_PASSES, } from '@agent-device/contracts/scroll-command'; @@ -160,7 +161,7 @@ const scrollFields = { 'Repeat scroll passes until this selector is visible on screen, then stop. Not valid with the top/bottom edge directions, which carry their own stop condition.', ), maxPasses: integerField( - 'Maximum scroll gestures for an until search (default 12). Requires until.', + `Maximum scroll gestures for an until search (default ${DEFAULT_SCROLL_UNTIL_PASSES}). Requires until.`, { min: 1, max: SCROLL_UNTIL_MAX_PASSES, diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index f0cfa08a93..f15acdbb8f 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -1,3 +1,7 @@ +import { + DEFAULT_SCROLL_UNTIL_PASSES, + SCROLL_UNTIL_MAX_PASSES, +} from '@agent-device/contracts/scroll-command'; import { listCliCommandNames } from '@agent-device/command-registry/catalog'; import { getReplayBackendDeclaration } from '@agent-device/replay-port/replay-backend-declarations'; import { normalizeCliCommandAlias } from '@agent-device/command-registry/cli-command-aliases'; @@ -208,7 +212,7 @@ Shapes: agent-device gesture pinch 0.5 200 400 agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is 12 gestures. For a longer list, add --max-passes (1–200); it requires --until and does not extend request timeouts. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until and does not extend request timeouts. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. From 31b34fb89aa903bc3c34931de0c6b4f8c0433ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:39:30 +0200 Subject: [PATCH 03/10] fix(scroll): bind pass cancellation to the live request --- packages/daemon/src/core/context.ts | 2 + .../__tests__/scroll-runtime.test.ts | 55 ++++++++++++------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/packages/daemon/src/core/context.ts b/packages/daemon/src/core/context.ts index 782fce235e..0748c89ee8 100644 --- a/packages/daemon/src/core/context.ts +++ b/packages/daemon/src/core/context.ts @@ -6,6 +6,7 @@ import { type ScreenshotRuntimeFlags, } from '@agent-device/contracts/capture'; import { getDiagnosticsMeta } from '@agent-device/host-kit/diagnostics'; +import { getRequestSignal } from '@agent-device/host-kit/request'; import { resolveRunnerLogicalLeaseContext } from './lease-context.ts'; import type { DaemonRequest } from './daemon-request.ts'; @@ -32,6 +33,7 @@ export function contextFromFlags( const effectiveRequestId = requestId ?? getDiagnosticsMeta().requestId; return { requestId: effectiveRequestId, + signal: getRequestSignal(effectiveRequestId), appBundleId, runnerLeaseContext: resolveRunnerLogicalLeaseContext({ meta }), logPath, diff --git a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts index ef4c4818cc..dbbfd4ac37 100644 --- a/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/scroll-runtime.test.ts @@ -14,7 +14,13 @@ import { resolveScrollRuntimePlan, type ScrollRuntimePlan, } from '@agent-device/contracts/platform-runtime-operations'; -import type { DaemonCommandContext } from '../../core/context.ts'; +import { contextFromFlags, type DaemonCommandContext } from '../../core/context.ts'; +import { + clearRequestAbortRegistration, + clearRequestCanceled, + markRequestCanceled, + registerRequestAbort, +} from '@agent-device/host-kit/request'; import { resolveBoundScrollRuntime } from '../scroll-runtime.ts'; import type { BindDeviceRuntime, @@ -625,34 +631,41 @@ test('bound scroll --until stops at a smaller explicit pass budget', async () => test.each(['capture', 'scroll'] as const)( 'bound scroll --until stops after cancellation during %s', async (phase) => { - const controller = new AbortController(); - const reason = new AppError('COMMAND_FAILED', 'request canceled', { - reason: 'request_canceled', - }); - let captures = 0; - let scrolls = 0; - await assert.rejects( - () => - runScroll( - ['down'], - { until: 'label=Email', maxPasses: 200, signal: controller.signal }, - { + const requestId = `scroll-until-canceled-${phase}`; + const registration = registerRequestAbort(requestId); + const context = contextFromFlags( + '/tmp/scroll-cancellation.log', + { until: 'label=Email', maxPasses: 200 }, + undefined, + undefined, + requestId, + ); + try { + let captures = 0; + let scrolls = 0; + await assert.rejects( + () => + runScroll(['down'], context, { captureSnapshot: async () => { captures += 1; - if (phase === 'capture') controller.abort(reason); + if (phase === 'capture') markRequestCanceled(requestId); return { nodes: untilNodes(2400 - scrolls * 100, true) }; }, scroll: async () => { scrolls += 1; - controller.abort(reason); + markRequestCanceled(requestId); return {}; }, - }, - ), - (error: unknown) => error instanceof AppError && error.details?.reason === 'request_canceled', - ); - assert.equal(captures, 1); - assert.equal(scrolls, phase === 'capture' ? 0 : 1); + }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'request_canceled', + ); + assert.equal(captures, 1); + assert.equal(scrolls, phase === 'capture' ? 0 : 1); + } finally { + clearRequestAbortRegistration(registration); + clearRequestCanceled(requestId); + } }, ); From 6aa73137e185287039343e078240c70ebd467a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:43:42 +0200 Subject: [PATCH 04/10] docs(evidence): record verified Android long-search gains --- ...ndroid-long-scroll-2026-10-19-results.json | 325 ++++++++++++++++++ .../android-long-scroll-2026-10-19.md | 75 ++++ 2 files changed, 400 insertions(+) create mode 100644 docs/evidence/android-long-scroll-2026-10-19-results.json create mode 100644 docs/evidence/android-long-scroll-2026-10-19.md diff --git a/docs/evidence/android-long-scroll-2026-10-19-results.json b/docs/evidence/android-long-scroll-2026-10-19-results.json new file mode 100644 index 0000000000..79ef07ba1a --- /dev/null +++ b/docs/evidence/android-long-scroll-2026-10-19-results.json @@ -0,0 +1,325 @@ +{ + "schemaVersion": 1, + "date": "2026-10-19", + "device": { + "name": "Pixel_7_CI", + "serial": "emulator-5554", + "androidApi": 36, + "helperVersion": "0.21.29-dev", + "backend": "android-helper" + }, + "apks": [ + { + "package": "dev.e2e.benchmark", + "sha256": "b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2" + }, + { + "package": "com.callstack.agentdevicelab", + "sha256": "4f332d39e76899b86e7006cda5d43ade6211733fe85ffe4552735e3e56c867b4" + } + ], + "profile": { + "kind": "historical full 42-case suite, not rerun for this change", + "startedAt": "2026-10-19T17:33:33.466Z", + "durationMs": 273989, + "operations": [ + { + "command": "snapshot", + "calls": 756, + "totalMs": 63493, + "medianMs": 53.0, + "maxMs": 592 + }, + { + "command": "open", + "calls": 43, + "totalMs": 38720, + "medianMs": 857, + "maxMs": 1853 + }, + { + "command": "scroll", + "calls": 69, + "totalMs": 34307, + "medianMs": 519, + "maxMs": 666 + }, + { + "command": "press", + "calls": 97, + "totalMs": 12901, + "medianMs": 63, + "maxMs": 469 + }, + { + "command": "fill", + "calls": 15, + "totalMs": 4635, + "medianMs": 254, + "maxMs": 506 + }, + { + "command": "longpress", + "calls": 3, + "totalMs": 3375, + "medianMs": 1253, + "maxMs": 1268 + } + ] + }, + "comparison": { + "order": "ABBA", + "includes": "target search, final snapshot and target press; app/menu setup and final semantic read excluded equally", + "target": "row-512; exact Found Row 0512 after press required for every successful sample", + "gestureDurationMs": 300, + "maxPasses": 90, + "samples": [ + { + "arm": "manual", + "sourceCommit": "507ef2376621d78aa76cf531bedc5abdf30e7a5a", + "workflowMs": 44850.218, + "passes": 55, + "exitCode": 0, + "startedAtUnix": 1792434973.6624231, + "finishedAtUnix": 1792435022.0873952, + "hostLoadBefore": [4.01806640625, 4.5986328125, 8.34912109375], + "hostLoadAfter": [4.53271484375, 4.56494140625, 8.13671875], + "rawSha256": "8482547a705c6a5f2fe54cf5584557e5d1b7e3810d9f87ede0affa3c4ccf3892", + "timings": [ + { + "name": "scroll", + "count": 55, + "sumMs": 28039.502, + "medianMs": 527.285 + }, + { + "name": "snapshot", + "count": 55, + "sumMs": 16714.394, + "medianMs": 300.552 + }, + { + "name": "targetPress", + "count": 1, + "sumMs": 96.322, + "medianMs": 96.322 + } + ] + }, + { + "arm": "until", + "sourceCommit": "b5f7e70590", + "workflowMs": 27529.686, + "passes": 55, + "exitCode": 0, + "startedAtUnix": 1792435022.087788, + "finishedAtUnix": 1792435053.1479082, + "hostLoadBefore": [4.53271484375, 4.56494140625, 8.13671875], + "hostLoadAfter": [5.7763671875, 4.85986328125, 8.119140625], + "rawSha256": "0e9dcbf186ec7c0e5623c6f78ed5f17888031fee5b8dfef384851f4034b37584", + "timings": [ + { + "name": "scrollUntil", + "ms": 27118.897459 + }, + { + "name": "snapshot", + "ms": 293.46962500000154 + }, + { + "name": "targetPress", + "ms": 117.31879200000185 + } + ] + }, + { + "arm": "until", + "sourceCommit": "b5f7e70590", + "workflowMs": 27632.46, + "passes": 55, + "exitCode": 0, + "startedAtUnix": 1792435053.148298, + "finishedAtUnix": 1792435083.249852, + "hostLoadBefore": [5.7763671875, 4.85986328125, 8.119140625], + "hostLoadAfter": [6.412109375, 5.16015625, 8.11767578125], + "rawSha256": "1e3bb9c4ac8ec92a8aaf066a444d2a493ac8afd89b5c22aae741035f643c54c7", + "timings": [ + { + "name": "scrollUntil", + "ms": 27220.094417 + }, + { + "name": "snapshot", + "ms": 331.74420800000007 + }, + { + "name": "targetPress", + "ms": 80.6217080000024 + } + ] + }, + { + "arm": "manual", + "sourceCommit": "507ef2376621d78aa76cf531bedc5abdf30e7a5a", + "workflowMs": 44170.108, + "passes": 55, + "exitCode": 0, + "startedAtUnix": 1792435083.25021, + "finishedAtUnix": 1792435129.74676, + "hostLoadBefore": [6.412109375, 5.16015625, 8.11767578125], + "hostLoadAfter": [6.4677734375, 5.40185546875, 8.03857421875], + "rawSha256": "4acf858d3c9341e8eec747d44ed5310e13148c6199cc53c61d63fe13964c2828", + "timings": [ + { + "name": "scroll", + "count": 55, + "sumMs": 27539.891, + "medianMs": 516.871 + }, + { + "name": "snapshot", + "count": 55, + "sumMs": 16526.446, + "medianMs": 290.709 + }, + { + "name": "targetPress", + "count": 1, + "sumMs": 103.772, + "medianMs": 103.772 + } + ] + } + ], + "medianMs": { + "manual": 44510.163, + "until": 27581.073 + }, + "savedMs": 16929.09, + "reductionPercent": 38.03421254602011 + }, + "finalSourceVerification": { + "sourceCommit": "ce71f344a3", + "benchmarkRecording": { + "workflowMs": 27941.681124, + "maxPassesPreserved": 90, + "target": "Found Row 0512" + }, + "replay": { + "replayed": 6, + "healed": 0, + "session": "android-scroll-profile", + "sessionActive": true, + "artifactPaths": [], + "snapshotDiagnostics": { + "stats": { + "count": 7, + "p50Ms": 57, + "p95Ms": 272, + "maxMs": 272, + "slowThresholdMs": 1500, + "platform": "android", + "backends": { + "android": 7 + } + } + }, + "message": "Replayed 6 steps in 33.5s This session's daemon was kept alive because its script left the session active; pass --session android-scroll-profile on your next command to reach it.", + "hint": "This session's daemon was kept alive because its script left the session active; pass --session android-scroll-profile on your next command to reach it." + }, + "testApp": [ + { + "arm": "manual", + "elapsedMs": 5926.68275, + "passes": 7, + "title": "Seasonal Footer Pick", + "androidSnapshot": { + "backend": "android-helper", + "pixelDensity": 2.625, + "helperVersion": "0.21.29-dev", + "helperApiVersion": "2", + "helperTransport": "persistent-session", + "helperSessionReused": false, + "installReason": "current", + "waitForIdleTimeoutMs": 500, + "waitForIdleQuietMs": 100, + "timeoutMs": 2000, + "maxDepth": 128, + "maxNodes": 5000, + "rootPresent": true, + "cacheInvalidated": true, + "captureMode": "interactive-windows", + "windowCount": 2, + "missingRootWindowTypes": [], + "nodeCount": 267, + "helperTruncated": false, + "elapsedMs": 142 + } + }, + { + "arm": "until", + "elapsedMs": 4027.1003330000003, + "passes": 7, + "title": "Seasonal Footer Pick", + "result": { + "direction": "down", + "until": "id=\"details-seasonal-footer\"", + "passes": 7, + "x1": 540, + "y1": 1920, + "x2": 540, + "y2": 480, + "referenceWidth": 1080, + "referenceHeight": 2400, + "pixels": 1440, + "backend": "android-helper", + "helperVersion": "0.21.29-dev", + "installReason": "current", + "helperKind": "swipe", + "helperApiVersion": "2", + "injectedEvents": 21, + "elapsedMs": 351, + "helperTransport": "persistent-session", + "message": "Scrolled down 7 passes until id=\"details-seasonal-footer\" was visible" + }, + "androidSnapshot": { + "backend": "android-helper", + "pixelDensity": 2.625, + "helperVersion": "0.21.29-dev", + "helperApiVersion": "2", + "helperTransport": "persistent-session", + "helperSessionReused": false, + "installReason": "current", + "waitForIdleTimeoutMs": 500, + "waitForIdleQuietMs": 100, + "timeoutMs": 2000, + "maxDepth": 128, + "maxNodes": 5000, + "rootPresent": true, + "cacheInvalidated": true, + "captureMode": "interactive-windows", + "windowCount": 2, + "missingRootWindowTypes": [], + "nodeCount": 267, + "helperTruncated": false, + "elapsedMs": 140 + } + }, + { + "arm": "one-pass-refusal", + "error": { + "code": "COMMAND_FAILED", + "message": "scroll down spent its 1-pass budget without id=\"details-seasonal-footer\" becoming visible", + "reason": "scroll_until_pass_limit", + "passes": 1 + } + } + ] + }, + "rejectedDurationExperiment": { + "orderMs": [300, 150, 150, 300], + "passes": [55, 61, 61, 55], + "workflowMs": [47462.889248, 48826.80208400002, 45803.86145000001, 44127.14453900001], + "decision": "Shorter swipes reduced single-command latency but made less progress and did not improve this workflow." + } +} diff --git a/docs/evidence/android-long-scroll-2026-10-19.md b/docs/evidence/android-long-scroll-2026-10-19.md new file mode 100644 index 0000000000..d295cc063f --- /dev/null +++ b/docs/evidence/android-long-scroll-2026-10-19.md @@ -0,0 +1,75 @@ +# Android long-list search experiment, 2026-10-19 + +The largest actionable cost in the retained Android suite is repeated list search. An explicit longer budget lets the existing verified `scroll --until` loop reach row 512 in one request. The controlled workflow median fell from 44.510 s to 27.581 s: 16.929 s saved, or 38.0%. Both routes used 55 gestures and activated the exact target. + +This is an opt-in workflow improvement. Separate `scroll` and `snapshot` requests retain their behavior. The external e2e adapter must adopt the loop to obtain this gain. This experiment does not establish a new 42-case suite total or a comparison against the experimental e2e engine. + +[Machine-readable results](android-long-scroll-2026-10-19-results.json) include samples, APK digests, source revisions, host loads, helper metadata, and correctness results. + +## Where the suite spent time + +The retained 42-case run began at 17:33:33 UTC and passed in 273.989 s. These request totals are attribution from that earlier run, not measurements of the new head: + +| Operation | Calls | Total | Median | +| --- | ---: | ---: | ---: | +| snapshot | 756 | 63.493 s | 53 ms | +| open | 43 | 38.720 s | 857 ms | +| scroll | 69 | 34.307 s | 519 ms | +| press | 97 | 12.901 s | 63 ms | +| fill | 15 | 4.635 s | 254 ms | +| longpress | 3 | 3.375 s | 1,253 ms | + +These are request totals, not additive explanations of the entire run. Adapter checks, fixture setup and authored waits also contribute. Steady snapshots and taps are usually below the 100 ms optimization threshold. The expensive path combines a roughly half-second gesture command with another stabilized snapshot after every page. A required 1,200 ms long press and the e2e negative-assertion grace interval are authored semantics, not latency to remove. + +Temporary native phase instrumentation attributed scroll cost to acknowledged input delivery, the gesture duration, idle observation and repeated post-gesture capture. Viewport discovery was usually zero milliseconds and at most 24 ms in that sample. Instrumented timings are diagnostic only; all instrumentation was removed before the controlled comparison. + +## Controlled comparison + +Pixel_7_CI, Android API 36, emulator-5554, Android helper 0.21.29-dev. No task-owned builds or tests ran beside timing. Host load was retained for each sample. The installed APK hashes were freshly verified by pulling the installed packages. + +Base: `507ef2376621d78aa76cf531bedc5abdf30e7a5a`. Feature timing head: `b5f7e70590`. The later request-signal wiring fix at `ce71f344a3` was verified separately below. + +| Order | Route | Gestures | Search, snapshot and target press | +| --- | --- | ---: | ---: | +| A | separate public SDK scroll/snapshot requests, base | 55 | 44.850 s | +| B | public SDK scroll with until and maxPasses 90 | 55 | 27.530 s | +| B | public SDK scroll with until and maxPasses 90 | 55 | 27.632 s | +| A | separate public SDK scroll/snapshot requests, base | 55 | 44.170 s | + +Each run relaunched `dev.e2e.benchmark`, opened Huge Virtualized List, searched for `row-512`, pressed its fresh ref and asserted `Found Row 0512`. App/menu setup and the final semantic read are outside both timing intervals. The external loop also checked that observed row ranges overlap between pages. All four processes exited zero; no failed or setup-only run enters the table. + +The loop reuses each fresh capture to decide target visibility, readable capture, end of content and no progress. It avoids the extra public per-page snapshot/stabilization path. It does not omit the loop's safety observations or speed up native input delivery. Gesture duration remains 300 ms and input acknowledgements are unchanged. + +The changed public call is: + +```js +await client.interactions.scroll({ + direction: 'down', + until: 'id="row-512"', + maxPasses: 90, +}); +``` + +The comparison route repeatedly calls `client.interactions.scroll({ direction: 'down' })` and `client.capture.snapshot({ interactiveOnly: true })` until that snapshot contains `row-512`. Both routes take a final interactive snapshot and press the target with that snapshot's generation-pinned ref. This reproduces the request boundary under comparison without including app launch. + +## Rejected primitive changes + +A 300/150/150/300 ms ABBA experiment made shorter individual gestures faster, but the shorter gestures needed 61 passes instead of 55. Whole-search medians were 45.795 s at 300 ms and 47.315 s at 150 ms. All four exact-target assertions passed. The default duration was therefore retained. + +An asynchronous intermediate-MOVE prototype did not show a reliable workflow gain. A controlled release-tail prototype moved cost into the following capture. Neither is included in the change. There is no claim that unrelated host load alone caused their results. + +## Safety and final-source verification + +The default budget stays at 12 gestures. Explicit budgets accept integers from 1 to 200 and require an until selector. Request timeouts remain independent and can end a search before it spends its gesture budget. Cancellation reads the real registered request signal through context construction and checks it before the next capture or gesture. + +At `ce71f344a3`: + +- The live benchmark search, final snapshot and correct target press took 27.942 s. Recording preserved `--max-passes 90`. +- The recorded `.ad` replay passed all six steps in 33.5 s with zero healing. The harness closed its surviving session. +- The installed Release test-app opened its catalog, reached `details-seasonal-footer`, pressed it and read the exact product name `Seasonal Footer Pick`. Both routes used seven gestures. Separate requests took 5.927 s; the loop took 4.027 s. This single pair corroborates behavior and is not a statistical estimate. +- A test-app search with maxPasses 1 failed with typed `scroll_until_pass_limit` and passes 1. +- Live test-app snapshots reported `android-helper`, version 0.21.29-dev, with invalidated accessibility cache. + +The new runtime cases failed on unchanged base: six failures covering a budget above 12, a smaller budget and invalid inputs. Recording admission and round-trip tests failed twice before the recording fix. Cancellation tests that use the real context builder and request registry failed twice before request-signal wiring, then passed. Focused recording/scroll/visibility tests passed 141 cases; the final context/socket/HTTP/scroll group passed 50 cases. These overlapping groups are not summed. + +The pass loop's sparse-capture refusal, edge detection, no-progress detection and visibility predicates remain in place. No gesture retry or recovery tap was added. Live verification covers the local Android helper, not Appium, HarmonyOS or Apple devices. The serialized affected gate and hosted CI remain separate validation obligations. From 691f36f664396f6d226e7a098c2bed7afd3bc239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:45:07 +0200 Subject: [PATCH 05/10] refactor(help): give gesture guidance its own schema module --- src/commands/schema/cli-help-gestures.ts | 41 +++++++++++++++++++++++ src/commands/schema/cli-help.ts | 42 ++---------------------- 2 files changed, 43 insertions(+), 40 deletions(-) create mode 100644 src/commands/schema/cli-help-gestures.ts diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts new file mode 100644 index 0000000000..c84fa57993 --- /dev/null +++ b/src/commands/schema/cli-help-gestures.ts @@ -0,0 +1,41 @@ +import { + DEFAULT_SCROLL_UNTIL_PASSES, + SCROLL_UNTIL_MAX_PASSES, +} from '@agent-device/contracts/scroll-command'; + +export const gestureHelpTopic = { + summary: 'Full multi-touch gesture shapes and platform quirks', + body: `agent-device help gestures + +Full command shapes and platform quirks for touch/pointer gestures beyond scroll/swipe. Read this when a task needs multi-touch, a repeated gesture series, or exact per-platform verification. + +Shapes: + agent-device longpress 300 500 800 + agent-device longpress @e12 800 + agent-device swipe 320 500 40 500 --count 8 --pause-ms 30 --pattern ping-pong + agent-device gesture pan 200 420 0 -80 500 + agent-device gesture pan 200 420 80 -40 700 --pointer-count 2 + agent-device scroll down --until 'id=submit' + agent-device gesture fling right 200 420 180 + agent-device gesture pinch 0.5 200 400 + agent-device gesture rotate 35 200 420 + agent-device gesture transform 200 420 80 -40 2 35 700 + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until and does not extend request timeouts. + longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. + For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. + +Platform quirks: + iOS transform/pinch/rotate use private XCTest synthesis for a continuous two-finger pan/scale/rotation path; verify app metrics instead of assuming requested values map exactly to recognizer output. + Android transform injects a geometric two-finger path; app recognizers may report non-exact pan/scale/rotation -- verify semantic app state or coarse per-component effects instead of exact numeric deltas unless the app exposes stable metrics. If Android needs exact values, prefer isolated gesture pan --pointer-count 2, gesture pinch, or gesture rotate over one combined transform: + agent-device gesture transform 200 420 80 -40 2 35 700 --platform android + agent-device wait text "pan changed yes" 3000 --platform android + tvOS coordinate pan and fling preserve only the dominant direction as a remote swipe; authored endpoints and duration are not preserved. + Gesture planning prefers the active-app frame; a backend without a gesture viewport resolver falls back to the visible snapshot union, which can be less accurate near edges. + Rare iOS accessibility gap: a row shown disabled/hittable:false where press reports success but no UI change, or a collapsed composite control with no child refs -- run snapshot -i --json, compute the target center from rects, press x y, then diff snapshot -i. Coordinates are fallback-only; document why you used them. + +macOS: + Context menus are secondary clicks, not long presses: agent-device click @e66 --button secondary --platform macos, then snapshot -i. + For fast desktop list traversal, prefer fixed pixel wheel steps and batch them when no snapshot is needed between passes: + agent-device scroll down --pixels 200 --duration-ms 50 --platform macos + agent-device batch --steps '[{"command":"scroll","input":{"direction":"down","pixels":200,"durationMs":50}},{"command":"scroll","input":{"direction":"down","pixels":200,"durationMs":50}}]' --platform macos`, +}; diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index f15acdbb8f..a51b85ef8f 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -1,7 +1,4 @@ -import { - DEFAULT_SCROLL_UNTIL_PASSES, - SCROLL_UNTIL_MAX_PASSES, -} from '@agent-device/contracts/scroll-command'; +import { gestureHelpTopic } from './cli-help-gestures.ts'; import { listCliCommandNames } from '@agent-device/command-registry/catalog'; import { getReplayBackendDeclaration } from '@agent-device/replay-port/replay-backend-declarations'; import { normalizeCliCommandAlias } from '@agent-device/command-registry/cli-command-aliases'; @@ -195,42 +192,7 @@ Recording: record contact-sheet [--out ] reads an exported MP4 back and writes one PNG holding the frames where the screen visibly changed, each labeled with its elapsed time. It is how you read a recording you cannot play, and it needs no session or device; the frames come out of that file, so the sheet cannot describe a screen the video does not contain. The sample grid is bounded and spread over the whole clip, so it reports coverage, not a review: a flash that opens and closes between two sample times is not in it. Needs a macOS host and an MP4 (a WebM recording is refused). Tracing: trace start ./trace.log, trace stop ./trace.log (path is positional, not --path).`, }, - gestures: { - summary: 'Full multi-touch gesture shapes and platform quirks', - body: `agent-device help gestures - -Full command shapes and platform quirks for touch/pointer gestures beyond scroll/swipe. Read this when a task needs multi-touch, a repeated gesture series, or exact per-platform verification. - -Shapes: - agent-device longpress 300 500 800 - agent-device longpress @e12 800 - agent-device swipe 320 500 40 500 --count 8 --pause-ms 30 --pattern ping-pong - agent-device gesture pan 200 420 0 -80 500 - agent-device gesture pan 200 420 80 -40 700 --pointer-count 2 - agent-device scroll down --until 'id=submit' - agent-device gesture fling right 200 420 180 - agent-device gesture pinch 0.5 200 400 - agent-device gesture rotate 35 200 420 - agent-device gesture transform 200 420 80 -40 2 35 700 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until and does not extend request timeouts. - longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. - For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. - -Platform quirks: - iOS transform/pinch/rotate use private XCTest synthesis for a continuous two-finger pan/scale/rotation path; verify app metrics instead of assuming requested values map exactly to recognizer output. - Android transform injects a geometric two-finger path; app recognizers may report non-exact pan/scale/rotation -- verify semantic app state or coarse per-component effects instead of exact numeric deltas unless the app exposes stable metrics. If Android needs exact values, prefer isolated gesture pan --pointer-count 2, gesture pinch, or gesture rotate over one combined transform: - agent-device gesture transform 200 420 80 -40 2 35 700 --platform android - agent-device wait text "pan changed yes" 3000 --platform android - tvOS coordinate pan and fling preserve only the dominant direction as a remote swipe; authored endpoints and duration are not preserved. - Gesture planning prefers the active-app frame; a backend without a gesture viewport resolver falls back to the visible snapshot union, which can be less accurate near edges. - Rare iOS accessibility gap: a row shown disabled/hittable:false where press reports success but no UI change, or a collapsed composite control with no child refs -- run snapshot -i --json, compute the target center from rects, press x y, then diff snapshot -i. Coordinates are fallback-only; document why you used them. - -macOS: - Context menus are secondary clicks, not long presses: agent-device click @e66 --button secondary --platform macos, then snapshot -i. - For fast desktop list traversal, prefer fixed pixel wheel steps and batch them when no snapshot is needed between passes: - agent-device scroll down --pixels 200 --duration-ms 50 --platform macos - agent-device batch --steps '[{"command":"scroll","input":{"direction":"down","pixels":200,"durationMs":50}},{"command":"scroll","input":{"direction":"down","pixels":200,"durationMs":50}}]' --platform macos`, - }, + gestures: gestureHelpTopic, tv: { summary: 'Android TV, tvOS, and Vega VVD focus-first remote navigation', body: `agent-device help tv From 84523341f147818a421ab62e4704f76d24220599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:51:15 +0200 Subject: [PATCH 06/10] test(scroll): cover explicit budgets through provider workflows --- ...ndroid-long-scroll-2026-10-19-results.json | 17 +++++++ .../android-long-scroll-2026-10-19.md | 4 +- packages/daemon/src/execution/scroll-until.ts | 7 ++- .../provider-scenarios/scroll-until.test.ts | 48 +++++++++++++++++-- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/docs/evidence/android-long-scroll-2026-10-19-results.json b/docs/evidence/android-long-scroll-2026-10-19-results.json index 79ef07ba1a..c1e0a8a11b 100644 --- a/docs/evidence/android-long-scroll-2026-10-19-results.json +++ b/docs/evidence/android-long-scroll-2026-10-19-results.json @@ -321,5 +321,22 @@ "passes": [55, 61, 61, 55], "workflowMs": [47462.889248, 48826.80208400002, 45803.86145000001, 44127.14453900001], "decision": "Shorter swipes reduced single-command latency but made less progress and did not improve this workflow." + }, + "providerIntegration": { + "command": "pnpm exec vitest run --project provider-integration test/integration/provider-scenarios/scroll-until.test.ts", + "baseCommit": "507ef2376621d78aa76cf531bedc5abdf30e7a5a", + "base": { + "passed": 3, + "failed": 2, + "failures": [ + "search limited to 12 despite maxPasses 14", + "maxPasses 1 ignored; missing expected rejection" + ] + }, + "feature": { + "passed": 5, + "failed": 0 + }, + "specSha256": "9cc972d09643dda62e0afe12fc55c8a4607cac104b7270da39beebab7013eed9" } } diff --git a/docs/evidence/android-long-scroll-2026-10-19.md b/docs/evidence/android-long-scroll-2026-10-19.md index d295cc063f..5e0f977a4d 100644 --- a/docs/evidence/android-long-scroll-2026-10-19.md +++ b/docs/evidence/android-long-scroll-2026-10-19.md @@ -50,7 +50,7 @@ await client.interactions.scroll({ }); ``` -The comparison route repeatedly calls `client.interactions.scroll({ direction: 'down' })` and `client.capture.snapshot({ interactiveOnly: true })` until that snapshot contains `row-512`. Both routes take a final interactive snapshot and press the target with that snapshot's generation-pinned ref. This reproduces the request boundary under comparison without including app launch. +The comparison route repeatedly calls `client.interactions.scroll({ direction: 'down' })` and `client.capture.snapshot({ interactiveOnly: true })` until that snapshot contains `row-512`. It reuses the last loop snapshot for the target press. The until route takes an interactive snapshot after the search. Both press the target with that snapshot's generation-pinned ref. This reproduces the request boundary under comparison without including app launch. ## Rejected primitive changes @@ -72,4 +72,6 @@ At `ce71f344a3`: The new runtime cases failed on unchanged base: six failures covering a budget above 12, a smaller budget and invalid inputs. Recording admission and round-trip tests failed twice before the recording fix. Cancellation tests that use the real context builder and request registry failed twice before request-signal wiring, then passed. Focused recording/scroll/visibility tests passed 141 cases; the final context/socket/HTTP/scroll group passed 50 cases. These overlapping groups are not summed. +Provider integration now tests arrival after 13 gestures with maxPasses 14 and refusal after one gesture with maxPasses 1. The same test file reports two failures and three passes on unchanged base production code: the longer search stops at 12, and the smaller budget is ignored. The feature passes all five cases. The visible page label changes with each capture so the longer search proves real progress through the existing no-progress detector. + The pass loop's sparse-capture refusal, edge detection, no-progress detection and visibility predicates remain in place. No gesture retry or recovery tap was added. Live verification covers the local Android helper, not Appium, HarmonyOS or Apple devices. The serialized affected gate and hosted CI remain separate validation obligations. diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index 4286ea8347..8c4e22cc38 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -1,6 +1,9 @@ import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; -import { DEFAULT_SCROLL_UNTIL_PASSES } from '@agent-device/contracts/scroll-command'; +import { + DEFAULT_SCROLL_UNTIL_PASSES, + SCROLL_UNTIL_MAX_PASSES, +} from '@agent-device/contracts/scroll-command'; import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; @@ -188,7 +191,7 @@ function scrollUntilNotFoundError( selector, direction, passes, - hint: `Raise the step with an amount (scroll ${direction} 0.8 --until ), or run snapshot -i to confirm the selector matches something on this screen.`, + hint: `Allow more gestures with --max-passes (up to ${SCROLL_UNTIL_MAX_PASSES}), raise the step with an amount (scroll ${direction} 0.8 --until ), or run snapshot -i to inspect the current screen.`, }, ); } diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts index 14bed0c9d8..d7d485c517 100644 --- a/test/integration/provider-scenarios/scroll-until.test.ts +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -7,18 +7,18 @@ import { withProviderScenarioResource } from './harness.ts'; /** * `scroll --until ` through the real daemon, provider admission, and capture path. * - * The row climbs one screen per capture, so it is off-screen for the first captures and on screen - * from the third: the loop's stop condition is observed rather than asserted. Keyed on captures + * The row climbs one screen per capture and arrives after the configured number of passes. + * The loop's stop condition is observed rather than asserted. Keyed on captures * rather than on injected gestures because the Android gesture path runs through the persistent * helper, not an adb shell command the world can count. */ /** Two screens below the fold, climbing one screen per capture: visible on the third capture. */ const ARRIVAL_PASSES = 2; -function climbingRow(): () => number { +function climbingRow(arrivalPasses = ARRIVAL_PASSES): () => number { let captures = 0; return () => { - const top = Math.max(200, 1400 - captures * 600); + const top = Math.max(200, 200 + (arrivalPasses - captures) * 600); captures += 1; return top; }; @@ -33,7 +33,7 @@ function climbingHierarchy(targetTop: () => number): () => string { '', '', ' ', - ' ', + ` `, ` `, ' ', '', @@ -67,6 +67,44 @@ test('Provider-backed integration scroll --until stops on the capture that bring ); }); +test.each([ + { maxPasses: 14, arrivalPasses: 13 }, + { maxPasses: 1, arrivalPasses: 2 }, +])( + 'Provider-backed integration scroll honors maxPasses $maxPasses', + async ({ maxPasses, arrivalPasses }) => { + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + snapshotXml: climbingHierarchy(climbingRow(arrivalPasses)), + }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + const search = () => + client.interactions.scroll({ + direction: 'down', + until: 'text=Terms', + maxPasses, + ...world.selection, + }); + if (maxPasses < arrivalPasses) { + await assert.rejects(search, (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_pass_limit'); + assert.equal(error.details?.passes, maxPasses); + return true; + }); + } else { + const result = await search(); + assert.equal(result.until, 'text=Terms'); + assert.equal(result.passes, arrivalPasses); + } + }, + ); + }, +); + test('Provider-backed integration scroll --until reports the end of the content as a typed failure', async () => { await withProviderScenarioResource( async () => From 8fb545bdb50451cbb6ec28d37b293862c6a87da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:51:15 +0200 Subject: [PATCH 07/10] chore(gates): classify the scroll search budget as device observable --- scripts/integration-progress-model.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 78146bde38..339f3569b1 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -167,6 +167,7 @@ function summarizeProviderScenarioFlagCoverage(files) { ['jitterPx', 'press jitter'], ['pixels', 'scroll distance'], ['until', 'scroll-until-visible stop condition'], + ['maxPasses', 'scroll-until-visible gesture budget'], ['doubleTap', 'double tap gesture'], ['clickButton', 'desktop mouse button selection', ['button']], ['backMode', 'explicit app/system back behavior', ['mode']], From 61dce4a9b006d93ac6ab1e29364d5bb05fbfb3af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 21:52:49 +0200 Subject: [PATCH 08/10] fix(cli): keep scroll execution out of eager flag metadata --- docs/evidence/android-long-scroll-2026-10-19.md | 10 ++++++++++ .../command-registry/src/flag-definitions-action.ts | 9 +++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/evidence/android-long-scroll-2026-10-19.md b/docs/evidence/android-long-scroll-2026-10-19.md index 5e0f977a4d..3e42398ae0 100644 --- a/docs/evidence/android-long-scroll-2026-10-19.md +++ b/docs/evidence/android-long-scroll-2026-10-19.md @@ -75,3 +75,13 @@ The new runtime cases failed on unchanged base: six failures covering a budget a Provider integration now tests arrival after 13 gestures with maxPasses 14 and refusal after one gesture with maxPasses 1. The same test file reports two failures and three passes on unchanged base production code: the longer search stops at 12, and the smaller budget is ignored. The feature passes all five cases. The visible page label changes with each capture so the longer search proves real progress through the existing no-progress detector. The pass loop's sparse-capture refusal, edge detection, no-progress detection and visibility predicates remain in place. No gesture retry or recovery tap was added. Live verification covers the local Android helper, not Appium, HarmonyOS or Apple devices. The serialized affected gate and hosted CI remain separate validation obligations. + +## Shared platform and provider applicability + +Source inspection confirms that the search policy already lives above platform execution. `resolveScrollRuntimePlan` requires both `scrollDirection` and `captureSnapshot`; `executeUntilScroll` applies the same budget, visibility, readability, edge, progress and request-cancellation decisions to the admitted binding. No Android branch owns this policy. + +Apple, web and Linux local runtimes use `bindLocalScrollInteractor`. WebDriver and Limrun bindings use `bindProviderScrollInteractor`. Both delegate one gesture through the same `bindScrollDirection` owner, preserving the selected backend's options and request signal. These bindings establish a common adoption path, not measured performance parity. Backend snapshots must still supply usable visibility and scroll geometry. + +The transferable optimization is to let that bounded loop inspect its captures instead of adding a public snapshot/stabilization request after every page. The external adapter can adopt the existing shared API for eligible local or provider targets. Android's 300 ms pacing, idle handling and input transport are not a cross-platform default to copy. The rejected 150 ms experiment also shows why whole-search progress matters more than individual gesture latency. + +The first hosted Coverage job failed an existing import-budget assertion before coverage thresholds ran: flag metadata pulled `scroll-command.ts` and its execution helpers into the eager flag-registry closure, increasing it from 30 to 31 modules. A type-only policy link now checks the static schema bound against `SCROLL_UNTIL_MAX_PASSES` without evaluating that module. The existing eager-import test reproduces the failure before the fix and passes afterward. No tests were added for this correction. diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 07f5ceb5d1..bdd3d67aba 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -1,9 +1,6 @@ import { RESPONSE_LEVELS } from '@agent-device/kernel/contracts'; import { RECORDING_SCOPE_VALUES } from '@agent-device/contracts/recording'; -import { - DEFAULT_SCROLL_UNTIL_PASSES, - SCROLL_UNTIL_MAX_PASSES, -} from '@agent-device/contracts/scroll-command'; +import type { SCROLL_UNTIL_MAX_PASSES } from '@agent-device/contracts/scroll-command'; import type { FlagDefinition } from './flag-types.ts'; export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ @@ -184,9 +181,9 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ names: ['--max-passes'], type: 'int', min: 1, - max: SCROLL_UNTIL_MAX_PASSES, + max: 200 satisfies typeof SCROLL_UNTIL_MAX_PASSES, usageLabel: '--max-passes ', - usageDescription: `Scroll --until: maximum gestures (default ${DEFAULT_SCROLL_UNTIL_PASSES}, up to ${SCROLL_UNTIL_MAX_PASSES})`, + usageDescription: 'Scroll --until: maximum gestures', projectConfig: false, recorded: true, }, From dd287d874a7a4bae1216dfdaccf280a0a33f23ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 20 Oct 2026 10:45:22 +0200 Subject: [PATCH 09/10] fix(scroll): share request cancellation across verified passes --- .../src/snapshot/scroll-edge-state.ts | 12 +++++++++++- packages/contracts/src/scroll-command.ts | 2 +- .../daemon/src/execution/scroll-runtime.ts | 19 ++++++++++--------- packages/daemon/src/execution/scroll-until.ts | 2 ++ src/commands/schema/cli-help-gestures.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index a6b7c529b9..fc6bceb0ab 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -168,6 +168,7 @@ function pushScrollSurfaceSignature( * gesture so an already-satisfied stop costs no scroll. */ export async function runScrollPasses(params: { + signal?: AbortSignal; observe: () => Promise; refuse: (observation: TObservation) => AppError | undefined; /** True ends the loop with this pass count; a terminal outcome of its own throws instead. */ @@ -191,10 +192,14 @@ export async function runScrollPasses(params: { let result: TResult | undefined; const recentSignatures: string[] = []; while (true) { + params.signal?.throwIfAborted(); const observation = await observe(); + params.signal?.throwIfAborted(); const refusal = refuse(observation); if (refusal) throw refusal; - if (await shouldStop(observation, passes)) return { passes, result }; + const stopped = await shouldStop(observation, passes); + params.signal?.throwIfAborted(); + if (stopped) return { passes, result }; if (passes >= passLimit) throw params.passLimitError(passes); pushScrollSurfaceSignature( recentSignatures, @@ -202,13 +207,16 @@ export async function runScrollPasses(params: { SCROLL_EDGE_STUCK_WINDOW, ); if (scrollSurfaceIsStuck(recentSignatures)) throw params.noProgressError(passes); + params.signal?.throwIfAborted(); result = await scroll(); passes += 1; + params.signal?.throwIfAborted(); await settleAfterPass?.(); } } export async function runScrollEdgePasses(params: { + signal?: AbortSignal; edge: ScrollEdge; captureState: (scope?: string) => Promise; scroll: () => Promise; @@ -225,6 +233,7 @@ export async function runScrollEdgePasses(params: { if (!discovered) { discovered = true; if (!isScrollEdgeRefused(observation) && observation.scope) { + params.signal?.throwIfAborted(); observation = await captureState(observation.scope); } } @@ -232,6 +241,7 @@ export async function runScrollEdgePasses(params: { return observation; }; return await runScrollPasses({ + signal: params.signal, observe, refuse: (observation) => isScrollEdgeRefused(observation) diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index 08454bc30e..e5c8be1b0d 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -89,7 +89,7 @@ export function assertScrollUntilCompatible( ): void { if (input.maxPasses !== undefined) { if (input.until === undefined) { - throw new AppError('INVALID_ARGS', 'scroll --max-passes requires --until', { + throw new AppError('INVALID_ARGS', 'scroll maxPasses requires until', { reason: 'scroll_max_passes_requires_until', }); } diff --git a/packages/daemon/src/execution/scroll-runtime.ts b/packages/daemon/src/execution/scroll-runtime.ts index 9298364fe0..78bfc62d03 100644 --- a/packages/daemon/src/execution/scroll-runtime.ts +++ b/packages/daemon/src/execution/scroll-runtime.ts @@ -1,3 +1,4 @@ +import { sleep } from '@agent-device/host-kit/retry'; import { assertExclusiveScrollDistanceInputs, assertScrollUntilCompatible, @@ -324,6 +325,7 @@ async function executeEdgeScroll( context: DaemonCommandContext, ): Promise> { const edgeResult = await runScrollEdgePasses({ + signal: context.signal, edge, captureState: async (scope) => await captureEdgeState(runtime, edge, scope, context), scroll: async () => await scrollOnce(runtime, target, options, context), @@ -333,6 +335,7 @@ async function executeEdgeScroll( (await runtime.operations.captureSnapshot(scrollCaptureInput(context, { scope }))) .nodes ?? [], edge, + context.signal, ); }, }); @@ -353,14 +356,9 @@ async function executeUntilScroll( direction: target.direction, platform: publicPlatformString(device), passLimit: context.maxPasses, - capture: async () => { - context.signal?.throwIfAborted(); - return await runtime.operations.captureSnapshot(scrollCaptureInput(context)); - }, - scroll: async () => { - context.signal?.throwIfAborted(); - return await scrollOnce(runtime, target, options, context); - }, + signal: context.signal, + capture: async () => await runtime.operations.captureSnapshot(scrollCaptureInput(context)), + scroll: async () => await scrollOnce(runtime, target, options, context), }); return withSuccessText( { @@ -391,6 +389,7 @@ async function captureEdgeState( edge, scope, captureNodes: async (snapshotScope) => { + context.signal?.throwIfAborted(); lastCapture = await runtime.operations.captureSnapshot( scrollCaptureInput(context, { scope: snapshotScope }), ); @@ -424,16 +423,18 @@ const SCROLL_REST_POLL_MS = 120; async function pollForScrollRest( captureNodes: () => Promise, edge: ScrollEdge, + signal?: AbortSignal, timeoutMs = SCROLL_REST_TIMEOUT_MS, pollMs = SCROLL_REST_POLL_MS, ): Promise { let previous: string | undefined; const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { + signal?.throwIfAborted(); const fingerprint = await scrollSurfaceFingerprint(await captureNodes(), edge); if (fingerprint === previous) return; previous = fingerprint; - await new Promise((resolve) => setTimeout(resolve, pollMs)); + await sleep(pollMs, signal); } } diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index 8c4e22cc38..1d35917101 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -47,6 +47,7 @@ export async function runScrollUntilVisible(params: { selector: string; direction: ScrollDirection; platform: Platform | PublicPlatform; + signal?: AbortSignal; passLimit?: number; capture: () => Promise; scroll: () => Promise; @@ -56,6 +57,7 @@ export async function runScrollUntilVisible(params: { let scrolled = false; try { const { passes, result } = await runScrollPasses({ + signal: params.signal, observe: capture, refuse: (captured) => { const refusal = scrollCaptureRefusal(captured); diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index c84fa57993..bea672a04f 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -20,7 +20,7 @@ Shapes: agent-device gesture pinch 0.5 200 400 agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until and does not extend request timeouts. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index bf6bf0dab8..7eaa93d798 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -495,7 +495,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' ### Scroll - `scroll [amount]` takes the finger path as a fraction of the viewport (`0.5` is half), or `--pixels ` for a fixed distance. Large distances are clamped. The app's scroll physics decide the final offset, so a scroll does not land on an exact position. -- `scroll --until ` repeats passes until the selector is on screen and fails if the content runs out first. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend request timeouts. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay & E2E](/docs/replay-e2e). +- `scroll --until ` repeats passes until the selector is on screen and fails if the content runs out first. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay & E2E](/docs/replay-e2e). - Default snapshot text summarizes off-screen interactive content instead of listing refs. `scroll --settle` (and `back --settle`) waits for the UI to go quiet and returns the diff with fresh refs, so you don't need another `snapshot -i`. Settle is best-effort and never fails the action. - On Android, settle ignores an allowlist of system-bar containers and shade/keyguard status headers; other SystemUI overlays remain content. Appium settle also applies package rules for keyboard and system chrome. - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. From f80700537a4bf04a7c28cc4706cbd23e998b25a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 20 Oct 2026 10:50:59 +0200 Subject: [PATCH 10/10] docs(evidence): record rebased scroll and cancellation verification --- .../evidence/android-long-scroll-2026-10-19.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/evidence/android-long-scroll-2026-10-19.md b/docs/evidence/android-long-scroll-2026-10-19.md index 3e42398ae0..6e11372005 100644 --- a/docs/evidence/android-long-scroll-2026-10-19.md +++ b/docs/evidence/android-long-scroll-2026-10-19.md @@ -85,3 +85,21 @@ Apple, web and Linux local runtimes use `bindLocalScrollInteractor`. WebDriver a The transferable optimization is to let that bounded loop inspect its captures instead of adding a public snapshot/stabilization request after every page. The external adapter can adopt the existing shared API for eligible local or provider targets. Android's 300 ms pacing, idle handling and input transport are not a cross-platform default to copy. The rejected 150 ms experiment also shows why whole-search progress matters more than individual gesture latency. The first hosted Coverage job failed an existing import-budget assertion before coverage thresholds ran: flag metadata pulled `scroll-command.ts` and its execution helpers into the eager flag-registry closure, increasing it from 30 to 31 modules. A type-only policy link now checks the static schema bound against `SCROLL_UNTIL_MAX_PASSES` without evaluating that module. The existing eager-import test reproduces the failure before the fix and passes afterward. No tests were added for this correction. + +## Review follow-up, 2026-10-20 + +Production head `dd287d874a7a4bae1216dfdaccf280a0a33f23ce` is rebased onto `620228b8ca67a69a108fa4fc5041fdd88880b792`. The range comparison preserves all earlier changes. Its only conflict moved main's physical-iOS gesture guidance into the extracted help topic. + +Cancellation now belongs to `runScrollPasses`, shared by target searches and edge scrolling. It checks before each observation and gesture, and after awaited work. Edge discovery retries and settling captures also check cancellation. The settling sleep wakes on abort. This preserves the request's typed cancellation reason and prevents another pass from starting. + +The signal-consumer audit distinguishes context types. Only the scroll runtime newly reads the signal populated by `contextFromFlags`. `runtimeExecutionFromContext` excludes that field. Existing Apple runner/native-helper contexts get their signal from platform request bindings. Backend capture contexts get theirs from the command surface. Limrun allocation gets its separate `LeaseLifecycleContext.signal` from the lease handler. Maestro and replay contexts also have their own construction paths. These consumers do not acquire new cancellation behavior from this change. Native macOS helper shutdown already sends SIGTERM with a one-second grace so its handler can release a held button. + +Live verification used the same Pixel_7_CI and APK hashes above. The emulator restored an older development-build snapshot; the checksum-verified Release fixture and benchmark APK were restored before verification. The PR's built SDK and isolated state directory `/private/tmp/agent-device-scroll-review-20261020` exercised this source; the Device panel's installed CLI predates the flag. + +- Exact benchmark search: 55 gestures, then a fresh target press and exact `Found Row 0512` assertion. Search, snapshot and press took 28.229 s in one run. This is a smoke measurement, not a new controlled comparison. +- Our Release app: both routes reached and opened `Seasonal Footer Pick` in seven gestures. A one-pass search failed with `scroll_until_pass_limit` and `passes: 1`. +- Canceling target search and `scroll bottom` after 1.6 seconds returned `request_canceled`. Request logs show three gestures for the target search and one for the edge search before cancellation. Both accepted a readable follow-up snapshot; all sessions closed. +- Existing checks passed: 916 focused scroll/eager-import cases; 83 shared-loop, macOS helper and Limrun cases; eight Apple runner cancellation cases. No tests were added or changed in this follow-up. +- Independent Opus review found no defects in the rebase, cancellation, signal-consumer audit, API-neutral validation text, or the 90-second timeout guidance. + +Raw outputs and request logs remain in the task's temporary artifacts. The earlier controlled ABBA measurements remain attributed to their original heads above.