diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 36fa2ee80b..10a91c89d2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -176,6 +176,17 @@ extension RunnerTests { ) } + private func plannedGestureResponse( + plan: RunnerGesturePlan, + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + outcome: RunnerInteractionOutcome + ) -> Response { + if let response = unsupportedResponse(for: outcome) { + return response + } + return gestureResponse(message: plan.intent, timing: timing) + } + #if AGENT_DEVICE_RUNNER_UNIT_TESTS func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { let response = gestureResponse( @@ -2056,7 +2067,8 @@ extension RunnerTests { error: ErrorPayload(code: "INVALID_ARGS", message: validationError) ) } - if plannedGestureExecution(for: plan) == .fastSwipe { + switch plannedGestureExecution(for: plan) { + case .fastSwipe: // Validation above guarantees a non-empty, single-pointer path for this execution kind. let first = plan.pointers[0].samples.first!.point let last = plan.pointers[0].samples.last!.point @@ -2074,14 +2086,12 @@ extension RunnerTests { synthesizedProfile: .fastSwipe ) ) + case .sampled: + let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + sampledPlannedGesture(app: activeApp, plan: plan) + } + return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) } - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { - sampledPlannedGesture(app: activeApp, plan: plan) - } - if let response = unsupportedResponse(for: outcome) { - return response - } - return gestureResponse(message: plan.intent, timing: timing) case .gestureViewport: let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame) guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else { diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index cce2d70868..a3a8d2e3f5 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -54,8 +54,10 @@ components atomically. Intent remains on the plan even when aliases share an exe The planner owns deterministic multi-touch geometry. Contacts start at -90 degrees, except Android pinch starts horizontally because a vertical pinch is captured by common vertical app scroll containers before the pinch recognizer activates. The same explicit planning profile preserves the -proven frame-count convention: Android rounds while Apple truncates the duration/16 ms frame count. -These are planner inputs, not adapter-generated trajectories. The larger of pinch's initial and final spans is 40% of the +proven frame-count convention for dense two-contact trajectories: Android rounds while Apple +truncates the duration/16 ms frame count. The Android transport lowerer uses that same Android +sampling profile for one-contact endpoint plans. These are planner inputs, not adapter-generated +trajectories. The larger of pinch's initial and final spans is 40% of the viewport's shorter side, preserving the proven Apple pinch geometry; other two-contact intents use 25% to keep translation and rotation trajectories compact. The other span follows from the requested scale, and both must satisfy a 48-point reliability floor. Combined transforms progress translation, scale, and rotation @@ -70,28 +72,31 @@ Platform adapters consume the canonical plan: - Android's `executeAndroidTouchPlan` adapter seam sends planned touch, including gesture plans plus the physical movement for scroll and long-press, to provider-native touch injection when - available, otherwise to the bundled instrumentation helper. The helper injects the exact planned - pointer samples. A stationary long-press needs no viewport on the helper path; the executor adds - the paired provider-owned viewport only for provider-native touch. Android touch execution never - falls back to `adb input swipe`. Public scroll durations below one 16 ms planner frame normalize - to that physical minimum and report the executed duration. Scroll evidence reports absolute - injected coordinates against zero-origin extents that include the viewport offset. Because - Android permits only one instrumentation owner of `UiAutomation`, snapshot capture, gesture - viewport resolution, and planned-touch injection share one bundled automation helper: a live - persistent helper session executes touch commands directly, and without one the same helper runs - one-shot. Nothing stops the snapshot session around gestures anymore (amended 2026-07, - issue #1275; previously a separate one-shot multi-touch helper forced a session stop/restart - around every local gesture). + available, otherwise to the bundled instrumentation helper. One-contact endpoint plans lower in + `src/platforms/android/touch-plan.ts` to 16 ms linear transport samples before either injection + path; two-contact plans retain their exact planned samples. A stationary long-press needs no + viewport on the helper path; the executor adds the paired provider-owned viewport only for + provider-native touch. Android touch execution never falls back to `adb input swipe`. Public + scroll durations below one 16 ms planner frame normalize to that physical minimum and report the + executed duration. Scroll evidence reports absolute injected coordinates against zero-origin + extents that include the viewport offset. Because Android permits only one instrumentation owner + of `UiAutomation`, snapshot capture, gesture viewport resolution, and planned-touch injection + share one bundled automation helper: a live persistent helper session executes touch commands + directly, and without one the same helper runs one-shot. Nothing stops the snapshot session + around gestures anymore (amended 2026-07, issue #1275; previously a separate one-shot + multi-touch helper forced a session stop/restart around every local gesture). - iOS lowers one-contact endpoint-hold plans to the established fast-swipe synthesis profile. That profile reaches the endpoint in 100 ms, then holds there for the planned duration before lifting, - matching Maestro's XCTest driver. Timed-pan and two-contact plans convert every point to native - orientation and feed the exact planned arrays to the private XCTest event bridge. Android and - WebDriver continue to execute the plan samples across the authored duration, matching their - native Maestro drivers. macOS lowers a one-contact plan to its drag executor and tvOS lowers it to remote - direction. Core admission and the Apple adapter both consume the same shared multi-touch support - policy; multi-touch remains capability-gated to iOS simulators. -- WebDriver lowers a supported plan to synchronized W3C pointer action sources. Multi-touch remains - capability-gated until a provider proves it. + matching Maestro's XCTest driver. One-contact plans are linear and therefore carry only their + start and end samples, with the authored duration between them. Two-contact plans convert every + planned point to native orientation and feed the exact arrays to the private XCTest event bridge. + macOS lowers a one-contact plan to its drag executor and tvOS lowers it to remote direction. Core + admission and the Apple adapter both consume the same shared multi-touch support policy; + multi-touch remains capability-gated to iOS simulators. +- WebDriver lowers a supported plan to synchronized W3C pointer action sources. A one-contact + endpoint plan becomes pointer down, one timed W3C `pointerMove` from start to end, and pointer up; + the driver owns interpolation across that W3C tick. Multi-touch remains capability-gated until a + provider proves it. The `Interactor` and backend expose one compositional `performGesture(plan)` primitive instead of a method per semantic alias. The old scalar Apple and Android multi-touch executors and the @@ -139,8 +144,13 @@ selectors or refs and therefore cannot claim element-targeting guarantees. - On bare ADB, Android scroll and long-press require the bundled automation helper (the snapshot helper APK) and `UiAutomation`; helper installation or runtime failure is surfaced directly rather than degrading to an approximate `adb input swipe`. -- Pointer plans are larger than scalar requests but bounded by duration and the 16 ms sample - cadence; deleting duplicate scalar executors offsets the package cost. +- Canonical one-contact plans contain only two samples: a 10-second pan is two shared-plan samples + instead of 626. Android expands that plan only at the transport boundary, while two-contact + plans remain cadence-bounded because their synchronized geometry is part of their contract. +- Unit tests cover canonical plan shape, Android lowering, helper/provider payloads, and WebDriver + action construction. They cannot prove timing or event delivery inside the private XCTest bridge, + so iOS timing changes require live simulator evidence that observes the requested content change + and records the runner start/end uptime delta alongside the requested duration. ## Alternatives Considered diff --git a/packages/contracts/src/gesture-plan-types.ts b/packages/contracts/src/gesture-plan-types.ts index 94bf7f7dd3..11eee2d055 100644 --- a/packages/contracts/src/gesture-plan-types.ts +++ b/packages/contracts/src/gesture-plan-types.ts @@ -46,13 +46,18 @@ export type PointerTrajectory = { samples: readonly PointerTrajectorySample[]; }; +export type SinglePointerTrajectory = { + pointerId: 0; + samples: readonly [PointerTrajectorySample, PointerTrajectorySample]; +}; + export type SinglePointerGesturePlan = { topology: 'single'; intent: 'fling' | 'pan'; executionProfile: GestureExecutionProfile; durationMs: number; viewport: Rect; - pointers: readonly [PointerTrajectory]; + pointers: readonly [SinglePointerTrajectory]; }; export type MultiTouchGesturePlan = { diff --git a/packages/contracts/src/gesture-plan.ts b/packages/contracts/src/gesture-plan.ts index 2115b2f174..2f4e915409 100644 --- a/packages/contracts/src/gesture-plan.ts +++ b/packages/contracts/src/gesture-plan.ts @@ -14,6 +14,7 @@ import type { MultiTouchGesturePlan, PointerTrajectory, SinglePointerGesturePlan, + SinglePointerTrajectory, } from './gesture-plan-types.ts'; export * from './gesture-plan-types.ts'; @@ -33,21 +34,23 @@ const DEFAULT_MULTI_TOUCH_DURATION_MS = 300; const MAX_ROTATION_DEGREES_PER_SAMPLE = 3; const MAX_ROTATION_DEFAULT_DURATION_MS = 2_400; +export type GestureSamplingProfile = 'default' | 'android'; + type GesturePlatformProfile = { pinchAxisDegrees: number; - frameCount: (rawFrameCount: number) => number; + samplingProfile: GestureSamplingProfile; }; const DEFAULT_GESTURE_PLATFORM_PROFILE: GesturePlatformProfile = { pinchAxisDegrees: GESTURE_INITIAL_ANGLE_DEGREES, - frameCount: Math.floor, + samplingProfile: 'default', }; const GESTURE_PLATFORM_PROFILES = { ios: DEFAULT_GESTURE_PLATFORM_PROFILE, macos: DEFAULT_GESTURE_PLATFORM_PROFILE, android: { pinchAxisDegrees: GESTURE_HORIZONTAL_ANGLE_DEGREES, - frameCount: Math.round, + samplingProfile: 'android', }, vega: DEFAULT_GESTURE_PLATFORM_PROFILE, linux: DEFAULT_GESTURE_PLATFORM_PROFILE, @@ -64,7 +67,7 @@ export function buildGesturePlan( const profile = gesturePlatformProfile(platform); switch (input.intent) { case 'fling': - return buildFlingPlan(input, frame, profile); + return buildFlingPlan(input, frame); case 'pan': return buildPanPlan(input, frame, profile); case 'pinch': @@ -121,18 +124,17 @@ export function singlePointerPlanEndpoints(plan: SinglePointerGesturePlan): { start: Point; end: Point; } { - const start = plan.pointers[0].samples[0]?.point; - const end = plan.pointers[0].samples.at(-1)?.point; - if (!start || !end) { - throw new AppError('INVALID_ARGS', 'single-pointer gesture plan requires samples'); - } - return { start, end }; + const [ + { + samples: [start, end], + }, + ] = plan.pointers; + return { start: start.point, end: end.point }; } function buildFlingPlan( input: Extract, viewport: Rect, - profile: GesturePlatformProfile, ): SinglePointerGesturePlan { if ('preset' in input) { const { from, to } = presetGestureEndpoints(input.preset, viewport); @@ -143,7 +145,6 @@ function buildFlingPlan( GESTURE_FLING_DURATION_MS, viewport, 'endpoint-hold', - profile, ); } if ('from' in input) { @@ -154,7 +155,6 @@ function buildFlingPlan( GESTURE_FLING_DURATION_MS, viewport, 'endpoint-hold', - profile, ); } const start = finitePoint(input.origin, 'gesture fling origin'); @@ -169,7 +169,6 @@ function buildFlingPlan( GESTURE_FLING_DURATION_MS, viewport, 'endpoint-hold', - profile, ); } @@ -193,7 +192,6 @@ function buildPanPlan( durationMs, viewport, input.executionProfile ?? 'timed-pan', - profile, ); } if (input.pointerCount !== 2) { @@ -220,14 +218,13 @@ function buildSinglePointerPlan( durationMs: number, viewport: Rect, executionProfile: GestureExecutionProfile, - profile: GesturePlatformProfile, ): SinglePointerGesturePlan { const start = finitePoint(from, `gesture ${intent} start`); const end = finitePoint(to, `gesture ${intent} end`); - const samples = sampleOffsets(durationMs, profile).map((offsetMs) => ({ - offsetMs, - point: interpolatePoint(start, end, offsetMs / durationMs), - })); + const samples: SinglePointerTrajectory['samples'] = [ + { offsetMs: 0, point: start }, + { offsetMs: durationMs, point: end }, + ]; assertSamplesInViewport(samples, viewport, { intent, pointerId: 0 }); return { topology: 'single', @@ -269,7 +266,7 @@ function buildTransformPlan( } const initialRadius = initialSpan / 2; - const offsets = sampleOffsets(motion.durationMs, profile); + const offsets = sampleGestureOffsets(motion.durationMs, profile.samplingProfile); const trajectory = (pointerId: 0 | 1, side: 1 | -1): PointerTrajectory => { const samples = offsets.map((offsetMs) => ({ offsetMs, @@ -309,7 +306,7 @@ function transformPointAt(options: { profile: GesturePlatformProfile; }): Point { const progress = options.offsetMs / options.durationMs; - const centroid = interpolatePoint(options.start, options.end, progress); + const centroid = interpolateGesturePoint(options.start, options.end, progress); const radius = options.initialRadius * (1 + (options.scale - 1) * progress); const angle = degreesToRadians( initialAngleForIntent(options.intent, options.profile) + options.rotationDegrees * progress, @@ -331,9 +328,18 @@ function initialSpanRatioForIntent(intent: MultiTouchGesturePlan['intent']): num return intent === 'pinch' ? GESTURE_PINCH_INITIAL_SPAN_RATIO : GESTURE_INITIAL_SPAN_RATIO; } -function sampleOffsets(durationMs: number, profile: GesturePlatformProfile): number[] { +export function sampleGestureOffsets( + durationMs: number, + profile: GestureSamplingProfile = 'default', +): number[] { + if (!Number.isFinite(durationMs) || durationMs <= 0) { + throw new AppError('INVALID_ARGS', 'gesture sample duration must be a positive finite number'); + } const rawFrameCount = durationMs / GESTURE_SAMPLE_INTERVAL_MS; - const frameCount = Math.max(3, profile.frameCount(rawFrameCount)); + const frameCount = Math.max( + 3, + profile === 'android' ? Math.round(rawFrameCount) : Math.floor(rawFrameCount), + ); return Array.from({ length: frameCount + 1 }, (_, index) => Math.round((durationMs * index) / frameCount), ); @@ -446,7 +452,7 @@ function addPoints(left: Point, right: Point): Point { return { x: left.x + right.x, y: left.y + right.y }; } -function interpolatePoint(start: Point, end: Point, progress: number): Point { +export function interpolateGesturePoint(start: Point, end: Point, progress: number): Point { return { x: start.x + (end.x - start.x) * progress, y: start.y + (end.y - start.y) * progress, diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts new file mode 100644 index 0000000000..3597af1d73 --- /dev/null +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { buildGesturePlan } from '@agent-device/contracts/interaction'; +import { createCloudWebDriverCapabilities } from './capabilities.ts'; +import type { WebDriverClient, W3CActionSequence } from './webdriver-client.ts'; +import { createWebDriverInteractor } from './webdriver-interactor.ts'; + +test('endpoint plans become one timed W3C pointer move', async () => { + const performed: W3CActionSequence[][] = []; + let released = false; + const client = { + performActions: async (actions: W3CActionSequence[]) => { + performed.push(actions); + }, + releaseActions: async () => { + released = true; + }, + } as unknown as WebDriverClient; + const interactor = createWebDriverInteractor({ + client, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }); + const plan = buildGesturePlan( + { + intent: 'pan', + origin: { x: 100, y: 200 }, + delta: { x: 100, y: 200 }, + durationMs: 500, + }, + { x: 0, y: 0, width: 400, height: 800 }, + ); + + assert.ok(interactor.performGesture); + assert.deepEqual(await interactor.performGesture(plan), { backend: 'webdriver-w3c-actions' }); + assert.equal(released, true); + assert.deepEqual(performed, [ + [ + { + type: 'pointer', + id: 'gesture-pointer-0', + parameters: { pointerType: 'touch' }, + actions: [ + { type: 'pointerMove', duration: 0, x: 100, y: 200 }, + { type: 'pointerDown', button: 0 }, + { type: 'pointerMove', duration: 500, x: 200, y: 400 }, + { type: 'pointerUp', button: 0 }, + ], + }, + ], + ]); +}); diff --git a/scripts/layering/facade-symbols.ts b/scripts/layering/facade-symbols.ts index ec8c9328ad..2619a55ec2 100644 --- a/scripts/layering/facade-symbols.ts +++ b/scripts/layering/facade-symbols.ts @@ -275,6 +275,7 @@ export const FACADE_SYMBOLS: readonly (readonly [string, readonly string[]])[] = 'GesturePlan', 'GesturePointerCount', 'GestureReferenceFrame', + 'GestureSamplingProfile', 'GestureSemanticInput', 'GuaranteeEnforcement', 'HomeCommandResult', @@ -334,6 +335,7 @@ export const FACADE_SYMBOLS: readonly (readonly [string, readonly string[]])[] = 'SettleParams', 'SettleTailEntry', 'SinglePointerGesturePlan', + 'SinglePointerTrajectory', 'SnapshotOptions', 'SnapshotResult', 'SwipeGesturePayload', @@ -367,6 +369,7 @@ export const FACADE_SYMBOLS: readonly (readonly [string, readonly string[]])[] = 'getClickButtonValidationError', 'honoredScrollDurationMs', 'inferGestureReferenceFrame', + 'interpolateGesturePoint', 'normalizePublicGesture', 'normalizePublicSwipeMotion', 'normalizeScrollDurationMs', @@ -374,6 +377,7 @@ export const FACADE_SYMBOLS: readonly (readonly [string, readonly string[]])[] = 'parseTvRemoteButton', 'readGesturePayload', 'resolveClickButton', + 'sampleGestureOffsets', 'singlePointerPlanEndpoints', 'swipePayloadFromPositionals', 'toAndroidTvRemoteKeyevent', diff --git a/src/core/__tests__/gesture-plan.test.ts b/src/core/__tests__/gesture-plan.test.ts index 27f0dfe00a..5fc71465a7 100644 --- a/src/core/__tests__/gesture-plan.test.ts +++ b/src/core/__tests__/gesture-plan.test.ts @@ -29,6 +29,10 @@ describe('single-pointer plans', () => { assert.equal(plan.durationMs, 500); assert.deepEqual(plan.pointers[0].samples[0]?.point, { x: 100, y: 200 }); assert.deepEqual(plan.pointers[0].samples.at(-1)?.point, { x: 60, y: 225 }); + assert.deepEqual( + plan.pointers[0].samples.map(({ offsetMs }) => offsetMs), + [0, 500], + ); }); test('fling has one fixed internal schedule and no public duration', () => { diff --git a/src/platforms/android/__tests__/touch-executor.test.ts b/src/platforms/android/__tests__/touch-executor.test.ts index e09c1628e6..bc3c5ade05 100644 --- a/src/platforms/android/__tests__/touch-executor.test.ts +++ b/src/platforms/android/__tests__/touch-executor.test.ts @@ -31,6 +31,40 @@ test('provider-native touch receives the plan as its only source of truth', asyn assert.deepEqual(result, { backend: 'provider-native-touch', injected: true }); }); +test('provider-native touch receives linear one-pointer samples at the Android transport cadence', async () => { + const plan = buildGesturePlan( + { + intent: 'pan', + origin: { x: 200, y: 300 }, + delta: { x: 100, y: -80 }, + durationMs: 64, + }, + viewport, + ); + const calls: unknown[] = []; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => viewport, + touch: async (request) => { + calls.push(request); + return {}; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await executeAndroidTouchPlan(ANDROID_EMULATOR, plan), + ); + + const loweredPlan = calls[0] as typeof plan; + assert.deepEqual( + loweredPlan.pointers[0].samples.map(({ offsetMs }) => offsetMs), + [0, 16, 32, 48, 64], + ); + assert.deepEqual(loweredPlan.pointers[0].samples[2]?.point, { x: 250, y: 260 }); +}); + test('provider touch viewport bypasses local helper transport and is validated', async () => { let calls = 0; await withAndroidAdbProvider( diff --git a/src/platforms/android/__tests__/touch-helper-session.test.ts b/src/platforms/android/__tests__/touch-helper-session.test.ts index 34dce4b2b4..262ab7dc9d 100644 --- a/src/platforms/android/__tests__/touch-helper-session.test.ts +++ b/src/platforms/android/__tests__/touch-helper-session.test.ts @@ -22,6 +22,7 @@ import { getAndroidSnapshotHelperSessionDeviceKey, resetAndroidSnapshotHelperSessions, } from '../snapshot-helper-session.ts'; +import { lowerAndroidTouchPlan } from '../touch-plan.ts'; import { executeAndroidTouchHelperPlan, readAndroidTouchHelperViewport } from '../touch-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; import { @@ -194,7 +195,7 @@ test('touch helper does not run one-shot while snapshot retirement is unconfirme }), }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ), (error: unknown) => (error as { details?: { reason?: string } }).details?.reason === @@ -266,7 +267,7 @@ test('gesture uses the persistent snapshot-helper session and does not stop it', }), }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ); assert.equal(result.helperTransport, 'persistent-session'); @@ -319,7 +320,7 @@ test('an APK replacement stops the stale session and the gesture runs one-shot', }), }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ); assert.equal(result.installReason, 'outdated'); @@ -387,7 +388,7 @@ test('a provider artifact that mismatches the live session helper stops it and r snapshotHelperArtifact: providerArtifact, }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ); assert.equal(result.installReason, 'current'); @@ -458,7 +459,7 @@ test('a same-version artifact with a different sha stops the live session and ru snapshotHelperArtifact: artifactA, }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ); assert.equal(result.installReason, 'current'); @@ -497,7 +498,7 @@ test('a structured ok=false session gesture response throws but leaves the sessi }), }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ), (error: unknown) => { assert.ok(error instanceof AppError); @@ -531,7 +532,7 @@ test('a malformed session gesture response stops the session and does not fall b }), }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ), ); diff --git a/src/platforms/android/__tests__/touch-helper.test.ts b/src/platforms/android/__tests__/touch-helper.test.ts index 2505d548e6..5a9812b95e 100644 --- a/src/platforms/android/__tests__/touch-helper.test.ts +++ b/src/platforms/android/__tests__/touch-helper.test.ts @@ -4,6 +4,8 @@ import { buildGesturePlan } from '@agent-device/contracts/interaction'; import { AppError } from '@agent-device/kernel/errors'; import { withAndroidAdbProvider } from '../adb-executor.ts'; import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { executeAndroidTouchPlan } from '../touch-executor.ts'; +import { lowerAndroidTouchPlan } from '../touch-plan.ts'; import { ANDROID_TOUCH_PLAN_PROTOCOL, executeAndroidTouchHelperPlan, @@ -48,7 +50,7 @@ afterEach(async () => { }); test('single-pointer plans normalize to a swipe request', () => { - const request = normalizeAndroidTouchHelperGestureRequest(flingPlan()); + const request = normalizeAndroidTouchHelperGestureRequest(lowerAndroidTouchPlan(flingPlan())); assert.equal(request.kind, 'swipe'); assert.equal(request.pointers.length, 1); assert.equal(request.durationMs, 100); @@ -160,7 +162,7 @@ test('one-shot gesture instruments the snapshot-helper runner with the touch-pla }), }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ); assert.deepEqual(capturedArgs?.slice(0, 9), [ @@ -180,6 +182,10 @@ test('one-shot gesture instruments the snapshot-helper runner with the touch-pla const payload = JSON.parse(Buffer.from(capturedArgs![9]!, 'base64').toString('utf8')); assert.equal(payload.protocol, ANDROID_TOUCH_PLAN_PROTOCOL); assert.equal(payload.kind, 'swipe'); + assert.deepEqual( + payload.pointers[0].samples.map(({ offsetMs }: { offsetMs: number }) => offsetMs), + [0, 17, 33, 50, 67, 83, 100], + ); assert.equal(result.backend, 'android-helper'); assert.equal(result.helperVersion, manifest.version); @@ -190,6 +196,49 @@ test('one-shot gesture instruments the snapshot-helper runner with the touch-pla assert.equal(result.elapsedMs, 12); }); +test('executor sends lowered endpoint samples to the helper transport', async () => { + const device = makeIsolatedDevice(); + let capturedSamples: Array<{ offsetMs: number; x: number; y: number }> | undefined; + await withAndroidAdbProvider( + { + exec: currentVersionAdb(async (args) => { + const payloadBase64 = args[args.indexOf('payloadBase64') + 1]; + const payload = JSON.parse(Buffer.from(payloadBase64!, 'base64').toString('utf8')) as { + pointers: Array<{ samples: Array<{ offsetMs: number; x: number; y: number }> }>; + }; + capturedSamples = payload.pointers[0]?.samples; + return { + exitCode: 0, + stdout: [resultRecord({ ok: 'true', kind: 'swipe' }), 'INSTRUMENTATION_CODE: 0'].join( + '\n', + ), + stderr: '', + }; + }), + }, + { serial: device.id }, + async () => + await executeAndroidTouchPlan( + device, + buildGesturePlan( + { + intent: 'pan', + origin: { x: 200, y: 300 }, + delta: { x: 100, y: -80 }, + durationMs: 64, + }, + viewport, + ), + ), + ); + + assert.deepEqual( + capturedSamples?.map(({ offsetMs }) => offsetMs), + [0, 16, 32, 48, 64], + ); + assert.deepEqual(capturedSamples?.[2], { offsetMs: 32, x: 250, y: 260 }); +}); + test('a provider-supplied snapshotHelperArtifact overrides the bundled artifact for touch', async () => { const device = makeIsolatedDevice(); const providerArtifact = { @@ -236,7 +285,7 @@ test('a provider-supplied snapshotHelperArtifact overrides the bundled artifact snapshotHelperArtifact: providerArtifact, }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ); assert.equal(vi.mocked(resolveAndroidHelperArtifact).mock.calls.length, 0); @@ -290,7 +339,7 @@ test('one-shot gesture failure propagates as a structured COMMAND_FAILED error', })), }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ), (error: unknown) => { assert.ok(error instanceof AppError); @@ -308,7 +357,7 @@ test('unparseable output with a zero exit code reports a parse failure', async ( withAndroidAdbProvider( { exec: currentVersionAdb(async () => ({ exitCode: 0, stdout: 'garbage', stderr: '' })) }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ), { message: 'Android automation helper output could not be parsed' }, ); @@ -320,7 +369,7 @@ test('unparseable output with a non-zero exit code reports a helper failure', as withAndroidAdbProvider( { exec: currentVersionAdb(async () => ({ exitCode: 1, stdout: '', stderr: 'boom' })) }, { serial: device.id }, - async () => await executeAndroidTouchHelperPlan(device, flingPlan()), + async () => await executeAndroidTouchHelperPlan(device, lowerAndroidTouchPlan(flingPlan())), ), { message: 'Android automation helper failed before returning parseable output' }, ); diff --git a/src/platforms/android/__tests__/touch-plan.test.ts b/src/platforms/android/__tests__/touch-plan.test.ts new file mode 100644 index 0000000000..f87e0362ea --- /dev/null +++ b/src/platforms/android/__tests__/touch-plan.test.ts @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { buildGesturePlan } from '@agent-device/contracts/interaction'; +import { lowerAndroidTouchPlan, type AndroidLoweredTouchPlan } from '../touch-plan.ts'; +import { longPressPlan } from './touch-helper.fixtures.ts'; + +const viewport = { x: 0, y: 0, width: 400, height: 800 }; + +test('lowers a 500 ms endpoint plan with the Android rounded cadence', () => { + const plan = buildGesturePlan( + { + intent: 'pan', + origin: { x: 100, y: 200 }, + delta: { x: 100, y: 200 }, + durationMs: 500, + }, + viewport, + ); + const lowered = lowerAndroidTouchPlan(plan); + + assert.equal(lowered.topology, 'single'); + assert.equal(lowered.pointers[0].samples.length, 32); + assert.deepEqual( + lowered.pointers[0].samples.map(({ offsetMs }) => offsetMs), + [ + 0, 16, 32, 48, 65, 81, 97, 113, 129, 145, 161, 177, 194, 210, 226, 242, 258, 274, 290, 306, + 323, 339, 355, 371, 387, 403, 419, 435, 452, 468, 484, 500, + ], + ); +}); + +test('preserves the pre-endpoint Android sample schedule and interpolation', () => { + const plan = buildGesturePlan( + { intent: 'fling', from: { x: 300, y: 400 }, to: { x: 100, y: 400 } }, + viewport, + ); + const lowered = lowerAndroidTouchPlan(plan); + + assert.deepEqual(lowered.pointers[0].samples, [ + { offsetMs: 0, point: { x: 300, y: 400 } }, + { offsetMs: 17, point: { x: 266, y: 400 } }, + { offsetMs: 33, point: { x: 234, y: 400 } }, + { offsetMs: 50, point: { x: 200, y: 400 } }, + { offsetMs: 67, point: { x: 166, y: 400 } }, + { offsetMs: 83, point: { x: 134, y: 400 } }, + { offsetMs: 100, point: { x: 100, y: 400 } }, + ]); +}); + +test('does not lower long-press or two-pointer plans', () => { + const longPress = longPressPlan(); + assert.equal(lowerAndroidTouchPlan(longPress), longPress); + + const twoPointer = buildGesturePlan( + { + intent: 'pan', + pointerCount: 2, + origin: { x: 200, y: 300 }, + delta: { x: 20, y: 10 }, + durationMs: 32, + }, + viewport, + ); + assert.equal(lowerAndroidTouchPlan(twoPointer), twoPointer); +}); + +test('lowered endpoint plans are valid Android transport plans', () => { + const plan = buildGesturePlan( + { + intent: 'pan', + origin: { x: 100, y: 200 }, + delta: { x: 100, y: 200 }, + durationMs: 64, + }, + viewport, + ); + const lowered: AndroidLoweredTouchPlan = lowerAndroidTouchPlan(plan); + assert.equal(lowered.pointers[0].samples.length, 5); +}); diff --git a/src/platforms/android/touch-executor.ts b/src/platforms/android/touch-executor.ts index f96658c2e3..eaf99fddfe 100644 --- a/src/platforms/android/touch-executor.ts +++ b/src/platforms/android/touch-executor.ts @@ -3,25 +3,26 @@ import type { Rect } from '@agent-device/kernel/snapshot'; import { resolveAndroidTouchProvider } from './adb-executor.ts'; import { executeAndroidTouchHelperPlan, readAndroidTouchHelperViewport } from './touch-helper.ts'; import { validateAndroidGestureViewport } from './gesture-viewport.ts'; -import type { AndroidTouchPlan } from './touch-plan.ts'; +import { lowerAndroidTouchPlan, type AndroidTouchPlan } from './touch-plan.ts'; export async function executeAndroidTouchPlan( device: DeviceInfo, plan: AndroidTouchPlan, ): Promise> { + const loweredPlan = lowerAndroidTouchPlan(plan); const provider = resolveAndroidTouchProvider(device); if (provider) { const providerPlan = - plan.intent === 'longPress' + loweredPlan.intent === 'longPress' ? { - ...plan, + ...loweredPlan, viewport: validateAndroidGestureViewport(await provider.gestureViewport()), } - : plan; + : loweredPlan; const result = (await provider.touch(providerPlan)) ?? {}; return { backend: 'provider-native-touch', ...result }; } - return await executeAndroidTouchHelperPlan(device, plan); + return await executeAndroidTouchHelperPlan(device, loweredPlan); } export async function readAndroidGestureViewport(device: DeviceInfo): Promise { diff --git a/src/platforms/android/touch-helper.ts b/src/platforms/android/touch-helper.ts index e339cc68ce..8a6f5f4f80 100644 --- a/src/platforms/android/touch-helper.ts +++ b/src/platforms/android/touch-helper.ts @@ -10,7 +10,7 @@ import { readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { validateAndroidGestureViewport } from './gesture-viewport.ts'; -import type { AndroidTouchPlan } from './touch-plan.ts'; +import type { AndroidLoweredTouchPlan } from './touch-plan.ts'; import { resolveAndroidHelperArtifact } from './helper-package-install.ts'; import { parseAndroidSnapshotHelperManifest } from './snapshot-helper-artifact.ts'; import { ensureAndroidSnapshotHelper } from './snapshot-helper-install.ts'; @@ -54,7 +54,7 @@ type PreparedAndroidTouchHelper = { export async function executeAndroidTouchHelperPlan( device: DeviceInfo, - plan: AndroidTouchPlan, + plan: AndroidLoweredTouchPlan, ): Promise> { const prepared = await prepareAndroidTouchHelper(device); const request = normalizeAndroidTouchHelperGestureRequest(plan); @@ -209,7 +209,7 @@ async function resolveAndroidTouchHelperArtifact(): Promise & { + pointers: readonly [AndroidTransportSinglePointerTrajectory]; +}; + +export type AndroidTransportGesturePlan = + | AndroidTransportSinglePointerGesturePlan + | MultiTouchGesturePlan; + +export type AndroidLoweredTouchPlan = AndroidTransportGesturePlan | AndroidLongPressTouchPlan; + export type AndroidProviderTouchPlan = - | GesturePlan + | AndroidTransportGesturePlan | (AndroidLongPressTouchPlan & { viewport: Rect }); + +export function lowerAndroidTouchPlan(plan: AndroidTouchPlan): AndroidLoweredTouchPlan { + if (plan.topology === 'two' || plan.intent === 'longPress') return plan; + + const [ + { + pointerId, + samples: [start, end], + }, + ] = plan.pointers; + const samples = sampleGestureOffsets(plan.durationMs, 'android').map((offsetMs) => ({ + offsetMs, + point: interpolateGesturePoint(start.point, end.point, offsetMs / plan.durationMs), + })); + + return { + ...plan, + pointers: [{ pointerId, samples }], + }; +}