Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,17 @@ extension RunnerTests {
)
}

private func plannedGestureResponse(
plan: RunnerGesturePlan,
timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
outcome: RunnerInteractionOutcome
) -> Response {
if let response = unsupportedResponse(for: outcome) {
return response
}
return gestureResponse(message: plan.intent, timing: timing)
}

#if AGENT_DEVICE_RUNNER_UNIT_TESTS
func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() {
let response = gestureResponse(
Expand Down Expand Up @@ -2056,7 +2067,8 @@ extension RunnerTests {
error: ErrorPayload(code: "INVALID_ARGS", message: validationError)
)
}
if plannedGestureExecution(for: plan) == .fastSwipe {
switch plannedGestureExecution(for: plan) {
case .fastSwipe:
// Validation above guarantees a non-empty, single-pointer path for this execution kind.
let first = plan.pointers[0].samples.first!.point
let last = plan.pointers[0].samples.last!.point
Expand All @@ -2074,14 +2086,12 @@ extension RunnerTests {
synthesizedProfile: .fastSwipe
)
)
case .sampled:
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
sampledPlannedGesture(app: activeApp, plan: plan)
}
return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome)
}
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
sampledPlannedGesture(app: activeApp, plan: plan)
}
if let response = unsupportedResponse(for: outcome) {
return response
}
return gestureResponse(message: plan.intent, timing: timing)
case .gestureViewport:
let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else {
Expand Down
58 changes: 34 additions & 24 deletions docs/adr/0013-unified-gesture-plans.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,10 @@ components atomically. Intent remains on the plan even when aliases share an exe
The planner owns deterministic multi-touch geometry. Contacts start at -90 degrees, except Android
pinch starts horizontally because a vertical pinch is captured by common vertical app scroll
containers before the pinch recognizer activates. The same explicit planning profile preserves the
proven frame-count convention: Android rounds while Apple truncates the duration/16 ms frame count.
These are planner inputs, not adapter-generated trajectories. The larger of pinch's initial and final spans is 40% of the
proven frame-count convention for dense two-contact trajectories: Android rounds while Apple
truncates the duration/16 ms frame count. The Android transport lowerer uses that same Android
sampling profile for one-contact endpoint plans. These are planner inputs, not adapter-generated
trajectories. The larger of pinch's initial and final spans is 40% of the
viewport's shorter side, preserving the proven Apple pinch geometry; other two-contact intents use
25% to keep translation and rotation trajectories compact. The other span follows from the requested scale, and both must
satisfy a 48-point reliability floor. Combined transforms progress translation, scale, and rotation
Expand All @@ -70,28 +72,31 @@ Platform adapters consume the canonical plan:

- Android's `executeAndroidTouchPlan` adapter seam sends planned touch, including gesture plans plus
the physical movement for scroll and long-press, to provider-native touch injection when
available, otherwise to the bundled instrumentation helper. The helper injects the exact planned
pointer samples. A stationary long-press needs no viewport on the helper path; the executor adds
the paired provider-owned viewport only for provider-native touch. Android touch execution never
falls back to `adb input swipe`. Public scroll durations below one 16 ms planner frame normalize
to that physical minimum and report the executed duration. Scroll evidence reports absolute
injected coordinates against zero-origin extents that include the viewport offset. Because
Android permits only one instrumentation owner of `UiAutomation`, snapshot capture, gesture
viewport resolution, and planned-touch injection share one bundled automation helper: a live
persistent helper session executes touch commands directly, and without one the same helper runs
one-shot. Nothing stops the snapshot session around gestures anymore (amended 2026-07,
issue #1275; previously a separate one-shot multi-touch helper forced a session stop/restart
around every local gesture).
available, otherwise to the bundled instrumentation helper. One-contact endpoint plans lower in
`src/platforms/android/touch-plan.ts` to 16 ms linear transport samples before either injection
path; two-contact plans retain their exact planned samples. A stationary long-press needs no
viewport on the helper path; the executor adds the paired provider-owned viewport only for
provider-native touch. Android touch execution never falls back to `adb input swipe`. Public
scroll durations below one 16 ms planner frame normalize to that physical minimum and report the
executed duration. Scroll evidence reports absolute injected coordinates against zero-origin
extents that include the viewport offset. Because Android permits only one instrumentation owner
of `UiAutomation`, snapshot capture, gesture viewport resolution, and planned-touch injection
share one bundled automation helper: a live persistent helper session executes touch commands
directly, and without one the same helper runs one-shot. Nothing stops the snapshot session
around gestures anymore (amended 2026-07, issue #1275; previously a separate one-shot
multi-touch helper forced a session stop/restart around every local gesture).
- iOS lowers one-contact endpoint-hold plans to the established fast-swipe synthesis profile. That
profile reaches the endpoint in 100 ms, then holds there for the planned duration before lifting,
matching Maestro's XCTest driver. Timed-pan and two-contact plans convert every point to native
orientation and feed the exact planned arrays to the private XCTest event bridge. Android and
WebDriver continue to execute the plan samples across the authored duration, matching their
native Maestro drivers. macOS lowers a one-contact plan to its drag executor and tvOS lowers it to remote
direction. Core admission and the Apple adapter both consume the same shared multi-touch support
policy; multi-touch remains capability-gated to iOS simulators.
- WebDriver lowers a supported plan to synchronized W3C pointer action sources. Multi-touch remains
capability-gated until a provider proves it.
matching Maestro's XCTest driver. One-contact plans are linear and therefore carry only their
start and end samples, with the authored duration between them. Two-contact plans convert every
planned point to native orientation and feed the exact arrays to the private XCTest event bridge.
macOS lowers a one-contact plan to its drag executor and tvOS lowers it to remote direction. Core
admission and the Apple adapter both consume the same shared multi-touch support policy;
multi-touch remains capability-gated to iOS simulators.
- WebDriver lowers a supported plan to synchronized W3C pointer action sources. A one-contact
endpoint plan becomes pointer down, one timed W3C `pointerMove` from start to end, and pointer up;
the driver owns interpolation across that W3C tick. Multi-touch remains capability-gated until a
provider proves it.

The `Interactor` and backend expose one compositional `performGesture(plan)` primitive instead of a
method per semantic alias. The old scalar Apple and Android multi-touch executors and the
Expand Down Expand Up @@ -139,8 +144,13 @@ selectors or refs and therefore cannot claim element-targeting guarantees.
- On bare ADB, Android scroll and long-press require the bundled automation helper (the snapshot
helper APK) and `UiAutomation`; helper installation or runtime failure is surfaced directly
rather than degrading to an approximate `adb input swipe`.
- Pointer plans are larger than scalar requests but bounded by duration and the 16 ms sample
cadence; deleting duplicate scalar executors offsets the package cost.
- Canonical one-contact plans contain only two samples: a 10-second pan is two shared-plan samples
instead of 626. Android expands that plan only at the transport boundary, while two-contact
plans remain cadence-bounded because their synchronized geometry is part of their contract.
- Unit tests cover canonical plan shape, Android lowering, helper/provider payloads, and WebDriver
action construction. They cannot prove timing or event delivery inside the private XCTest bridge,
so iOS timing changes require live simulator evidence that observes the requested content change
and records the runner start/end uptime delta alongside the requested duration.

## Alternatives Considered

Expand Down
7 changes: 6 additions & 1 deletion packages/contracts/src/gesture-plan-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,18 @@ export type PointerTrajectory = {
samples: readonly PointerTrajectorySample[];
};

export type SinglePointerTrajectory = {
pointerId: 0;
samples: readonly [PointerTrajectorySample, PointerTrajectorySample];
};

export type SinglePointerGesturePlan = {
topology: 'single';
intent: 'fling' | 'pan';
executionProfile: GestureExecutionProfile;
durationMs: number;
viewport: Rect;
pointers: readonly [PointerTrajectory];
pointers: readonly [SinglePointerTrajectory];
};

export type MultiTouchGesturePlan = {
Expand Down
56 changes: 31 additions & 25 deletions packages/contracts/src/gesture-plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type {
MultiTouchGesturePlan,
PointerTrajectory,
SinglePointerGesturePlan,
SinglePointerTrajectory,
} from './gesture-plan-types.ts';

export * from './gesture-plan-types.ts';
Expand All @@ -33,21 +34,23 @@ const DEFAULT_MULTI_TOUCH_DURATION_MS = 300;
const MAX_ROTATION_DEGREES_PER_SAMPLE = 3;
const MAX_ROTATION_DEFAULT_DURATION_MS = 2_400;

export type GestureSamplingProfile = 'default' | 'android';

type GesturePlatformProfile = {
pinchAxisDegrees: number;
frameCount: (rawFrameCount: number) => number;
samplingProfile: GestureSamplingProfile;
};

const DEFAULT_GESTURE_PLATFORM_PROFILE: GesturePlatformProfile = {
pinchAxisDegrees: GESTURE_INITIAL_ANGLE_DEGREES,
frameCount: Math.floor,
samplingProfile: 'default',
};
const GESTURE_PLATFORM_PROFILES = {
ios: DEFAULT_GESTURE_PLATFORM_PROFILE,
macos: DEFAULT_GESTURE_PLATFORM_PROFILE,
android: {
pinchAxisDegrees: GESTURE_HORIZONTAL_ANGLE_DEGREES,
frameCount: Math.round,
samplingProfile: 'android',
},
vega: DEFAULT_GESTURE_PLATFORM_PROFILE,
linux: DEFAULT_GESTURE_PLATFORM_PROFILE,
Expand All @@ -64,7 +67,7 @@ export function buildGesturePlan(
const profile = gesturePlatformProfile(platform);
switch (input.intent) {
case 'fling':
return buildFlingPlan(input, frame, profile);
return buildFlingPlan(input, frame);
case 'pan':
return buildPanPlan(input, frame, profile);
case 'pinch':
Expand Down Expand Up @@ -121,18 +124,17 @@ export function singlePointerPlanEndpoints(plan: SinglePointerGesturePlan): {
start: Point;
end: Point;
} {
const start = plan.pointers[0].samples[0]?.point;
const end = plan.pointers[0].samples.at(-1)?.point;
if (!start || !end) {
throw new AppError('INVALID_ARGS', 'single-pointer gesture plan requires samples');
}
return { start, end };
const [
{
samples: [start, end],
},
] = plan.pointers;
return { start: start.point, end: end.point };
}

function buildFlingPlan(
input: Extract<GestureSemanticInput, { intent: 'fling' }>,
viewport: Rect,
profile: GesturePlatformProfile,
): SinglePointerGesturePlan {
if ('preset' in input) {
const { from, to } = presetGestureEndpoints(input.preset, viewport);
Expand All @@ -143,7 +145,6 @@ function buildFlingPlan(
GESTURE_FLING_DURATION_MS,
viewport,
'endpoint-hold',
profile,
);
}
if ('from' in input) {
Expand All @@ -154,7 +155,6 @@ function buildFlingPlan(
GESTURE_FLING_DURATION_MS,
viewport,
'endpoint-hold',
profile,
);
}
const start = finitePoint(input.origin, 'gesture fling origin');
Expand All @@ -169,7 +169,6 @@ function buildFlingPlan(
GESTURE_FLING_DURATION_MS,
viewport,
'endpoint-hold',
profile,
);
}

Expand All @@ -193,7 +192,6 @@ function buildPanPlan(
durationMs,
viewport,
input.executionProfile ?? 'timed-pan',
profile,
);
}
if (input.pointerCount !== 2) {
Expand All @@ -220,14 +218,13 @@ function buildSinglePointerPlan(
durationMs: number,
viewport: Rect,
executionProfile: GestureExecutionProfile,
profile: GesturePlatformProfile,
): SinglePointerGesturePlan {
const start = finitePoint(from, `gesture ${intent} start`);
const end = finitePoint(to, `gesture ${intent} end`);
const samples = sampleOffsets(durationMs, profile).map((offsetMs) => ({
offsetMs,
point: interpolatePoint(start, end, offsetMs / durationMs),
}));
const samples: SinglePointerTrajectory['samples'] = [
{ offsetMs: 0, point: start },
{ offsetMs: durationMs, point: end },
];
assertSamplesInViewport(samples, viewport, { intent, pointerId: 0 });
return {
topology: 'single',
Expand Down Expand Up @@ -269,7 +266,7 @@ function buildTransformPlan(
}
const initialRadius = initialSpan / 2;

const offsets = sampleOffsets(motion.durationMs, profile);
const offsets = sampleGestureOffsets(motion.durationMs, profile.samplingProfile);
const trajectory = (pointerId: 0 | 1, side: 1 | -1): PointerTrajectory => {
const samples = offsets.map((offsetMs) => ({
offsetMs,
Expand Down Expand Up @@ -309,7 +306,7 @@ function transformPointAt(options: {
profile: GesturePlatformProfile;
}): Point {
const progress = options.offsetMs / options.durationMs;
const centroid = interpolatePoint(options.start, options.end, progress);
const centroid = interpolateGesturePoint(options.start, options.end, progress);
const radius = options.initialRadius * (1 + (options.scale - 1) * progress);
const angle = degreesToRadians(
initialAngleForIntent(options.intent, options.profile) + options.rotationDegrees * progress,
Expand All @@ -331,9 +328,18 @@ function initialSpanRatioForIntent(intent: MultiTouchGesturePlan['intent']): num
return intent === 'pinch' ? GESTURE_PINCH_INITIAL_SPAN_RATIO : GESTURE_INITIAL_SPAN_RATIO;
}

function sampleOffsets(durationMs: number, profile: GesturePlatformProfile): number[] {
export function sampleGestureOffsets(
durationMs: number,
profile: GestureSamplingProfile = 'default',
): number[] {
if (!Number.isFinite(durationMs) || durationMs <= 0) {
throw new AppError('INVALID_ARGS', 'gesture sample duration must be a positive finite number');
}
const rawFrameCount = durationMs / GESTURE_SAMPLE_INTERVAL_MS;
const frameCount = Math.max(3, profile.frameCount(rawFrameCount));
const frameCount = Math.max(
3,
profile === 'android' ? Math.round(rawFrameCount) : Math.floor(rawFrameCount),
);
return Array.from({ length: frameCount + 1 }, (_, index) =>
Math.round((durationMs * index) / frameCount),
);
Expand Down Expand Up @@ -446,7 +452,7 @@ function addPoints(left: Point, right: Point): Point {
return { x: left.x + right.x, y: left.y + right.y };
}

function interpolatePoint(start: Point, end: Point, progress: number): Point {
export function interpolateGesturePoint(start: Point, end: Point, progress: number): Point {
return {
x: start.x + (end.x - start.x) * progress,
y: start.y + (end.y - start.y) * progress,
Expand Down
52 changes: 52 additions & 0 deletions packages/provider-webdriver/src/webdriver-interactor.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import { buildGesturePlan } from '@agent-device/contracts/interaction';
import { createCloudWebDriverCapabilities } from './capabilities.ts';
import type { WebDriverClient, W3CActionSequence } from './webdriver-client.ts';
import { createWebDriverInteractor } from './webdriver-interactor.ts';

test('endpoint plans become one timed W3C pointer move', async () => {
const performed: W3CActionSequence[][] = [];
let released = false;
const client = {
performActions: async (actions: W3CActionSequence[]) => {
performed.push(actions);
},
releaseActions: async () => {
released = true;
},
} as unknown as WebDriverClient;
const interactor = createWebDriverInteractor({
client,
backend: 'android',
capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }),
});
const plan = buildGesturePlan(
{
intent: 'pan',
origin: { x: 100, y: 200 },
delta: { x: 100, y: 200 },
durationMs: 500,
},
{ x: 0, y: 0, width: 400, height: 800 },
);

assert.ok(interactor.performGesture);
assert.deepEqual(await interactor.performGesture(plan), { backend: 'webdriver-w3c-actions' });
assert.equal(released, true);
assert.deepEqual(performed, [
[
{
type: 'pointer',
id: 'gesture-pointer-0',
parameters: { pointerType: 'touch' },
actions: [
{ type: 'pointerMove', duration: 0, x: 100, y: 200 },
{ type: 'pointerDown', button: 0 },
{ type: 'pointerMove', duration: 500, x: 200, y: 400 },
{ type: 'pointerUp', button: 0 },
],
},
],
]);
});
Loading
Loading