From 3b636f020cd0fd4a848194f8fc3aab0f4ec2a5d1 Mon Sep 17 00:00:00 2001 From: Thiago Brezinski Date: Mon, 3 Aug 2026 14:38:23 +0100 Subject: [PATCH 1/7] feat: add selector-targeted drag gestures --- docs/adr/0013-unified-gesture-plans.md | 18 +- packages/contracts/src/client-gesture.ts | 8 + packages/contracts/src/gesture-input.test.ts | 34 ++++ packages/contracts/src/gesture-input.ts | 45 ++++- .../src/gesture-normalization.test.ts | 57 ++++++ .../contracts/src/gesture-normalization.ts | 45 ++++- packages/contracts/src/gesture-plan-types.ts | 11 ++ packages/contracts/src/gesture-plan.test.ts | 83 +++++++++ packages/contracts/src/gesture-plan.ts | 85 +++++++++ packages/contracts/src/interaction.ts | 2 +- src/__tests__/client.test.ts | 31 ++++ .../test-utils/property-arbitraries.ts | 6 +- src/agent-device-client.ts | 6 + .../parser/__tests__/cli-help-topics.test.ts | 2 +- src/cli/parser/cli-help.ts | 2 +- src/client/client-types.ts | 2 + src/commands/interaction/gesture.test.ts | 16 ++ src/commands/interaction/index.ts | 25 ++- src/commands/interaction/metadata.ts | 13 +- .../runtime/__tests__/test-utils/index.ts | 31 ++++ .../interaction/runtime/gesture-command.ts | 173 +++++++++++++++++- .../interaction/runtime/gestures.test.ts | 76 ++++++++ .../__tests__/gesture-capabilities.test.ts | 14 ++ src/core/capabilities.ts | 25 ++- .../__tests__/parity.test.ts | 1 + src/core/command-descriptor/registry.ts | 1 + .../__tests__/session-script-writer.test.ts | 19 ++ .../interaction-gesture-drag.test.ts | 164 +++++++++++++++++ .../session-replay-target-token.test.ts | 16 ++ src/daemon/handlers/interaction-gesture.ts | 129 ++++++++++--- src/daemon/handlers/interaction-ref-policy.ts | 23 ++- .../handlers/session-replay-target-token.ts | 1 + src/daemon/session-script-writer.ts | 13 ++ src/mcp/__tests__/command-tools.test.ts | 36 ++++ website/docs/docs/client-api.md | 2 +- website/docs/docs/commands.md | 3 + 36 files changed, 1156 insertions(+), 62 deletions(-) create mode 100644 packages/contracts/src/gesture-plan.test.ts create mode 100644 src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index cce2d70868..483bd35a8d 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -33,7 +33,8 @@ while still routing through the canonical `pan` input. The runtime plans canonical intent in `packages/contracts/src/gesture-plan.ts`. Contact topology is separate from motion: -- one contact: pan or fling with a complete pointer trajectory and an explicit execution profile; +- one contact: pan, fling, or target-authored drag with a complete pointer trajectory and an explicit + execution profile; - two contacts: pan, pinch, rotate, or transform with two complete, synchronized trajectories. `swipe` without a duration remains public sugar for a fixed-duration fling. Timed public forms @@ -126,8 +127,19 @@ Public two-finger pan is additive: `pointerCount?: 1 | 2` on pan and CLI `kind`, `durationMs`, `pointerCount`, `from`, and `to` fields, followed by backend evidence. Recording/replay keeps its existing public command identity and session semantics. -ADR 0011's element dispatch-path matrix remains unchanged: coordinate gestures do not resolve -selectors or refs and therefore cannot claim element-targeting guarantees. +Target-authored drag is additive: `gesture drag ` accepts a selector or pinned +snapshot ref at each endpoint, resolves both endpoints before device injection, then lowers their center +points to one uninterrupted single-pointer plan. The plan holds at the source, moves over the authored +duration, optionally holds at the destination, and releases. Both endpoint-resolution disclosures and +portable selector chains are returned to the caller. Recordings replace session-local refs at both +endpoints with those selector chains; the source additionally carries the action's `target-v1` identity +evidence because it is the element whose mutation is initiated. Ref admission happens for both endpoints +before either is dispatched, and the usual mutation boundary expires the frame after the gesture. + +ADR 0011's coordinate path remains unchanged: coordinate-authored gestures do not resolve selectors or +refs and therefore cannot claim element-targeting guarantees. Target-authored drag explicitly runs the +shared selector/ref resolution preflight before it enters the coordinate gesture executor; its endpoint +resolution and recording contracts are tested at that composition seam. ## Consequences diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 998f547143..823f318e4f 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -81,6 +81,14 @@ export type PanOptions = DeviceCommandBaseOptions & { durationMs?: number; }; +export type DragOptions = DeviceCommandBaseOptions & { + source: string; + destination: string; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; +}; + export type FlingOptions = DeviceCommandBaseOptions & { direction: ScrollDirection; x: number; diff --git a/packages/contracts/src/gesture-input.test.ts b/packages/contracts/src/gesture-input.test.ts index c59b3f4bca..b18c5952ef 100644 --- a/packages/contracts/src/gesture-input.test.ts +++ b/packages/contracts/src/gesture-input.test.ts @@ -47,3 +47,37 @@ test('rotate does not accept velocity', () => { code: 'INVALID_ARGS', }); }); + +test('drag requires two non-empty targets and validates each timing phase', () => { + for (const input of [ + { kind: 'drag', source: '', destination: 'id="drop-target"' }, + { kind: 'drag', source: 'id="drag-source"', destination: ' ' }, + { kind: 'drag', source: 'id="drag-source"', destination: 'id="drop-target"', sourceHoldMs: 0 }, + { kind: 'drag', source: 'id="drag-source"', destination: 'id="drop-target"', moveMs: 15 }, + { + kind: 'drag', + source: 'id="drag-source"', + destination: 'id="drop-target"', + destinationHoldMs: -1, + }, + ]) { + assert.throws(() => readGesturePayload(input), { code: 'INVALID_ARGS' }); + } + + assert.deepEqual( + readGesturePayload({ + kind: 'drag', + source: 'id="drag-source"', + destination: '@e2~s42', + destinationHoldMs: 0, + }), + { + kind: 'drag', + source: 'id="drag-source"', + destination: '@e2~s42', + sourceHoldMs: undefined, + moveMs: undefined, + destinationHoldMs: 0, + }, + ); +}); diff --git a/packages/contracts/src/gesture-input.ts b/packages/contracts/src/gesture-input.ts index ad9fb62244..e48f0acfdb 100644 --- a/packages/contracts/src/gesture-input.ts +++ b/packages/contracts/src/gesture-input.ts @@ -13,7 +13,24 @@ import { type GesturePointerCount, } from './gesture-plan-types.ts'; -export const GESTURE_KINDS = ['pan', 'fling', 'swipe', 'pinch', 'rotate', 'transform'] as const; +export const COORDINATE_GESTURE_KINDS = [ + 'pan', + 'fling', + 'swipe', + 'pinch', + 'rotate', + 'transform', +] as const; +export const GESTURE_KINDS = [...COORDINATE_GESTURE_KINDS, 'drag'] as const; + +export type DragGesturePayload = { + kind: 'drag'; + source: string; + destination: string; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; +}; export type PanGesturePayload = { kind: 'pan'; @@ -62,7 +79,10 @@ export type GesturePayload = | SwipeGesturePayload | PinchGesturePayload | RotateGesturePayload - | TransformGesturePayload; + | TransformGesturePayload + | DragGesturePayload; + +export type CoordinateGesturePayload = Exclude; export function readGesturePayload(input: unknown): GesturePayload { const record = readRecord(input); @@ -78,6 +98,19 @@ export function readGesturePayload(input: unknown): GesturePayload { durationMs: readOptionalGestureDuration(record), }; } + if (kind === 'drag') { + return { + kind, + source: readNonEmptyString(record, 'source'), + destination: readNonEmptyString(record, 'destination'), + sourceHoldMs: readOptionalInteger(record, 'sourceHoldMs', { min: 1, max: 10_000 }), + moveMs: readOptionalInteger(record, 'moveMs', { min: 16, max: 10_000 }), + destinationHoldMs: readOptionalInteger(record, 'destinationHoldMs', { + min: 0, + max: 10_000, + }), + }; + } if (record.pointerCount !== undefined) { throw new AppError('INVALID_ARGS', 'pointerCount is supported only for gesture pan'); } @@ -173,6 +206,14 @@ function readNumber(record: Record, key: string): number { return value; } +function readNonEmptyString(record: Record, key: string): string { + const value = record[key]; + if (typeof value !== 'string' || value.trim().length === 0) { + throw new AppError('INVALID_ARGS', `Expected ${key} to be a non-empty string.`); + } + return value; +} + function readEnum( record: Record, key: string, diff --git a/packages/contracts/src/gesture-normalization.test.ts b/packages/contracts/src/gesture-normalization.test.ts index 1ac2bfb136..bf1b98a702 100644 --- a/packages/contracts/src/gesture-normalization.test.ts +++ b/packages/contracts/src/gesture-normalization.test.ts @@ -4,6 +4,7 @@ import { describeReplayGestureArityError, gesturePayloadFromPositionals, gesturePayloadToPositionals, + normalizeGestureCommandInput, normalizePublicGesture, normalizePublicSwipeMotion, swipePayloadFromPositionals, @@ -58,6 +59,62 @@ test('gesture recording codec round-trips fling with distance', () => { assert.deepEqual(gesturePayloadFromPositionals(gesturePayloadToPositionals(payload)), payload); }); +test('gesture recording codec round-trips selector-authored hold drag timings', () => { + const payload = { + kind: 'drag' as const, + source: 'id="drag-source"', + destination: '@e2~s42', + sourceHoldMs: 800, + moveMs: 700, + destinationHoldMs: 250, + }; + assert.deepEqual(gesturePayloadFromPositionals(gesturePayloadToPositionals(payload)), payload); +}); + +test('drag payloads normalize to one target-authored runtime command', () => { + assert.deepEqual( + normalizeGestureCommandInput({ + kind: 'drag', + source: 'id="drag-source"', + destination: 'id="drop-target"', + sourceHoldMs: 700, + moveMs: 450, + destinationHoldMs: 100, + }), + { + intent: 'drag', + source: 'id="drag-source"', + destination: 'id="drop-target"', + sourceHoldMs: 700, + moveMs: 450, + destinationHoldMs: 100, + }, + ); +}); + +test('gesture drag positional syntax requires both endpoints and rejects trailing arguments', () => { + assert.throws(() => gesturePayloadFromPositionals(['drag', 'id="drag-source"']), { + code: 'INVALID_ARGS', + }); + assert.throws( + () => + gesturePayloadFromPositionals([ + 'drag', + 'id="drag-source"', + 'id="drop-target"', + '800', + '500', + '0', + 'extra', + ]), + { + code: 'INVALID_ARGS', + message: + 'gesture drag accepts at most 5 arguments: source destination [sourceHoldMs] [moveMs] [destinationHoldMs].', + }, + ); +}); + test('a retired trailing positional reports its migration, not a bare usage line', () => { assert.throws(() => swipePayloadFromPositionals(['197', '650', '197', '300', '300']), { code: 'INVALID_ARGS', diff --git a/packages/contracts/src/gesture-normalization.ts b/packages/contracts/src/gesture-normalization.ts index 73e26f96cb..57e655efe2 100644 --- a/packages/contracts/src/gesture-normalization.ts +++ b/packages/contracts/src/gesture-normalization.ts @@ -1,7 +1,7 @@ import type { Point } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { readGesturePayload, type GESTURE_KINDS, type GesturePayload } from './gesture-input.ts'; -import type { GestureSemanticInput } from './gesture-plan-types.ts'; +import type { GestureCommandInput, GestureSemanticInput } from './gesture-plan-types.ts'; export type NormalizedPublicGesture = { gesture: GestureSemanticInput; @@ -81,6 +81,11 @@ const PUBLIC_GESTURE_SYNTAX: Record = { max: 7, usage: 'gesture transform accepts at most 7 arguments: x y dx dy scale degrees [durationMs]', }, + 'gesture drag': { + max: 5, + usage: + 'gesture drag accepts at most 5 arguments: source destination [sourceHoldMs] [moveMs] [destinationHoldMs]', + }, }; /** `swipe x1 y1 x2 y2 durationMs` translates to the equivalent timed pan. */ @@ -256,6 +261,17 @@ export function gesturePayloadFromPositionals( durationMs: optionalPositionNumber(args[6]), }); } + case 'drag': { + assertGestureArity('gesture drag', args); + return readGesturePayload({ + kind, + source: args[0], + destination: args[1], + sourceHoldMs: optionalPositionNumber(args[2]), + moveMs: optionalPositionNumber(args[3]), + destinationHoldMs: optionalPositionNumber(args[4]), + }); + } default: return readGesturePayload({ kind }); } @@ -294,6 +310,15 @@ export function gesturePayloadToPositionals(input: GesturePayload): string[] { input.degrees, input.durationMs, ]); + case 'drag': + return compact([ + input.kind, + input.source, + input.destination, + input.sourceHoldMs, + input.moveMs, + input.destinationHoldMs, + ]); } } @@ -355,9 +380,27 @@ export function normalizePublicGesture(input: GesturePayload): NormalizedPublicG durationMs: input.durationMs, }, }; + case 'drag': + throw new AppError( + 'INVALID_ARGS', + 'gesture drag targets must be resolved before coordinate normalization', + ); } } +/** Converts every public gesture payload into the runtime's semantic command shape. */ +export function normalizeGestureCommandInput(input: GesturePayload): GestureCommandInput { + if (input.kind !== 'drag') return normalizePublicGesture(input).gesture; + return { + intent: 'drag', + source: input.source, + destination: input.destination, + sourceHoldMs: input.sourceHoldMs, + moveMs: input.moveMs, + destinationHoldMs: input.destinationHoldMs, + }; +} + export function normalizePublicSwipeMotion(input: { from: Point; to: Point; diff --git a/packages/contracts/src/gesture-plan-types.ts b/packages/contracts/src/gesture-plan-types.ts index 94bf7f7dd3..f379d5b7f4 100644 --- a/packages/contracts/src/gesture-plan-types.ts +++ b/packages/contracts/src/gesture-plan-types.ts @@ -8,6 +8,15 @@ export const GESTURE_DURATION_MAX_MS = 10_000; export type GestureIntent = 'fling' | 'pan' | 'pinch' | 'rotate' | 'transform'; +export type DragGestureInput = { + intent: 'drag'; + source: string; + destination: string; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; +}; + /** Selects one-pointer release timing without changing semantic gesture intent. */ export type GestureExecutionProfile = 'endpoint-hold' | 'timed-pan'; @@ -39,6 +48,8 @@ export type GestureSemanticInput = durationMs?: number; }; +export type GestureCommandInput = GestureSemanticInput | DragGestureInput; + export type PointerTrajectorySample = { offsetMs: number; point: Point }; export type PointerTrajectory = { diff --git a/packages/contracts/src/gesture-plan.test.ts b/packages/contracts/src/gesture-plan.test.ts new file mode 100644 index 0000000000..677d229641 --- /dev/null +++ b/packages/contracts/src/gesture-plan.test.ts @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import fc from 'fast-check'; +import { test } from 'vitest'; +import { buildDragGesturePlan } from './gesture-plan.ts'; + +const DRAG_PROPERTY_RUNS = 100; + +test('hold drag keeps one pointer down through source hold, movement, and destination hold', () => { + const plan = buildDragGesturePlan( + { + from: { x: 20, y: 30 }, + to: { x: 120, y: 230 }, + sourceHoldMs: 800, + moveMs: 700, + destinationHoldMs: 250, + }, + { x: 0, y: 0, width: 400, height: 800 }, + 'ios', + ); + + assert.equal(plan.topology, 'single'); + assert.equal(plan.durationMs, 1_750); + assert.deepEqual(plan.pointers[0]?.samples[0], { offsetMs: 0, point: { x: 20, y: 30 } }); + assert.deepEqual(plan.pointers[0]?.samples[1], { offsetMs: 800, point: { x: 20, y: 30 } }); + assert.deepEqual(plan.pointers[0]?.samples.at(-2), { + offsetMs: 1_500, + point: { x: 120, y: 230 }, + }); + assert.deepEqual(plan.pointers[0]?.samples.at(-1), { + offsetMs: 1_750, + point: { x: 120, y: 230 }, + }); +}); + +test('hold drag rejects a combined duration above the backend ceiling', () => { + assert.throws( + () => + buildDragGesturePlan( + { + from: { x: 20, y: 30 }, + to: { x: 120, y: 230 }, + sourceHoldMs: 5_000, + moveMs: 5_000, + destinationHoldMs: 1, + }, + { x: 0, y: 0, width: 400, height: 800 }, + ), + { code: 'INVALID_ARGS', message: 'gesture drag total duration must be at most 10000' }, + ); +}); + +test('every drag plan preserves ordered samples and exact hold/move endpoints', () => { + const point = fc.record({ + x: fc.integer({ min: 2, max: 398 }), + y: fc.integer({ min: 2, max: 798 }), + }); + fc.assert( + fc.property( + point, + point, + fc.integer({ min: 1, max: 3_000 }), + fc.integer({ min: 16, max: 3_000 }), + fc.integer({ min: 0, max: 3_000 }), + (from, to, sourceHoldMs, moveMs, destinationHoldMs) => { + const plan = buildDragGesturePlan( + { from, to, sourceHoldMs, moveMs, destinationHoldMs }, + { x: 0, y: 0, width: 400, height: 800 }, + ); + const samples = plan.pointers[0].samples; + assert.equal(samples[0]?.offsetMs, 0); + assert.deepEqual(samples[0]?.point, { x: from.x, y: from.y }); + assert.equal(samples[1]?.offsetMs, sourceHoldMs); + assert.deepEqual(samples[1]?.point, { x: from.x, y: from.y }); + assert.equal(samples.at(-1)?.offsetMs, sourceHoldMs + moveMs + destinationHoldMs); + assert.deepEqual(samples.at(-1)?.point, { x: to.x, y: to.y }); + for (let index = 1; index < samples.length; index += 1) { + assert.ok(samples[index]!.offsetMs >= samples[index - 1]!.offsetMs); + } + }, + ), + { numRuns: DRAG_PROPERTY_RUNS }, + ); +}); diff --git a/packages/contracts/src/gesture-plan.ts b/packages/contracts/src/gesture-plan.ts index 2115b2f174..dccb588e06 100644 --- a/packages/contracts/src/gesture-plan.ts +++ b/packages/contracts/src/gesture-plan.ts @@ -30,6 +30,9 @@ const GESTURE_VIEWPORT_INSET_PX = 1; const DEFAULT_PAN_DURATION_MS = 500; export const GESTURE_FLING_DURATION_MS = 100; const DEFAULT_MULTI_TOUCH_DURATION_MS = 300; +export const DEFAULT_DRAG_SOURCE_HOLD_MS = 800; +export const DEFAULT_DRAG_MOVE_MS = 500; +export const DEFAULT_DRAG_DESTINATION_HOLD_MS = 0; const MAX_ROTATION_DEGREES_PER_SAMPLE = 3; const MAX_ROTATION_DEFAULT_DURATION_MS = 2_400; @@ -129,6 +132,66 @@ export function singlePointerPlanEndpoints(plan: SinglePointerGesturePlan): { return { start, end }; } +/** + * Plans one uninterrupted pointer contact: activate at the source, move to the + * destination, optionally hold there, then release. Element targets are + * resolved by the interaction runtime before this portable planning seam. + */ +export function buildDragGesturePlan( + input: { + from: Point; + to: Point; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; + }, + viewport: Rect, + platform?: PublicPlatform, +): SinglePointerGesturePlan { + const frame = normalizeViewport(viewport); + const profile = gesturePlatformProfile(platform); + const start = finitePoint(input.from, 'gesture drag source'); + const end = finitePoint(input.to, 'gesture drag destination'); + const sourceHoldMs = normalizePositiveDuration( + input.sourceHoldMs, + DEFAULT_DRAG_SOURCE_HOLD_MS, + 'gesture drag sourceHoldMs', + ); + const moveMs = normalizeDuration(input.moveMs, DEFAULT_DRAG_MOVE_MS, 'gesture drag moveMs'); + const destinationHoldMs = normalizeNonNegativeDuration( + input.destinationHoldMs, + DEFAULT_DRAG_DESTINATION_HOLD_MS, + 'gesture drag destinationHoldMs', + ); + const durationMs = sourceHoldMs + moveMs + destinationHoldMs; + if (durationMs > GESTURE_DURATION_MAX_MS) { + throw new AppError( + 'INVALID_ARGS', + `gesture drag total duration must be at most ${GESTURE_DURATION_MAX_MS}`, + ); + } + const samples = [ + { offsetMs: 0, point: start }, + { offsetMs: sourceHoldMs, point: start }, + ...sampleOffsets(moveMs, profile) + .slice(1) + .map((moveOffsetMs) => ({ + offsetMs: sourceHoldMs + moveOffsetMs, + point: interpolatePoint(start, end, moveOffsetMs / moveMs), + })), + ...(destinationHoldMs > 0 ? [{ offsetMs: durationMs, point: end }] : []), + ]; + assertSamplesInViewport(samples, frame, { intent: 'drag', pointerId: 0 }); + return { + topology: 'single', + intent: 'pan', + executionProfile: 'timed-pan', + durationMs, + viewport: frame, + pointers: [{ pointerId: 0, samples }], + }; +} + function buildFlingPlan( input: Extract, viewport: Rect, @@ -402,6 +465,28 @@ function normalizeDuration(value: number | undefined, fallback: number, field: s return durationMs; } +function normalizePositiveDuration(value: number | undefined, fallback: number, field: string) { + const durationMs = value ?? fallback; + if (!Number.isInteger(durationMs) || durationMs < 1 || durationMs > GESTURE_DURATION_MAX_MS) { + throw new AppError( + 'INVALID_ARGS', + `${field} must be an integer between 1 and ${GESTURE_DURATION_MAX_MS}`, + ); + } + return durationMs; +} + +function normalizeNonNegativeDuration(value: number | undefined, fallback: number, field: string) { + const durationMs = value ?? fallback; + if (!Number.isInteger(durationMs) || durationMs < 0 || durationMs > GESTURE_DURATION_MAX_MS) { + throw new AppError( + 'INVALID_ARGS', + `${field} must be an integer between 0 and ${GESTURE_DURATION_MAX_MS}`, + ); + } + return durationMs; +} + function defaultTransformDuration(rotationDegrees: number): number { const rotationDuration = Math.ceil(Math.abs(rotationDegrees) / MAX_ROTATION_DEGREES_PER_SAMPLE) * diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 48d4a41788..9014aef51b 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -50,7 +50,7 @@ export type ResolutionDiagnosticEntry = { /** * ADR 0012 decision 2: pre-action disclosure of how the acting path resolved - * its target, on every press/click/fill/longpress response. Never ref-issuing. + * its target, including each endpoint of a target-authored drag. Never ref-issuing. * `direct-ios`/`not-observed` = the XCTest fast path has no daemon tree to * report from; `ref`/`label-fallback` = a stale `@ref` recovered via * first-match label lookup, never exact ref provenance; `alternatives` holds diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 7f3ba0a3f2..3f53392284 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -767,6 +767,37 @@ test('interactions.pan projects one- and two-finger requests through typed gestu ); }); +test('interactions.drag projects generic endpoints and timing phases through structured input', async () => { + const setup = createTransport(async () => ({ ok: true, data: { message: 'Dragged' } })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.interactions.drag({ + source: 'id="drag-source"', + destination: '@e2~s42', + sourceHoldMs: 700, + moveMs: 600, + destinationHoldMs: 200, + }); + + assert.deepEqual( + setup.calls.map(({ command, positionals, input }) => ({ command, positionals, input })), + [ + { + command: 'gesture', + positionals: [], + input: { + kind: 'drag', + source: 'id="drag-source"', + destination: '@e2~s42', + sourceHoldMs: 700, + moveMs: 600, + destinationHoldMs: 200, + }, + }, + ], + ); +}); + // fallow-ignore-next-line complexity test('replay.run serializes client-collected AD_VAR shell env into daemon request', async () => { const previousAppId = process.env.AD_VAR_APP_ID; diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 6e51316cf6..8e67ce5a53 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -1,5 +1,5 @@ import { - GESTURE_KINDS, + COORDINATE_GESTURE_KINDS, SCROLL_DIRECTIONS, SWIPE_PRESETS, type GesturePayload, @@ -228,7 +228,7 @@ function pointInViewportArb(viewport: Rect): fc.Arbitrary { */ function gesturePayloadArbByKind( viewport: Rect, -): Record<(typeof GESTURE_KINDS)[number], fc.Arbitrary> { +): Record<(typeof COORDINATE_GESTURE_KINDS)[number], fc.Arbitrary> { const origin = pointInViewportArb(viewport); const delta = fc.record({ x: fc.integer({ min: -viewport.width, max: viewport.width }), @@ -281,7 +281,7 @@ export const gestureInViewportArb: fc.Arbitrary<{ viewport: Rect; gesture: Gestu const byKind = gesturePayloadArbByKind(viewport); return fc.record({ viewport: fc.constant(viewport), - gesture: fc.oneof(...GESTURE_KINDS.map((kind) => byKind[kind])), + gesture: fc.oneof(...COORDINATE_GESTURE_KINDS.map((kind) => byKind[kind])), }); }); diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index f95eeec288..f936a00383 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -15,6 +15,7 @@ import type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + DragOptions, FlingOptions, InternalRequestOptions, Lease, @@ -400,6 +401,7 @@ export function createAgentDeviceClient( longPress: async (options) => await executeCommand('longpress', options), swipe: async (options) => await executeCommand('swipe', options), pan: async (options) => await executeCommand('gesture', panGestureInput(options)), + drag: async (options) => await executeCommand('gesture', dragGestureInput(options)), fling: async (options) => await executeCommand('gesture', flingGestureInput(options)), swipeGesture: async (options) => await executeCommand('gesture', swipePresetGestureInput(options)), @@ -452,6 +454,10 @@ function panGestureInput(options: PanOptions): InternalRequestOptions & Record { + return { ...options, kind: 'drag' }; +} + function flingGestureInput( options: FlingOptions, ): InternalRequestOptions & Record { diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 576ddf97d1..5ac5f13c7e 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -19,7 +19,7 @@ test('usage includes concise top-level commands', async () => { assert.match(usageText, /clipboard read \| clipboard write /); assert.match(usageText, /keyboard \[action\]/); assert.match(usageText, /trigger-app-event\s{2,}Invoke app-defined automation\/test events/); - assert.match(usageText, /gesture \.\.\./); + assert.match(usageText, /gesture \.\.\./); assert.doesNotMatch( usageText, /install-from-source \| install-from-source --github-actions-artifact/, diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 979e6c6850..49c3b3997c 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -213,7 +213,7 @@ Command shape: Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output. If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @, @ref, @Label_Name, or @eN placeholders. Close means agent-device close. App-owned back means back; system back means back --system. - Taps are press or click; tap is an alias for press. On Android TV, tvOS, and Vega OS, read help tv and use tv-remote press up|down|left|right|select to move D-pad/remote focus before activating controls; use tv-remote longpress