diff --git a/test/integration/android-emulator-e2e/live-assertions.ts b/test/integration/android-emulator-e2e/live-assertions.ts index 0e8cd19464..970bd001ee 100644 --- a/test/integration/android-emulator-e2e/live-assertions.ts +++ b/test/integration/android-emulator-e2e/live-assertions.ts @@ -8,26 +8,35 @@ import { assertMp4File, assertNonEmptyFile, createLiveDeviceAssertions, + filesDiffer, } from '../live-device-e2e/assertions.ts'; import type { CliJsonResult } from '../cli-json.ts'; import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; -import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; +import { type AndroidLiveHarness, androidLiveHarness, type LiveContext } from './live-harness.ts'; -export { assertFilesDiffer, assertJsonContains, assertMp4File, assertNonEmptyFile }; +export { assertFilesDiffer, assertJsonContains, assertMp4File, assertNonEmptyFile, filesDiffer }; -export const { assertElementText, assertWaitSelector, assertWaitText, capturePng } = - createLiveDeviceAssertions( - runStep, - verifyCommand, +/** + * Builds the assertion helpers bound to one harness's `runStep`/`verifyCommand`, so a harness + * constructed with a scripted `runCli` observes the same steps the live lane runs. + */ +export function createAndroidLiveAssertions(harness: AndroidLiveHarness = androidLiveHarness) { + return createLiveDeviceAssertions( + harness.runStep, + harness.verifyCommand, PUBLIC_COMMANDS.wait, ); +} + +export const { assertElementText, assertWaitSelector, assertWaitText, capturePng } = + createAndroidLiveAssertions(); export async function scrollToVisibleSelector( context: LiveContext, selector: string, ): Promise { for (let attempt = 0; attempt < 6; attempt += 1) { - const result = await runStep( + const result = await androidLiveHarness.runStep( context, `check ${selector} visibility (${attempt + 1})`, ['is', 'visible', selector], @@ -43,7 +52,7 @@ export async function scrollToVisibleSelector( `could not observe ${selector}: ${JSON.stringify(result.json)}`, ); if (attempt < 5) { - await runStep(context, `scroll toward ${selector} (${attempt + 1})`, [ + await androidLiveHarness.runStep(context, `scroll toward ${selector} (${attempt + 1})`, [ 'scroll', 'down', '0.25', diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.test.ts b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts new file mode 100644 index 0000000000..812e42576e --- /dev/null +++ b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts @@ -0,0 +1,237 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; + +import { mkdtempForTestSync } from '@agent-device/test-kit'; + +import type { CliJsonResult } from '../cli-json.ts'; +import { createLiveDeviceContext, type LiveRunCli } from '../live-device-e2e/runtime.ts'; +import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; +import { createAndroidLiveAssertions, filesDiffer } from './live-assertions.ts'; +import { createAndroidLiveHarness, type LiveContext } from './live-harness.ts'; +import { createHomeAndRecentsRestorationScenario } from './live-automation-scenario.ts'; + +const APP_ID = 'com.callstack.agentdevicelab'; +const LAUNCHER = 'com.google.android.apps.nexuslauncher'; + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const FOREGROUND = Buffer.concat([PNG_SIGNATURE, Buffer.from('fixture-foreground')]); +const HOME = Buffer.concat([PNG_SIGNATURE, Buffer.from('launcher-home')]); +const RECENTS = Buffer.concat([PNG_SIGNATURE, Buffer.from('recents-surface')]); + +/** + * A scripted device behind the harness-owned `runCli` injection. `screenshot` behaves like a CLI + * runner under a step timeout: the capture would finish after `workMs`, but a forwarded + * `options.timeoutMs` kills it first and returns a failure without writing the image. The + * forwarded timeout of every screenshot call is recorded in `timeouts`. + */ +function scriptedDevice(options: { script: Record; workMs: number }): { + runCli: LiveRunCli; + timeouts: Record; +} { + const attempts = new Map(); + const timeouts: Record = {}; + let foregroundPackage = APP_ID; + const runCli: LiveRunCli = (args, _env, cliOptions): Promise => { + const command = args[0]; + if (command === 'home') { + foregroundPackage = LAUNCHER; + return Promise.resolve({ + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: {} }, + }); + } + if (command === 'open') { + foregroundPackage = APP_ID; + return Promise.resolve({ + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: {} }, + }); + } + if (command === 'appstate') { + return Promise.resolve({ + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: { package: foregroundPackage } }, + }); + } + if (command === 'screenshot') { + const output = args[1]!; + const base = path.basename(output); + const timeoutMs = cliOptions?.timeoutMs; + (timeouts[base] ??= []).push(timeoutMs); + return new Promise((resolve) => { + let done = false; + const finish = setTimeout(() => { + if (done) return; + done = true; + const attempt = (attempts.get(base) ?? 0) + 1; + attempts.set(base, attempt); + const images = options.script[base] ?? [FOREGROUND]; + fs.writeFileSync(output, images[Math.min(attempt - 1, images.length - 1)]!); + resolve({ status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }); + }, options.workMs); + if (timeoutMs !== undefined) { + setTimeout(() => { + if (done) return; + done = true; + clearTimeout(finish); + resolve({ + status: 1, + stdout: '', + stderr: `screenshot killed after ${timeoutMs}ms`, + json: { success: false, error: { code: 'timeout' } }, + }); + }, timeoutMs); + } + }); + } + return Promise.resolve({ + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: {} }, + }); + }; + return { runCli, timeouts }; +} + +function makeContext(): LiveContext { + return { + ...createLiveDeviceContext({ + artifactRoot: mkdtempForTestSync('home-recents-scenario-'), + session: 'scenario-test', + }), + appId: APP_ID, + appPath: '/tmp/fixture.apk', + serial: 'emulator-5554', + tier: 'smoke', + }; +} + +/** Runs the Home/Recents restoration scenario through the harness entry with a scripted device. */ +function runScenarioWithDevice(device: { runCli: LiveRunCli }) { + const harness = createAndroidLiveHarness({ runCli: device.runCli }); + const assertions = createAndroidLiveAssertions(harness); + const context = makeContext(); + const scenario = createHomeAndRecentsRestorationScenario({ + ...assertions, + runStep: harness.runStep, + verifyBehavior: harness.verifyBehavior, + verifyCommand: harness.verifyCommand, + }); + return { context, promise: harness.runScenario(context, scenario) }; +} + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +async function settle(promise: Promise): Promise> { + const settled = Promise.allSettled([promise]).then(([result]) => result); + await vi.runAllTimersAsync(); + return settled; +} + +test('Home and Recents pass through the harness scenario entry after delayed captures, with forwarded capture timeouts', async () => { + // Quick captures; the first Home capture still shows the fixture and the first Recents capture + // still shows Home, so a single-capture route fails both pixel comparisons here. + const device = scriptedDevice({ + script: { + 'foreground.png': [FOREGROUND], + 'home.png': [FOREGROUND, HOME], + 'recents.png': [HOME, RECENTS], + }, + workMs: 100, + }); + const { context, promise } = runScenarioWithDevice(device); + const result = await settle(promise); + + expect(result.status).toBe('fulfilled'); + const screenshotSteps = context.stepHistory + .filter((record) => record.commandName === 'screenshot') + .map((record) => record.step); + expect(screenshotSteps).toContain('capture Android Home surface (attempt 2)'); + expect(screenshotSteps).toContain('capture Android Recents surface (attempt 2)'); + // The default capture forwarded a bounded timeout to the CLI runner on every wait attempt. + for (const base of ['home.png', 'recents.png']) { + const attemptTimeouts = device.timeouts[base] ?? []; + expect(attemptTimeouts.length).toBe(2); + for (const timeoutMs of attemptTimeouts) { + expect(typeof timeoutMs).toBe('number'); + expect(timeoutMs).toBeLessThanOrEqual(5_000); + expect(timeoutMs).toBeGreaterThan(0); + } + } + expect( + filesDiffer( + path.join(context.artifactDir, 'foreground.png'), + path.join(context.artifactDir, 'home.png'), + ), + ).toBe(true); + expect( + filesDiffer( + path.join(context.artifactDir, 'home.png'), + path.join(context.artifactDir, 'recents.png'), + ), + ).toBe(true); + expect(context.behaviorEvidence['home-recents-restoration']?.length).toBeGreaterThan(0); + expect(context.completedScenarios).toContain('smoke:home-recents-restoration'); +}); + +test('a wedged capture is cut at the forwarded remaining timeout and the scenario fails at the wait deadline', async () => { + // Every capture would finish after 60s; only the forwarded step timeout can cut it, and the + // wait must still fail at its own 10s deadline with the capture failures as the last observation. + const device = scriptedDevice({ + script: { 'foreground.png': [FOREGROUND], 'home.png': [FOREGROUND], 'recents.png': [RECENTS] }, + workMs: 60_000, + }); + const { context, promise } = runScenarioWithDevice(device); + const result = await settle(promise); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /Android Home never replaced the fixture pixels within 10000ms.*capture \d+ failed/s, + ), + }, + }); + // The remaining deadline (not a constant) was forwarded to the wedged captures: timeouts are + // set, bounded by the per-attempt cap, and shrink as the wait's deadline approaches. + const homeTimeouts = device.timeouts['home.png'] ?? []; + expect(homeTimeouts.length).toBeGreaterThan(1); + for (const timeoutMs of homeTimeouts) expect(typeof timeoutMs).toBe('number'); + expect(Math.max(...homeTimeouts.map((t) => t!))).toBeLessThanOrEqual(5_000); + expect(Math.min(...homeTimeouts.map((t) => t!))).toBeLessThan(5_000); + expect( + context.stepHistory.filter((record) => record.commandName === 'screenshot').length, + ).toBeGreaterThanOrEqual(homeTimeouts.length); +}); + +test('the scenario fails when Recents never differs from the Home surface', async () => { + const device = scriptedDevice({ + script: { 'foreground.png': [FOREGROUND], 'home.png': [HOME], 'recents.png': [HOME] }, + workMs: 100, + }); + const { promise } = runScenarioWithDevice(device); + const result = await settle(promise); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /Android Recents never differed from the Home pixels within 10000ms/s, + ), + }, + }); +}); diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index 8d9b86ff8d..809879662b 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { LiveScenario } from '../live-device-e2e/runtime.ts'; import { assertElementText, assertDiffLine, @@ -14,12 +15,62 @@ import { requireAndroidResourceId, scrollToVisibleSelector, } from './live-assertions.ts'; -import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; +import { + type AndroidLiveHarness, + type LiveContext, + runStep, + verifyBehavior, + verifyCommand, +} from './live-harness.ts'; import { assertHumanSnapshotCompaction } from './live-snapshot-compaction.ts'; +import { waitForSurfaceChange } from './live-surface-transition.ts'; const C = PUBLIC_COMMANDS; -export async function assertAutomationSystem(context: LiveContext): Promise { +/** + * The harness-bound steps one scenario runs through. The smoke scenario uses the live harness's + * steps; a harness-level regression constructs its own harness (with a scripted `runCli`) and + * passes its steps here, so the scenario runs through the harness entry instead of a direct call. + */ +export type AutomationScenarioSteps = { + assertElementText: typeof assertElementText; + assertWaitSelector: typeof assertWaitSelector; + assertWaitText: typeof assertWaitText; + capturePng: typeof capturePng; + runStep: AndroidLiveHarness['runStep']; + verifyBehavior: AndroidLiveHarness['verifyBehavior']; + verifyCommand: AndroidLiveHarness['verifyCommand']; +}; + +const automationScenarioSteps: AutomationScenarioSteps = { + assertElementText, + assertWaitSelector, + assertWaitText, + capturePng, + runStep, + verifyBehavior, + verifyCommand, +}; + +/** + * Scenario entry for the Home/Recents restoration lane. `smoke:automation-system` embeds this + * lane inline; harness-level regressions register the returned scenario with + * `createAndroidLiveHarness(...).runScenario` so the lane runs through the harness entry. + */ +export function createHomeAndRecentsRestorationScenario( + steps: AutomationScenarioSteps = automationScenarioSteps, +): LiveScenario { + return { + id: 'smoke:home-recents-restoration', + run: (context) => assertHomeAndRecentsRestoration(context, steps), + }; +} + +export async function assertAutomationSystem( + context: LiveContext, + steps: AutomationScenarioSteps = automationScenarioSteps, +): Promise { + const { assertElementText, assertWaitText, runStep, verifyBehavior, verifyCommand } = steps; await assertElementText(context, 'id="automation-event-name"', 'cold.start'); await assertElementText( context, @@ -189,7 +240,7 @@ export async function assertAutomationSystem(context: LiveContext): Promise { +async function assertHomeAndRecentsRestoration( + context: LiveContext, + steps: AutomationScenarioSteps, +): Promise { + const { assertWaitSelector, capturePng, runStep, verifyBehavior, verifyCommand } = steps; const foreground = path.join(context.artifactDir, 'foreground.png'); await capturePng(context, 'capture Android fixture foreground', foreground); await runStep(context, 'send Android fixture Home', ['home']); const homeState = await runStep(context, 'read Android Home foreground state', ['appstate']); assert.notEqual(homeState.json?.data?.package, context.appId, JSON.stringify(homeState.json)); const home = path.join(context.artifactDir, 'home.png'); - await capturePng(context, 'capture Android Home surface', home); + // Home is dispatched before the launcher has drawn, so one capture can still show the fixture. + await waitForSurfaceChange( + context, + foreground, + home, + 'capture Android Home surface', + 'Android Home never replaced the fixture pixels', + { capturePng }, + ); assertFilesDiffer(foreground, home, 'Android Home should replace fixture pixels'); verifyCommand(context, C.home, 'Home changes Android foreground package and system pixels'); await runStep(context, 'open Android Recents', ['app-switcher']); const recents = path.join(context.artifactDir, 'recents.png'); - await capturePng(context, 'capture Android Recents surface', recents); + // The Recents transition can still be animating when the switcher is dispatched. + await waitForSurfaceChange( + context, + home, + recents, + 'capture Android Recents surface', + 'Android Recents never differed from the Home pixels', + { capturePng }, + ); assertFilesDiffer(home, recents, 'Android Recents should differ from Home pixels'); verifyCommand(context, C.appSwitcher, 'Recents pixels differ from Home system surface'); diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts index 035de4833b..26092cd050 100644 --- a/test/integration/android-emulator-e2e/live-harness.ts +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { createLiveDeviceContext, createLiveDeviceHarness, + type LiveRunCli, requiredEnv, type LiveDeviceContext, } from '../live-device-e2e/runtime.ts'; @@ -38,27 +39,38 @@ export function createContext(): LiveContext { }; } -const harness = createLiveDeviceHarness({ - behaviorsForScenario: liveBehaviorsForScenario, - commandsForScenario: liveCommandsForScenario, - deviceEvidence: (context) => readAndroidDeviceEvidence(context), - // Seven adb reads, two of them dumpsys: the default bound would drop the whole file, - // including the crash stack, on a loaded runner. - deviceEvidenceTimeoutMs: 25_000, - commonFlags: (context, args) => [ - ...args, - '--platform', - 'android', - '--serial', - context.serial, - '--session', - context.session, - '--daemon-server-mode', - 'dual', - ...(args.includes('--json') ? [] : ['--json']), - ], - writeCoverageReport, -}); +/** + * Builds the Android live harness. `overrides.runCli` is the harness-owned CLI runner injection: + * harness-level regressions script a device through it and run a scenario via `runScenario`. + */ +export function createAndroidLiveHarness(overrides: { runCli?: LiveRunCli } = {}) { + return createLiveDeviceHarness({ + behaviorsForScenario: liveBehaviorsForScenario, + commandsForScenario: liveCommandsForScenario, + deviceEvidence: (context) => readAndroidDeviceEvidence(context), + // Seven adb reads, two of them dumpsys: the default bound would drop the whole file, + // including the crash stack, on a loaded runner. + deviceEvidenceTimeoutMs: 25_000, + commonFlags: (context, args) => [ + ...args, + '--platform', + 'android', + '--serial', + context.serial, + '--session', + context.session, + '--daemon-server-mode', + 'dual', + ...(args.includes('--json') ? [] : ['--json']), + ], + writeCoverageReport, + ...(overrides.runCli === undefined ? {} : { runCli: overrides.runCli }), + }); +} + +export const androidLiveHarness = createAndroidLiveHarness(); + +export type AndroidLiveHarness = ReturnType; export const { runScenario, @@ -67,7 +79,7 @@ export const { verifyBehavior, verifyCommand, verifyNestedCommand: verifyNestedReplayCommand, -} = harness; +} = androidLiveHarness; export async function cleanupSession(context: LiveContext): Promise { const failures: unknown[] = []; diff --git a/test/integration/android-emulator-e2e/live-surface-transition.test.ts b/test/integration/android-emulator-e2e/live-surface-transition.test.ts new file mode 100644 index 0000000000..eb12d4348a --- /dev/null +++ b/test/integration/android-emulator-e2e/live-surface-transition.test.ts @@ -0,0 +1,197 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; + +import { mkdtempForTestSync } from '@agent-device/test-kit'; + +import type { LiveContext } from './live-harness.ts'; +import { type SurfaceCapture, waitForSurfaceChange } from './live-surface-transition.ts'; + +const CONTEXT = {} as LiveContext; + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const FIXTURE_SURFACE = Buffer.concat([PNG_SIGNATURE, Buffer.from('fixture-foreground')]); +const HOME_SURFACE = Buffer.concat([PNG_SIGNATURE, Buffer.from('home-surface')]); + +let artifactDir: string; +let baselinePath: string; +let outputPath: string; + +beforeEach(() => { + vi.useFakeTimers(); + artifactDir = mkdtempForTestSync('surface-transition-'); + baselinePath = path.join(artifactDir, 'baseline.png'); + outputPath = path.join(artifactDir, 'observed.png'); + fs.writeFileSync(baselinePath, FIXTURE_SURFACE); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +async function settle(promise: Promise): Promise> { + const settled = Promise.allSettled([promise]).then(([result]) => result); + await vi.runAllTimersAsync(); + return settled; +} + +/** A capture script: entry `attempt - 1` is written, the last entry repeats; an Error is thrown. */ +function scriptedCapture(script: (Buffer | Error)[]): { + capture: SurfaceCapture; + attempts: () => number; +} { + let attempts = 0; + const capture: SurfaceCapture = async (output, attempt) => { + attempts = attempt; + const image = script[Math.min(attempt - 1, script.length - 1)]!; + if (image instanceof Error) throw image; + fs.writeFileSync(output, image); + }; + return { attempts: () => attempts, capture }; +} + +test('a delayed transition satisfies the wait only after a capture differs, keeping that final image', async () => { + const { attempts, capture } = scriptedCapture([FIXTURE_SURFACE, FIXTURE_SURFACE, HOME_SURFACE]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + { capture }, + ), + ); + + expect(result.status).toBe('fulfilled'); + expect(attempts()).toBe(3); + expect(fs.readFileSync(outputPath).equals(HOME_SURFACE)).toBe(true); +}); + +test('a capture that fails transiently is retried until the surface changes', async () => { + const { attempts, capture } = scriptedCapture([ + new Error('screenshot: device offline'), + FIXTURE_SURFACE, + HOME_SURFACE, + ]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + { capture }, + ), + ); + + expect(result.status).toBe('fulfilled'); + expect(attempts()).toBe(3); + expect(fs.readFileSync(outputPath).equals(HOME_SURFACE)).toBe(true); +}); + +test('a surface that never changes fails at the deadline and keeps the last observed image', async () => { + const { attempts, capture } = scriptedCapture([FIXTURE_SURFACE]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + { capture }, + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /surface never changed within 10000ms.*capture \d+ still matched/s, + ), + }, + }); + expect(attempts()).toBe(40); + expect(fs.readFileSync(outputPath).equals(FIXTURE_SURFACE)).toBe(true); +}); + +test('a wait whose captures keep failing reports the last capture error at the deadline', async () => { + const { capture } = scriptedCapture([new Error('screenshot: device offline')]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + { capture }, + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching(/surface never changed within 10000ms.*device offline/s), + }, + }); +}); + +test('no capture outlives the remaining deadline', async () => { + const start = Date.now(); + let failedAt = 0; + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + { + capture: (_output, _attempt, remainingMs) => + new Promise((_resolve, reject) => + setTimeout(() => { + failedAt = Date.now(); + reject(new Error('capture timed out')); + }, remainingMs), + ), + }, + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching(/surface never changed within 10000ms.*capture timed out/s), + }, + }); + expect(failedAt - start).toBeLessThanOrEqual(10_000); +}); + +test('a capture that resolves after the deadline is rejected as a late observation', async () => { + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + { + capture: async (output, _attempt, remainingMs) => { + await new Promise((resolve) => setTimeout(resolve, remainingMs + 500)); + fs.writeFileSync(output, HOME_SURFACE); + }, + }, + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /surface never changed within 10000ms.*resolved after the deadline/s, + ), + }, + }); + // The final observed image is still kept, even when it arrives late. + expect(fs.readFileSync(outputPath).equals(HOME_SURFACE)).toBe(true); +}); diff --git a/test/integration/android-emulator-e2e/live-surface-transition.ts b/test/integration/android-emulator-e2e/live-surface-transition.ts new file mode 100644 index 0000000000..151e8eeaea --- /dev/null +++ b/test/integration/android-emulator-e2e/live-surface-transition.ts @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +import { capturePng as defaultPngCapture, filesDiffer } from './live-assertions.ts'; +import type { LiveContext } from './live-harness.ts'; +import type { RunStepOptions } from '../live-device-e2e/runtime.ts'; + +const SURFACE_TRANSITION_TIMEOUT_MS = 10_000; +const SURFACE_TRANSITION_POLL_MS = 250; +const SURFACE_CAPTURE_TIMEOUT_MS = 5_000; + +/** A PNG capture step: `screenshot --scale 0.5` with harness step options. */ +export type PngCapture = ( + context: LiveContext, + step: string, + outputPath: string, + options: RunStepOptions, +) => Promise; + +/** + * One screenshot attempt of a surface wait. Writes the observed image to `outputPath`. + * `remainingMs` is the wait's remaining deadline when the attempt starts, so a capture can be + * bounded to it instead of overrunning the wait. + */ +export type SurfaceCapture = ( + outputPath: string, + attempt: number, + remainingMs: number, +) => Promise; + +export type SurfaceTransitionOptions = { + /** + * Injects one attempt. Tests that inject `capture` bypass the default capture and therefore + * do not observe its timeout forwarding. + */ + capture?: SurfaceCapture; + /** + * The PNG capture the default attempts run through. Scenarios pass the capture bound to their + * harness so the default capture's `timeoutMs` reaches the CLI runner of that harness. + */ + capturePng?: PngCapture; +}; + +/** + * Waits until a fresh screenshot differs from the baseline surface, keeping the final observed + * image at `outputPath`. System navigation (Home, Recents) is dispatched before the new surface + * has drawn, so a single capture right after the command can still show the previous surface. + * Each attempt removes and overwrites `outputPath`, so the file always holds the last observed + * image and a failed capture cannot leave a stale one behind. The default capture is bounded by + * the remaining deadline (capped per attempt), a capture that resolves after the deadline is + * rejected as a late observation, and the wait fails at the deadline if navigation never + * produced the change. + */ +export async function waitForSurfaceChange( + context: LiveContext, + baselinePath: string, + outputPath: string, + captureStep: string, + failure: string, + options: SurfaceTransitionOptions = {}, +): Promise { + const capturePng = options.capturePng ?? defaultPngCapture; + const capture = options.capture ?? defaultCapture(capturePng, context, captureStep); + const deadline = Date.now() + SURFACE_TRANSITION_TIMEOUT_MS; + let lastObservation = 'no capture finished'; + let attempt = 0; + for ( + let remaining = SURFACE_TRANSITION_TIMEOUT_MS; + remaining > 0; + remaining = deadline - Date.now() + ) { + attempt += 1; + fs.rmSync(outputPath, { force: true }); + try { + await capture(outputPath, attempt, remaining); + if (Date.now() >= deadline) { + lastObservation = `capture ${attempt} resolved after the deadline`; + } else if (filesDiffer(baselinePath, outputPath)) { + return; + } else { + lastObservation = `capture ${attempt} still matched ${baselinePath}`; + } + } catch (error) { + lastObservation = `capture ${attempt} failed: ${error instanceof Error ? error.message : String(error)}`; + } + const pause = Math.min(SURFACE_TRANSITION_POLL_MS, deadline - Date.now()); + if (pause > 0) await new Promise((resolve) => setTimeout(resolve, pause)); + } + assert.fail(`${failure} within ${SURFACE_TRANSITION_TIMEOUT_MS}ms (${lastObservation})`); +} + +function defaultCapture( + capturePng: PngCapture, + context: LiveContext, + captureStep: string, +): SurfaceCapture { + return (output, attempt, remainingMs) => + capturePng(context, `${captureStep} (attempt ${attempt})`, output, { + // The wait owns the miss policy: a failed attempt is retried under the deadline and only + // the deadline verdict reports it, so per-attempt failure evidence stays deferred. + allowFailure: true, + timeoutMs: Math.min(remainingMs, SURFACE_CAPTURE_TIMEOUT_MS), + }); +} diff --git a/test/integration/live-device-e2e/assertions.ts b/test/integration/live-device-e2e/assertions.ts index fcba5ddba5..b0267ca84f 100644 --- a/test/integration/live-device-e2e/assertions.ts +++ b/test/integration/live-device-e2e/assertions.ts @@ -5,13 +5,13 @@ import { assertPngFile } from '../provider-scenarios/assertions.ts'; import { isPlayableVideo } from '@agent-device/capture-kit/recording-video'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import type { CliJsonResult } from '../cli-json.ts'; -import type { LiveDeviceContext } from './runtime.ts'; +import type { LiveDeviceContext, RunStepOptions } from './runtime.ts'; type RunStep = ( context: Context, step: string, args: string[], - options?: { allowFailure?: boolean }, + options?: RunStepOptions, ) => Promise; export function createLiveDeviceAssertions< @@ -65,8 +65,13 @@ export function createLiveDeviceAssertions< ); } - async function capturePng(context: Context, step: string, outputPath: string): Promise { - await runStep(context, step, ['screenshot', outputPath, '--scale', '0.5']); + async function capturePng( + context: Context, + step: string, + outputPath: string, + options: RunStepOptions = {}, + ): Promise { + await runStep(context, step, ['screenshot', outputPath, '--scale', '0.5'], options); assertPngFile(outputPath); } @@ -78,8 +83,12 @@ export function assertJsonContains(result: CliJsonResult, expected: string, mess assert.ok(serialized.includes(expected), `${message}\nreceived: ${serialized}`); } +export function filesDiffer(first: string, second: string): boolean { + return !fs.readFileSync(first).equals(fs.readFileSync(second)); +} + export function assertFilesDiffer(first: string, second: string, message: string): void { - assert.notDeepEqual(fs.readFileSync(first), fs.readFileSync(second), message); + assert.ok(filesDiffer(first, second), message); } export function assertNonEmptyFile(filePath: string, name: string): void { diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index c1c387befe..18ee1af7db 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -22,6 +22,12 @@ export type ScenarioTiming = { id: string; }; +export type LiveRunCli = ( + args: string[], + env: NodeJS.ProcessEnv, + options?: { timeoutMs?: number }, +) => Promise; + export type LiveDeviceContext = { artifactDir: string; behaviorEvidence: Partial>; @@ -45,11 +51,8 @@ type HarnessOptions = { behaviorsForScenario: (scenarioId: string) => readonly BehaviorId[]; commandsForScenario: (scenarioId: string) => readonly string[]; commonFlags: (context: Context, args: readonly string[]) => string[]; - runCli?: ( - args: string[], - env: NodeJS.ProcessEnv, - options?: { timeoutMs?: number }, - ) => Promise; + /** CLI runner injection: the seam tests use to script a device for a scenario. */ + runCli?: LiveRunCli; /** * Platform-owned device facts for a failed step (rotation state, whether the app process is * alive, crash logs), read outside agent-device so they describe the device even when the CLI @@ -61,7 +64,7 @@ type HarnessOptions = { writeCoverageReport: (context: Context) => void; }; -type RunStepOptions = { +export type RunStepOptions = { allowFailure?: boolean; commonFlags?: boolean; expectFailure?: boolean; diff --git a/vitest.config.ts b/vitest.config.ts index 77dacc6226..709c41fd61 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -244,6 +244,12 @@ export default defineConfig({ // seam, so the crash/process/activity selectors need no emulator to be pinned. 'test/integration/android-emulator-e2e/device-evidence.test.ts', 'test/integration/android-emulator-e2e/live-ime-window.test.ts', + // The Home/Recents surface-transition wait (#4710): replays scripted captures through + // the capture seam under fake timers, so the deadline and delayed change need no emulator. + 'test/integration/android-emulator-e2e/live-surface-transition.test.ts', + // The same wait driven through the real scenario wiring with a scripted device (#4710 + // review): proves Home and Recents poll via the scenario, not just the helper. + 'test/integration/android-emulator-e2e/live-automation-scenario.test.ts', // The frozen replay-compat corpus (#1417): parse-only, no device or // subprocess work, so it belongs in the fast lane next to the // grammar it guards.