From 13b6f71b8b98d6f1697a6c9c8f65dc67f885dbe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 19:35:57 +0100 Subject: [PATCH 1/4] fix(android-e2e): wait for the Home/Recents surface change before pixel assertions The Android smoke scenario captured one screenshot immediately after home and app-switcher and asserted it differed from the previous surface. When the launcher or Recents transition had not drawn yet, the capture still showed the previous surface and the identical-image assertion failed intermittently on main and unrelated PRs (#4710). assertHomeAndRecentsRestoration now polls screenshots through the new waitForSurfaceChange helper at that seam: each attempt overwrites the artifact so the final observed image is kept, the Home foreground-package proof and both assertFilesDiffer pixel comparisons are retained, and the wait fails at its 10s deadline if navigation never produces the change. No fixed sleep, no retry of the smoke test, no weakened assertion. live-surface-transition.test.ts pins the contract deterministically under fake timers: a delayed transition satisfies the wait only once a capture differs (keeping that final image), transient capture failures are retried, and a surface that never changes (or captures that keep failing) fails at the deadline with the last observation. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../android-emulator-e2e/live-assertions.ts | 3 +- .../live-automation-scenario.ts | 19 ++- .../live-surface-transition.test.ts | 137 ++++++++++++++++++ .../live-surface-transition.ts | 48 ++++++ .../integration/live-device-e2e/assertions.ts | 6 +- vitest.config.ts | 3 + 6 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 test/integration/android-emulator-e2e/live-surface-transition.test.ts create mode 100644 test/integration/android-emulator-e2e/live-surface-transition.ts diff --git a/test/integration/android-emulator-e2e/live-assertions.ts b/test/integration/android-emulator-e2e/live-assertions.ts index 0e8cd19464..ac1dd08cfd 100644 --- a/test/integration/android-emulator-e2e/live-assertions.ts +++ b/test/integration/android-emulator-e2e/live-assertions.ts @@ -8,12 +8,13 @@ import { assertMp4File, assertNonEmptyFile, createLiveDeviceAssertions, + filesDiffer, } from '../live-device-e2e/assertions.ts'; import type { CliJsonResult } from '../cli-json.ts'; import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; -export { assertFilesDiffer, assertJsonContains, assertMp4File, assertNonEmptyFile }; +export { assertFilesDiffer, assertJsonContains, assertMp4File, assertNonEmptyFile, filesDiffer }; export const { assertElementText, assertWaitSelector, assertWaitText, capturePng } = createLiveDeviceAssertions( diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index 8d9b86ff8d..5a07434800 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -16,6 +16,7 @@ import { } from './live-assertions.ts'; import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; import { assertHumanSnapshotCompaction } from './live-snapshot-compaction.ts'; +import { waitForSurfaceChange } from './live-surface-transition.ts'; const C = PUBLIC_COMMANDS; @@ -227,13 +228,27 @@ async function assertHomeAndRecentsRestoration(context: LiveContext): Promise { + vi.useFakeTimers(); + artifactDir = mkdtempForTestSync('surface-transition-'); + baselinePath = path.join(artifactDir, 'baseline.png'); + outputPath = path.join(artifactDir, 'observed.png'); + fs.writeFileSync(baselinePath, FIXTURE_SURFACE); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +async function settle(promise: Promise): Promise> { + const settled = Promise.allSettled([promise]).then(([result]) => result); + await vi.runAllTimersAsync(); + return settled; +} + +/** A capture script: entry `attempt - 1` is written, the last entry repeats; an Error is thrown. */ +function scriptedCapture(script: (Buffer | Error)[]): { + capture: SurfaceCapture; + attempts: () => number; +} { + let attempts = 0; + const capture: SurfaceCapture = async (output, attempt) => { + attempts = attempt; + const image = script[Math.min(attempt - 1, script.length - 1)]!; + if (image instanceof Error) throw image; + fs.writeFileSync(output, image); + }; + return { attempts: () => attempts, capture }; +} + +test('a delayed transition satisfies the wait only after a capture differs, keeping that final image', async () => { + const { attempts, capture } = scriptedCapture([FIXTURE_SURFACE, FIXTURE_SURFACE, HOME_SURFACE]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + capture, + ), + ); + + expect(result.status).toBe('fulfilled'); + expect(attempts()).toBe(3); + expect(fs.readFileSync(outputPath).equals(HOME_SURFACE)).toBe(true); +}); + +test('a capture that fails transiently is retried until the surface changes', async () => { + const { attempts, capture } = scriptedCapture([ + new Error('screenshot: device offline'), + FIXTURE_SURFACE, + HOME_SURFACE, + ]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + capture, + ), + ); + + expect(result.status).toBe('fulfilled'); + expect(attempts()).toBe(3); + expect(fs.readFileSync(outputPath).equals(HOME_SURFACE)).toBe(true); +}); + +test('a surface that never changes fails at the deadline and keeps the last observed image', async () => { + const { attempts, capture } = scriptedCapture([FIXTURE_SURFACE]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + capture, + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /surface never changed within 10000ms.*capture \d+ still matched/s, + ), + }, + }); + expect(attempts()).toBe(40); + expect(fs.readFileSync(outputPath).equals(FIXTURE_SURFACE)).toBe(true); +}); + +test('a wait whose captures keep failing reports the last capture error at the deadline', async () => { + const { capture } = scriptedCapture([new Error('screenshot: device offline')]); + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + capture, + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching(/surface never changed within 10000ms.*device offline/s), + }, + }); +}); diff --git a/test/integration/android-emulator-e2e/live-surface-transition.ts b/test/integration/android-emulator-e2e/live-surface-transition.ts new file mode 100644 index 0000000000..5f74fb4bf5 --- /dev/null +++ b/test/integration/android-emulator-e2e/live-surface-transition.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; + +import { capturePng, filesDiffer } from './live-assertions.ts'; +import type { LiveContext } from './live-harness.ts'; + +const SURFACE_TRANSITION_TIMEOUT_MS = 10_000; +const SURFACE_TRANSITION_POLL_MS = 250; + +/** One screenshot attempt of a surface wait. Writes the observed image to `outputPath`. */ +export type SurfaceCapture = (outputPath: string, attempt: number) => Promise; + +/** + * Waits until a fresh screenshot differs from the baseline surface, keeping the final observed + * image at `outputPath`. System navigation (Home, Recents) is dispatched before the new surface + * has drawn, so a single capture right after the command can still show the previous surface. + * Each attempt overwrites `outputPath`, so the file always holds the last observed image, and the + * wait fails at the deadline if navigation never produced the change. + */ +export async function waitForSurfaceChange( + context: LiveContext, + baselinePath: string, + outputPath: string, + captureStep: string, + failure: string, + capture: SurfaceCapture = (output, attempt) => + capturePng(context, `${captureStep} (attempt ${attempt})`, output), +): Promise { + const deadline = Date.now() + SURFACE_TRANSITION_TIMEOUT_MS; + let lastObservation = 'no capture finished'; + let attempt = 0; + for ( + let remaining = SURFACE_TRANSITION_TIMEOUT_MS; + remaining > 0; + remaining = deadline - Date.now() + ) { + attempt += 1; + try { + await capture(outputPath, attempt); + if (filesDiffer(baselinePath, outputPath)) return; + lastObservation = `capture ${attempt} still matched ${baselinePath}`; + } catch (error) { + lastObservation = `capture ${attempt} failed: ${error instanceof Error ? error.message : String(error)}`; + } + const pause = Math.min(SURFACE_TRANSITION_POLL_MS, deadline - Date.now()); + if (pause > 0) await new Promise((resolve) => setTimeout(resolve, pause)); + } + assert.fail(`${failure} within ${SURFACE_TRANSITION_TIMEOUT_MS}ms (${lastObservation})`); +} diff --git a/test/integration/live-device-e2e/assertions.ts b/test/integration/live-device-e2e/assertions.ts index fcba5ddba5..b50b4fd6a5 100644 --- a/test/integration/live-device-e2e/assertions.ts +++ b/test/integration/live-device-e2e/assertions.ts @@ -78,8 +78,12 @@ export function assertJsonContains(result: CliJsonResult, expected: string, mess assert.ok(serialized.includes(expected), `${message}\nreceived: ${serialized}`); } +export function filesDiffer(first: string, second: string): boolean { + return !fs.readFileSync(first).equals(fs.readFileSync(second)); +} + export function assertFilesDiffer(first: string, second: string, message: string): void { - assert.notDeepEqual(fs.readFileSync(first), fs.readFileSync(second), message); + assert.ok(filesDiffer(first, second), message); } export function assertNonEmptyFile(filePath: string, name: string): void { diff --git a/vitest.config.ts b/vitest.config.ts index 77dacc6226..eca419321e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -244,6 +244,9 @@ export default defineConfig({ // seam, so the crash/process/activity selectors need no emulator to be pinned. 'test/integration/android-emulator-e2e/device-evidence.test.ts', 'test/integration/android-emulator-e2e/live-ime-window.test.ts', + // The Home/Recents surface-transition wait (#4710): replays scripted captures through + // the capture seam under fake timers, so the deadline and delayed change need no emulator. + 'test/integration/android-emulator-e2e/live-surface-transition.test.ts', // The frozen replay-compat corpus (#1417): parse-only, no device or // subprocess work, so it belongs in the fast lane next to the // grammar it guards. From 8516300c956fa37f99a2a9a27a35fdbe9a28f9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 21:09:08 +0100 Subject: [PATCH 2/4] test(android-e2e): drive Home/Recents through the scenario harness and bound captures by the wait deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #4716: the helper-level tests alone would still pass if the scenario went back to single captures, so the regression now also runs assertHomeAndRecentsRestoration through the real harness wiring (context runCli override) against a scripted device whose first Home capture still shows the fixture and whose first Recents capture still shows Home. The scenario passes only via the polling wait, and fails at the 10s deadline when either surface never changes — a single-capture route fails both pixel comparisons on the first identical capture. waitForSurfaceChange now passes the remaining deadline into each capture (capped at 5s, allowFailure so per-attempt failure evidence stays deferred to the deadline verdict), removes the stale artifact before each attempt, and rejects a capture that resolves after the deadline as a late observation. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../live-automation-scenario.test.ts | 164 ++++++++++++++++++ .../live-automation-scenario.ts | 2 +- .../live-surface-transition.test.ts | 56 ++++++ .../live-surface-transition.ts | 45 ++++- .../integration/live-device-e2e/assertions.ts | 13 +- test/integration/live-device-e2e/runtime.ts | 18 +- vitest.config.ts | 3 + 7 files changed, 284 insertions(+), 17 deletions(-) create mode 100644 test/integration/android-emulator-e2e/live-automation-scenario.test.ts diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.test.ts b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts new file mode 100644 index 0000000000..798014102d --- /dev/null +++ b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts @@ -0,0 +1,164 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; + +import { mkdtempForTestSync } from '@agent-device/test-kit'; + +import type { CliJsonResult } from '../cli-json.ts'; +import { createLiveDeviceContext, type LiveRunCli } from '../live-device-e2e/runtime.ts'; +import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; +import { assertHomeAndRecentsRestoration } from './live-automation-scenario.ts'; +import { filesDiffer } from './live-assertions.ts'; +import type { LiveContext } from './live-harness.ts'; + +const APP_ID = 'com.callstack.agentdevicelab'; +const LAUNCHER = 'com.google.android.apps.nexuslauncher'; + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const FOREGROUND = Buffer.concat([PNG_SIGNATURE, Buffer.from('fixture-foreground')]); +const HOME = Buffer.concat([PNG_SIGNATURE, Buffer.from('launcher-home')]); +const RECENTS = Buffer.concat([PNG_SIGNATURE, Buffer.from('recents-surface')]); + +/** + * A scripted device behind the context's `runCli` override: `screenshot` writes the next scripted + * image for that artifact (the last entry repeats), `home`/`open` drive the foreground package, + * and every other command succeeds. The scenario runs through the real harness wiring + * (commonFlags, step history, command/behavior evidence), not a helper call. + */ +function scriptedDevice(script: Record) { + const attempts = new Map(); + let foregroundPackage = APP_ID; + const runCli: LiveRunCli = async (args): Promise => { + const command = args[0]; + if (command === 'home') { + foregroundPackage = LAUNCHER; + return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + } + if (command === 'open') { + foregroundPackage = APP_ID; + return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + } + if (command === 'appstate') { + return { + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: { package: foregroundPackage } }, + }; + } + if (command === 'screenshot') { + const output = args[1]!; + const base = path.basename(output); + const attempt = (attempts.get(base) ?? 0) + 1; + attempts.set(base, attempt); + const images = script[base] ?? [FOREGROUND]; + fs.writeFileSync(output, images[Math.min(attempt - 1, images.length - 1)]!); + return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + } + return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + }; + return { runCli }; +} + +function makeContext(runCli: LiveRunCli): LiveContext { + return { + ...createLiveDeviceContext({ + artifactRoot: mkdtempForTestSync('home-recents-scenario-'), + session: 'scenario-test', + }), + appId: APP_ID, + appPath: '/tmp/fixture.apk', + serial: 'emulator-5554', + tier: 'smoke', + runCli, + }; +} + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +async function settle(promise: Promise): Promise> { + const settled = Promise.allSettled([promise]).then(([result]) => result); + await vi.runAllTimersAsync(); + return settled; +} + +test('Home and Recents go through the scenario wait and pass only after the delayed captures differ', async () => { + // The first Home capture still shows the fixture and the first Recents capture still shows + // Home: a single-capture route would fail both pixel comparisons here. + const { runCli } = scriptedDevice({ + 'foreground.png': [FOREGROUND], + 'home.png': [FOREGROUND, HOME], + 'recents.png': [HOME, RECENTS], + }); + const context = makeContext(runCli); + context.currentScenario = 'smoke:automation-system'; + const result = await settle(assertHomeAndRecentsRestoration(context)); + + expect(result.status).toBe('fulfilled'); + const screenshotSteps = context.stepHistory + .filter((record) => record.commandName === 'screenshot') + .map((record) => record.step); + expect(screenshotSteps).toContain('capture Android Home surface (attempt 2)'); + expect(screenshotSteps).toContain('capture Android Recents surface (attempt 2)'); + expect( + filesDiffer( + path.join(context.artifactDir, 'foreground.png'), + path.join(context.artifactDir, 'home.png'), + ), + ).toBe(true); + expect( + filesDiffer( + path.join(context.artifactDir, 'home.png'), + path.join(context.artifactDir, 'recents.png'), + ), + ).toBe(true); + expect(context.behaviorEvidence['home-recents-restoration']?.length).toBeGreaterThan(0); + const appStates = context.stepHistory.filter((record) => record.commandName === 'appstate'); + expect(appStates.length).toBe(2); +}); + +test('the scenario fails when Home never replaces the fixture pixels', async () => { + const { runCli } = scriptedDevice({ + 'foreground.png': [FOREGROUND], + 'home.png': [FOREGROUND], + 'recents.png': [RECENTS], + }); + const context = makeContext(runCli); + context.currentScenario = 'smoke:automation-system'; + const result = await settle(assertHomeAndRecentsRestoration(context)); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /Android Home never replaced the fixture pixels within 10000ms/s, + ), + }, + }); +}); + +test('the scenario fails when Recents never differs from the Home surface', async () => { + const { runCli } = scriptedDevice({ + 'foreground.png': [FOREGROUND], + 'home.png': [HOME], + 'recents.png': [HOME], + }); + const context = makeContext(runCli); + context.currentScenario = 'smoke:automation-system'; + const result = await settle(assertHomeAndRecentsRestoration(context)); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /Android Recents never differed from the Home pixels within 10000ms/s, + ), + }, + }); +}); diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index 5a07434800..65916831cb 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -221,7 +221,7 @@ async function assertOrientationFixtureState( await assertElementText(context, 'id="automation-window"', expectedWindow); } -async function assertHomeAndRecentsRestoration(context: LiveContext): Promise { +export async function assertHomeAndRecentsRestoration(context: LiveContext): Promise { const foreground = path.join(context.artifactDir, 'foreground.png'); await capturePng(context, 'capture Android fixture foreground', foreground); await runStep(context, 'send Android fixture Home', ['home']); diff --git a/test/integration/android-emulator-e2e/live-surface-transition.test.ts b/test/integration/android-emulator-e2e/live-surface-transition.test.ts index 6c648342d3..15da7ee6e3 100644 --- a/test/integration/android-emulator-e2e/live-surface-transition.test.ts +++ b/test/integration/android-emulator-e2e/live-surface-transition.test.ts @@ -135,3 +135,59 @@ test('a wait whose captures keep failing reports the last capture error at the d }, }); }); + +test('no capture outlives the remaining deadline', async () => { + const start = Date.now(); + let failedAt = 0; + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + (_output, _attempt, remainingMs) => + new Promise((_resolve, reject) => + setTimeout(() => { + failedAt = Date.now(); + reject(new Error('capture timed out')); + }, remainingMs), + ), + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching(/surface never changed within 10000ms.*capture timed out/s), + }, + }); + expect(failedAt - start).toBeLessThanOrEqual(10_000); +}); + +test('a capture that resolves after the deadline is rejected as a late observation', async () => { + const result = await settle( + waitForSurfaceChange( + CONTEXT, + baselinePath, + outputPath, + 'capture surface', + 'surface never changed', + async (output, _attempt, remainingMs) => { + await new Promise((resolve) => setTimeout(resolve, remainingMs + 500)); + fs.writeFileSync(output, HOME_SURFACE); + }, + ), + ); + + expect(result).toMatchObject({ + status: 'rejected', + reason: { + message: expect.stringMatching( + /surface never changed within 10000ms.*resolved after the deadline/s, + ), + }, + }); + // The final observed image is still kept, even when it arrives late. + expect(fs.readFileSync(outputPath).equals(HOME_SURFACE)).toBe(true); +}); diff --git a/test/integration/android-emulator-e2e/live-surface-transition.ts b/test/integration/android-emulator-e2e/live-surface-transition.ts index 5f74fb4bf5..3cabfe251e 100644 --- a/test/integration/android-emulator-e2e/live-surface-transition.ts +++ b/test/integration/android-emulator-e2e/live-surface-transition.ts @@ -1,20 +1,32 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; import { capturePng, filesDiffer } from './live-assertions.ts'; import type { LiveContext } from './live-harness.ts'; const SURFACE_TRANSITION_TIMEOUT_MS = 10_000; const SURFACE_TRANSITION_POLL_MS = 250; +const SURFACE_CAPTURE_TIMEOUT_MS = 5_000; -/** One screenshot attempt of a surface wait. Writes the observed image to `outputPath`. */ -export type SurfaceCapture = (outputPath: string, attempt: number) => Promise; +/** + * One screenshot attempt of a surface wait. Writes the observed image to `outputPath`. + * `remainingMs` is the wait's remaining deadline when the attempt starts, so a capture can be + * bounded to it instead of overrunning the wait. + */ +export type SurfaceCapture = ( + outputPath: string, + attempt: number, + remainingMs: number, +) => Promise; /** * Waits until a fresh screenshot differs from the baseline surface, keeping the final observed * image at `outputPath`. System navigation (Home, Recents) is dispatched before the new surface * has drawn, so a single capture right after the command can still show the previous surface. - * Each attempt overwrites `outputPath`, so the file always holds the last observed image, and the - * wait fails at the deadline if navigation never produced the change. + * Each attempt removes and overwrites `outputPath`, so the file always holds the last observed + * image and a failed capture cannot leave a stale one behind. A capture that resolves after the + * deadline is rejected as a late observation, and the wait fails at the deadline if navigation + * never produced the change. */ export async function waitForSurfaceChange( context: LiveContext, @@ -22,8 +34,7 @@ export async function waitForSurfaceChange( outputPath: string, captureStep: string, failure: string, - capture: SurfaceCapture = (output, attempt) => - capturePng(context, `${captureStep} (attempt ${attempt})`, output), + capture?: SurfaceCapture, ): Promise { const deadline = Date.now() + SURFACE_TRANSITION_TIMEOUT_MS; let lastObservation = 'no capture finished'; @@ -34,10 +45,16 @@ export async function waitForSurfaceChange( remaining = deadline - Date.now() ) { attempt += 1; + fs.rmSync(outputPath, { force: true }); try { - await capture(outputPath, attempt); - if (filesDiffer(baselinePath, outputPath)) return; - lastObservation = `capture ${attempt} still matched ${baselinePath}`; + await (capture ?? defaultCapture(context, captureStep))(outputPath, attempt, remaining); + if (Date.now() >= deadline) { + lastObservation = `capture ${attempt} resolved after the deadline`; + } else if (filesDiffer(baselinePath, outputPath)) { + return; + } else { + lastObservation = `capture ${attempt} still matched ${baselinePath}`; + } } catch (error) { lastObservation = `capture ${attempt} failed: ${error instanceof Error ? error.message : String(error)}`; } @@ -46,3 +63,13 @@ export async function waitForSurfaceChange( } assert.fail(`${failure} within ${SURFACE_TRANSITION_TIMEOUT_MS}ms (${lastObservation})`); } + +function defaultCapture(context: LiveContext, captureStep: string): SurfaceCapture { + return (output, attempt, remainingMs) => + capturePng(context, `${captureStep} (attempt ${attempt})`, output, { + // The wait owns the miss policy: a failed attempt is retried under the deadline and only + // the deadline verdict reports it, so per-attempt failure evidence stays deferred. + allowFailure: true, + timeoutMs: Math.min(remainingMs, SURFACE_CAPTURE_TIMEOUT_MS), + }); +} diff --git a/test/integration/live-device-e2e/assertions.ts b/test/integration/live-device-e2e/assertions.ts index b50b4fd6a5..b0267ca84f 100644 --- a/test/integration/live-device-e2e/assertions.ts +++ b/test/integration/live-device-e2e/assertions.ts @@ -5,13 +5,13 @@ import { assertPngFile } from '../provider-scenarios/assertions.ts'; import { isPlayableVideo } from '@agent-device/capture-kit/recording-video'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import type { CliJsonResult } from '../cli-json.ts'; -import type { LiveDeviceContext } from './runtime.ts'; +import type { LiveDeviceContext, RunStepOptions } from './runtime.ts'; type RunStep = ( context: Context, step: string, args: string[], - options?: { allowFailure?: boolean }, + options?: RunStepOptions, ) => Promise; export function createLiveDeviceAssertions< @@ -65,8 +65,13 @@ export function createLiveDeviceAssertions< ); } - async function capturePng(context: Context, step: string, outputPath: string): Promise { - await runStep(context, step, ['screenshot', outputPath, '--scale', '0.5']); + async function capturePng( + context: Context, + step: string, + outputPath: string, + options: RunStepOptions = {}, + ): Promise { + await runStep(context, step, ['screenshot', outputPath, '--scale', '0.5'], options); assertPngFile(outputPath); } diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index c1c387befe..a75224b206 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -22,6 +22,12 @@ export type ScenarioTiming = { id: string; }; +export type LiveRunCli = ( + args: string[], + env: NodeJS.ProcessEnv, + options?: { timeoutMs?: number }, +) => Promise; + export type LiveDeviceContext = { artifactDir: string; behaviorEvidence: Partial>; @@ -34,6 +40,11 @@ export type LiveDeviceContext = { startedAtMs: number; stepHistory: StepRecord[]; timings: ScenarioTiming[]; + /** + * Per-context CLI override, ahead of the harness-wide `runCli`. Lets one scenario run against a + * scripted device (delayed captures, scripted appstate) without touching the shared harness. + */ + runCli?: LiveRunCli; }; export type LiveScenario = { @@ -61,7 +72,7 @@ type HarnessOptions = { writeCoverageReport: (context: Context) => void; }; -type RunStepOptions = { +export type RunStepOptions = { allowFailure?: boolean; commonFlags?: boolean; expectFailure?: boolean; @@ -235,7 +246,8 @@ export function createLiveDeviceHarness< ): Promise<{ result: CliJsonResult; fullArgs: string[] }> { const fullArgs = buildStepArgs(context, args, stepOptions); const startedAt = Date.now(); - const result = await (options.runCli ?? runBuiltCliJson)(fullArgs, context.env, { + const runCli = context.runCli ?? options.runCli ?? runBuiltCliJson; + const result = await runCli(fullArgs, context.env, { timeoutMs: stepOptions.timeoutMs, }); const failedAsExpected = stepOptions.expectFailure === true && result.status !== 0; @@ -294,7 +306,7 @@ export function createLiveDeviceHarness< } function captureFailedStepEvidence(context: Context): Promise { - const runCli = options.runCli ?? runBuiltCliJson; + const runCli = context.runCli ?? options.runCli ?? runBuiltCliJson; const deviceEvidence = options.deviceEvidence; return collectFailedStepEvidence({ stem: path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`), diff --git a/vitest.config.ts b/vitest.config.ts index eca419321e..709c41fd61 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -247,6 +247,9 @@ export default defineConfig({ // The Home/Recents surface-transition wait (#4710): replays scripted captures through // the capture seam under fake timers, so the deadline and delayed change need no emulator. 'test/integration/android-emulator-e2e/live-surface-transition.test.ts', + // The same wait driven through the real scenario wiring with a scripted device (#4710 + // review): proves Home and Recents poll via the scenario, not just the helper. + 'test/integration/android-emulator-e2e/live-automation-scenario.test.ts', // The frozen replay-compat corpus (#1417): parse-only, no device or // subprocess work, so it belongs in the fast lane next to the // grammar it guards. From f3b8ce23145b1ccc376f2d7031e828c87d2b4eff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 22:52:22 +0100 Subject: [PATCH 3/4] test(android-e2e): drive the Home/Recents lane through the harness entry and pin the capture timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #4716 at 8516300: the scenario test faked the route via a context runCli override and a direct internal export. The Android harness is now constructible with its existing runner injection (createAndroidLiveHarness), the assertion helpers are bound per harness (createAndroidLiveAssertions), and the Home/Recents lane runs as a registered scenario (runScenario) with steps passed in — the context override and the scenario-internal export are gone. The scripted CLI double observes and enforces the forwarded step timeout like a real runner: captures finish after 60s unless the forwarded timeoutMs kills them. With the timeout, the wedged-capture test fails at the 10s wait deadline with 'capture N failed' and the recorded timeouts show the remaining deadline (capped at 5s, shrinking); removing timeoutMs from the default capture makes both scenario tests fail (verified locally), so the regression detects the removal. waitForSurfaceChange takes the bound capturePng so the default capture's timeout reaches the harness's CLI runner. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../android-emulator-e2e/live-assertions.ts | 22 ++- .../live-automation-scenario.test.ts | 173 +++++++++++++----- .../live-automation-scenario.ts | 64 ++++++- .../android-emulator-e2e/live-harness.ts | 56 +++--- .../live-surface-transition.test.ts | 32 ++-- .../live-surface-transition.ts | 43 ++++- test/integration/live-device-e2e/runtime.ts | 17 +- 7 files changed, 290 insertions(+), 117 deletions(-) diff --git a/test/integration/android-emulator-e2e/live-assertions.ts b/test/integration/android-emulator-e2e/live-assertions.ts index ac1dd08cfd..970bd001ee 100644 --- a/test/integration/android-emulator-e2e/live-assertions.ts +++ b/test/integration/android-emulator-e2e/live-assertions.ts @@ -12,23 +12,31 @@ import { } from '../live-device-e2e/assertions.ts'; import type { CliJsonResult } from '../cli-json.ts'; import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; -import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; +import { type AndroidLiveHarness, androidLiveHarness, type LiveContext } from './live-harness.ts'; export { assertFilesDiffer, assertJsonContains, assertMp4File, assertNonEmptyFile, filesDiffer }; -export const { assertElementText, assertWaitSelector, assertWaitText, capturePng } = - createLiveDeviceAssertions( - runStep, - verifyCommand, +/** + * Builds the assertion helpers bound to one harness's `runStep`/`verifyCommand`, so a harness + * constructed with a scripted `runCli` observes the same steps the live lane runs. + */ +export function createAndroidLiveAssertions(harness: AndroidLiveHarness = androidLiveHarness) { + return createLiveDeviceAssertions( + harness.runStep, + harness.verifyCommand, PUBLIC_COMMANDS.wait, ); +} + +export const { assertElementText, assertWaitSelector, assertWaitText, capturePng } = + createAndroidLiveAssertions(); export async function scrollToVisibleSelector( context: LiveContext, selector: string, ): Promise { for (let attempt = 0; attempt < 6; attempt += 1) { - const result = await runStep( + const result = await androidLiveHarness.runStep( context, `check ${selector} visibility (${attempt + 1})`, ['is', 'visible', selector], @@ -44,7 +52,7 @@ export async function scrollToVisibleSelector( `could not observe ${selector}: ${JSON.stringify(result.json)}`, ); if (attempt < 5) { - await runStep(context, `scroll toward ${selector} (${attempt + 1})`, [ + await androidLiveHarness.runStep(context, `scroll toward ${selector} (${attempt + 1})`, [ 'scroll', 'down', '0.25', diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.test.ts b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts index 798014102d..812e42576e 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.test.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts @@ -7,9 +7,9 @@ import { mkdtempForTestSync } from '@agent-device/test-kit'; import type { CliJsonResult } from '../cli-json.ts'; import { createLiveDeviceContext, type LiveRunCli } from '../live-device-e2e/runtime.ts'; import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; -import { assertHomeAndRecentsRestoration } from './live-automation-scenario.ts'; -import { filesDiffer } from './live-assertions.ts'; -import type { LiveContext } from './live-harness.ts'; +import { createAndroidLiveAssertions, filesDiffer } from './live-assertions.ts'; +import { createAndroidLiveHarness, type LiveContext } from './live-harness.ts'; +import { createHomeAndRecentsRestorationScenario } from './live-automation-scenario.ts'; const APP_ID = 'com.callstack.agentdevicelab'; const LAUNCHER = 'com.google.android.apps.nexuslauncher'; @@ -20,47 +20,88 @@ const HOME = Buffer.concat([PNG_SIGNATURE, Buffer.from('launcher-home')]); const RECENTS = Buffer.concat([PNG_SIGNATURE, Buffer.from('recents-surface')]); /** - * A scripted device behind the context's `runCli` override: `screenshot` writes the next scripted - * image for that artifact (the last entry repeats), `home`/`open` drive the foreground package, - * and every other command succeeds. The scenario runs through the real harness wiring - * (commonFlags, step history, command/behavior evidence), not a helper call. + * A scripted device behind the harness-owned `runCli` injection. `screenshot` behaves like a CLI + * runner under a step timeout: the capture would finish after `workMs`, but a forwarded + * `options.timeoutMs` kills it first and returns a failure without writing the image. The + * forwarded timeout of every screenshot call is recorded in `timeouts`. */ -function scriptedDevice(script: Record) { +function scriptedDevice(options: { script: Record; workMs: number }): { + runCli: LiveRunCli; + timeouts: Record; +} { const attempts = new Map(); + const timeouts: Record = {}; let foregroundPackage = APP_ID; - const runCli: LiveRunCli = async (args): Promise => { + const runCli: LiveRunCli = (args, _env, cliOptions): Promise => { const command = args[0]; if (command === 'home') { foregroundPackage = LAUNCHER; - return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + return Promise.resolve({ + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: {} }, + }); } if (command === 'open') { foregroundPackage = APP_ID; - return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + return Promise.resolve({ + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: {} }, + }); } if (command === 'appstate') { - return { + return Promise.resolve({ status: 0, stdout: '', stderr: '', json: { success: true, data: { package: foregroundPackage } }, - }; + }); } if (command === 'screenshot') { const output = args[1]!; const base = path.basename(output); - const attempt = (attempts.get(base) ?? 0) + 1; - attempts.set(base, attempt); - const images = script[base] ?? [FOREGROUND]; - fs.writeFileSync(output, images[Math.min(attempt - 1, images.length - 1)]!); - return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + const timeoutMs = cliOptions?.timeoutMs; + (timeouts[base] ??= []).push(timeoutMs); + return new Promise((resolve) => { + let done = false; + const finish = setTimeout(() => { + if (done) return; + done = true; + const attempt = (attempts.get(base) ?? 0) + 1; + attempts.set(base, attempt); + const images = options.script[base] ?? [FOREGROUND]; + fs.writeFileSync(output, images[Math.min(attempt - 1, images.length - 1)]!); + resolve({ status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }); + }, options.workMs); + if (timeoutMs !== undefined) { + setTimeout(() => { + if (done) return; + done = true; + clearTimeout(finish); + resolve({ + status: 1, + stdout: '', + stderr: `screenshot killed after ${timeoutMs}ms`, + json: { success: false, error: { code: 'timeout' } }, + }); + }, timeoutMs); + } + }); } - return { status: 0, stdout: '', stderr: '', json: { success: true, data: {} } }; + return Promise.resolve({ + status: 0, + stdout: '', + stderr: '', + json: { success: true, data: {} }, + }); }; - return { runCli }; + return { runCli, timeouts }; } -function makeContext(runCli: LiveRunCli): LiveContext { +function makeContext(): LiveContext { return { ...createLiveDeviceContext({ artifactRoot: mkdtempForTestSync('home-recents-scenario-'), @@ -70,10 +111,23 @@ function makeContext(runCli: LiveRunCli): LiveContext { appPath: '/tmp/fixture.apk', serial: 'emulator-5554', tier: 'smoke', - runCli, }; } +/** Runs the Home/Recents restoration scenario through the harness entry with a scripted device. */ +function runScenarioWithDevice(device: { runCli: LiveRunCli }) { + const harness = createAndroidLiveHarness({ runCli: device.runCli }); + const assertions = createAndroidLiveAssertions(harness); + const context = makeContext(); + const scenario = createHomeAndRecentsRestorationScenario({ + ...assertions, + runStep: harness.runStep, + verifyBehavior: harness.verifyBehavior, + verifyCommand: harness.verifyCommand, + }); + return { context, promise: harness.runScenario(context, scenario) }; +} + beforeEach(() => { vi.useFakeTimers(); }); @@ -88,17 +142,19 @@ async function settle(promise: Promise): Promise> return settled; } -test('Home and Recents go through the scenario wait and pass only after the delayed captures differ', async () => { - // The first Home capture still shows the fixture and the first Recents capture still shows - // Home: a single-capture route would fail both pixel comparisons here. - const { runCli } = scriptedDevice({ - 'foreground.png': [FOREGROUND], - 'home.png': [FOREGROUND, HOME], - 'recents.png': [HOME, RECENTS], +test('Home and Recents pass through the harness scenario entry after delayed captures, with forwarded capture timeouts', async () => { + // Quick captures; the first Home capture still shows the fixture and the first Recents capture + // still shows Home, so a single-capture route fails both pixel comparisons here. + const device = scriptedDevice({ + script: { + 'foreground.png': [FOREGROUND], + 'home.png': [FOREGROUND, HOME], + 'recents.png': [HOME, RECENTS], + }, + workMs: 100, }); - const context = makeContext(runCli); - context.currentScenario = 'smoke:automation-system'; - const result = await settle(assertHomeAndRecentsRestoration(context)); + const { context, promise } = runScenarioWithDevice(device); + const result = await settle(promise); expect(result.status).toBe('fulfilled'); const screenshotSteps = context.stepHistory @@ -106,6 +162,16 @@ test('Home and Recents go through the scenario wait and pass only after the dela .map((record) => record.step); expect(screenshotSteps).toContain('capture Android Home surface (attempt 2)'); expect(screenshotSteps).toContain('capture Android Recents surface (attempt 2)'); + // The default capture forwarded a bounded timeout to the CLI runner on every wait attempt. + for (const base of ['home.png', 'recents.png']) { + const attemptTimeouts = device.timeouts[base] ?? []; + expect(attemptTimeouts.length).toBe(2); + for (const timeoutMs of attemptTimeouts) { + expect(typeof timeoutMs).toBe('number'); + expect(timeoutMs).toBeLessThanOrEqual(5_000); + expect(timeoutMs).toBeGreaterThan(0); + } + } expect( filesDiffer( path.join(context.artifactDir, 'foreground.png'), @@ -119,39 +185,46 @@ test('Home and Recents go through the scenario wait and pass only after the dela ), ).toBe(true); expect(context.behaviorEvidence['home-recents-restoration']?.length).toBeGreaterThan(0); - const appStates = context.stepHistory.filter((record) => record.commandName === 'appstate'); - expect(appStates.length).toBe(2); + expect(context.completedScenarios).toContain('smoke:home-recents-restoration'); }); -test('the scenario fails when Home never replaces the fixture pixels', async () => { - const { runCli } = scriptedDevice({ - 'foreground.png': [FOREGROUND], - 'home.png': [FOREGROUND], - 'recents.png': [RECENTS], +test('a wedged capture is cut at the forwarded remaining timeout and the scenario fails at the wait deadline', async () => { + // Every capture would finish after 60s; only the forwarded step timeout can cut it, and the + // wait must still fail at its own 10s deadline with the capture failures as the last observation. + const device = scriptedDevice({ + script: { 'foreground.png': [FOREGROUND], 'home.png': [FOREGROUND], 'recents.png': [RECENTS] }, + workMs: 60_000, }); - const context = makeContext(runCli); - context.currentScenario = 'smoke:automation-system'; - const result = await settle(assertHomeAndRecentsRestoration(context)); + const { context, promise } = runScenarioWithDevice(device); + const result = await settle(promise); expect(result).toMatchObject({ status: 'rejected', reason: { message: expect.stringMatching( - /Android Home never replaced the fixture pixels within 10000ms/s, + /Android Home never replaced the fixture pixels within 10000ms.*capture \d+ failed/s, ), }, }); + // The remaining deadline (not a constant) was forwarded to the wedged captures: timeouts are + // set, bounded by the per-attempt cap, and shrink as the wait's deadline approaches. + const homeTimeouts = device.timeouts['home.png'] ?? []; + expect(homeTimeouts.length).toBeGreaterThan(1); + for (const timeoutMs of homeTimeouts) expect(typeof timeoutMs).toBe('number'); + expect(Math.max(...homeTimeouts.map((t) => t!))).toBeLessThanOrEqual(5_000); + expect(Math.min(...homeTimeouts.map((t) => t!))).toBeLessThan(5_000); + expect( + context.stepHistory.filter((record) => record.commandName === 'screenshot').length, + ).toBeGreaterThanOrEqual(homeTimeouts.length); }); test('the scenario fails when Recents never differs from the Home surface', async () => { - const { runCli } = scriptedDevice({ - 'foreground.png': [FOREGROUND], - 'home.png': [HOME], - 'recents.png': [HOME], + const device = scriptedDevice({ + script: { 'foreground.png': [FOREGROUND], 'home.png': [HOME], 'recents.png': [HOME] }, + workMs: 100, }); - const context = makeContext(runCli); - context.currentScenario = 'smoke:automation-system'; - const result = await settle(assertHomeAndRecentsRestoration(context)); + const { promise } = runScenarioWithDevice(device); + const result = await settle(promise); expect(result).toMatchObject({ status: 'rejected', diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index 65916831cb..fe220de11a 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { LiveScenario } from '../live-device-e2e/runtime.ts'; import { assertElementText, assertDiffLine, @@ -14,13 +15,62 @@ import { requireAndroidResourceId, scrollToVisibleSelector, } from './live-assertions.ts'; -import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; +import { + type AndroidLiveHarness, + type LiveContext, + runStep, + verifyBehavior, + verifyCommand, +} from './live-harness.ts'; import { assertHumanSnapshotCompaction } from './live-snapshot-compaction.ts'; import { waitForSurfaceChange } from './live-surface-transition.ts'; const C = PUBLIC_COMMANDS; -export async function assertAutomationSystem(context: LiveContext): Promise { +/** + * The harness-bound steps one scenario runs through. The smoke scenario uses the live harness's + * steps; a harness-level regression constructs its own harness (with a scripted `runCli`) and + * passes its steps here, so the scenario runs through the harness entry instead of a direct call. + */ +export type AutomationScenarioSteps = { + assertElementText: typeof assertElementText; + assertWaitSelector: typeof assertWaitSelector; + assertWaitText: typeof assertWaitText; + capturePng: typeof capturePng; + runStep: AndroidLiveHarness['runStep']; + verifyBehavior: AndroidLiveHarness['verifyBehavior']; + verifyCommand: AndroidLiveHarness['verifyCommand']; +}; + +export const automationScenarioSteps: AutomationScenarioSteps = { + assertElementText, + assertWaitSelector, + assertWaitText, + capturePng, + runStep, + verifyBehavior, + verifyCommand, +}; + +/** + * Scenario entry for the Home/Recents restoration lane. `smoke:automation-system` embeds this + * lane inline; harness-level regressions register the returned scenario with + * `createAndroidLiveHarness(...).runScenario` so the lane runs through the harness entry. + */ +export function createHomeAndRecentsRestorationScenario( + steps: AutomationScenarioSteps = automationScenarioSteps, +): LiveScenario { + return { + id: 'smoke:home-recents-restoration', + run: (context) => assertHomeAndRecentsRestoration(context, steps), + }; +} + +export async function assertAutomationSystem( + context: LiveContext, + steps: AutomationScenarioSteps = automationScenarioSteps, +): Promise { + const { assertElementText, assertWaitText, runStep, verifyBehavior, verifyCommand } = steps; await assertElementText(context, 'id="automation-event-name"', 'cold.start'); await assertElementText( context, @@ -190,7 +240,7 @@ export async function assertAutomationSystem(context: LiveContext): Promise { +async function assertHomeAndRecentsRestoration( + context: LiveContext, + steps: AutomationScenarioSteps, +): Promise { + const { assertWaitSelector, capturePng, runStep, verifyBehavior, verifyCommand } = steps; const foreground = path.join(context.artifactDir, 'foreground.png'); await capturePng(context, 'capture Android fixture foreground', foreground); await runStep(context, 'send Android fixture Home', ['home']); @@ -235,6 +289,7 @@ export async function assertHomeAndRecentsRestoration(context: LiveContext): Pro home, 'capture Android Home surface', 'Android Home never replaced the fixture pixels', + { capturePng }, ); assertFilesDiffer(foreground, home, 'Android Home should replace fixture pixels'); verifyCommand(context, C.home, 'Home changes Android foreground package and system pixels'); @@ -248,6 +303,7 @@ export async function assertHomeAndRecentsRestoration(context: LiveContext): Pro recents, 'capture Android Recents surface', 'Android Recents never differed from the Home pixels', + { capturePng }, ); assertFilesDiffer(home, recents, 'Android Recents should differ from Home pixels'); verifyCommand(context, C.appSwitcher, 'Recents pixels differ from Home system surface'); diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts index 035de4833b..26092cd050 100644 --- a/test/integration/android-emulator-e2e/live-harness.ts +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { createLiveDeviceContext, createLiveDeviceHarness, + type LiveRunCli, requiredEnv, type LiveDeviceContext, } from '../live-device-e2e/runtime.ts'; @@ -38,27 +39,38 @@ export function createContext(): LiveContext { }; } -const harness = createLiveDeviceHarness({ - behaviorsForScenario: liveBehaviorsForScenario, - commandsForScenario: liveCommandsForScenario, - deviceEvidence: (context) => readAndroidDeviceEvidence(context), - // Seven adb reads, two of them dumpsys: the default bound would drop the whole file, - // including the crash stack, on a loaded runner. - deviceEvidenceTimeoutMs: 25_000, - commonFlags: (context, args) => [ - ...args, - '--platform', - 'android', - '--serial', - context.serial, - '--session', - context.session, - '--daemon-server-mode', - 'dual', - ...(args.includes('--json') ? [] : ['--json']), - ], - writeCoverageReport, -}); +/** + * Builds the Android live harness. `overrides.runCli` is the harness-owned CLI runner injection: + * harness-level regressions script a device through it and run a scenario via `runScenario`. + */ +export function createAndroidLiveHarness(overrides: { runCli?: LiveRunCli } = {}) { + return createLiveDeviceHarness({ + behaviorsForScenario: liveBehaviorsForScenario, + commandsForScenario: liveCommandsForScenario, + deviceEvidence: (context) => readAndroidDeviceEvidence(context), + // Seven adb reads, two of them dumpsys: the default bound would drop the whole file, + // including the crash stack, on a loaded runner. + deviceEvidenceTimeoutMs: 25_000, + commonFlags: (context, args) => [ + ...args, + '--platform', + 'android', + '--serial', + context.serial, + '--session', + context.session, + '--daemon-server-mode', + 'dual', + ...(args.includes('--json') ? [] : ['--json']), + ], + writeCoverageReport, + ...(overrides.runCli === undefined ? {} : { runCli: overrides.runCli }), + }); +} + +export const androidLiveHarness = createAndroidLiveHarness(); + +export type AndroidLiveHarness = ReturnType; export const { runScenario, @@ -67,7 +79,7 @@ export const { verifyBehavior, verifyCommand, verifyNestedCommand: verifyNestedReplayCommand, -} = harness; +} = androidLiveHarness; export async function cleanupSession(context: LiveContext): Promise { const failures: unknown[] = []; diff --git a/test/integration/android-emulator-e2e/live-surface-transition.test.ts b/test/integration/android-emulator-e2e/live-surface-transition.test.ts index 15da7ee6e3..eb12d4348a 100644 --- a/test/integration/android-emulator-e2e/live-surface-transition.test.ts +++ b/test/integration/android-emulator-e2e/live-surface-transition.test.ts @@ -59,7 +59,7 @@ test('a delayed transition satisfies the wait only after a capture differs, keep outputPath, 'capture surface', 'surface never changed', - capture, + { capture }, ), ); @@ -81,7 +81,7 @@ test('a capture that fails transiently is retried until the surface changes', as outputPath, 'capture surface', 'surface never changed', - capture, + { capture }, ), ); @@ -99,7 +99,7 @@ test('a surface that never changes fails at the deadline and keeps the last obse outputPath, 'capture surface', 'surface never changed', - capture, + { capture }, ), ); @@ -124,7 +124,7 @@ test('a wait whose captures keep failing reports the last capture error at the d outputPath, 'capture surface', 'surface never changed', - capture, + { capture }, ), ); @@ -146,13 +146,15 @@ test('no capture outlives the remaining deadline', async () => { outputPath, 'capture surface', 'surface never changed', - (_output, _attempt, remainingMs) => - new Promise((_resolve, reject) => - setTimeout(() => { - failedAt = Date.now(); - reject(new Error('capture timed out')); - }, remainingMs), - ), + { + capture: (_output, _attempt, remainingMs) => + new Promise((_resolve, reject) => + setTimeout(() => { + failedAt = Date.now(); + reject(new Error('capture timed out')); + }, remainingMs), + ), + }, ), ); @@ -173,9 +175,11 @@ test('a capture that resolves after the deadline is rejected as a late observati outputPath, 'capture surface', 'surface never changed', - async (output, _attempt, remainingMs) => { - await new Promise((resolve) => setTimeout(resolve, remainingMs + 500)); - fs.writeFileSync(output, HOME_SURFACE); + { + capture: async (output, _attempt, remainingMs) => { + await new Promise((resolve) => setTimeout(resolve, remainingMs + 500)); + fs.writeFileSync(output, HOME_SURFACE); + }, }, ), ); diff --git a/test/integration/android-emulator-e2e/live-surface-transition.ts b/test/integration/android-emulator-e2e/live-surface-transition.ts index 3cabfe251e..151e8eeaea 100644 --- a/test/integration/android-emulator-e2e/live-surface-transition.ts +++ b/test/integration/android-emulator-e2e/live-surface-transition.ts @@ -1,13 +1,22 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; -import { capturePng, filesDiffer } from './live-assertions.ts'; +import { capturePng as defaultPngCapture, filesDiffer } from './live-assertions.ts'; import type { LiveContext } from './live-harness.ts'; +import type { RunStepOptions } from '../live-device-e2e/runtime.ts'; const SURFACE_TRANSITION_TIMEOUT_MS = 10_000; const SURFACE_TRANSITION_POLL_MS = 250; const SURFACE_CAPTURE_TIMEOUT_MS = 5_000; +/** A PNG capture step: `screenshot --scale 0.5` with harness step options. */ +export type PngCapture = ( + context: LiveContext, + step: string, + outputPath: string, + options: RunStepOptions, +) => Promise; + /** * One screenshot attempt of a surface wait. Writes the observed image to `outputPath`. * `remainingMs` is the wait's remaining deadline when the attempt starts, so a capture can be @@ -19,14 +28,28 @@ export type SurfaceCapture = ( remainingMs: number, ) => Promise; +export type SurfaceTransitionOptions = { + /** + * Injects one attempt. Tests that inject `capture` bypass the default capture and therefore + * do not observe its timeout forwarding. + */ + capture?: SurfaceCapture; + /** + * The PNG capture the default attempts run through. Scenarios pass the capture bound to their + * harness so the default capture's `timeoutMs` reaches the CLI runner of that harness. + */ + capturePng?: PngCapture; +}; + /** * Waits until a fresh screenshot differs from the baseline surface, keeping the final observed * image at `outputPath`. System navigation (Home, Recents) is dispatched before the new surface * has drawn, so a single capture right after the command can still show the previous surface. * Each attempt removes and overwrites `outputPath`, so the file always holds the last observed - * image and a failed capture cannot leave a stale one behind. A capture that resolves after the - * deadline is rejected as a late observation, and the wait fails at the deadline if navigation - * never produced the change. + * image and a failed capture cannot leave a stale one behind. The default capture is bounded by + * the remaining deadline (capped per attempt), a capture that resolves after the deadline is + * rejected as a late observation, and the wait fails at the deadline if navigation never + * produced the change. */ export async function waitForSurfaceChange( context: LiveContext, @@ -34,8 +57,10 @@ export async function waitForSurfaceChange( outputPath: string, captureStep: string, failure: string, - capture?: SurfaceCapture, + options: SurfaceTransitionOptions = {}, ): Promise { + const capturePng = options.capturePng ?? defaultPngCapture; + const capture = options.capture ?? defaultCapture(capturePng, context, captureStep); const deadline = Date.now() + SURFACE_TRANSITION_TIMEOUT_MS; let lastObservation = 'no capture finished'; let attempt = 0; @@ -47,7 +72,7 @@ export async function waitForSurfaceChange( attempt += 1; fs.rmSync(outputPath, { force: true }); try { - await (capture ?? defaultCapture(context, captureStep))(outputPath, attempt, remaining); + await capture(outputPath, attempt, remaining); if (Date.now() >= deadline) { lastObservation = `capture ${attempt} resolved after the deadline`; } else if (filesDiffer(baselinePath, outputPath)) { @@ -64,7 +89,11 @@ export async function waitForSurfaceChange( assert.fail(`${failure} within ${SURFACE_TRANSITION_TIMEOUT_MS}ms (${lastObservation})`); } -function defaultCapture(context: LiveContext, captureStep: string): SurfaceCapture { +function defaultCapture( + capturePng: PngCapture, + context: LiveContext, + captureStep: string, +): SurfaceCapture { return (output, attempt, remainingMs) => capturePng(context, `${captureStep} (attempt ${attempt})`, output, { // The wait owns the miss policy: a failed attempt is retried under the deadline and only diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index a75224b206..18ee1af7db 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -40,11 +40,6 @@ export type LiveDeviceContext = { startedAtMs: number; stepHistory: StepRecord[]; timings: ScenarioTiming[]; - /** - * Per-context CLI override, ahead of the harness-wide `runCli`. Lets one scenario run against a - * scripted device (delayed captures, scripted appstate) without touching the shared harness. - */ - runCli?: LiveRunCli; }; export type LiveScenario = { @@ -56,11 +51,8 @@ type HarnessOptions = { behaviorsForScenario: (scenarioId: string) => readonly BehaviorId[]; commandsForScenario: (scenarioId: string) => readonly string[]; commonFlags: (context: Context, args: readonly string[]) => string[]; - runCli?: ( - args: string[], - env: NodeJS.ProcessEnv, - options?: { timeoutMs?: number }, - ) => Promise; + /** CLI runner injection: the seam tests use to script a device for a scenario. */ + runCli?: LiveRunCli; /** * Platform-owned device facts for a failed step (rotation state, whether the app process is * alive, crash logs), read outside agent-device so they describe the device even when the CLI @@ -246,8 +238,7 @@ export function createLiveDeviceHarness< ): Promise<{ result: CliJsonResult; fullArgs: string[] }> { const fullArgs = buildStepArgs(context, args, stepOptions); const startedAt = Date.now(); - const runCli = context.runCli ?? options.runCli ?? runBuiltCliJson; - const result = await runCli(fullArgs, context.env, { + const result = await (options.runCli ?? runBuiltCliJson)(fullArgs, context.env, { timeoutMs: stepOptions.timeoutMs, }); const failedAsExpected = stepOptions.expectFailure === true && result.status !== 0; @@ -306,7 +297,7 @@ export function createLiveDeviceHarness< } function captureFailedStepEvidence(context: Context): Promise { - const runCli = context.runCli ?? options.runCli ?? runBuiltCliJson; + const runCli = options.runCli ?? runBuiltCliJson; const deviceEvidence = options.deviceEvidence; return collectFailedStepEvidence({ stem: path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`), From 55f5b877e31a676107f47d5457d2d5179a317b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 22:55:17 +0100 Subject: [PATCH 4/4] test(android-e2e): keep the default scenario steps module-local fallow flags automationScenarioSteps as an unused export; only the scenario module consumes it as the default steps parameter. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../android-emulator-e2e/live-automation-scenario.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index fe220de11a..809879662b 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -42,7 +42,7 @@ export type AutomationScenarioSteps = { verifyCommand: AndroidLiveHarness['verifyCommand']; }; -export const automationScenarioSteps: AutomationScenarioSteps = { +const automationScenarioSteps: AutomationScenarioSteps = { assertElementText, assertWaitSelector, assertWaitText,