From f69536cc6b90b56af72511cd994c34f04caa4abc Mon Sep 17 00:00:00 2001 From: Jochen Delabie Date: Tue, 10 Nov 2026 13:42:12 +0100 Subject: [PATCH 1/4] Element screenshots, iOS goBack, snapshot tuning, bounds visibility, iOS fill fix - screenshot({ clip, scale }): crop to the element (points -> pixels via the screen scale) and downscale, with a dependency-free PNG transform. Falls back to the full screen for PNG formats Appium does not produce. - pressButton('BACK') on iOS: left-edge swipe instead of throwing. - snapshotTuning option: waitForIdleTimeout 0 (+ animationCoolOffTimeout 0 on iOS) via /appium/settings once per session. Measured on production: fill+assert 7.3s -> 3.5s on an Android emulator, 3.4s -> 2.95s on an iOS simulator. - visibility: 'bounds' option for mobilecli-like isVisible. - iOS clearText/fill: clear through GET /element/active + element clear. XCUITest's source has no focused attribute and RN fields put the caret at the start, so the backspace path never cleared anything. --- src/actions.ts | 15 ++ src/driver.ts | 88 +++++++++- src/hierarchy.ts | 39 +++-- src/options.ts | 20 +++ src/png.ts | 209 +++++++++++++++++++++++ test/fake-hub/fake-hub.ts | 10 +- test/integration/driver-features.test.ts | 113 ++++++++++++ test/unit/png.test.ts | 99 +++++++++++ 8 files changed, 571 insertions(+), 22 deletions(-) create mode 100644 src/png.ts create mode 100644 test/integration/driver-features.test.ts create mode 100644 test/unit/png.test.ts diff --git a/src/actions.ts b/src/actions.ts index d1edc1b..8c0e9a0 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -75,6 +75,21 @@ export function swipeActions( ])]; } +/** + * iOS system back gesture: a quick swipe from the left screen edge to past the + * middle, at mid-height. UINavigationController's interactive pop recognizer + * needs the touch to start within a few points of the edge. + */ +export function edgeSwipeBackActions(screen: ScreenSize): PointerSource[] { + const y = screen.height / 2; + return [pointerSource('finger1', [ + move(2, y), + down(), + move(screen.width * 0.6, y, 250), + up(), + ])]; +} + /** Multi-finger gesture: each pointer path becomes one W3C input source, * with `time` offsets converted to per-segment move durations. */ export function gestureActions(gestures: GestureSequence): PointerSource[] { diff --git a/src/driver.ts b/src/driver.ts index 9ae9dd1..467f5ef 100644 --- a/src/driver.ts +++ b/src/driver.ts @@ -3,6 +3,7 @@ import createDebug from 'debug'; import { NoDeviceAvailableError, type AllocatedDevice, + type Bounds, type AllocationCriteria, type AppInfo, type ConnectionConfig, @@ -28,6 +29,7 @@ import { } from '@mobilewright/protocol'; import { doubleTapActions, + edgeSwipeBackActions, gestureActions, longPressActions, swipeActions, @@ -42,6 +44,7 @@ import { parseChord, toAndroidKeyPress, toW3CKeyActions, typeTextActions } from import { TestingBotTestObserver } from './observer.js'; import { requireCredentials, resolveOptions, type ResolvedOptions, type TestingBotDriverOptions } from './options.js'; import { RestApiClient, type UserInfo } from './rest-api.js'; +import { transformPng } from './png.js'; import { RunLog } from './run-log.js'; import { setActiveDriver, type ThrottlePreset, type ThrottleProfile } from './session-commands.js'; import { TunnelManager } from './tunnel.js'; @@ -105,6 +108,12 @@ const ANDROID_BUTTON_KEYCODES: Record = { LOCK: 26, // same as POWER — locks the screen }; +/** Default snapshot tuning per platform (see the `snapshotTuning` option). */ +const SNAPSHOT_TUNING: Record> = { + android: { waitForIdleTimeout: 0 }, + ios: { waitForIdleTimeout: 0, animationCoolOffTimeout: 0 }, +}; + // XCUITest's mobile: pressButton supports exactly these names on iOS. const IOS_BUTTON_NAMES: Partial> = { HOME: 'home', @@ -282,6 +291,7 @@ export class TestingBotDriver implements MobilewrightDriver { } const platform = criteria.platform ?? (String(caps['platformName']).toLowerCase() === 'ios' ? 'ios' : 'android'); + await this.applySnapshotTuning(sessionId, platform); const session: AllocatedSession = { sessionId, platform, @@ -457,10 +467,27 @@ export class TestingBotDriver implements MobilewrightDriver { const capabilities = buildCapabilities(criteria, this.options, pinned, appUrls); const { sessionId } = await this.hub.newSession(capabilities, { timeout: this.options.allocationTimeout }); this.runLog.append({ type: 'session', sessionId }); + await this.applySnapshotTuning(sessionId, config.platform); debug('started per-test session %s — https://testingbot.com/members/tests/%s', sessionId, sessionId); return sessionId; } + /** + * Turn off the backend's wait-for-idle before each page-source read (see the + * `snapshotTuning` option). Once per session, best-effort: a server that + * rejects a setting still leaves a working session. + */ + private async applySnapshotTuning(sessionId: string, platform: Platform): Promise { + if (this.options.snapshotTuning === false) return; + const settings = { ...SNAPSHOT_TUNING[platform], ...this.options.snapshotTuning }; + try { + await this.hub.post(sessionId, '/appium/settings', { settings }); + debug('snapshot tuning applied to %s: %o', sessionId, settings); + } catch (err) { + debug('snapshot tuning rejected on %s (%s), keeping server defaults', sessionId, err); + } + } + // ─── TestingBot-specific session commands ───────────────────── /** Run any Appium/TestingBot execute-script command on the live session. */ @@ -513,7 +540,7 @@ export class TestingBotDriver implements MobilewrightDriver { const { sessionId, platform } = this.session(); const xml = await this.hub.source(sessionId); const { parseHierarchy } = await import('./hierarchy.js'); - return parseHierarchy(xml, platform); + return parseHierarchy(xml, platform, this.options.visibility); } // ─── MobilewrightSession: input ──────────────────────────────── @@ -568,8 +595,20 @@ export class TestingBotDriver implements MobilewrightDriver { await this.pressKeys(['ctrl+a', 'backspace']); return; } - // iOS has no working select-all chord; delete the focused field's value - // one backspace per character (hierarchy tells us how many). + // iOS: clear the focused field through its element reference. Backspaces + // only delete what is left of the caret, and tapping a field (React Native + // in particular) can put the caret at the start, so key-based deletion + // leaves the value untouched. + const active = await this.activeElementId(sessionId); + if (active) { + try { + await this.hub.post(sessionId, `/element/${active}/clear`, {}); + return; + } catch (err) { + debug('clearText: element clear failed (%s), falling back to backspaces', err); + } + } + // Fallback: one backspace per character of the focused field's value. const { parseHierarchy } = await import('./hierarchy.js'); const nodes = parseHierarchy(await this.hub.source(sessionId), 'ios'); const focused = findFocused(nodes); @@ -581,10 +620,20 @@ export class TestingBotDriver implements MobilewrightDriver { } } + /** WebDriver id of the focused element (GET /element/active), if any. */ + private async activeElementId(sessionId: string): Promise { + try { + const ref = await this.hub.get | null>(sessionId, '/element/active'); + return ref?.['element-6066-11e4-a52e-4f304ffac5c0'] ?? ref?.['ELEMENT']; + } catch (err) { + debug('no active element (%s)', err); + return undefined; + } + } + /** - * Neither Appium driver can replace a field's value without an element - * reference, and the protocol only hands us the focused field — so clear it - * the same way `clearText()` does, then type the new value. + * The protocol only hands us the focused field, so clear it the same way + * `clearText()` does, then type the new value. */ async fill(text: string): Promise { await this.clearText(); @@ -607,6 +656,13 @@ export class TestingBotDriver implements MobilewrightDriver { await this.hub.execute(sessionId, 'mobile: pressKey', [{ keycode: ANDROID_BUTTON_KEYCODES[button] }]); return; } + if (button === 'BACK') { + // iOS has no back button; screen.goBack() lands here. Use the system + // edge-swipe-back gesture that UINavigationController responds to. + const screen = await this.getScreenSize(); + await this.hub.performActions(sessionId, edgeSwipeBackActions(screen)); + return; + } const name = IOS_BUTTON_NAMES[button]; if (!name) { throw new Error(`TestingBotDriver: hardware button "${button}" is not supported on iOS`); @@ -616,11 +672,25 @@ export class TestingBotDriver implements MobilewrightDriver { // ─── MobilewrightSession: screen ─────────────────────────────── - async screenshot(opts?: ScreenshotOptions): Promise { + async screenshot(opts?: ScreenshotOptions & { clip?: Bounds; scale?: number }): Promise { const { sessionId } = this.session(); // TestingBot/Appium produce PNG; a jpeg format request falls back to PNG. - const base64 = await this.hub.screenshot(sessionId); - const buffer = Buffer.from(base64, 'base64'); + let buffer: Buffer = Buffer.from(await this.hub.screenshot(sessionId), 'base64'); + const factor = opts?.scale !== undefined && opts.scale > 0 && opts.scale < 1 ? opts.scale : 1; + if (opts?.clip || factor < 1) { + // `clip` is in screen points (locator.screenshot() passes element + // bounds); the PNG is in physical pixels. + const ratio = opts?.clip ? (await this.getScreenSize()).scale : 1; + const rect = opts?.clip && { + x: opts.clip.x * ratio, + y: opts.clip.y * ratio, + width: opts.clip.width * ratio, + height: opts.clip.height * ratio, + }; + const transformed = transformPng(buffer, rect, factor); + if (transformed) buffer = transformed; + else debug('screenshot: could not crop/scale this PNG, returning the full screen'); + } if (opts?.path) await writeFile(opts.path, buffer); return buffer; } diff --git a/src/hierarchy.ts b/src/hierarchy.ts index 9e7c937..9b61635 100644 --- a/src/hierarchy.ts +++ b/src/hierarchy.ts @@ -8,7 +8,15 @@ import type { Bounds, Platform, ViewNode } from '@mobilewright/protocol'; * role engine normalizes them itself (bareTypeName), and getByType matches * case-insensitively on the raw value. */ -export function parseHierarchy(xml: string, platform: Platform): ViewNode[] { +/** + * How `isVisible` is judged. 'native' trusts the automation backend's own + * flag; 'bounds' treats any element with a non-empty frame as visible. XCUITest's + * `visible` is stricter than local mobilecli (an element drawn under a sibling + * reports invisible), so 'bounds' matches what suites developed locally expect. + */ +export type VisibilityMode = 'native' | 'bounds'; + +export function parseHierarchy(xml: string, platform: Platform, visibility: VisibilityMode = 'native'): ViewNode[] { const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '', @@ -17,7 +25,9 @@ export function parseHierarchy(xml: string, platform: Platform): ViewNode[] { parseAttributeValue: false, }); const doc = parser.parse(xml) as OrderedNode[]; - const mapper = platform === 'ios' ? mapIosNode : mapAndroidNode; + const mapper: NodeMapper = platform === 'ios' + ? (tag, attrs, children) => mapIosNode(tag, attrs, children, visibility) + : (tag, attrs, children) => mapAndroidNode(tag, attrs, children, visibility); return doc.flatMap((n) => convert(n, mapper)); } @@ -44,14 +54,14 @@ function convert(node: OrderedNode, mapper: NodeMapper): ViewNode[] { return mapped ? [mapped] : children; } -function mapAndroidNode(tag: string, attrs: Record, children: ViewNode[]): ViewNode | null { +function mapAndroidNode(tag: string, attrs: Record, children: ViewNode[], visibility: VisibilityMode): ViewNode | null { const type = attrs['class'] ?? tag; const bounds = parseAndroidBounds(attrs['bounds']); const node: ViewNode = { type, bounds, children, - isVisible: attrs['displayed'] !== 'false', + isVisible: visibility === 'bounds' ? hasArea(bounds) : attrs['displayed'] !== 'false', isEnabled: attrs['enabled'] !== 'false', raw: attrs, }; @@ -65,18 +75,19 @@ function mapAndroidNode(tag: string, attrs: Record, children: Vi return node; } -function mapIosNode(tag: string, attrs: Record, children: ViewNode[]): ViewNode | null { +function mapIosNode(tag: string, attrs: Record, children: ViewNode[], visibility: VisibilityMode): ViewNode | null { const type = attrs['type'] ?? tag; + const bounds: Bounds = { + x: num(attrs['x']), + y: num(attrs['y']), + width: num(attrs['width']), + height: num(attrs['height']), + }; const node: ViewNode = { type, - bounds: { - x: num(attrs['x']), - y: num(attrs['y']), - width: num(attrs['width']), - height: num(attrs['height']), - }, + bounds, children, - isVisible: attrs['visible'] !== 'false', + isVisible: visibility === 'bounds' ? hasArea(bounds) : attrs['visible'] !== 'false', isEnabled: attrs['enabled'] !== 'false', raw: attrs, }; @@ -89,6 +100,10 @@ function mapIosNode(tag: string, attrs: Record, children: ViewNo return node; } +function hasArea(bounds: Bounds): boolean { + return bounds.width > 0 && bounds.height > 0; +} + /** Android bounds attribute: "[x1,y1][x2,y2]". */ function parseAndroidBounds(value: string | undefined): Bounds { const match = value?.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/); diff --git a/src/options.ts b/src/options.ts index 0a4b68e..7eef91e 100644 --- a/src/options.ts +++ b/src/options.ts @@ -114,6 +114,22 @@ export interface TestingBotDriverOptions { * on every test. Default: false (mobilewright's pool reuses sessions). */ sessionPerTest?: boolean; + /** + * Appium settings applied once per session to speed up hierarchy polling. + * mobilewright reads the hierarchy every 100ms and waits for two identical + * reads, so the backend's own wait-for-idle before each read is wasted time. + * Default: `{ waitForIdleTimeout: 0 }`, plus `animationCoolOffTimeout: 0` on + * iOS. Entries here are merged over the defaults; `false` keeps the server's + * defaults. + */ + snapshotTuning?: false | Record; + /** + * How element visibility is judged. 'native' (default) uses the Appium + * backend's flag; 'bounds' counts any element with a non-empty frame as + * visible, matching local mobilecli. Use 'bounds' if `toBeVisible()` fails on + * iOS for elements that are on screen but overlapped by a sibling. + */ + visibility?: 'native' | 'bounds'; } export interface ResolvedOptions { @@ -147,6 +163,8 @@ export interface ResolvedOptions { tbOptions: Record; testResults: 'on' | 'off'; sessionPerTest: boolean; + snapshotTuning: false | Record; + visibility: 'native' | 'bounds'; } /** @@ -188,6 +206,8 @@ export function resolveOptions(options: TestingBotDriverOptions = {}): ResolvedO tbOptions: options.tbOptions ?? {}, testResults: options.testResults ?? 'on', sessionPerTest: options.sessionPerTest ?? false, + snapshotTuning: options.snapshotTuning === false ? false : (options.snapshotTuning ?? {}), + visibility: options.visibility ?? 'native', }; } diff --git a/src/png.ts b/src/png.ts new file mode 100644 index 0000000..09b502c --- /dev/null +++ b/src/png.ts @@ -0,0 +1,209 @@ +import { deflateSync, inflateSync } from 'node:zlib'; + +/** + * Minimal PNG crop/downscale for `screenshot({ clip, scale })`. Appium returns + * 8-bit RGB or RGBA screenshots, so that is all this decodes. Anything else + * (palette, 16-bit, grayscale, interlaced) returns `undefined`, and the caller + * falls back to the uncropped image instead of failing the screenshot. + */ + +const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +export interface PixelRect { + x: number; + y: number; + width: number; + height: number; +} + +interface Decoded { + width: number; + height: number; + /** Bytes per pixel: 3 (RGB) or 4 (RGBA). */ + bpp: number; + colorType: number; + /** Unfiltered pixel rows, tightly packed. */ + pixels: Buffer; +} + +export function pngSize(png: Buffer): { width: number; height: number } | undefined { + if (png.length < 24 || !png.subarray(0, 8).equals(SIGNATURE)) return undefined; + return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) }; +} + +/** + * Crop `png` to `rect` (in image pixels, clamped to the image) and then + * downscale by `factor` (0 < factor <= 1). Returns undefined when the PNG + * format is unsupported or the rect lies entirely outside the image. + */ +export function transformPng(png: Buffer, rect: PixelRect | undefined, factor = 1): Buffer | undefined { + const img = decode(png); + if (!img) return undefined; + + let x0 = 0, y0 = 0, w = img.width, h = img.height; + if (rect) { + x0 = clamp(Math.floor(rect.x), 0, img.width); + y0 = clamp(Math.floor(rect.y), 0, img.height); + const x1 = clamp(Math.ceil(rect.x + rect.width), 0, img.width); + const y1 = clamp(Math.ceil(rect.y + rect.height), 0, img.height); + w = x1 - x0; + h = y1 - y0; + if (w <= 0 || h <= 0) return undefined; + } + + const { bpp } = img; + const srcStride = img.width * bpp; + let out: Buffer = Buffer.alloc(w * h * bpp); + for (let row = 0; row < h; row++) { + const start = (y0 + row) * srcStride + x0 * bpp; + img.pixels.copy(out, row * w * bpp, start, start + w * bpp); + } + + if (factor > 0 && factor < 1) { + const dw = Math.max(1, Math.round(w * factor)); + const dh = Math.max(1, Math.round(h * factor)); + out = downscale(out, w, h, dw, dh, bpp); + w = dw; + h = dh; + } + return encode(out, w, h, bpp, img.colorType); +} + +function decode(png: Buffer): Decoded | undefined { + if (png.length < 33 || !png.subarray(0, 8).equals(SIGNATURE)) return undefined; + let offset = 8; + let width = 0, height = 0, colorType = -1, bitDepth = 0, interlace = 0; + const idat: Buffer[] = []; + while (offset + 8 <= png.length) { + const length = png.readUInt32BE(offset); + const type = png.toString('latin1', offset + 4, offset + 8); + const data = png.subarray(offset + 8, offset + 8 + length); + if (type === 'IHDR') { + width = data.readUInt32BE(0); + height = data.readUInt32BE(4); + bitDepth = data[8]!; + colorType = data[9]!; + interlace = data[12]!; + } else if (type === 'IDAT') { + idat.push(data); + } else if (type === 'IEND') { + break; + } + offset += 12 + length; + } + const bpp = colorType === 6 ? 4 : colorType === 2 ? 3 : 0; + if (!bpp || bitDepth !== 8 || interlace !== 0 || !width || !height || !idat.length) return undefined; + + const raw = inflateSync(Buffer.concat(idat)); + const stride = width * bpp; + if (raw.length < height * (stride + 1)) return undefined; + const pixels = Buffer.alloc(height * stride); + for (let y = 0; y < height; y++) { + const filter = raw[y * (stride + 1)]!; + const src = y * (stride + 1) + 1; + const dst = y * stride; + const prev = y > 0 ? dst - stride : -1; + for (let i = 0; i < stride; i++) { + const left = i >= bpp ? pixels[dst + i - bpp]! : 0; + const up = prev >= 0 ? pixels[prev + i]! : 0; + const upLeft = prev >= 0 && i >= bpp ? pixels[prev + i - bpp]! : 0; + let pred: number; + switch (filter) { + case 0: pred = 0; break; + case 1: pred = left; break; + case 2: pred = up; break; + case 3: pred = (left + up) >> 1; break; + case 4: pred = paeth(left, up, upLeft); break; + default: return undefined; + } + pixels[dst + i] = (raw[src + i]! + pred) & 0xff; + } + } + return { width, height, bpp, colorType, pixels }; +} + +function encode(pixels: Buffer, width: number, height: number, bpp: number, colorType: number): Buffer { + const stride = width * bpp; + // Filter type 1 (Sub) on every row: cheap and compresses screenshots well. + const raw = Buffer.alloc(height * (stride + 1)); + for (let y = 0; y < height; y++) { + const dst = y * (stride + 1); + const src = y * stride; + raw[dst] = 1; + for (let i = 0; i < stride; i++) { + const left = i >= bpp ? pixels[src + i - bpp]! : 0; + raw[dst + 1 + i] = (pixels[src + i]! - left) & 0xff; + } + } + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); + ihdr.writeUInt32BE(height, 4); + ihdr[8] = 8; + ihdr[9] = colorType; + return Buffer.concat([ + SIGNATURE, + chunk('IHDR', ihdr), + chunk('IDAT', deflateSync(raw)), + chunk('IEND', Buffer.alloc(0)), + ]); +} + +/** Box-filter downscale: each output pixel averages the source pixels it covers. */ +function downscale(src: Buffer, sw: number, sh: number, dw: number, dh: number, bpp: number): Buffer { + const out = Buffer.alloc(dw * dh * bpp); + const sum = new Array(bpp); + for (let dy = 0; dy < dh; dy++) { + const ys = Math.floor((dy * sh) / dh); + const ye = Math.max(ys + 1, Math.floor(((dy + 1) * sh) / dh)); + for (let dx = 0; dx < dw; dx++) { + const xs = Math.floor((dx * sw) / dw); + const xe = Math.max(xs + 1, Math.floor(((dx + 1) * sw) / dw)); + sum.fill(0); + for (let y = ys; y < ye; y++) { + for (let x = xs; x < xe; x++) { + const p = (y * sw + x) * bpp; + for (let c = 0; c < bpp; c++) sum[c]! += src[p + c]!; + } + } + const n = (ye - ys) * (xe - xs); + const o = (dy * dw + dx) * bpp; + for (let c = 0; c < bpp; c++) out[o + c] = Math.round(sum[c]! / n); + } + } + return out; +} + +function chunk(type: string, data: Buffer): Buffer { + const head = Buffer.alloc(8); + head.writeUInt32BE(data.length, 0); + head.write(type, 4, 'latin1'); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(Buffer.concat([head.subarray(4), data])) >>> 0, 0); + return Buffer.concat([head, data, crc]); +} + +let crcTable: Int32Array | undefined; +function crc32(buf: Buffer): number { + if (!crcTable) { + crcTable = new Int32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + crcTable[n] = c; + } + } + let crc = -1; + for (let i = 0; i < buf.length; i++) crc = crcTable[(crc ^ buf[i]!) & 0xff]! ^ (crc >>> 8); + return crc ^ -1; +} + +function paeth(a: number, b: number, c: number): number { + const p = a + b - c; + const pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + if (pa <= pb && pa <= pc) return a; + return pb <= pc ? b : c; +} + +function clamp(v: number, lo: number, hi: number): number { + return Math.min(Math.max(v, lo), hi); +} diff --git a/test/fake-hub/fake-hub.ts b/test/fake-hub/fake-hub.ts index b3a6b7b..0c8ab34 100644 --- a/test/fake-hub/fake-hub.ts +++ b/test/fake-hub/fake-hub.ts @@ -42,6 +42,10 @@ export class FakeHub { launcherActivityMissing = false; /** Extra delay (ms) before answering POST /session. */ allocationDelay = 0; + /** PNG served by /screenshot (defaults to a 1x1 image). */ + screenshotPng: Buffer | undefined; + /** W3C element id GET /element/active returns; undefined = no focused element. */ + activeElementId: string | undefined; private server: Server | undefined; private port = 0; @@ -133,7 +137,11 @@ export class FakeHub { case '/window/rect': return wd({ x: 0, y: 0, width: 393, height: 852 }); case '/screenshot': - return wd(TINY_PNG.toString('base64')); + return wd((this.screenshotPng ?? TINY_PNG).toString('base64')); + case '/element/active': + return this.activeElementId + ? wd({ 'element-6066-11e4-a52e-4f304ffac5c0': this.activeElementId }) + : wd({ error: 'no such element', message: 'no focused element' }, 404); case '/source': { const platform = String(session.capabilities['platformName'] ?? 'iOS'); const fixture = platform.toLowerCase() === 'android' ? 'android-page-source.xml' : 'ios-page-source.xml'; diff --git a/test/integration/driver-features.test.ts b/test/integration/driver-features.test.ts new file mode 100644 index 0000000..f430f4e --- /dev/null +++ b/test/integration/driver-features.test.ts @@ -0,0 +1,113 @@ +import { deflateSync } from 'node:zlib'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { TestingBotDriver } from '../../src/driver.js'; +import { pngSize } from '../../src/png.js'; +import { FakeHub } from '../fake-hub/fake-hub.js'; + +/** Solid RGBA PNG of the given pixel size (CRCs zeroed; the driver never checks them). */ +function solidPng(width: number, height: number): Buffer { + const row = Buffer.concat([Buffer.from([0]), Buffer.alloc(width * 4, 0x7f)]); + const chunk = (type: string, data: Buffer) => { + const len = Buffer.alloc(4); len.writeUInt32BE(data.length); + return Buffer.concat([len, Buffer.from(type, 'latin1'), data, Buffer.alloc(4)]); + }; + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); ihdr.writeUInt32BE(height, 4); ihdr[8] = 8; ihdr[9] = 6; + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk('IHDR', ihdr), chunk('IDAT', deflateSync(Buffer.concat(Array(height).fill(row)))), chunk('IEND', Buffer.alloc(0)), + ]); +} + +describe('driver features against FakeHub', () => { + let hub: FakeHub; + const options = (extra: Record = {}) => ({ + key: 'k', secret: 's', hubUrl: hub.hubUrl, apiUrl: hub.apiUrl, + allocationTimeout: 5_000, commandTimeout: 2_000, + apps: { ios: 'tb://ios', android: 'tb://android' } as const, + ...extra, + }); + + beforeEach(async () => { + hub = new FakeHub(); + await hub.start(); + }); + afterEach(async () => { + await hub.stop(); + }); + + async function connected(platform: 'ios' | 'android', extra: Record = {}) { + const driver = new TestingBotDriver(options(extra)); + await driver.prepare(); + const allocated = await driver.allocate({ platform }, new Set()); + await driver.connect({ platform, deviceId: allocated.deviceId }); + return { driver, sessionId: allocated.deviceId }; + } + + const settingsCalls = () => hub.requests.filter((r) => r.path.endsWith('/appium/settings')); + + it('applies per-platform snapshot tuning once at allocation', async () => { + await connected('ios'); + await connected('android', { snapshotTuning: { snapshotMaxDepth: 40 } }); + expect(settingsCalls().map((r) => r.body)).toEqual([ + { settings: { waitForIdleTimeout: 0, animationCoolOffTimeout: 0 } }, + { settings: { waitForIdleTimeout: 0, snapshotMaxDepth: 40 } }, + ]); + }); + + it('skips snapshot tuning when disabled', async () => { + await connected('ios', { snapshotTuning: false }); + expect(settingsCalls()).toHaveLength(0); + }); + + it('crops screenshot({ clip }) from points to pixels using the screen scale', async () => { + hub.screenshotPng = solidPng(393 * 3, 852 * 3); // /window/rect is 393x852 → scale 3 + const { driver } = await connected('ios'); + const element = await driver.screenshot({ clip: { x: 10, y: 20, width: 100, height: 50 } } as never); + expect(pngSize(element)).toEqual({ width: 300, height: 150 }); + const half = await driver.screenshot({ scale: 0.5 } as never); + expect(pngSize(half)).toEqual({ width: 590, height: 1278 }); + }); + + it('turns BACK on iOS into a left-edge swipe', async () => { + const { driver } = await connected('ios'); + await driver.pressButton('BACK'); + const actions = hub.requests.filter((r) => r.path.endsWith('/actions')).at(-1)!.body as { + actions: { actions: { type: string; x?: number }[] }[]; + }; + const moves = actions.actions[0]!.actions.filter((a) => a.type === 'pointerMove'); + expect(moves[0]!.x).toBeLessThanOrEqual(5); + expect(moves.at(-1)!.x).toBeGreaterThan(393 / 2); + }); + + it('clears the focused iOS field through its element reference', async () => { + hub.activeElementId = 'el-42'; + const { driver } = await connected('ios'); + await driver.fill('hello'); + const paths = hub.requests.map((r) => `${r.method} ${r.path.replace(/^\/wd\/hub\/session\/[^/]+/, '')}`); + expect(paths).toContain('GET /element/active'); + expect(paths).toContain('POST /element/el-42/clear'); + }); + + it('falls back to backspaces when there is no active element', async () => { + const { driver } = await connected('ios'); + await driver.clearText(); + expect(hub.requests.some((r) => r.path.endsWith('/clear'))).toBe(false); + expect(hub.requests.some((r) => r.path.endsWith('/actions'))).toBe(true); + }); + + it("judges visibility by bounds with visibility: 'bounds'", async () => { + const { driver: native } = await connected('ios'); + const { driver: bounds } = await connected('ios', { visibility: 'bounds' }); + const flat = (nodes: { isVisible: boolean; bounds: { width: number; height: number }; children: never[] }[]): typeof nodes => + nodes.flatMap((n) => [n, ...flat(n.children)]); + const n = flat((await native.getViewHierarchy()) as never); + const b = flat((await bounds.getViewHierarchy()) as never); + b.forEach((node) => expect(node.isVisible).toBe(node.bounds.width > 0 && node.bounds.height > 0)); + expect(n.length).toBe(b.length); + // The fixture's "Offscreen" label is visible="false" but has a frame. + const idx = n.findIndex((node) => (node as { label?: string }).label === 'Offscreen'); + expect(n[idx]!.isVisible).toBe(false); + expect(b[idx]!.isVisible).toBe(true); + }); +}); diff --git a/test/unit/png.test.ts b/test/unit/png.test.ts new file mode 100644 index 0000000..2d33328 --- /dev/null +++ b/test/unit/png.test.ts @@ -0,0 +1,99 @@ +import { deflateSync, inflateSync } from 'node:zlib'; +import { describe, expect, it } from 'vitest'; +import { pngSize, transformPng } from '../../src/png.js'; + +/** Build a test PNG whose pixel (x, y) is [x, y, x ^ y, 255], using every filter type. */ +function makePng(width: number, height: number, bpp = 4, filters = [0, 1, 2, 3, 4]): Buffer { + const pixel = (x: number, y: number) => [x & 0xff, y & 0xff, (x ^ y) & 0xff, 255].slice(0, bpp); + const stride = width * bpp; + const rows: number[][] = []; + for (let y = 0; y < height; y++) { + const row: number[] = []; + for (let x = 0; x < width; x++) row.push(...pixel(x, y)); + rows.push(row); + } + const raw: number[] = []; + const paeth = (a: number, b: number, c: number) => { + const p = a + b - c, pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + return pa <= pb && pa <= pc ? a : pb <= pc ? b : c; + }; + for (let y = 0; y < height; y++) { + const f = filters[y % filters.length]!; + raw.push(f); + for (let i = 0; i < stride; i++) { + const cur = rows[y]![i]!; + const left = i >= bpp ? rows[y]![i - bpp]! : 0; + const up = y > 0 ? rows[y - 1]![i]! : 0; + const ul = y > 0 && i >= bpp ? rows[y - 1]![i - bpp]! : 0; + const pred = [0, left, up, (left + up) >> 1, paeth(left, up, ul)][f]!; + raw.push((cur - pred) & 0xff); + } + } + const chunk = (type: string, data: Buffer) => { + const len = Buffer.alloc(4); len.writeUInt32BE(data.length); + return Buffer.concat([len, Buffer.from(type, 'latin1'), data, Buffer.alloc(4)]); // CRC unchecked by decoder + }; + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); ihdr.writeUInt32BE(height, 4); ihdr[8] = 8; ihdr[9] = bpp === 4 ? 6 : 2; + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk('IHDR', ihdr), chunk('IDAT', deflateSync(Buffer.from(raw))), chunk('IEND', Buffer.alloc(0)), + ]); +} + +/** Decode our own (Sub-filtered) output back to pixel rows. */ +function readPixels(png: Buffer, bpp: number): number[][] { + const { width, height } = pngSize(png)!; + let off = 8; const idat: Buffer[] = []; + while (off < png.length) { + const len = png.readUInt32BE(off); const type = png.toString('latin1', off + 4, off + 8); + if (type === 'IDAT') idat.push(png.subarray(off + 8, off + 8 + len)); + off += 12 + len; + } + const raw = inflateSync(Buffer.concat(idat)); const stride = width * bpp; const rows: number[][] = []; + for (let y = 0; y < height; y++) { + expect(raw[y * (stride + 1)]).toBe(1); + const row: number[] = []; + for (let i = 0; i < stride; i++) row.push((raw[y * (stride + 1) + 1 + i]! + (i >= bpp ? row[i - bpp]! : 0)) & 0xff); + rows.push(row); + } + return rows; +} + +describe('transformPng', () => { + it('crops to the rect across all five filter types (RGBA)', () => { + const out = transformPng(makePng(40, 30), { x: 5, y: 7, width: 10, height: 6 })!; + expect(pngSize(out)).toEqual({ width: 10, height: 6 }); + const rows = readPixels(out, 4); + expect(rows[0]!.slice(0, 4)).toEqual([5, 7, 5 ^ 7, 255]); + expect(rows[5]!.slice(-4)).toEqual([14, 12, 14 ^ 12, 255]); + }); + + it('crops RGB images', () => { + const out = transformPng(makePng(20, 20, 3), { x: 2, y: 3, width: 4, height: 4 })!; + expect(readPixels(out, 3)[0]!.slice(0, 3)).toEqual([2, 3, 2 ^ 3]); + }); + + it('clamps a rect that overhangs the image and rounds fractional edges outward', () => { + const out = transformPng(makePng(20, 20), { x: 15.5, y: -3, width: 10, height: 5.2 })!; + expect(pngSize(out)).toEqual({ width: 5, height: 3 }); + }); + + it('returns undefined for a rect fully outside the image', () => { + expect(transformPng(makePng(10, 10), { x: 50, y: 50, width: 5, height: 5 })).toBeUndefined(); + }); + + it('downscales by box-averaging', () => { + const out = transformPng(makePng(8, 8), undefined, 0.5)!; + expect(pngSize(out)).toEqual({ width: 4, height: 4 }); + // Output (0,0) averages source x,y in {0,1}: R = (0+1+0+1)/4 = 0.5 → 1 (round half up). + expect(readPixels(out, 4)[0]!.slice(0, 2)).toEqual([1, 1]); + }); + + it('returns undefined for non-PNG or unsupported formats', () => { + expect(transformPng(Buffer.from('not a png'), undefined)).toBeUndefined(); + const palette = makePng(4, 4); + palette[8 + 8 + 9] = 3; // IHDR color type → palette + expect(transformPng(palette, { x: 0, y: 0, width: 2, height: 2 })).toBeUndefined(); + }); +}); From 6f01e138bde97d980e6fd8085b5e0b9549cba6a7 Mon Sep 17 00:00:00 2001 From: Jochen Delabie Date: Tue, 10 Nov 2026 13:46:52 +0100 Subject: [PATCH 2/4] Report each test from test.info() inside the worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit connect()/disconnect() run inside the test (the device fixture is test-scoped), so the driver reads Playwright's test.info() from the user's own @playwright/test copy instead of reconstructing test<->session links from wall-clock intervals afterwards. - connect(): add a TestingBot session link annotation to the Playwright report; in sessionPerTest mode name the session after the test ("describe › title (retry N)") and correct the fixture's device.model / device.osVersion annotations for rotated sessions. - disconnect(): record the attempt's outcome in the run log; in sessionPerTest mode push the verdict while the session is still alive. - observer: sessions whose every span carries an outcome get an exact name and verdict (pooled sessions fail if any attempt on them failed); the timing heuristic remains the fallback when test.info() is unavailable. Verified on production: per-test sessions named and judged correctly including retries, pooled session reports "1 of 2 tests failed". --- src/driver.ts | 60 ++++++++++++++- src/observer.ts | 41 ++++++++++- src/run-log.ts | 9 ++- src/test-info.ts | 98 +++++++++++++++++++++++++ test/integration/pool-lifecycle.test.ts | 45 ++++++++++++ test/unit/run-log.test.ts | 4 +- test/unit/test-info.test.ts | 53 +++++++++++++ 7 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 src/test-info.ts create mode 100644 test/unit/test-info.test.ts diff --git a/src/driver.ts b/src/driver.ts index 467f5ef..bc68d96 100644 --- a/src/driver.ts +++ b/src/driver.ts @@ -46,6 +46,8 @@ import { requireCredentials, resolveOptions, type ResolvedOptions, type TestingB import { RestApiClient, type UserInfo } from './rest-api.js'; import { transformPng } from './png.js'; import { RunLog } from './run-log.js'; +import { attemptName, attemptOutcome, currentTestInfo } from './test-info.js'; +import { exactVerdict } from './observer.js'; import { setActiveDriver, type ThrottlePreset, type ThrottleProfile } from './session-commands.js'; import { TunnelManager } from './tunnel.js'; import { AppiumWebViewBridge } from './webview.js'; @@ -371,6 +373,7 @@ export class TestingBotDriver implements MobilewrightDriver { await this.hub.get(poolDeviceId, '/orientation', { timeout }); this.active = { sessionId: poolDeviceId, platform: config.platform, deviceType: config.deviceType, connectedAt: Date.now() }; setActiveDriver(this); + this.annotateTest(poolDeviceId); debug('connected to session %s', poolDeviceId); return { deviceId: poolDeviceId, platform: config.platform }; } @@ -390,9 +393,16 @@ export class TestingBotDriver implements MobilewrightDriver { } this.rotatedDeviceIds.add(poolDeviceId); } - sessionId ??= await this.startFreshSession(config, poolDeviceId); + let rotated: { model?: string; osVersion?: string } | undefined; + if (!sessionId) { + const fresh = await this.startFreshSession(config, poolDeviceId); + sessionId = fresh.sessionId; + rotated = { model: fresh.model, osVersion: fresh.osVersion }; + } this.active = { sessionId, platform: config.platform, deviceType: config.deviceType, connectedAt: Date.now() }; setActiveDriver(this); + await this.nameSessionAfterTest(sessionId); + this.annotateTest(sessionId, rotated); debug('connected to session %s (slot %s)', sessionId, poolDeviceId); return { deviceId: poolDeviceId, platform: config.platform }; } @@ -407,12 +417,15 @@ export class TestingBotDriver implements MobilewrightDriver { // session it cannot attribute gets no verdict at all. Nothing in teardown // — a hub round-trip, a hung request, the worker dying — may come between // the test ending and this line. + const info = currentTestInfo(); + const outcome = info && attemptOutcome(info); this.runLog.append({ type: 'session', sessionId: session.sessionId, startedAt: session.connectedAt, endedAt: Date.now(), workerIndex: workerIndex(), + ...(outcome ? { test: outcome } : {}), }); if (session.throttled) { // A pooled session outlives the test: reset network conditions so they @@ -421,6 +434,13 @@ export class TestingBotDriver implements MobilewrightDriver { debug('could not reset throttling on %s: %s', session.sessionId, err); }); } + if (this.options.sessionPerTest && outcome && this.options.testResults !== 'off') { + // This session hosted exactly this attempt: report its verdict now, while + // the session is alive, instead of relying on the run-end observer alone. + await this.api.updateTest(session.sessionId, exactVerdict([outcome])).catch((err) => { + debug('could not report verdict for %s: %s', session.sessionId, err); + }); + } if (!this.options.sessionPerTest) { // Soft detach only: the pool re-grants this slot to the next test and // ends the session via release(). A pooled session that hosted exactly @@ -436,7 +456,10 @@ export class TestingBotDriver implements MobilewrightDriver { } /** Create a new hub session for this slot from the connect() config. */ - private async startFreshSession(config: ConnectionConfig, poolDeviceId: string): Promise { + private async startFreshSession( + config: ConnectionConfig, + poolDeviceId: string, + ): Promise<{ sessionId: string; model?: string; osVersion?: string }> { // mobilewright's connect() passes only platform/deviceId/deviceType, so // the slot's deviceName/osVersion filter comes from the coordinator's // allocate() record. Values on the config win when a caller provides them. @@ -465,11 +488,40 @@ export class TestingBotDriver implements MobilewrightDriver { } } const capabilities = buildCapabilities(criteria, this.options, pinned, appUrls); - const { sessionId } = await this.hub.newSession(capabilities, { timeout: this.options.allocationTimeout }); + const { sessionId, capabilities: caps } = await this.hub.newSession(capabilities, { timeout: this.options.allocationTimeout }); this.runLog.append({ type: 'session', sessionId }); await this.applySnapshotTuning(sessionId, config.platform); debug('started per-test session %s — https://testingbot.com/members/tests/%s', sessionId, sessionId); - return sessionId; + return { + sessionId, + model: pinned?.deviceName ?? str(caps['appium:deviceName'] ?? caps['deviceName']), + osVersion: pinned?.platformVersion ?? str(caps['appium:platformVersion'] ?? caps['platformVersion']), + }; + } + + /** + * Link the running test to its TestingBot session in Playwright's report. + * A rotated per-test session can land on a different device than the pool + * slot's first one, so the fixture's device annotations are corrected too. + */ + private annotateTest(sessionId: string, device?: { model?: string; osVersion?: string }): void { + const info = currentTestInfo(); + if (!info) return; + info.annotations.push({ type: 'TestingBot', description: `https://testingbot.com/members/tests/${sessionId}` }); + for (const annotation of info.annotations) { + if (annotation.type === 'device.model' && device?.model) annotation.description = device.model; + if (annotation.type === 'device.osVersion' && device?.osVersion) annotation.description = device.osVersion; + } + } + + /** sessionPerTest: name the session after its test as soon as it starts. */ + private async nameSessionAfterTest(sessionId: string): Promise { + if (this.options.testResults === 'off') return; + const info = currentTestInfo(); + if (!info) return; + await this.api.updateTest(sessionId, { name: attemptName(info) }).catch((err) => { + debug('could not name session %s: %s', sessionId, err); + }); } /** diff --git a/src/observer.ts b/src/observer.ts index de9ba79..3359232 100644 --- a/src/observer.ts +++ b/src/observer.ts @@ -7,7 +7,8 @@ import type { TestResultInfo, TestRunInfo, } from '@mobilewright/protocol'; -import type { RestApiClient } from './rest-api.js'; +import type { RestApiClient, TestUpdate } from './rest-api.js'; +import type { AttemptOutcome } from './test-info.js'; const debug = createDebug('testingbot:observer'); @@ -20,6 +21,8 @@ export interface ReportableSession { endedAt?: number; /** Worker process that ran this session's test, when unambiguous. */ workerIndex?: number; + /** Test attempts the worker recorded from test.info(), one per span. */ + tests?: AttemptOutcome[]; } export interface GitInfo { @@ -129,8 +132,22 @@ export class TestingBotTestObserver implements TestObserver { const sessions = this.context.sessions(); debug('run ended (%s), reporting to %d session(s)%s', result.status, sessions.length, git ? ` with git ${git.commit?.slice(0, 8) ?? '?'} (${git.branch ?? 'no branch'})` : ''); - const attributed = this.attribute(sessions, outcomes); + // Sessions whose every span recorded its test attempt need no heuristic. + const exact = new Map(sessions + .filter((s) => s.spans > 0 && s.tests?.length === s.spans) + .map((s) => [s.sessionId, exactVerdict(s.tests!)])); + const attributed = this.attribute(sessions.filter((s) => !exact.has(s.sessionId)), outcomes); for (const session of sessions) { + const verdict = exact.get(session.sessionId); + if (verdict) { + try { + await this.api.updateTest(session.sessionId, { ...verdict, build: this.context.build, extra, groups }); + debug('reported %s -> exact %o', session.sessionId, verdict); + } catch (err) { + console.warn(`TestingBotDriver: could not report result for session ${session.sessionId}: ${String(err)}`); + } + continue; + } const test = attributed.get(session.sessionId); // An unattributable session must never inherit the run's failure: on a // failed run that would stamp another test's verdict onto a session @@ -303,6 +320,26 @@ export class TestingBotTestObserver implements TestObserver { } } +/** + * Name and verdict from the attempts recorded inside the session itself. One + * attempt names the session after it; a pooled session that hosted several + * keeps its name and fails if any attempt on it failed. Skipped attempts carry + * no verdict. + */ +export function exactVerdict(tests: AttemptOutcome[]): Pick { + const ran = tests.filter((t) => t.status !== 'skipped'); + const failed = ran.filter((t) => t.status === 'failed'); + const name = tests.length === 1 ? tests[0]!.title : undefined; + if (ran.length === 0) return { ...(name ? { name } : {}), statusMessage: 'skipped' }; + const statusMessage = failed.length === 0 + ? (ran.length === 1 ? 'passed' : `${ran.length} tests passed`) + : (ran.length === 1 + ? `${failed[0]!.error ?? 'failed'}` + : `${failed.length} of ${ran.length} tests failed. ${failed.map((t) => `${t.title}: ${t.error ?? 'failed'}`).join(' | ')}` + ).slice(0, 500); + return { ...(name ? { name } : {}), success: failed.length === 0, statusMessage }; +} + /** * Playwright's JSON report for this run, when mobilewright exposes one. * Documented upstream as a transitional escape hatch, so every reader below is diff --git a/src/run-log.ts b/src/run-log.ts index 36036dc..bc7df14 100644 --- a/src/run-log.ts +++ b/src/run-log.ts @@ -2,6 +2,7 @@ import { appendFileSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { createHash } from 'node:crypto'; +import type { AttemptOutcome } from './test-info.js'; /** * Tiny JSONL file shared between the coordinator process (allocator half, @@ -24,6 +25,9 @@ export interface SessionRecord { * reports the same index per test, so a session can be matched against the * tests of its own worker instead of every test in the run. */ workerIndex?: number; + /** The test attempt that ran in this connect/disconnect cycle, read from + * Playwright's test.info() inside the worker — an exact join, when present. */ + test?: AttemptOutcome; } export interface AppRecord { @@ -68,6 +72,8 @@ export interface MergedSession { * recorded cycle agrees, since a pooled slot re-granted across workers has * no single owner. */ workerIndex?: number; + /** Test attempts recorded for this session (one per span, when known). */ + tests: AttemptOutcome[]; } export class RunLog { @@ -117,7 +123,8 @@ export class RunLog { const workersSeen = new Map>(); for (const record of this.read()) { if (record.type !== 'session') continue; - const merged = byId.get(record.sessionId) ?? { sessionId: record.sessionId, spans: 0 }; + const merged = byId.get(record.sessionId) ?? { sessionId: record.sessionId, spans: 0, tests: [] }; + if (record.test) merged.tests.push(record.test); if (record.startedAt !== undefined && record.endedAt !== undefined) { merged.spans += 1; merged.startedAt = Math.min(merged.startedAt ?? record.startedAt, record.startedAt); diff --git a/src/test-info.ts b/src/test-info.ts new file mode 100644 index 0000000..f66e882 --- /dev/null +++ b/src/test-info.ts @@ -0,0 +1,98 @@ +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import createDebug from 'debug'; + +const debug = createDebug('testingbot:test-info'); + +/** The slice of Playwright's TestInfo the driver reads. */ +export interface CurrentTestInfo { + testId: string; + title: string; + /** [file, ...describe titles, title] */ + titlePath: string[]; + retry: number; + status?: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; + expectedStatus: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; + errors: { message?: string }[]; + annotations: { type: string; description?: string }[]; + project: { name: string }; +} + +/** Outcome of one test attempt, as seen from inside the worker at disconnect(). */ +export interface AttemptOutcome { + id: string; + title: string; + retry: number; + status: 'passed' | 'failed' | 'skipped'; + error?: string; +} + +type PlaywrightTest = { test: { info(): CurrentTestInfo } }; +let playwrightTest: PlaywrightTest | null | undefined; + +/** + * Load the `@playwright/test` copy the user's tests actually run on: + * `test.info()` reads state held by that module instance, so it is resolved + * the way `@mobilewright/test` resolves it, falling back to the project's own. + */ +function loadPlaywrightTest(): PlaywrightTest | null { + if (playwrightTest !== undefined) return playwrightTest; + const fromProject = createRequire(join(process.cwd(), 'package.json')); + const origins = [ + () => createRequire(fromProject.resolve('@mobilewright/test')), + () => fromProject, + ]; + for (const origin of origins) { + try { + playwrightTest = origin()('@playwright/test') as PlaywrightTest; + return playwrightTest; + } catch { + // try the next origin + } + } + debug('@playwright/test not resolvable — per-test reporting falls back to the run observer'); + playwrightTest = null; + return null; +} + +/** The test running in this worker right now, or undefined outside a test. */ +export function currentTestInfo(): CurrentTestInfo | undefined { + try { + return loadPlaywrightTest()?.test.info(); + } catch { + return undefined; // test.info() throws outside a running test + } +} + +/** Dashboard name for a test attempt: describe › title, marked when retried. */ +export function attemptName(info: Pick): string { + const path = info.titlePath.length > 1 ? info.titlePath.slice(1) : [info.title]; + return path.join(' › ') + (info.retry > 0 ? ` (retry ${info.retry})` : ''); +} + +/** The attempt's outcome, valid once the test body has finished (fixture teardown). */ +export function attemptOutcome(info: CurrentTestInfo): AttemptOutcome | undefined { + if (!info.status) return undefined; + const status = info.status === 'skipped' + ? 'skipped' + : info.status === info.expectedStatus ? 'passed' : 'failed'; + const message = info.errors.find((e) => e.message)?.message; + return { + id: info.testId, + title: attemptName(info), + retry: info.retry, + status, + ...(status === 'failed' ? { error: firstLine(message) ?? info.status } : {}), + }; +} + +function firstLine(message: string | undefined): string | undefined { + // Playwright error messages carry ANSI colour codes. + // eslint-disable-next-line no-control-regex + return message?.replace(/\u001b\[[0-9;]*m/g, '').split('\n').find((l) => l.trim())?.trim().slice(0, 200); +} + +/** Test seam: replace the resolver (undefined restores the real one). */ +export function setPlaywrightTestForTesting(value: PlaywrightTest | null | undefined): void { + playwrightTest = value; +} diff --git a/test/integration/pool-lifecycle.test.ts b/test/integration/pool-lifecycle.test.ts index 6117a90..371fbd3 100644 --- a/test/integration/pool-lifecycle.test.ts +++ b/test/integration/pool-lifecycle.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { NoDeviceAvailableError } from '@mobilewright/protocol'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { TestingBotDriver } from '../../src/driver.js'; +import { setPlaywrightTestForTesting, type CurrentTestInfo } from '../../src/test-info.js'; import { FakeHub } from '../fake-hub/fake-hub.js'; /** @@ -315,6 +316,50 @@ describe('sessionPerTest against FakeHub', () => { expect(hub.testUpdates.get(fresh.id)).toMatchObject({ 'test[success]': '1' }); }); + it('names, annotates and reports each rotated session from test.info()', async () => { + let current: CurrentTestInfo | undefined; + setPlaywrightTestForTesting({ test: { info: () => { if (!current) throw new Error('no test'); return current; } } }); + const testInfo = (title: string, retry = 0): CurrentTestInfo => ({ + testId: title, title, titlePath: ['f.test.ts', title], retry, expectedStatus: 'passed', errors: [], + annotations: [{ type: 'device.model', description: 'first-slot-device' }], project: { name: 'ios' }, + }); + try { + const coordinator = new TestingBotDriver(options()); + await coordinator.prepare(); + const allocated = await coordinator.allocate({ platform: 'ios', deviceType: 'simulator' }, new Set()); + const worker = new TestingBotDriver(options()); + + current = testInfo('passes'); + await worker.connect({ platform: 'ios', deviceId: allocated.deviceId, deviceType: 'simulator' }); + expect(hub.testUpdates.get(allocated.deviceId)).toMatchObject({ 'test[name]': 'passes' }); + expect(current.annotations).toContainEqual({ type: 'TestingBot', description: `https://testingbot.com/members/tests/${allocated.deviceId}` }); + current.status = 'passed'; + await worker.disconnect(); + expect(hub.testUpdates.get(allocated.deviceId)).toMatchObject({ 'test[success]': '1' }); + + current = testInfo('fails', 1); + await worker.connect({ platform: 'ios', deviceId: allocated.deviceId, deviceType: 'simulator' }); + const second = hub.liveSessions()[0]!.id; + expect(hub.testUpdates.get(second)).toMatchObject({ 'test[name]': 'fails (retry 1)' }); + current.status = 'failed'; + current.errors = [{ message: 'Error: nope\n at x' }]; + await worker.disconnect(); + expect(hub.testUpdates.get(second)).toMatchObject({ 'test[success]': '0', 'test[status_message]': 'Error: nope' }); + current = undefined; + + await coordinator.release(allocated.deviceId); + await coordinator.dispose(); + // The observer agrees with the in-test verdicts — no run aggregate leaks in. + coordinator.observer!.onRunStart?.({ totalTests: 2 }); + coordinator.observer!.onTestEnd?.({ id: 'x', title: 'x', titlePath: [] }, { status: 'failed', retry: 0, duration: 1, errors: [], steps: [] }); + await coordinator.observer!.onRunEnd?.({ status: 'failed', startTime: new Date(0), duration: 1 }); + expect(hub.testUpdates.get(allocated.deviceId)).toMatchObject({ 'test[success]': '1', 'test[name]': 'passes' }); + expect(hub.testUpdates.get(second)).toMatchObject({ 'test[success]': '0' }); + } finally { + setPlaywrightTestForTesting(undefined); + } + }); + it('keeps the slot\'s deviceName/osVersion filter on rotated sessions', async () => { // mobilewright's connect() forwards only platform/deviceId/deviceType, so // the rotated session must recover the filter from the allocate() record — diff --git a/test/unit/run-log.test.ts b/test/unit/run-log.test.ts index 6a66363..ccafe86 100644 --- a/test/unit/run-log.test.ts +++ b/test/unit/run-log.test.ts @@ -12,7 +12,7 @@ describe('RunLog.sessions', () => { const log = runLog(); log.append({ type: 'session', sessionId: 's1', startedAt: 100, endedAt: 200 }); log.append({ type: 'session', sessionId: 's1', startedAt: 300, endedAt: 400 }); - expect(log.sessions()).toEqual([{ sessionId: 's1', spans: 2, startedAt: 100, endedAt: 400 }]); + expect(log.sessions()).toEqual([{ sessionId: 's1', spans: 2, startedAt: 100, endedAt: 400, tests: [] }]); }); it('keeps the worker index a session was recorded from', () => { @@ -39,7 +39,7 @@ describe('RunLog.sessions', () => { it('records an untimed session with no spans and survives a truncated line', () => { const log = runLog(); log.append({ type: 'session', sessionId: 's1' }); - expect(log.sessions()).toEqual([{ sessionId: 's1', spans: 0 }]); + expect(log.sessions()).toEqual([{ sessionId: 's1', spans: 0, tests: [] }]); log.truncate(); expect(log.sessions()).toEqual([]); }); diff --git a/test/unit/test-info.test.ts b/test/unit/test-info.test.ts new file mode 100644 index 0000000..534e9a3 --- /dev/null +++ b/test/unit/test-info.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { exactVerdict } from '../../src/observer.js'; +import { attemptName, attemptOutcome, type CurrentTestInfo } from '../../src/test-info.js'; + +const info = (over: Partial = {}): CurrentTestInfo => ({ + testId: 't1', title: 'adds numbers', titlePath: ['calc.test.ts', 'calculator', 'adds numbers'], + retry: 0, status: 'passed', expectedStatus: 'passed', errors: [], annotations: [], project: { name: 'android' }, + ...over, +}); + +describe('attemptName', () => { + it('drops the file and joins describe blocks', () => { + expect(attemptName(info())).toBe('calculator › adds numbers'); + }); + it('marks retries', () => { + expect(attemptName(info({ retry: 2 }))).toBe('calculator › adds numbers (retry 2)'); + }); +}); + +describe('attemptOutcome', () => { + it('is undefined while the test body still runs', () => { + expect(attemptOutcome(info({ status: undefined }))).toBeUndefined(); + }); + it('treats an expected failure (test.fail) as passed', () => { + expect(attemptOutcome(info({ status: 'failed', expectedStatus: 'failed' }))!.status).toBe('passed'); + }); + it('reports the first error line without ANSI codes', () => { + const out = attemptOutcome(info({ + status: 'failed', + errors: [{ message: '\u001b[31mExpectError: Expected "42"\u001b[39m\n at foo.ts:3' }], + }))!; + expect(out).toMatchObject({ status: 'failed', error: 'ExpectError: Expected "42"' }); + }); + it('maps timeouts to failed and skips to skipped', () => { + expect(attemptOutcome(info({ status: 'timedOut' }))).toMatchObject({ status: 'failed', error: 'timedOut' }); + expect(attemptOutcome(info({ status: 'skipped' }))!.status).toBe('skipped'); + }); +}); + +describe('exactVerdict', () => { + const a = { id: 'a', title: 'A', retry: 0, status: 'passed' as const }; + const b = { id: 'b', title: 'B', retry: 0, status: 'failed' as const, error: 'boom' }; + it('names a single-attempt session after its test', () => { + expect(exactVerdict([b])).toEqual({ name: 'B', success: false, statusMessage: 'boom' }); + }); + it('fails a pooled session if any attempt failed, keeping its name', () => { + expect(exactVerdict([a, b])).toEqual({ success: false, statusMessage: '1 of 2 tests failed. B: boom' }); + expect(exactVerdict([a, a])).toEqual({ success: true, statusMessage: '2 tests passed' }); + }); + it('gives skipped-only sessions no verdict', () => { + expect(exactVerdict([{ ...a, status: 'skipped' }])).toEqual({ name: 'A', statusMessage: 'skipped' }); + }); +}); From 0477d26f4d1bb2c3e647342e2b6488c76d00d004 Mon Sep 17 00:00:00 2001 From: Jochen Delabie Date: Tue, 10 Nov 2026 13:54:21 +0100 Subject: [PATCH 3/4] Retry 'no device available' in allocate() while the run holds no session mobilewright's pool re-queues a NoDeviceAvailableError and only retries when one of its own slots is released, so with an empty pool a fast hub rejection (capacity, pinned device busy, plan limit) idled the test until its allocation timeout. allocate() now retries with exponential backoff (allocationRetryDelay, default 5s, capped at 30s) until the pool's signal aborts; with a session held it still hands the error back to the pool. --- src/driver.ts | 40 +++++++++++++++++++++++++ src/options.ts | 8 +++++ test/integration/pool-lifecycle.test.ts | 21 +++++++++++++ 3 files changed, 69 insertions(+) diff --git a/src/driver.ts b/src/driver.ts index bc68d96..26580a5 100644 --- a/src/driver.ts +++ b/src/driver.ts @@ -204,7 +204,37 @@ export class TestingBotDriver implements MobilewrightDriver { await this.tunnel.start(this.options); } + /** + * mobilewright's pool re-queues a NoDeviceAvailableError and only retries + * when one of its own slots is released. With no session held by this run, + * nothing is ever released, so the test would idle until its allocation + * timeout even if a device frees up seconds later. Retry here instead, with + * backoff, until the pool's signal aborts (or allocationTimeout without one). + */ async allocate( + criteria: AllocationCriteria, + takenDeviceIds: ReadonlySet, + signal?: AbortSignal, + ): Promise { + const deadline = signal ? Infinity : Date.now() + this.options.allocationTimeout; + for (let attempt = 1; ; attempt++) { + try { + return await this.allocateOnce(criteria, takenDeviceIds, signal); + } catch (err) { + if (!(err instanceof NoDeviceAvailableError) || signal?.aborted) throw err; + if (this.allocated.size > 0) { + debug('no device available; this run holds %d session(s), handing back to the pool', this.allocated.size); + throw err; + } + const delay = Math.min(this.options.allocationRetryDelay * 2 ** (attempt - 1), 30_000); + if (Date.now() + delay > deadline) throw err; + debug('no device available (%s); retry %d in %dms', err.message, attempt, delay); + if (!(await sleep(delay, signal))) throw err; + } + } + } + + private async allocateOnce( criteria: AllocationCriteria, _takenDeviceIds: ReadonlySet, signal?: AbortSignal, @@ -1062,6 +1092,16 @@ function describeAllocation(criteria: AllocationCriteria): string { } /** Values usable directly as appium:app without an upload. */ +/** Resolves true after `ms`, or false as soon as `signal` aborts. */ +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(false); + const onAbort = () => { clearTimeout(timer); resolve(false); }; + const timer = setTimeout(() => { signal?.removeEventListener('abort', onAbort); resolve(true); }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + function isRemoteApp(value: string): boolean { return /^(tb|https?):\/\//i.test(value); } diff --git a/src/options.ts b/src/options.ts index 7eef91e..b7d5ccd 100644 --- a/src/options.ts +++ b/src/options.ts @@ -130,6 +130,12 @@ export interface TestingBotDriverOptions { * iOS for elements that are on screen but overlapped by a sibling. */ visibility?: 'native' | 'bounds'; + /** + * First delay, in ms, before retrying an allocation that failed with "no + * device available" while this run holds no session (doubles per attempt, + * capped at 30s). Default: 5000. + */ + allocationRetryDelay?: number; } export interface ResolvedOptions { @@ -165,6 +171,7 @@ export interface ResolvedOptions { sessionPerTest: boolean; snapshotTuning: false | Record; visibility: 'native' | 'bounds'; + allocationRetryDelay: number; } /** @@ -208,6 +215,7 @@ export function resolveOptions(options: TestingBotDriverOptions = {}): ResolvedO sessionPerTest: options.sessionPerTest ?? false, snapshotTuning: options.snapshotTuning === false ? false : (options.snapshotTuning ?? {}), visibility: options.visibility ?? 'native', + allocationRetryDelay: options.allocationRetryDelay ?? 5_000, }; } diff --git a/test/integration/pool-lifecycle.test.ts b/test/integration/pool-lifecycle.test.ts index 371fbd3..0690113 100644 --- a/test/integration/pool-lifecycle.test.ts +++ b/test/integration/pool-lifecycle.test.ts @@ -70,16 +70,37 @@ describe('pool lifecycle against FakeHub', () => { expect(hub.liveSessions()).toHaveLength(0); }); + it('retries "no device available" itself while the run holds no session', async () => { + const driver = new TestingBotDriver({ ...driverOptions(), allocationRetryDelay: 20 }); + hub.busyCount = 2; + const allocated = await driver.allocate({ platform: 'android' }, new Set(), new AbortController().signal); + expect(hub.requests.filter((r) => r.method === 'POST' && r.path === '/wd/hub/session')).toHaveLength(3); + await driver.release(allocated.deviceId); + }); + + it('stops retrying when the pool aborts', async () => { + const driver = new TestingBotDriver({ ...driverOptions(), allocationRetryDelay: 60_000 }); + hub.busyCount = 1; + const controller = new AbortController(); + const pending = driver.allocate({ platform: 'android' }, new Set(), controller.signal); + setTimeout(() => controller.abort(), 50); + await expect(pending).rejects.toBeInstanceOf(NoDeviceAvailableError); + expect(hub.liveSessions()).toHaveLength(0); + }); + it('session methods before connect() fail with a clear error', async () => { const worker = new TestingBotDriver(driverOptions()); await expect(worker.tap(1, 1)).rejects.toThrow('No active session. Call connect() first.'); }); it('maps hub "busy" errors to NoDeviceAvailableError so the pool re-queues', async () => { + // A held session means the pool will retry on release — hand the error back. + const held = await coordinator.allocate({ platform: 'ios' }, new Set()); hub.busyCount = 1; await expect( coordinator.allocate({ platform: 'android' }, new Set()), ).rejects.toBeInstanceOf(NoDeviceAvailableError); + await coordinator.release(held.deviceId); // Next attempt (the pool's retry) succeeds. const allocated = await coordinator.allocate({ platform: 'android' }, new Set()); expect(hub.liveSessions()).toHaveLength(1); From bd9ad577346dc46307882e6a0da7dbdf9a58ec6c Mon Sep 17 00:00:00 2001 From: Jochen Delabie Date: Tue, 10 Nov 2026 14:22:51 +0100 Subject: [PATCH 4/4] Document v0.7 features and add the live production check suite e2e/live.config.ts runs against TestingBot's public sample app (no local builds): hierarchy timing, fill + sum, element screenshots, iOS goBack in Settings, and per-test reporting, with env toggles for sessionPerTest, forced failures, retries, visibility and snapshot tuning. README: exact reporting, new options, iOS BACK and element screenshots, and the mixed-platform pool caveat. CHANGELOG: 0.7.0 (unreleased). --- .gitignore | 2 ++ CHANGELOG.md | 36 ++++++++++++++++++++++++++ README.md | 34 +++++++++++++++++++----- e2e/live.config.ts | 33 ++++++++++++++++++++++++ e2e/live/features.test.ts | 50 ++++++++++++++++++++++++++++++++++++ e2e/live/fill-repeat.test.ts | 26 +++++++++++++++++++ e2e/live/report.test.ts | 11 ++++++++ 7 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 e2e/live.config.ts create mode 100644 e2e/live/features.test.ts create mode 100644 e2e/live/fill-repeat.test.ts create mode 100644 e2e/live/report.test.ts diff --git a/.gitignore b/.gitignore index 2a50cde..677e3ce 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dist/ e2e/mobilewright-report/ e2e/test-results/ .DS_Store +e2e/live-report.json +e2e/live-html/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 35f38fe..9255d49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Changelog +## 0.7.0 — unreleased + +### Added +- **Exact per-test reporting.** The driver reads Playwright's `test.info()` + inside `connect()`/`disconnect()` (which run inside the test), so each + session's test and outcome are known exactly instead of reconstructed from + timestamps. With `sessionPerTest` each session is named after its test as it + starts (`describe › title (retry N)`) and gets its verdict when the test + ends; pooled sessions fail if any test on them failed. Every test in the + Playwright report links to its TestingBot session. The timing-based join + remains as the fallback. +- **Element screenshots.** `locator.screenshot()` is cropped to the element + (points → pixels via the screen scale) and `screenshot({ scale })` + downscales; previously both returned the full screen. +- **`screen.goBack()` on iOS** performs the system left-edge back swipe + instead of throwing. +- **`snapshotTuning`** option, on by default: `waitForIdleTimeout: 0` (plus + `animationCoolOffTimeout: 0` on iOS) once per session. Measured on + production: fill + assert 7.3s → 3.5s on an Android emulator, 3.4s → 2.95s + on an iOS simulator. `false` keeps the server defaults. +- **`visibility: 'bounds'`** option: judge `isVisible` by frame size, like + local mobilecli. +- **`allocationRetryDelay`** option: a "no device available" error while the + run holds no session is retried with backoff (5s doubling to 30s) until + the pool gives up. mobilewright's pool only retries when one of its own + slots is released, so such a test previously idled until its timeout. + +### Fixed +- `fill()` / `clearText()` on iOS now clear the focused field through its + element reference. XCUITest's page source has no focused flag, and React + Native fields put the caret at the start, so the old backspace fallback + never removed anything (`fill('7')` on a field holding `0` gave `70`). +- Rotated `sessionPerTest` sessions now correct the test's `device.model` / + `device.osVersion` annotations, which described the pool slot's first + session. + ## 0.6.0 — 2026-10-28 Tracks `@mobilewright/protocol` 0.0.67. diff --git a/README.md b/README.md index b2ab646..7b91b0f 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,11 @@ Regexes and version ranges work on both real and virtual devices: real devices r ## Sessions, naming, and reporting -- By default a device slot's TestingBot session is reused across tests (fast: you pay device startup once). Sessions that hosted exactly one test are named after that test with its own pass/fail; sessions that hosted several get a run summary (e.g. `12/12 tests passed`). -- Set **`sessionPerTest: true`** for one session, video, and named verdict **per test** — the clearest dashboard, at the cost of device startup on every test. -- Tests are matched to sessions by **worker identity first**: the driver stamps Playwright's worker index on each session as it records it, and the run report reports the same index per test, so a test that ran in another worker process is never a candidate. Timing separates the tests within one worker, using the worker's own timestamps from the report when it is available. +- By default a device slot's TestingBot session is reused across tests (fast: you pay device startup once). Sessions that hosted exactly one test are named after that test with its own pass/fail; sessions that hosted several pass only if every test on them passed (e.g. `1 of 3 tests failed. checkout › pays: ...`). +- Set **`sessionPerTest: true`** for one session, video, and named verdict **per test** — the clearest dashboard, at the cost of device startup on every test. The session is named after the test as soon as it starts (`describe › title`, plus `(retry N)` on retries), and its verdict is reported the moment the test ends, while the session is still alive. +- Each test in Playwright's HTML report gets a **`TestingBot`** annotation linking to the session (and its video) that test ran on. +- Tests are matched to sessions **exactly**: `connect()`/`disconnect()` run inside the test, so the driver reads Playwright's `test.info()` there and records which test attempt ran on which session, with its outcome. The heuristic below is only the fallback when `@playwright/test` cannot be resolved from the project. +- Fallback: tests are matched to sessions by **worker identity first**: the driver stamps Playwright's worker index on each session as it records it, and the run report reports the same index per test, so a test that ran in another worker process is never a candidate. Timing separates the tests within one worker, using the worker's own timestamps from the report when it is available. - A session is only ever given a verdict the driver can stand behind: its own test's, or the run summary when it genuinely hosted several tests (or was the run's only session). When a session cannot be tied to a test — an ambiguous overlap under heavy parallelism, or a worker that died before recording its span — it is reported with build, extra and tags but **no** pass/fail, rather than inheriting a failure from a test that ran somewhere else. `DEBUG=testingbot:observer` names the session and the reason. - Git metadata is reported automatically when mobilewright captures it: the commit hash, branch, author and subject land in the session's **extra** field (alongside your own `extra`, never replacing it), and the branch is added as a **tag** so the dashboard can filter by it. Enable it with `captureGitInfo: { commit: true }` in `mobilewright.config.ts` — Playwright turns it on by itself in CI. - Failure messages are reported to the session's status message; `build` groups sessions on the dashboard and **defaults automatically from CI environments** (GitHub Actions, GitLab, CircleCI, Buildkite, Bitrise, Travis, Azure DevOps, Jenkins, TeamCity → e.g. `owner/repo #123`), or from a `TESTINGBOT_BUILD` variable. @@ -115,6 +117,14 @@ new TestingBotDriver({ build: 'ci-1234', // default: auto-detected from CI env / TESTINGBOT_BUILD testResults: 'on', // 'off' disables pass/fail reporting + snapshotTuning: {}, // Appium settings applied once per session; default turns off + // the wait-for-idle before each hierarchy read (much faster + // assertions while the UI animates). false = server defaults + visibility: 'native', // 'bounds' = any element with a non-empty frame is visible + // (mobilecli-like; helps iOS toBeVisible() on overlapped views) + allocationRetryDelay: 5_000,// first backoff (ms) when no device is free and the run holds + // no session yet; doubles up to 30s until the pool gives up + allocationTimeout: 395_000, // ms to wait for a device (TestingBot queues busy devices) idleTimeout: 230, // seconds before TestingBot reaps an idle session maxDuration: 1800, // max session length in seconds @@ -246,6 +256,8 @@ jobs: | `cannot install "..." mid-session` | TestingBot has no mid-session installs — every app in `installApps` must also be listed in the driver's `apps` option (first entry = app under test, the rest install as helper apps). | | Modifier chords fail on iOS | XCUITest cannot hold modifier keys; chords are Android-only (`clearText()` handles iOS differently). | | Sessions show the run summary instead of a test name | The session hosted several tests; `sessionPerTest: true` gives each test its own session. | +| An iOS project never gets a device in a mixed Android + iOS run | mobilewright's pool keeps idle slots of the other platform. Run each platform in its own `mobilewright test --project ...` invocation, or raise `workers`. | +| `toBeVisible()` fails on iOS for an element that is on screen | XCUITest reports elements drawn under a sibling as invisible. Set `visibility: 'bounds'`. | | A session shows no pass/fail at all | The driver could not tell which test ran in it and declined to guess. Run with `DEBUG=testingbot:observer` — it logs the session id and why. | | `DEBUG=testingbot:*` | Prints every allocation, hub command, upload, and report the driver performs. | @@ -265,11 +277,11 @@ jobs: ## Current limitations -- One TestingBot session may host several mobilewright tests unless `sessionPerTest: true`; multi-test sessions report the run-level verdict. -- The worker process that owns a session is never told which test it is running, so the session-to-test join is reconstructed after the run from Playwright's worker index plus timing. Without the run report (`jsonReport()`) it degrades to timing alone. Either way it is deliberately conservative and leaves a session unjudged rather than risk naming it after the wrong test. +- One TestingBot session may host several mobilewright tests unless `sessionPerTest: true`; a multi-test session keeps one name and fails if any test on it failed. +- **Run Android and iOS projects in separate invocations** (`--project android`, then `--project ios`) unless `workers` exceeds the number of projects. mobilewright's device pool never evicts an idle slot of one platform to make room for the other, so with every slot holding an Android session an iOS test waits until it times out. - Modifier key chords (`pressKeys(['ctrl+a'])`) work on Android only — XCUITest cannot hold modifier keys. -- `pressButton` on iOS supports `HOME`, `VOLUME_UP`, `VOLUME_DOWN`; `listApps()` reports the foreground app only. -- Screenshots are always PNG, and `screenshot({ clip })` is not honoured yet — the full screen comes back uncropped. `applyDeviceSettings` turns Android animations off best-effort via `mobile: shell` (a no-op on iOS). +- `pressButton` on iOS supports `HOME`, `VOLUME_UP`, `VOLUME_DOWN`, and `BACK` (performed as the system left-edge swipe, so `screen.goBack()` works in navigation stacks); `listApps()` reports the foreground app only. +- Screenshots are always PNG. `locator.screenshot()` is cropped to the element and `screenshot({ scale })` downscales. `applyDeviceSettings` turns Android animations off best-effort via `mobile: shell` (a no-op on iOS). - `setGeolocation` needs the platform driver's location extension to be available: emulators and simulators always are, real devices depend on the OS version TestingBot's Appium build supports (iOS 17+ for XCUITest). - `setFoldState` (protocol 0.0.62) throws: Appium has no hinge control for TestingBot's foldables. - Real iOS devices need a test-signed `.ipa`; simulator builds are rejected on real devices with a clear error. @@ -285,6 +297,14 @@ npm run build Live e2e tests against a real TestingBot account live in `e2e/` (see `e2e/mobilewright.config.ts`). +`e2e/live.config.ts` runs feature checks (hierarchy speed, fill, element screenshots, iOS back navigation, per-test reporting) against TestingBot's public sample app, so it needs no local builds: + +```bash +cd e2e && npm install --no-save mobilewright@latest @mobilewright/test@latest +TESTINGBOT_KEY=... TESTINGBOT_SECRET=... npx mobilewright test -c live.config.ts --project android-emulator +# E2E_SESSION_PER_TEST=1, E2E_FORCE_FAIL=1, E2E_RETRIES=1, E2E_VISIBILITY=bounds, E2E_NO_TUNING=1 toggle scenarios +``` + ### Releasing Update `CHANGELOG.md` first — it ships inside the published tarball. diff --git a/e2e/live.config.ts b/e2e/live.config.ts new file mode 100644 index 0000000..dd8d91d --- /dev/null +++ b/e2e/live.config.ts @@ -0,0 +1,33 @@ +import { defineConfig } from 'mobilewright'; +import { TestingBotDriver } from '@testingbot/mobilewright-driver'; + +/** + * Production feature checks against TestingBot's public sample app (a React + * Native calculator). Needs TESTINGBOT_KEY / TESTINGBOT_SECRET. + * npx mobilewright test -c live.config.ts --project android-emulator + */ +const ANDROID_APP = 'https://testingbot.com/appium/sample.apk'; +const IOS_SIM_APP = 'https://testingbot.com/appium/sample.zip'; + +export default defineConfig({ + testDir: './live', + timeout: 300_000, + retries: Number(process.env['E2E_RETRIES'] ?? 0), + workers: Number(process.env['E2E_WORKERS'] ?? 1), + use: { actionTimeout: 30_000 }, + expect: { timeout: 30_000 }, + driver: new TestingBotDriver({ + build: `mobilewright-live-${new Date().toISOString().slice(0, 10)}`, + sessionPerTest: process.env['E2E_SESSION_PER_TEST'] === '1', + apps: { android: ANDROID_APP, 'ios-simulator': IOS_SIM_APP }, + ...(process.env['E2E_NO_TUNING'] === '1' ? { snapshotTuning: false as const } : {}), + ...(process.env['E2E_VISIBILITY'] === 'bounds' ? { visibility: 'bounds' as const } : {}), + }), + projects: [ + { name: 'android-emulator', use: { platform: 'android', deviceType: 'emulator', bundleId: 'com.sampleapp' } }, + { name: 'android-real', use: { platform: 'android', deviceType: 'real', osVersion: '>=12', bundleId: 'com.sampleapp' } }, + { name: 'android-galaxy', use: { platform: 'android', deviceType: 'real', deviceName: /^Galaxy S2[345]$/, bundleId: 'com.sampleapp' } }, + { name: 'ios-simulator', use: { platform: 'ios', deviceType: 'simulator', bundleId: 'org.reactjs.native.example.SampleApp' } }, + ], + reporter: [['list'], ['json', { outputFile: 'live-report.json' }], ['html', { outputFolder: 'live-html', open: 'never' }]], +}); diff --git a/e2e/live/features.test.ts b/e2e/live/features.test.ts new file mode 100644 index 0000000..94579bf --- /dev/null +++ b/e2e/live/features.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from '@mobilewright/test'; + +const pngSize = (b: Buffer) => ({ width: b.readUInt32BE(16), height: b.readUInt32BE(20) }); + +test('hierarchy polling speed', async ({ device }) => { + const driver = device.driver; + await driver.getViewHierarchy(); // warm-up + const times: number[] = []; + for (let i = 0; i < 10; i++) { + const t = Date.now(); + await driver.getViewHierarchy(); + times.push(Date.now() - t); + } + times.sort((a, b) => a - b); + console.log(`[timing] tuning=${process.env['E2E_NO_TUNING'] === '1' ? 'off' : 'on'} median=${times[5]}ms min=${times[0]}ms max=${times[9]}ms all=${times.join(',')}`); +}); + +test('fill both inputs and read the sum', async ({ screen, platform }) => { + await screen.getByLabel('inputA').fill('12'); + await screen.getByLabel('inputB').fill('30'); + if (platform === 'android') await screen.goBack(); // tall keyboards hide the sum field + const sum = screen.getByLabel('sum'); + if (platform === 'ios') await expect(screen.getByType('XCUIElementTypeTextField').nth(2)).toHaveValue('42'); + else await expect(sum).toHaveText('42'); +}); + +test('element screenshot is cropped to the element', async ({ device, screen }) => { + const full = await screen.screenshot(); + const el = await screen.getByLabel('inputA').screenshot(); + const size = await device.driver.getScreenSize(); + const node = (await device.driver.getViewHierarchy()); + const find = (ns: any[]): any => ns.flatMap((n) => [n, ...[find(n.children ?? [])].flat()]); + const target = find(node).filter(Boolean).find((n: any) => n.label === 'inputA'); + console.log(`[clip] full=${JSON.stringify(pngSize(full))} element=${JSON.stringify(pngSize(el))} bounds=${JSON.stringify(target?.bounds)} scale=${size.scale}`); + expect(pngSize(el).width).toBeLessThan(pngSize(full).width / 2); + expect(Math.abs(pngSize(el).width - Math.round(target.bounds.width * size.scale))).toBeLessThanOrEqual(2); + expect(Math.abs(pngSize(el).height - Math.round(target.bounds.height * size.scale))).toBeLessThanOrEqual(2); + const half = await screen.screenshot({ scale: 0.5 }); + expect(Math.abs(pngSize(half).width - pngSize(full).width / 2)).toBeLessThanOrEqual(1); +}); + +test('goBack navigates back on iOS (Settings app)', async ({ device, screen, platform }) => { + test.skip(platform !== 'ios', 'iOS-only: Android BACK is a key event'); + await device.launchApp('com.apple.Preferences'); + await screen.getByText('General', { exact: true }).tap(); + await expect(screen.getByText('About', { exact: true })).toBeVisible(); + await screen.goBack(); + await expect(screen.getByText('About', { exact: true })).not.toBeVisible(); + await expect(screen.getByText('General', { exact: true })).toBeVisible(); +}); diff --git a/e2e/live/fill-repeat.test.ts b/e2e/live/fill-repeat.test.ts new file mode 100644 index 0000000..99a0542 --- /dev/null +++ b/e2e/live/fill-repeat.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from '@mobilewright/test'; +for (let i = 1; i <= 4; i++) { + test(`fill-repeat ${i}`, async ({ device, screen, platform }) => { + const t = Date.now(); + await screen.getByLabel('inputA').fill(String(i * 3)); + await screen.getByLabel('inputB').fill(String(i * 4)); + // Tall keyboards (Samsung) cover the sum field, and Android drops covered + // views from the hierarchy: dismiss the keyboard before reading the result. + if (platform === 'android') await screen.goBack(); + try { + if (platform === 'ios') await expect(screen.getByType('XCUIElementTypeTextField').nth(2)).toHaveValue(String(i * 7)); + else await expect(screen.getByLabel('sum')).toHaveText(String(i * 7), { timeout: 8_000 }); + } catch (err) { + const d: any = device.driver; + const flat = (ns: any[]): any[] => ns.flatMap((n) => [n, ...flat(n.children ?? [])]); + const nodes = flat(await d.getViewHierarchy()); + const hits = nodes.filter((n) => /sum|Total/.test(`${n.label ?? ''}${n.text ?? ''}`)); + const xml: string = await d.hub.source(d.currentSessionId()); + console.log(`[fail] model=${test.info().annotations.find((a) => a.type === 'device.model')?.description} nodes=${nodes.length} hits=${JSON.stringify(hits.map((n) => ({ t: n.type, l: n.label, x: n.text, v: n.isVisible, b: n.bounds })))}`); + console.log(`[fail] raw sum: ${xml.split('\n').filter((l) => /sum/.test(l)).map((l) => l.trim().slice(0, 260)).join('\n')}`); + console.log(`[fail] top: ${xml.split('\n').slice(0, 6).map((l) => l.trim().slice(0, 200)).join('\n')}`); + throw err; + } + console.log(`[fill] tuning=${process.env['E2E_NO_TUNING'] === '1' ? 'off' : 'on'} ${Date.now() - t}ms`); + }); +} diff --git a/e2e/live/report.test.ts b/e2e/live/report.test.ts new file mode 100644 index 0000000..87b31f2 --- /dev/null +++ b/e2e/live/report.test.ts @@ -0,0 +1,11 @@ +import { expect, test } from '@mobilewright/test'; + +test.describe('reporting', () => { + test('passes', async ({ screen }) => { + await expect(screen.getByLabel('inputA').first()).toBeVisible(); + }); + test('fails on purpose', async ({ screen }) => { + test.skip(process.env['E2E_FORCE_FAIL'] !== '1', 'only with E2E_FORCE_FAIL=1'); + await expect(screen.getByText('This text does not exist')).toBeVisible({ timeout: 3_000 }); + }); +});