From 78dd2034ee81a7c2db76354b4a1019b760fc7c6d Mon Sep 17 00:00:00 2001 From: Jochen Delabie Date: Wed, 2 Sep 2026 12:52:42 +0200 Subject: [PATCH] cli: --device-matrix runs every flow on each listed device (TB-375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --device-matrix "[:][:real]" (comma-separated or repeatable) submits one capability per cell in a single run request, so one project fans out into one run per device with its own results, live table rows and dashboard link. No cross-product: each cell is exactly one device. --real-device (or an .ipa app) applies to every cell; --device and --deviceVersion are rejected alongside a matrix. The CLI prints devices × flows before submitting. Also: status --wait no longer installs the cancel-on-interrupt handler. It watches a project it did not start, so Ctrl-C now detaches instead of stopping the runs (a killed watcher cancelled a live matrix run during verification). Pairs with the web change that validates all capabilities before creating any run, so an invalid cell never yields a partial matrix. --- README.md | 16 +++- src/cli.ts | 38 ++++++++- src/models/maestro_options.ts | 41 +++++++++- src/providers/maestro.ts | 85 ++++++++++++++++---- tests/cli.test.ts | 68 ++++++++++++++++ tests/providers/maestro.test.ts | 134 ++++++++++++++++++++++++++++++++ 6 files changed, 364 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 59c2861..718602c 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ testingbot maestro [options] | `--platform ` | Platform: Android or iOS | | `--deviceVersion ` | OS version (e.g., "14", "17.2") | | `--real-device` | Use a real device instead of emulator/simulator | +| `--device-matrix ` | Run every flow on each listed device in one go. Cells are `[:][:real]`, comma-separated or repeatable. Cannot be combined with `--device` or `--deviceVersion`; `--real-device` (or an `.ipa` app) applies to every cell | | `--orientation ` | Screen orientation: PORTRAIT or LANDSCAPE | | `--device-locale ` | Device locale (e.g., "en_US", "de_DE") | | `--timezone ` | Timezone (e.g., "America/New_York", "Europe/London") | @@ -238,6 +239,19 @@ testingbot maestro app.apk ./flows \ --repo-name "myapp" ``` +#### Device matrix + +Run the same flows across several devices in a single command. Each cell names exactly one device; there is no cross-product, because not every device exists in every OS version. Every flow runs once per device, so the cost is devices × flows, and the CLI prints that summary before submitting. + +```sh +testingbot maestro app.apk ./flows \ + --device-matrix "Pixel 9:14" \ + --device-matrix "Samsung Galaxy S24:14:real" \ + --device-matrix "Pixel 8" +``` + +Each device becomes its own run with its own results, live table rows and dashboard link; `--json` lists the `device` per run, and `--retry` re-runs only the flow that failed on the device it failed on. If any cell is not a valid device/OS combination the whole request is rejected and nothing runs, so a matrix never partially submits. + #### Organizing flows and subflows Every top-level flow you pass runs as its own test. A **subflow** (a reusable @@ -356,7 +370,7 @@ testingbot list --count 25 --offset 25 --json | Option | Description | |--------|-------------| -| `-w, --wait` | Block until every run has finished, showing the same live flow table as a foreground run | +| `-w, --wait` | Block until every run has finished, showing the same live flow table as a foreground run. Ctrl-C detaches without cancelling the runs | | `-q, --quiet` | Suppress progress output | Exit code is `0` while the project is still running (JSON `outcome: "running"`), `0`/`2` once it completed, `1` on errors. diff --git a/src/cli.ts b/src/cli.ts index 11820d7..053c25e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -24,7 +24,7 @@ import Maestro from './providers/maestro'; import Login from './providers/login'; import Credentials from './models/credentials'; import path from 'node:path'; -import type { RunMetadata } from './models/maestro_options'; +import type { DeviceMatrixCell, RunMetadata } from './models/maestro_options'; import TestingBotError from './models/testingbot_error'; import { redirectLogsToStderr } from './logger'; import { @@ -130,6 +130,36 @@ function parseKeyValues( return result; } +/** + * Parses `--device-matrix` cells of the form `[:][:real]`. + * Each cell is exactly one device; there is no cross-product, because not + * every device exists in every OS version. + */ +function parseDeviceMatrix( + cells: string[] | undefined, +): DeviceMatrixCell[] | undefined { + if (!cells || cells.length === 0) return undefined; + return cells.map((raw) => { + const parts = raw.split(':').map((p) => p.trim()); + const realIndex = parts.findIndex( + (p, i) => i > 0 && p.toLowerCase() === 'real', + ); + const realDevice = realIndex !== -1; + if (realDevice) parts.splice(realIndex, 1); + const [device, version, ...rest] = parts; + if (!device || rest.length > 0) { + throw new TestingBotError( + `Invalid --device-matrix cell "${raw}": expected "[:][:real]", e.g. "Pixel 9:14" or "iPhone 16:18.2:real".`, + ); + } + return { + device, + ...(version && { version }), + ...(realDevice && { realDevice: true }), + }; + }); +} + /** Drops unset fields; returns undefined when nothing is set. */ function buildRunMetadata(fields: RunMetadata): RunMetadata | undefined { const metadata: Record = {}; @@ -542,6 +572,11 @@ program '--real-device', 'Use a real device instead of an emulator/simulator.', ) + .option( + '--device-matrix ', + 'Run every flow on each listed device: "[:][:real]", comma-separated or repeatable (e.g. "Pixel 9:14,Samsung Galaxy S24:14:real"). Cannot be combined with --device or --deviceVersion.', + collectCommaSeparated, + ) .option( '--google-play', 'Use the Google Play Store-enabled version (Android emulator only).', @@ -836,6 +871,7 @@ program excludeFlows: args.excludeFlows, platformName: args.platform, version: args.deviceVersion, + deviceMatrix: parseDeviceMatrix(args.deviceMatrix), name: args.name, orientation: args.orientation, locale: args.deviceLocale, diff --git a/src/models/maestro_options.ts b/src/models/maestro_options.ts index 36e4d87..d8adca8 100644 --- a/src/models/maestro_options.ts +++ b/src/models/maestro_options.ts @@ -46,6 +46,13 @@ export interface MaestroRunOptions { version?: string; } +/** One device of a --device-matrix run. */ +export interface DeviceMatrixCell { + device: string; + version?: string; + realDevice?: boolean; +} + export const MAX_OTHER_APPS = 4; // Mirror devicecloud.dev: retries are capped at 2 (max 3 total runs per flow). @@ -77,6 +84,7 @@ export default class MaestroOptions { private _flows: string[]; private _otherApps: string[]; private _device?: string; + private _deviceMatrix?: DeviceMatrixCell[]; private _includeTags?: string[]; private _excludeTags?: string[]; private _excludeFlows?: string[]; @@ -121,6 +129,7 @@ export default class MaestroOptions { excludeFlows?: string[]; platformName?: 'Android' | 'iOS'; version?: string; + deviceMatrix?: DeviceMatrixCell[]; name?: string; orientation?: Orientation; locale?: string; @@ -161,6 +170,7 @@ export default class MaestroOptions { ); } this._device = device; + this._deviceMatrix = options?.deviceMatrix; this._includeTags = options?.includeTags; this._excludeTags = options?.excludeTags; this._excludeFlows = options?.excludeFlows; @@ -233,6 +243,11 @@ export default class MaestroOptions { return this._device; } + /** Devices of a --device-matrix run; undefined for a single-device run. */ + public get deviceMatrix(): DeviceMatrixCell[] | undefined { + return this._deviceMatrix; + } + public get includeTags(): string[] | undefined { return this._includeTags; } @@ -377,12 +392,32 @@ export default class MaestroOptions { return Object.keys(opts).length > 0 ? opts : undefined; } + /** + * One capability set per device: the matrix cells when --device-matrix was + * given, otherwise the single device from --device/--deviceVersion. + */ + public getCapabilitiesList( + detectedPlatform?: 'Android' | 'iOS', + ): MaestroCapabilities[] { + if (this._deviceMatrix && this._deviceMatrix.length > 0) { + return this._deviceMatrix.map((cell) => + this.getCapabilities(detectedPlatform, cell), + ); + } + return [this.getCapabilities(detectedPlatform)]; + } + public getCapabilities( detectedPlatform?: 'Android' | 'iOS', + cell?: DeviceMatrixCell, ): MaestroCapabilities { // Use provided platform, or detected platform, or default based on extension let platformName = this._platformName ?? detectedPlatform; - let deviceName = this._device; + let deviceName = cell?.device ?? this._device; + const version = cell ? cell.version : this._version; + // A cell's :real suffix adds to --real-device (or an .ipa app), which + // applies to the whole matrix. + const realDevice = this._realDevice || Boolean(cell?.realDevice); // Fallback to extension-based detection if no platform determined if (!platformName) { @@ -401,7 +436,7 @@ export default class MaestroOptions { platformName, }; - if (this._version) caps.version = this._version; + if (version) caps.version = version; if (this._name) caps.name = this._name; if (this._orientation) caps.orientation = this._orientation; if (this._locale) caps.locale = this._locale; @@ -410,7 +445,7 @@ export default class MaestroOptions { if (this._geoCountryCode) caps['testingbot.geoCountryCode'] = this._geoCountryCode; if (this._tunnelIdentifier) caps.tunnelIdentifier = this._tunnelIdentifier; - if (this._realDevice) caps.realDevice = 'true'; + if (realDevice) caps.realDevice = 'true'; if (this._groups && this._groups.length > 0) caps.groups = this._groups; if (this._googlePlayStore) caps.googlePlayStore = true; diff --git a/src/providers/maestro.ts b/src/providers/maestro.ts index 18d6fda..22e2a36 100644 --- a/src/providers/maestro.ts +++ b/src/providers/maestro.ts @@ -149,6 +149,8 @@ export default class Maestro extends BaseProvider { private updateKey: string | null = null; private socketFallbackWarned = false; private otherAppUrls: string[] = []; + // Top-level flows in the uploaded bundle, for the device-matrix summary. + private uploadedFlowCount: number | undefined = undefined; private flowAnimationFrame = 0; private flowAnimationTimer: NodeJS.Timeout | null = null; @@ -213,6 +215,27 @@ export default class Maestro extends BaseProvider { throw new TestingBotError(`flows option is required`); } + const matrix = this.options.deviceMatrix; + if (matrix && matrix.length > 0) { + // --real-device is allowed: it (or an .ipa app, which implies it) + // applies to every cell. Device and version must live in the cells. + if (this.options.device || this.options.version) { + throw new TestingBotError( + '--device-matrix cannot be combined with --device or --deviceVersion: list every device as a matrix cell instead.', + ); + } + const seen = new Set(); + for (const cell of matrix) { + const key = `${cell.device}|${cell.version ?? ''}|${cell.realDevice ? 'real' : ''}`; + if (seen.has(key)) { + throw new TestingBotError( + `--device-matrix lists "${cell.device}${cell.version ? `:${cell.version}` : ''}" more than once.`, + ); + } + seen.add(key); + } + } + if (this.options.report && !this.options.reportOutputDir) { throw new TestingBotError( `--report-output-dir is required when --report is specified`, @@ -324,7 +347,9 @@ export default class Maestro extends BaseProvider { this.detectedPlatform = await this.detectPlatform(); } - const capabilities = this.options.getCapabilities(this.detectedPlatform); + const capabilities = this.options.getCapabilitiesList( + this.detectedPlatform, + ); const maestroOptions = this.options.getMaestroOptions(); const metadata = this.options.metadata; @@ -367,7 +392,7 @@ export default class Maestro extends BaseProvider { }, ], runPayload: { - capabilities: [capabilities], + capabilities, ...(maestroOptions && { maestroOptions }), ...(this.options.shardSplit && { shardSplit: this.options.shardSplit, @@ -1026,7 +1051,8 @@ export default class Maestro extends BaseProvider { return true; } - const { allFlowFiles, baseDir } = result; + const { allFlowFiles, baseDir, topLevelFlowFiles } = result; + this.uploadedFlowCount = topLevelFlowFiles.length; const { zipPath, tmpDir } = await this.createFlowsZip( allFlowFiles, baseDir, @@ -2144,15 +2170,47 @@ export default class Maestro extends BaseProvider { return parts.slice(0, commonLength).join(path.sep) || path.sep; } + /** + * Before submitting a device matrix, spell out what is about to run: every + * flow executes once per device, so the cost is devices × flows. + */ + private logDeviceMatrixSummary( + capabilities: { + deviceName: string; + version?: string; + realDevice?: string; + }[], + ): void { + if (this.options.quiet || capabilities.length < 2) return; + const flows = this.uploadedFlowCount; + const total = + flows != null ? ` = ${flows * capabilities.length} flow runs` : ''; + logger.info( + `Device matrix: ${capabilities.length} devices${flows != null ? ` × ${flows} flows` : ''}${total}`, + ); + for (const cap of capabilities) { + const details = [ + cap.version ? `OS ${cap.version}` : null, + cap.realDevice === 'true' ? 'real device' : null, + ] + .filter(Boolean) + .join(', '); + logger.info(` • ${cap.deviceName}${details ? ` (${details})` : ''}`); + } + } + private async runTests() { try { - const capabilities = this.options.getCapabilities(this.detectedPlatform); + const capabilities = this.options.getCapabilitiesList( + this.detectedPlatform, + ); const maestroOptions = this.options.getMaestroOptions(); const metadata = this.options.metadata; + this.logDeviceMatrixSummary(capabilities); const response = await axios.post( `${this.URL}/${this.appId}/run`, { - capabilities: [capabilities], + capabilities, ...(maestroOptions && { maestroOptions }), ...(this.options.shardSplit && { shardSplit: this.options.shardSplit, @@ -2819,15 +2877,16 @@ export default class Maestro extends BaseProvider { this.appId = appId; try { if (options.wait) { - this.setupSignalHandlers(); - try { - if (!this.options.quiet) { - logger.info(`Waiting for project ${appId} to complete...`); - } - return await this.waitForCompletion(); - } finally { - this.removeSignalHandlers(); + // Deliberately no signal handlers: the foreground `maestro` command + // cancels its runs on Ctrl-C, but `status --wait` only watches a + // project it did not start. Interrupting the watcher must not stop + // the tests. + if (!this.options.quiet) { + logger.info( + `Waiting for project ${appId} to complete (Ctrl-C detaches; the runs keep going)...`, + ); } + return await this.waitForCompletion(); } const status = await this.getStatus(); diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 9d66b10..03d2605 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -1713,6 +1713,74 @@ describe('TestingBotCTL CLI', () => { }); }); + describe('--device-matrix', () => { + beforeEach(() => { + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockMaestroRun.mockResolvedValue({ + success: true, + outcome: 'passed', + runs: [], + }); + }); + + type Opts = { deviceMatrix?: unknown; device?: string }; + + test('parses comma-separated and repeated cells with version and real flag', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--device-matrix', + 'Pixel 9:14, Samsung Galaxy S24:14:real', + '--device-matrix', + 'Pixel 8', + '--device-matrix', + 'Pixel 7:REAL', + ]); + expect(lastConstructorOptions(Maestro).deviceMatrix).toEqual([ + { device: 'Pixel 9', version: '14' }, + { device: 'Samsung Galaxy S24', version: '14', realDevice: true }, + { device: 'Pixel 8' }, + { device: 'Pixel 7', realDevice: true }, + ]); + expect(lastConstructorOptions(Maestro).device).toBeUndefined(); + }); + + test('is undefined when the flag is absent', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + ]); + expect( + lastConstructorOptions(Maestro).deviceMatrix, + ).toBeUndefined(); + }); + + test('rejects malformed cells', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--device-matrix', + 'Pixel 9:14:extra:junk', + ]); + expect(mockMaestroRun).not.toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining( + 'Invalid --device-matrix cell "Pixel 9:14:extra:junk"', + ), + ); + expect(process.exitCode).toBe(1); + }); + }); + test('unknown command should show help', async () => { const exitSpy = jest .spyOn(process, 'exit') diff --git a/tests/providers/maestro.test.ts b/tests/providers/maestro.test.ts index 52f548a..e11471b 100644 --- a/tests/providers/maestro.test.ts +++ b/tests/providers/maestro.test.ts @@ -6910,9 +6910,12 @@ onFlowStart: runs: [run()], }); maestro['getStatus'] = jest.fn(); + maestro['setupSignalHandlers'] = jest.fn(); const result = await maestro.status(1234, { wait: true }); expect(maestro['waitForCompletion']).toHaveBeenCalledTimes(1); expect(maestro['getStatus']).not.toHaveBeenCalled(); + // Watching must never install the cancel-on-interrupt handler. + expect(maestro['setupSignalHandlers']).not.toHaveBeenCalled(); expect(result.outcome).toBe('passed'); }); @@ -7263,4 +7266,135 @@ onFlowStart: }); }); }); + + describe('device matrix', () => { + const matrixOptions = (extra: Record = {}) => + new MaestroOptions('app.apk', 'path/to/flows', undefined, { + deviceMatrix: [ + { device: 'Pixel 9', version: '14' }, + { device: 'Samsung Galaxy S24', version: '14', realDevice: true }, + ], + locale: 'de_DE', + ...extra, + }); + + it('getCapabilitiesList returns one capability per cell, sharing the common settings', () => { + const caps = matrixOptions().getCapabilitiesList('Android'); + expect(caps).toEqual([ + expect.objectContaining({ + deviceName: 'Pixel 9', + platformName: 'Android', + version: '14', + locale: 'de_DE', + }), + expect.objectContaining({ + deviceName: 'Samsung Galaxy S24', + version: '14', + realDevice: 'true', + locale: 'de_DE', + }), + ]); + expect(caps[0]).not.toHaveProperty('realDevice'); + }); + + it('getCapabilitiesList falls back to the single device without a matrix', () => { + const single = new MaestroOptions('app.apk', 'flows', 'Pixel 6', { + version: '13', + realDevice: true, + }); + expect(single.getCapabilitiesList('Android')).toEqual([ + expect.objectContaining({ + deviceName: 'Pixel 6', + version: '13', + realDevice: 'true', + }), + ]); + }); + + it('validate() rejects a matrix combined with --device', async () => { + const m = new Maestro( + mockCredentials, + new MaestroOptions('app.apk', 'flows', 'Pixel 6', { + deviceMatrix: [{ device: 'Pixel 9' }], + }), + ); + await expect(m['validate']()).rejects.toThrow( + '--device-matrix cannot be combined with --device', + ); + }); + + it('--real-device (or an .ipa app) applies to every cell', () => { + const ipa = new MaestroOptions('app.ipa', 'flows', undefined, { + deviceMatrix: [{ device: 'iPhone 16' }, { device: 'iPhone 15' }], + }); + expect(ipa.getCapabilitiesList('iOS').map((c) => c.realDevice)).toEqual([ + 'true', + 'true', + ]); + const explicit = new MaestroOptions('app.apk', 'flows', undefined, { + realDevice: true, + deviceMatrix: [{ device: 'Pixel 9' }], + }); + expect(explicit.getCapabilitiesList('Android')[0].realDevice).toBe( + 'true', + ); + }); + + it('validate() rejects duplicate cells', async () => { + const m = new Maestro( + mockCredentials, + new MaestroOptions('app.apk', 'flows', undefined, { + deviceMatrix: [ + { device: 'Pixel 9', version: '14' }, + { device: 'Pixel 9', version: '14' }, + ], + }), + ); + await expect(m['validate']()).rejects.toThrow('more than once'); + }); + + it('runTests() submits every cell in one request and logs the summary', async () => { + const m = new Maestro(mockCredentials, matrixOptions()); + m['appId'] = 1234; + m['detectedPlatform'] = 'Android'; + m['uploadedFlowCount'] = 3; + axios.post = jest + .fn() + .mockResolvedValue({ data: { success: true, runs: [] }, headers: {} }); + const logSpy = jest.spyOn(logger, 'info').mockImplementation(() => {}); + + await m['runTests'](); + + const payload = (axios.post as jest.Mock).mock.calls[0][1]; + expect(payload.capabilities).toHaveLength(2); + expect( + payload.capabilities.map((c: { deviceName: string }) => c.deviceName), + ).toEqual(['Pixel 9', 'Samsung Galaxy S24']); + const logged = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(logged).toContain( + 'Device matrix: 2 devices × 3 flows = 6 flow runs', + ); + expect(logged).toContain('Pixel 9 (OS 14)'); + expect(logged).toContain('Samsung Galaxy S24 (OS 14, real device)'); + logSpy.mockRestore(); + }); + + it('runTests() surfaces server-side capability rejection without partial success', async () => { + const m = new Maestro(mockCredentials, matrixOptions()); + m['appId'] = 1234; + axios.post = jest.fn().mockResolvedValue({ + data: { + success: false, + runs: [], + errors: [ + 'Invalid combination: {"deviceName":"Pixel 9","version":"14"}', + ], + }, + headers: {}, + }); + await expect(m['runTests']()).rejects.toThrow( + 'Running Maestro test failed', + ); + }); + }); });