cli: --device-matrix to run flows across several devices in one go (TB-375)

README.md

@@ -108,6 +108,7 @@ testingbot maestro <app> <flows...> [options]
 | `--platform <name>` | Platform: Android or iOS |
 | `--deviceVersion <version>` | OS version (e.g., "14", "17.2") |
 | `--real-device` | Use a real device instead of emulator/simulator |
+| `--device-matrix <cells>` | Run every flow on each listed device in one go. Cells are `<device>[:<version>][:real]`, comma-separated or repeatable. Cannot be combined with `--device` or `--deviceVersion`; `--real-device` (or an `.ipa` app) applies to every cell |
 | `--orientation <orientation>` | Screen orientation: PORTRAIT or LANDSCAPE |
 | `--device-locale <locale>` | Device locale (e.g., "en_US", "de_DE") |
 | `--timezone <timezone>` | Timezone (e.g., "America/New_York", "Europe/London") |
@@ -238,6 +239,19 @@ testingbot maestro app.apk ./flows \
   --repo-name "myapp"
 ```
 
+#### Device matrix
+
+Run the same flows across several devices in a single command. Each cell names exactly one device; there is no cross-product, because not every device exists in every OS version. Every flow runs once per device, so the cost is devices × flows, and the CLI prints that summary before submitting.
+
+```sh
+testingbot maestro app.apk ./flows \
+  --device-matrix "Pixel 9:14" \
+  --device-matrix "Samsung Galaxy S24:14:real" \
+  --device-matrix "Pixel 8"
+```
+
+Each device becomes its own run with its own results, live table rows and dashboard link; `--json` lists the `device` per run, and `--retry` re-runs only the flow that failed on the device it failed on. If any cell is not a valid device/OS combination the whole request is rejected and nothing runs, so a matrix never partially submits.
+
 #### Organizing flows and subflows
 
 Every top-level flow you pass runs as its own test. A **subflow** (a reusable
@@ -356,7 +370,7 @@ testingbot list --count 25 --offset 25 --json
 
 | Option | Description |
 |--------|-------------|
-| `-w, --wait` | Block until every run has finished, showing the same live flow table as a foreground run |
+| `-w, --wait` | Block until every run has finished, showing the same live flow table as a foreground run. Ctrl-C detaches without cancelling the runs |
 | `-q, --quiet` | Suppress progress output |
 
 Exit code is `0` while the project is still running (JSON `outcome: "running"`), `0`/`2` once it completed, `1` on errors.

src/cli.ts

@@ -24,7 +24,7 @@ import Maestro from './providers/maestro';
 import Login from './providers/login';
 import Credentials from './models/credentials';
 import path from 'node:path';
-import type { RunMetadata } from './models/maestro_options';
+import type { DeviceMatrixCell, RunMetadata } from './models/maestro_options';
 import TestingBotError from './models/testingbot_error';
 import { redirectLogsToStderr } from './logger';
 import {
@@ -130,6 +130,36 @@ function parseKeyValues(
   return result;
 }
 
+/**
+ * Parses `--device-matrix` cells of the form `<device>[:<version>][:real]`.
+ * Each cell is exactly one device; there is no cross-product, because not
+ * every device exists in every OS version.
+ */
+function parseDeviceMatrix(
+  cells: string[] | undefined,
+): DeviceMatrixCell[] | undefined {
+  if (!cells || cells.length === 0) return undefined;
+  return cells.map((raw) => {
+    const parts = raw.split(':').map((p) => p.trim());
+    const realIndex = parts.findIndex(
+      (p, i) => i > 0 && p.toLowerCase() === 'real',
+    );
+    const realDevice = realIndex !== -1;
+    if (realDevice) parts.splice(realIndex, 1);
+    const [device, version, ...rest] = parts;
+    if (!device || rest.length > 0) {
+      throw new TestingBotError(
+        `Invalid --device-matrix cell "${raw}": expected "<device>[:<version>][:real]", e.g. "Pixel 9:14" or "iPhone 16:18.2:real".`,
+      );
+    }
+    return {
+      device,
+      ...(version && { version }),
+      ...(realDevice && { realDevice: true }),
+    };
+  });
+}
+
 /** Drops unset fields; returns undefined when nothing is set. */
 function buildRunMetadata(fields: RunMetadata): RunMetadata | undefined {
   const metadata: Record<string, unknown> = {};
@@ -542,6 +572,11 @@ program
     '--real-device',
     'Use a real device instead of an emulator/simulator.',
   )
+  .option(
+    '--device-matrix <cells>',
+    'Run every flow on each listed device: "<device>[:<version>][:real]", comma-separated or repeatable (e.g. "Pixel 9:14,Samsung Galaxy S24:14:real"). Cannot be combined with --device or --deviceVersion.',
+    collectCommaSeparated,
+  )
   .option(
     '--google-play',
     'Use the Google Play Store-enabled version (Android emulator only).',
@@ -836,6 +871,7 @@ program
         excludeFlows: args.excludeFlows,
         platformName: args.platform,
         version: args.deviceVersion,
+        deviceMatrix: parseDeviceMatrix(args.deviceMatrix),
         name: args.name,
         orientation: args.orientation,
         locale: args.deviceLocale,

src/models/maestro_options.ts

@@ -46,6 +46,13 @@ export interface MaestroRunOptions {
   version?: string;
 }
 
+/** One device of a --device-matrix run. */
+export interface DeviceMatrixCell {
+  device: string;
+  version?: string;
+  realDevice?: boolean;
+}
+
 export const MAX_OTHER_APPS = 4;
 
 // Mirror devicecloud.dev: retries are capped at 2 (max 3 total runs per flow).
@@ -77,6 +84,7 @@ export default class MaestroOptions {
   private _flows: string[];
   private _otherApps: string[];
   private _device?: string;
+  private _deviceMatrix?: DeviceMatrixCell[];
   private _includeTags?: string[];
   private _excludeTags?: string[];
   private _excludeFlows?: string[];
@@ -121,6 +129,7 @@ export default class MaestroOptions {
       excludeFlows?: string[];
       platformName?: 'Android' | 'iOS';
       version?: string;
+      deviceMatrix?: DeviceMatrixCell[];
       name?: string;
       orientation?: Orientation;
       locale?: string;
@@ -161,6 +170,7 @@ export default class MaestroOptions {
       );
     }
     this._device = device;
+    this._deviceMatrix = options?.deviceMatrix;
     this._includeTags = options?.includeTags;
     this._excludeTags = options?.excludeTags;
     this._excludeFlows = options?.excludeFlows;
@@ -233,6 +243,11 @@ export default class MaestroOptions {
     return this._device;
   }
 
+  /** Devices of a --device-matrix run; undefined for a single-device run. */
+  public get deviceMatrix(): DeviceMatrixCell[] | undefined {
+    return this._deviceMatrix;
+  }
+
   public get includeTags(): string[] | undefined {
     return this._includeTags;
   }
@@ -377,12 +392,32 @@ export default class MaestroOptions {
     return Object.keys(opts).length > 0 ? opts : undefined;
   }
 
+  /**
+   * One capability set per device: the matrix cells when --device-matrix was
+   * given, otherwise the single device from --device/--deviceVersion.
+   */
+  public getCapabilitiesList(
+    detectedPlatform?: 'Android' | 'iOS',
+  ): MaestroCapabilities[] {
+    if (this._deviceMatrix && this._deviceMatrix.length > 0) {
+      return this._deviceMatrix.map((cell) =>
+        this.getCapabilities(detectedPlatform, cell),
+      );
+    }
+    return [this.getCapabilities(detectedPlatform)];
+  }
+
   public getCapabilities(
     detectedPlatform?: 'Android' | 'iOS',
+    cell?: DeviceMatrixCell,
   ): MaestroCapabilities {
     // Use provided platform, or detected platform, or default based on extension
     let platformName = this._platformName ?? detectedPlatform;
-    let deviceName = this._device;
+    let deviceName = cell?.device ?? this._device;
+    const version = cell ? cell.version : this._version;
+    // A cell's :real suffix adds to --real-device (or an .ipa app), which
+    // applies to the whole matrix.
+    const realDevice = this._realDevice || Boolean(cell?.realDevice);
 
     // Fallback to extension-based detection if no platform determined
     if (!platformName) {
@@ -401,7 +436,7 @@ export default class MaestroOptions {
       platformName,
     };
 
-    if (this._version) caps.version = this._version;
+    if (version) caps.version = version;
     if (this._name) caps.name = this._name;
     if (this._orientation) caps.orientation = this._orientation;
     if (this._locale) caps.locale = this._locale;
@@ -410,7 +445,7 @@ export default class MaestroOptions {
     if (this._geoCountryCode)
       caps['testingbot.geoCountryCode'] = this._geoCountryCode;
     if (this._tunnelIdentifier) caps.tunnelIdentifier = this._tunnelIdentifier;
-    if (this._realDevice) caps.realDevice = 'true';
+    if (realDevice) caps.realDevice = 'true';
     if (this._groups && this._groups.length > 0) caps.groups = this._groups;
     if (this._googlePlayStore) caps.googlePlayStore = true;
 

src/providers/maestro.ts

@@ -149,6 +149,8 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
   private updateKey: string | null = null;
   private socketFallbackWarned = false;
   private otherAppUrls: string[] = [];
+  // Top-level flows in the uploaded bundle, for the device-matrix summary.
+  private uploadedFlowCount: number | undefined = undefined;
 
   private flowAnimationFrame = 0;
   private flowAnimationTimer: NodeJS.Timeout | null = null;
@@ -213,6 +215,27 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
       throw new TestingBotError(`flows option is required`);
     }
 
+    const matrix = this.options.deviceMatrix;
+    if (matrix && matrix.length > 0) {
+      // --real-device is allowed: it (or an .ipa app, which implies it)
+      // applies to every cell. Device and version must live in the cells.
+      if (this.options.device || this.options.version) {
+        throw new TestingBotError(
+          '--device-matrix cannot be combined with --device or --deviceVersion: list every device as a matrix cell instead.',
+        );
+      }
+      const seen = new Set<string>();
+      for (const cell of matrix) {
+        const key = `${cell.device}|${cell.version ?? ''}|${cell.realDevice ? 'real' : ''}`;
+        if (seen.has(key)) {
+          throw new TestingBotError(
+            `--device-matrix lists "${cell.device}${cell.version ? `:${cell.version}` : ''}" more than once.`,
+          );
+        }
+        seen.add(key);
+      }
+    }
+
     if (this.options.report && !this.options.reportOutputDir) {
       throw new TestingBotError(
         `--report-output-dir is required when --report is specified`,
@@ -324,7 +347,9 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
         this.detectedPlatform = await this.detectPlatform();
       }
 
-      const capabilities = this.options.getCapabilities(this.detectedPlatform);
+      const capabilities = this.options.getCapabilitiesList(
+        this.detectedPlatform,
+      );
       const maestroOptions = this.options.getMaestroOptions();
       const metadata = this.options.metadata;
 
@@ -367,7 +392,7 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
           },
         ],
         runPayload: {
-          capabilities: [capabilities],
+          capabilities,
           ...(maestroOptions && { maestroOptions }),
           ...(this.options.shardSplit && {
             shardSplit: this.options.shardSplit,
@@ -1026,7 +1051,8 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
       return true;
     }
 
-    const { allFlowFiles, baseDir } = result;
+    const { allFlowFiles, baseDir, topLevelFlowFiles } = result;
+    this.uploadedFlowCount = topLevelFlowFiles.length;
     const { zipPath, tmpDir } = await this.createFlowsZip(
       allFlowFiles,
       baseDir,
@@ -2144,15 +2170,47 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
     return parts.slice(0, commonLength).join(path.sep) || path.sep;
   }
 
+  /**
+   * Before submitting a device matrix, spell out what is about to run: every
+   * flow executes once per device, so the cost is devices × flows.
+   */
+  private logDeviceMatrixSummary(
+    capabilities: {
+      deviceName: string;
+      version?: string;
+      realDevice?: string;
+    }[],
+  ): void {
+    if (this.options.quiet || capabilities.length < 2) return;
+    const flows = this.uploadedFlowCount;
+    const total =
+      flows != null ? ` = ${flows * capabilities.length} flow runs` : '';
+    logger.info(
+      `Device matrix: ${capabilities.length} devices${flows != null ? ` × ${flows} flows` : ''}${total}`,
+    );
+    for (const cap of capabilities) {
+      const details = [
+        cap.version ? `OS ${cap.version}` : null,
+        cap.realDevice === 'true' ? 'real device' : null,
+      ]
+        .filter(Boolean)
+        .join(', ');
+      logger.info(`  • ${cap.deviceName}${details ? ` (${details})` : ''}`);
+    }
+  }
+
   private async runTests() {
     try {
-      const capabilities = this.options.getCapabilities(this.detectedPlatform);
+      const capabilities = this.options.getCapabilitiesList(
+        this.detectedPlatform,
+      );
       const maestroOptions = this.options.getMaestroOptions();
       const metadata = this.options.metadata;
+      this.logDeviceMatrixSummary(capabilities);
       const response = await axios.post(
         `${this.URL}/${this.appId}/run`,
         {
-          capabilities: [capabilities],
+          capabilities,
           ...(maestroOptions && { maestroOptions }),
           ...(this.options.shardSplit && {
             shardSplit: this.options.shardSplit,
@@ -2819,15 +2877,16 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
     this.appId = appId;
     try {
       if (options.wait) {
-        this.setupSignalHandlers();
-        try {
-          if (!this.options.quiet) {
-            logger.info(`Waiting for project ${appId} to complete...`);
-          }
-          return await this.waitForCompletion();
-        } finally {
-          this.removeSignalHandlers();
+        // Deliberately no signal handlers: the foreground `maestro` command
+        // cancels its runs on Ctrl-C, but `status --wait` only watches a
+        // project it did not start. Interrupting the watcher must not stop
+        // the tests.
+        if (!this.options.quiet) {
+          logger.info(
+            `Waiting for project ${appId} to complete (Ctrl-C detaches; the runs keep going)...`,
+          );
         }
+        return await this.waitForCompletion();
       }
 
       const status = await this.getStatus();