diff --git a/README.md b/README.md index 80e6052..59c2861 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ testingbot maestro [options] | `--groups ` | Tag the test session with one or more groups (comma-separated). Groups appear on the test in the TestingBot dashboard | | `--include-tags ` | Only run flows with these tags (comma-separated) | | `--exclude-tags ` | Exclude flows with these tags (comma-separated) | +| `--exclude-flows ` | Flow files, directories or glob patterns to leave out of the run (comma-separated, repeatable). An excluded flow that another flow still invokes via `runFlow` is bundled as a subflow but never runs on its own | | `-e, --env ` | Environment variable for flows (can be repeated) | | `--config ` | Path to a custom Maestro config file (default: config.yaml in project root) | | `--maestro-version ` | Maestro version to use (e.g., "2.0.10") | @@ -148,7 +149,7 @@ testingbot maestro [options] | `--json` | Print results as a single JSON document on stdout (logs move to stderr). Implies `--quiet`. Exit code 2 when tests fail | | `--json-file` | Write results as JSON to a file (default: `_testingbot.json` in the current directory). Implies `--quiet`. Exit code stays 0 when tests fail so the pipeline can gate on the file | | `--json-file-name ` | Custom path for the JSON results file (requires `--json-file`) | -| `--report ` | Download report after completion: html or junit | +| `--report ` | Download report after completion: `html`, `html-detailed`, `junit` or `allure` | | `--report-output-dir ` | Directory to save reports (required with --report) | | `--download-artifacts [mode]` | Download test artifacts (logs, screenshots, video). Mode: `all` (default) or `failed` | | `--artifacts-output-dir ` | Directory to save artifacts zip (defaults to current directory) | @@ -167,10 +168,17 @@ testingbot maestro [options] | Option | Description | |--------|-------------| +| `--branch ` | Git branch this test run was built from | | `--commit-sha ` | Git commit SHA associated with this test run | | `--pull-request-id ` | Pull request ID this test run originated from | +| `--pr-url ` | Pull request URL this test run originated from | | `--repo-name ` | Repository name (e.g., GitHub repo slug) | | `--repo-owner ` | Repository owner (e.g., GitHub organization or username) | +| `-m, --metadata ` | Free-form metadata attached to the run and shown in the dashboard (repeatable, e.g. `-m team=mobile -m env=staging`) | + +**Allure reports:** `--report allure` converts each run's results into Allure result files under `/allure-results/`, one JSON per flow with its steps, status and failure details. Render them with `allure serve /allure-results` (requires the [Allure CLI](https://allurereport.org/docs/install/)). Results from several runs or shards accumulate in the same directory. + +**Migrating from Maestro Cloud:** the `maestro cloud` spelling of common flags is accepted as hidden aliases, so an existing command line runs unchanged: `--app-file`, `--flows `, `--apiKey`, `--device-model iPhone-17-Pro`, `--device-os iOS-18-2` / `android-34`, `--format JUNIT|HTML`, `--output ` (its directory becomes `--report-output-dir`) and `--test-suite-name`. The canonical flag wins when both are given. **Examples:** @@ -357,7 +365,7 @@ Exit code is `0` while the project is still running (JSON `outcome: "running"`), | Option | Description | |--------|-------------| -| `--report ` | Download report: `html`, `html-detailed` or `junit` | +| `--report ` | Download report: `html`, `html-detailed`, `junit` or `allure` | | `--report-output-dir ` | Directory to save reports (required with `--report`) | | `--download-artifacts [mode]` | Download logs, screenshots and video. Mode: `all` (default) or `failed` | | `--artifacts-output-dir ` | Directory to save the artifacts zip (defaults to current directory) | diff --git a/src/cli.ts b/src/cli.ts index b41f142..11820d7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,4 @@ -import { Command, InvalidArgumentError } from 'commander'; +import { Command, InvalidArgumentError, Option } from 'commander'; import logger, { enableDebugLogging } from './logger'; import Auth from './auth'; import Espresso from './providers/espresso'; @@ -23,6 +23,8 @@ import MaestroOptions, { import Maestro from './providers/maestro'; import Login from './providers/login'; import Credentials from './models/credentials'; +import path from 'node:path'; +import type { RunMetadata } from './models/maestro_options'; import TestingBotError from './models/testingbot_error'; import { redirectLogsToStderr } from './logger'; import { @@ -88,6 +90,147 @@ function parseProjectId(value: string): number { return id; } +/** + * Commander accumulator for repeatable flags. Returns a new array each time + * and takes no default, so nothing is shared between parses (a default `[]` + * would be mutated in place and leak values across invocations). + */ +function collectRepeatable(value: string, acc: string[] | undefined): string[] { + return [...(acc ?? []), value]; +} + +/** Like collectRepeatable, but each value may also be comma-separated. */ +function collectCommaSeparated( + value: string, + acc: string[] | undefined, +): string[] { + const parts = value + .split(',') + .map((part) => part.trim()) + .filter(Boolean); + return [...(acc ?? []), ...parts]; +} + +/** Parses `KEY=VALUE` entries; empty keys or a missing `=` are rejected. */ +function parseKeyValues( + entries: string[] | undefined, + flag: string, +): Record | undefined { + if (!entries || entries.length === 0) return undefined; + const result: Record = {}; + for (const entry of entries) { + const eq = entry.indexOf('='); + if (eq <= 0) { + throw new TestingBotError( + `Invalid ${flag} entry "${entry}": expected KEY=VALUE.`, + ); + } + result[entry.slice(0, eq).trim()] = entry.slice(eq + 1); + } + return result; +} + +/** Drops unset fields; returns undefined when nothing is set. */ +function buildRunMetadata(fields: RunMetadata): RunMetadata | undefined { + const metadata: Record = {}; + for (const [key, value] of Object.entries(fields)) { + if (value !== undefined && value !== '') metadata[key] = value; + } + return Object.keys(metadata).length > 0 + ? (metadata as RunMetadata) + : undefined; +} + +/** Android API level → OS version, for `--device-os android-34`. */ +const ANDROID_API_TO_VERSION: Record = { + '26': '8.0', + '27': '8.1', + '28': '9', + '29': '10', + '30': '11', + '31': '12', + '32': '12', + '33': '13', + '34': '14', + '35': '15', + '36': '16', +}; + +interface MaestroCloudAliasArgs { + appFile?: string; + flows?: string; + deviceModel?: string; + deviceOs?: string; + format?: string; + output?: string; + testSuiteName?: string; + app?: string; + device?: string; + platform?: 'Android' | 'iOS'; + deviceVersion?: string; + report?: ReportFormat; + reportOutputDir?: string; + name?: string; +} + +/** + * Maestro Cloud spells several flags differently (`maestro cloud --app-file + * app.zip --flows ./flows --device-os iOS-18-2 --format JUNIT`). These hidden + * aliases let such a command line run unchanged. Canonical flags win when + * both are given. Returns extra flow paths from `--flows`. + */ +function applyMaestroCloudAliases(args: MaestroCloudAliasArgs): string[] { + if (args.appFile && !args.app) args.app = args.appFile; + + const extraFlows = (args.flows ?? '') + .split(',') + .map((f) => f.trim()) + .filter(Boolean); + + if (args.deviceModel && !args.device) { + // Maestro Cloud model ids are dash/underscore-joined: iPhone-17-Pro, pixel_7 + args.device = args.deviceModel.replace(/[-_]+/g, ' '); + } + + if (args.deviceOs) { + const match = /^(ios|android)[-_]?(.*)$/i.exec(args.deviceOs.trim()); + if (!match) { + throw new TestingBotError( + `Invalid --device-os "${args.deviceOs}": expected iOS-[-] or android-.`, + ); + } + const isIos = match[1].toLowerCase() === 'ios'; + if (!args.platform) args.platform = isIos ? 'iOS' : 'Android'; + if (!args.deviceVersion && match[2]) { + const raw = match[2].replace(/-/g, '.'); + args.deviceVersion = isIos ? raw : (ANDROID_API_TO_VERSION[raw] ?? raw); + } + } + + if (args.format && !args.report) { + const format = args.format.toLowerCase(); + if (format === 'junit' || format === 'html') { + args.report = format; + } else if (format !== 'noop') { + throw new TestingBotError( + `Invalid --format "${args.format}": expected JUNIT, HTML or NOOP.`, + ); + } + } + + if (args.output && !args.reportOutputDir) { + args.reportOutputDir = path.dirname(path.resolve(args.output)); + if (args.report) { + logger.warn( + `--output is mapped to --report-output-dir ${args.reportOutputDir}; reports are named report_run_.`, + ); + } + } + + if (args.testSuiteName && !args.name) args.name = args.testSuiteName; + return extraFlows; +} + /** Emits JSON output (if requested) and sets the exit code for a finished run. */ async function finishCommand( output: JsonOutput, @@ -445,6 +588,11 @@ program 'Exclude flows with these tags (comma-separated).', (val) => val.split(',').map((t) => t.trim()), ) + .option( + '--exclude-flows ', + 'Flow files, directories or globs to leave out (comma-separated, repeatable).', + collectCommaSeparated, + ) // Environment variables .option( '-e, --env ', @@ -489,7 +637,7 @@ program // Report options .option( '--report ', - 'Download test report after completion: html, html-detailed, or junit.', + 'Download test report after completion: html, html-detailed, junit, or allure.', (val) => val.toLowerCase() as ReportFormat, ) .option( @@ -516,7 +664,17 @@ program (val) => parseInt(val, 10), ) // CI/CD metadata + .option('--branch ', 'Git branch this upload was built from.') .option('--commit-sha ', 'The commit SHA of this upload.') + .option( + '--pr-url ', + 'URL of the pull request this upload originated from.', + ) + .option( + '-m, --metadata ', + 'Arbitrary metadata to attach to the run, shown in the dashboard (repeatable).', + collectRepeatable, + ) .option( '--pull-request-id ', 'The ID of the pull request this upload originated from.', @@ -543,10 +701,59 @@ program '--json-file-name ', 'Custom path for the JSON results file (requires --json-file).', ) + .addOption( + new Option( + '--app-file ', + 'Alias of --app (Maestro Cloud compatibility).', + ).hideHelp(), + ) + .addOption( + new Option( + '--flows ', + 'Comma-separated flow paths (Maestro Cloud compatibility).', + ).hideHelp(), + ) + .addOption( + new Option( + '--apiKey ', + 'Alias of --api-key (Maestro Cloud compatibility).', + ).hideHelp(), + ) + .addOption( + new Option( + '--device-model ', + 'Alias of --device, e.g. iPhone-17-Pro (Maestro Cloud compatibility).', + ).hideHelp(), + ) + .addOption( + new Option( + '--device-os ', + 'Platform and OS version, e.g. iOS-18-2 or android-34 (Maestro Cloud compatibility).', + ).hideHelp(), + ) + .addOption( + new Option( + '--format ', + 'Alias of --report: JUNIT or HTML (Maestro Cloud compatibility).', + ).hideHelp(), + ) + .addOption( + new Option( + '--output ', + 'Report file path; its directory becomes --report-output-dir (Maestro Cloud compatibility).', + ).hideHelp(), + ) + .addOption( + new Option( + '--test-suite-name ', + 'Alias of --name (Maestro Cloud compatibility).', + ).hideHelp(), + ) .action(async (appFileArg, flowsArgs, args) => { let jsonOptions: JsonOutputOptions | undefined; try { jsonOptions = jsonOptionsFrom(args); + const aliasFlows = applyMaestroCloudAliases(args); let app: string; let flows: string[]; @@ -561,6 +768,7 @@ program app = appFileArg; flows = flowsArgs || []; } + flows = [...flows, ...aliasFlows]; const missing: string[] = []; if (!app && args.appBinaryId == null) @@ -612,19 +820,20 @@ program } } - const metadata = - args.commitSha || args.pullRequestId || args.repoName || args.repoOwner - ? { - commitSha: args.commitSha, - pullRequestId: args.pullRequestId, - repoName: args.repoName, - repoOwner: args.repoOwner, - } - : undefined; + const metadata = buildRunMetadata({ + commitSha: args.commitSha, + pullRequestId: args.pullRequestId, + pullRequestUrl: args.prUrl, + repoName: args.repoName, + repoOwner: args.repoOwner, + branch: args.branch, + custom: parseKeyValues(args.metadata, '--metadata'), + }); const options = new MaestroOptions(app, flows, args.device, { includeTags: args.includeTags, excludeTags: args.excludeTags, + excludeFlows: args.excludeFlows, platformName: args.platform, version: args.deviceVersion, name: args.name, @@ -1003,7 +1212,7 @@ withFlags( ) .option( '--report ', - 'Download test report: html, html-detailed, or junit.', + 'Download test report: html, html-detailed, junit, or allure.', (val) => val.toLowerCase() as ReportFormat, ) .option( diff --git a/src/models/maestro_options.ts b/src/models/maestro_options.ts index 20cab1f..36e4d87 100644 --- a/src/models/maestro_options.ts +++ b/src/models/maestro_options.ts @@ -9,14 +9,18 @@ export interface MaestroConfig { export type Orientation = 'PORTRAIT' | 'LANDSCAPE'; export type ThrottleNetwork = '4G' | '3G' | 'Edge' | 'airplane' | 'disable'; -export type ReportFormat = 'html' | 'html-detailed' | 'junit'; +export type ReportFormat = 'html' | 'html-detailed' | 'junit' | 'allure'; export type ArtifactDownloadMode = 'all' | 'failed'; export interface RunMetadata { commitSha?: string; pullRequestId?: string; + pullRequestUrl?: string; repoName?: string; repoOwner?: string; + branch?: string; + /** Free-form key/value pairs from --metadata, shown on the dashboard. */ + custom?: Record; } export interface MaestroCapabilities { @@ -75,6 +79,7 @@ export default class MaestroOptions { private _device?: string; private _includeTags?: string[]; private _excludeTags?: string[]; + private _excludeFlows?: string[]; private _platformName?: 'Android' | 'iOS'; private _version?: string; private _name?: string; @@ -113,6 +118,7 @@ export default class MaestroOptions { options?: { includeTags?: string[]; excludeTags?: string[]; + excludeFlows?: string[]; platformName?: 'Android' | 'iOS'; version?: string; name?: string; @@ -157,6 +163,7 @@ export default class MaestroOptions { this._device = device; this._includeTags = options?.includeTags; this._excludeTags = options?.excludeTags; + this._excludeFlows = options?.excludeFlows; this._platformName = options?.platformName; this._version = options?.version; this._name = options?.name; @@ -230,6 +237,11 @@ export default class MaestroOptions { return this._includeTags; } + /** Flow files, directories or globs to leave out of the bundle (--exclude-flows). */ + public get excludeFlows(): string[] | undefined { + return this._excludeFlows; + } + public get excludeTags(): string[] | undefined { return this._excludeTags; } diff --git a/src/providers/maestro.ts b/src/providers/maestro.ts index b95d87d..18d6fda 100644 --- a/src/providers/maestro.ts +++ b/src/providers/maestro.ts @@ -15,6 +15,7 @@ import { detectPlatformFromFile } from '../utils/file-type-detector'; import pc from 'picocolors'; import BaseProvider, { ProviderResult } from './base_provider'; import type { JsonFlowResult, JsonRunResult } from '../utils/json_output'; +import { junitToAllureResults, writeAllureResults } from '../utils/allure'; import { setTitle } from '../ui/terminal-title'; import { HTTP, SOCKET } from '../config/constants'; @@ -862,9 +863,13 @@ export default class Maestro extends BaseProvider { } } + const excluded = await this.applyFlowExclusions(allFlowFiles); + if (allFlowFiles.length === 0) { throw new TestingBotError( - `No flow files (.yaml, .yml) found in the provided paths`, + excluded > 0 + ? `--exclude-flows removed every flow (${excluded} excluded); nothing left to run` + : `No flow files (.yaml, .yml) found in the provided paths`, ); } @@ -1053,6 +1058,49 @@ export default class Maestro extends BaseProvider { * (config.yaml or config.yml). This identifies the project root so the zip * preserves the directory structure needed for relative paths like ../../screens/. */ + /** + * Drops flows matched by --exclude-flows from `files` in place. Entries may + * be files, directories (everything beneath is excluded) or glob patterns. + * Returns how many files were removed. Runs before dependency discovery, so + * an excluded flow that another flow still runFlow's is bundled as a + * subflow but never executes on its own. + */ + private async applyFlowExclusions(files: string[]): Promise { + const patterns = this.options.excludeFlows ?? []; + if (patterns.length === 0) return 0; + + const excludedFiles = new Set(); + const excludedDirs: string[] = []; + for (const pattern of patterns) { + if (/[*?[\]{}]/.test(pattern)) { + for (const match of await glob(pattern)) { + excludedFiles.add(path.resolve(match)); + } + continue; + } + const resolved = path.resolve(pattern); + const stat = await fs.promises.stat(resolved).catch(() => null); + if (stat?.isDirectory()) excludedDirs.push(resolved + path.sep); + else excludedFiles.add(resolved); + } + + const before = files.length; + const kept = files.filter((file) => { + const resolved = path.resolve(file); + return ( + !excludedFiles.has(resolved) && + !excludedDirs.some((dir) => resolved.startsWith(dir)) + ); + }); + files.splice(0, files.length, ...kept); + + const removed = before - kept.length; + if (removed > 0 && !this.options.quiet) { + logger.info(`Excluded ${removed} flow file(s) via --exclude-flows`); + } + return removed; + } + private async findMaestroProjectRoot( flowFiles: string[], ): Promise<{ dir: string; configPath: string } | null> { @@ -3394,6 +3442,8 @@ export default class Maestro extends BaseProvider { let reportKey: string; switch (reportFormat) { case 'junit': + case 'allure': + // Allure results are derived client-side from the JUnit XML. reportEndpoint = 'junit_report'; reportKey = 'junit_report'; break; @@ -3434,6 +3484,23 @@ export default class Maestro extends BaseProvider { continue; } + if (reportFormat === 'allure') { + const results = junitToAllureResults(reportContent, { + runId: run.id, + device: this.getRunDisplayName(run), + platform: run.capabilities.platformName, + osVersion: run.environment?.version ?? run.capabilities.version, + startedAt: run.flows?.[0]?.requested_at, + }); + const dir = await writeAllureResults(outputDir, results); + if (!this.options.quiet) { + logger.info( + ` Saved ${results.length} Allure result(s) for run ${run.id} to ${dir}`, + ); + } + continue; + } + const fileExtension = reportFormat === 'junit' ? 'xml' : 'html'; const fileName = `report_run_${run.id}.${fileExtension}`; const filePath = path.join(outputDir, fileName); @@ -3449,6 +3516,12 @@ export default class Maestro extends BaseProvider { ); } } + + if (reportFormat === 'allure' && !this.options.quiet) { + logger.info( + ` Render the report with: allure serve ${path.join(outputDir, 'allure-results')}`, + ); + } } private async getRunDetails(runId: number): Promise { diff --git a/src/utils/allure.ts b/src/utils/allure.ts new file mode 100644 index 0000000..243579e --- /dev/null +++ b/src/utils/allure.ts @@ -0,0 +1,253 @@ +import { randomUUID, createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +/** + * Converts the JUnit XML TestingBot returns for a Maestro run into Allure + * result files (`allure-results/*-result.json`), the input format of + * `allure generate` / `allure serve`. + * + * The XML is produced by Maestro itself (one per flow, with + * entries per command and an optional ), so a + * small purpose-built reader is enough and avoids pulling in an XML parser. + */ + +export type AllureStatus = 'passed' | 'failed' | 'broken' | 'skipped'; + +export interface AllureStep { + name: string; + status: AllureStatus; + stage: 'finished'; + start?: number; + stop?: number; +} + +export interface AllureResult { + uuid: string; + historyId: string; + name: string; + fullName: string; + status: AllureStatus; + stage: 'finished'; + start: number; + stop: number; + labels: { name: string; value: string }[]; + statusDetails?: { message: string; trace?: string }; + steps: AllureStep[]; +} + +export interface AllureContext { + runId: number; + device?: string; + platform?: string; + osVersion?: string; + /** Fallback when the XML carries no step timestamps. */ + startedAt?: string; +} + +const XML_ENTITIES: Record = { + '<': '<', + '>': '>', + '"': '"', + ''': "'", + '&': '&', +}; + +function decodeXml(value: string): string { + return value + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&#x([0-9a-fA-F]+);/g, (_, code) => + String.fromCodePoint(parseInt(code, 16)), + ) + .replace(/&(lt|gt|quot|apos|amp);/g, (match) => XML_ENTITIES[match]); +} + +function parseAttributes(tag: string): Record { + const attrs: Record = {}; + const re = /([\w:.-]+)\s*=\s*"([^"]*)"/g; + let match: RegExpExecArray | null; + while ((match = re.exec(tag)) !== null) { + attrs[match[1]] = decodeXml(match[2]); + } + return attrs; +} + +function stepStatus(status: string | undefined): AllureStatus { + switch ((status ?? '').toUpperCase()) { + case 'COMPLETED': + case 'SUCCESS': + case 'PASSED': + return 'passed'; + case 'FAILED': + case 'ERROR': + return 'failed'; + case 'SKIPPED': + case 'PENDING': + return 'skipped'; + default: + return 'broken'; + } +} + +/** Human-readable step name from Maestro's command name plus its details. */ +function stepName(value: string | undefined, details: string | undefined) { + const command = (value ?? 'step').replace(/Command$/, ''); + if (!details) return command; + try { + const parsed = JSON.parse(details) as Record; + const summary = Object.entries(parsed) + .filter(([key]) => key !== 'optional') + .map(([key, val]) => `${key}=${JSON.stringify(val)}`) + .join(', '); + return summary ? `${command} (${summary})` : command; + } catch { + return command; + } +} + +/** + * Parses TestingBot's Maestro JUnit XML into Allure results. Returns one + * result per . Unknown structure yields an empty array rather than + * throwing so a report download never fails the run. + */ +export function junitToAllureResults( + xml: string, + context: AllureContext, +): AllureResult[] { + const results: AllureResult[] = []; + const suiteRe = /]*)>([\s\S]*?)<\/testsuite>/g; + let suiteMatch: RegExpExecArray | null; + let sawSuite = false; + + const handleSuite = (suiteAttrs: Record, body: string) => { + const caseRe = /]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g; + let caseMatch: RegExpExecArray | null; + while ((caseMatch = caseRe.exec(body)) !== null) { + const attrs = parseAttributes(caseMatch[1]); + const inner = caseMatch[2] ?? ''; + results.push(buildResult(attrs, inner, suiteAttrs, context)); + } + }; + + while ((suiteMatch = suiteRe.exec(xml)) !== null) { + sawSuite = true; + handleSuite(parseAttributes(suiteMatch[1]), suiteMatch[2]); + } + if (!sawSuite) { + handleSuite({}, xml); + } + return results; +} + +function buildResult( + attrs: Record, + inner: string, + suiteAttrs: Record, + context: AllureContext, +): AllureResult { + const name = attrs.name || attrs.id || 'flow'; + const fullName = `${attrs.classname || name}#${name}`; + + const steps: AllureStep[] = []; + const propRe = /]*)\/?>/g; + let propMatch: RegExpExecArray | null; + while ((propMatch = propRe.exec(inner)) !== null) { + const p = parseAttributes(propMatch[1]); + if (p.name !== 'step') continue; + const ts = p.timestamp ? Number(p.timestamp) : undefined; + steps.push({ + name: stepName(p.value, p.details), + status: stepStatus(p.status), + stage: 'finished', + ...(ts != null && !Number.isNaN(ts) && { start: ts, stop: ts }), + }); + } + // Maestro emits steps in reverse or arbitrary order; sort by timestamp. + steps.sort((a, b) => (a.start ?? 0) - (b.start ?? 0)); + + const failureMatch = + /]*)>([\s\S]*?)<\/failure>/.exec(inner) ?? + /]*)>([\s\S]*?)<\/error>/.exec(inner); + const failureAttrs = failureMatch ? parseAttributes(failureMatch[1]) : {}; + const failureText = failureMatch ? decodeXml(failureMatch[2]).trim() : ''; + const skipped = / s.start) + .filter((t): t is number => t != null); + const durationMs = Math.round(Number(attrs.time || 0) * 1000); + const fallbackStart = context.startedAt + ? new Date(context.startedAt).getTime() + : Date.now() - durationMs; + const start = timestamps.length > 0 ? Math.min(...timestamps) : fallbackStart; + const stop = Math.max( + start + durationMs, + timestamps.length > 0 ? Math.max(...timestamps) : start, + ); + + const labels = [ + { name: 'framework', value: 'maestro' }, + { name: 'language', value: 'yaml' }, + { name: 'suite', value: suiteAttrs.name || 'Maestro' }, + { name: 'testClass', value: attrs.classname || name }, + { name: 'host', value: `testingbot-run-${context.runId}` }, + ...(context.device ? [{ name: 'device', value: context.device }] : []), + ...(context.platform + ? [{ name: 'platform', value: context.platform }] + : []), + ...(context.osVersion + ? [{ name: 'osVersion', value: context.osVersion }] + : []), + ]; + + const message = failureAttrs.message || failureText.split('\n')[0] || ''; + + return { + uuid: randomUUID(), + historyId: createHash('md5').update(fullName).digest('hex'), + name, + fullName, + status, + stage: 'finished', + start, + stop, + labels, + ...(failureMatch && { + statusDetails: { + message, + ...(failureText && failureText !== message && { trace: failureText }), + }, + }), + steps, + }; +} + +/** + * Writes results into `/allure-results`, one `-result.json` + * per test, and returns the directory. Existing files are left in place so + * several runs (or shards) can accumulate into one report. + */ +export async function writeAllureResults( + outputDir: string, + results: AllureResult[], +): Promise { + const dir = path.join(outputDir, 'allure-results'); + await fs.promises.mkdir(dir, { recursive: true }); + await Promise.all( + results.map((result) => + fs.promises.writeFile( + path.join(dir, `${result.uuid}-result.json`), + JSON.stringify(result, null, 2), + 'utf-8', + ), + ), + ); + return dir; +} diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 76cf115..9d66b10 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -1494,6 +1494,225 @@ describe('TestingBotCTL CLI', () => { }); }); + describe('drop-in flags: metadata, exclude-flows, Maestro Cloud aliases', () => { + beforeEach(() => { + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockMaestroRun.mockResolvedValue({ + success: true, + outcome: 'passed', + runs: [], + }); + }); + + type Opts = { + metadata?: Record; + excludeFlows?: string[]; + app: string; + flows: string[]; + device?: string; + platformName?: string; + version?: string; + report?: string; + reportOutputDir?: string; + name?: string; + }; + + test('--branch, --pr-url and --metadata land in run metadata', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--branch', + 'main', + '--pr-url', + 'https://github.com/o/r/pull/7', + '--commit-sha', + 'abc', + '-m', + 'team=mobile', + '--metadata', + 'env=staging=eu', + ]); + expect(lastConstructorOptions(Maestro).metadata).toEqual({ + commitSha: 'abc', + pullRequestUrl: 'https://github.com/o/r/pull/7', + branch: 'main', + custom: { team: 'mobile', env: 'staging=eu' }, + }); + }); + + test('metadata is omitted entirely when no CI flag is given', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + ]); + expect(lastConstructorOptions(Maestro).metadata).toBeUndefined(); + }); + + test('--metadata without KEY=VALUE is rejected', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '-m', + 'novalue', + ]); + expect(mockMaestroRun).not.toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Invalid --metadata entry "novalue"'), + ); + expect(process.exitCode).toBe(1); + }); + + test('--exclude-flows accepts comma-separated and repeated values', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--exclude-flows', + 'a.yaml, ./wip', + '--exclude-flows', + '**/slow-*.yaml', + ]); + expect(lastConstructorOptions(Maestro).excludeFlows).toEqual([ + 'a.yaml', + './wip', + '**/slow-*.yaml', + ]); + }); + + test('--report allure is accepted', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--report', + 'ALLURE', + '--report-output-dir', + './reports', + ]); + expect(lastConstructorOptions(Maestro).report).toBe('allure'); + }); + + test('a maestro cloud command line runs unchanged via hidden aliases', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + '--apiKey', + 'k', + '--app-file', + 'app.zip', + '--flows', + './flows,./smoke', + '--device-model', + 'iPhone-17-Pro', + '--device-os', + 'iOS-18-2', + '--format', + 'JUNIT', + '--output', + 'out/report.xml', + '--test-suite-name', + 'nightly', + ]); + expect(mockGetCredentials).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: 'k' }), + ); + const opts = lastConstructorOptions(Maestro); + expect(opts.app).toBe('app.zip'); + expect(opts.flows).toEqual(['./flows', './smoke']); + expect(opts.device).toBe('iPhone 17 Pro'); + expect(opts.platformName).toBe('iOS'); + expect(opts.version).toBe('18.2'); + expect(opts.report).toBe('junit'); + expect(opts.reportOutputDir).toMatch(/[\\/]out$/); + expect(opts.name).toBe('nightly'); + }); + + test('--device-os android-34 maps the API level to Android 14', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--device-os', + 'android-34', + ]); + const opts = lastConstructorOptions(Maestro); + expect(opts.platformName).toBe('Android'); + expect(opts.version).toBe('14'); + }); + + test('canonical flags win over aliases', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + '--app', + 'real.apk', + '--app-file', + 'alias.apk', + './flows', + '--device', + 'Pixel 9', + '--device-model', + 'pixel_7', + ]); + const opts = lastConstructorOptions(Maestro); + expect(opts.app).toBe('real.apk'); + expect(opts.device).toBe('Pixel 9'); + }); + + test('--format NOOP means no report; other values are rejected', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--format', + 'NOOP', + ]); + expect(lastConstructorOptions(Maestro).report).toBeUndefined(); + + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--format', + 'PDF', + ]); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Invalid --format "PDF"'), + ); + expect(process.exitCode).toBe(1); + }); + + test('aliases are hidden from help', () => { + const maestroCmd = program.commands.find((c) => c.name() === 'maestro')!; + const help = maestroCmd.helpInformation(); + expect(help).not.toContain('--app-file'); + expect(help).not.toContain('--device-os'); + expect(help).toContain('--exclude-flows'); + expect(help).toContain('--metadata'); + }); + }); + test('unknown command should show help', async () => { const exitSpy = jest .spyOn(process, 'exit') diff --git a/tests/providers/maestro.test.ts b/tests/providers/maestro.test.ts index 3cdf121..52f548a 100644 --- a/tests/providers/maestro.test.ts +++ b/tests/providers/maestro.test.ts @@ -7150,4 +7150,117 @@ onFlowStart: } }); }); + + describe('--exclude-flows', () => { + const files = () => [ + '/proj/flows/login.yaml', + '/proj/flows/checkout.yaml', + '/proj/flows/wip/draft.yaml', + '/proj/flows/slow-sync.yaml', + ]; + + const withExclusions = (excludeFlows: string[]) => + new Maestro( + mockCredentials, + new MaestroOptions('app.apk', '/proj/flows', undefined, { + excludeFlows, + }), + ); + + it('is a no-op without patterns', async () => { + const list = files(); + await expect(maestro['applyFlowExclusions'](list)).resolves.toBe(0); + expect(list).toHaveLength(4); + }); + + it('drops exact files and everything under a directory', async () => { + fs.promises.stat = jest.fn().mockImplementation(async (p: string) => ({ + isDirectory: () => String(p).endsWith('/wip'), + isFile: () => !String(p).endsWith('/wip'), + })); + const m = withExclusions(['/proj/flows/login.yaml', '/proj/flows/wip']); + const list = files(); + await expect(m['applyFlowExclusions'](list)).resolves.toBe(2); + expect(list).toEqual([ + '/proj/flows/checkout.yaml', + '/proj/flows/slow-sync.yaml', + ]); + }); + + it('expands glob patterns', async () => { + const { glob } = jest.requireMock('glob') as { glob: jest.Mock }; + glob.mockResolvedValueOnce(['/proj/flows/slow-sync.yaml']); + const m = withExclusions(['/proj/flows/slow-*.yaml']); + const list = files(); + await expect(m['applyFlowExclusions'](list)).resolves.toBe(1); + expect(glob).toHaveBeenCalledWith('/proj/flows/slow-*.yaml'); + expect(list).not.toContain('/proj/flows/slow-sync.yaml'); + }); + + it('ignores patterns that match nothing', async () => { + fs.promises.stat = jest.fn().mockRejectedValue(new Error('ENOENT')); + const m = withExclusions(['/proj/flows/missing.yaml']); + const list = files(); + await expect(m['applyFlowExclusions'](list)).resolves.toBe(0); + expect(list).toHaveLength(4); + }); + }); + + describe('--report allure', () => { + it('converts the junit report into allure-results', async () => { + const m = new Maestro( + mockCredentials, + MaestroOptions.forExistingProject({ + report: 'allure', + reportOutputDir: '/tmp/reports', + }), + ); + m['appId'] = 1234; + axios.get = jest.fn().mockResolvedValue({ + data: { + junit_report: + '', + }, + headers: {}, + }); + fs.promises.mkdir = jest.fn().mockResolvedValue(undefined); + fs.promises.writeFile = jest.fn().mockResolvedValue(undefined); + + await m['fetchReports']([ + { + id: 5678, + status: 'DONE', + capabilities: { + deviceName: 'Pixel 6', + platformName: 'Android', + version: '14', + }, + success: 1, + } as never, + ]); + + expect(axios.get).toHaveBeenCalledWith( + 'https://api.testingbot.com/v1/app-automate/maestro/1234/5678/junit_report', + expect.anything(), + ); + expect(fs.promises.mkdir).toHaveBeenCalledWith( + path.join('/tmp/reports', 'allure-results'), + { recursive: true }, + ); + expect(fs.promises.writeFile).toHaveBeenCalledTimes(1); + const [file, body] = (fs.promises.writeFile as jest.Mock).mock.calls[0]; + expect(String(file)).toMatch( + /allure-results[\\/][0-9a-f-]{36}-result\.json$/, + ); + expect(JSON.parse(String(body))).toMatchObject({ + name: 'login', + status: 'passed', + labels: expect.arrayContaining([ + { name: 'device', value: 'Pixel 6' }, + { name: 'platform', value: 'Android' }, + { name: 'osVersion', value: '14' }, + ]), + }); + }); + }); }); diff --git a/tests/utils/allure.test.ts b/tests/utils/allure.test.ts new file mode 100644 index 0000000..34768c6 --- /dev/null +++ b/tests/utils/allure.test.ts @@ -0,0 +1,129 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + junitToAllureResults, + writeAllureResults, +} from '../../src/utils/allure'; + +const PASSING_XML = ` + + + + + + + + +`; + +const FAILING_XML = ` + + + + + Element not found: Sign in + at tapOn (login.yaml:4) + + + +`; + +describe('junitToAllureResults', () => { + it('maps a passing Maestro testcase with ordered steps and labels', () => { + const results = junitToAllureResults(PASSING_XML, { + runId: 42, + device: 'iPhone 17 Pro', + platform: 'iOS', + osVersion: '26.5', + }); + expect(results).toHaveLength(1); + const result = results[0]; + expect(result).toMatchObject({ + name: 'flow_app', + fullName: 'flow_app#flow_app', + status: 'passed', + stage: 'finished', + }); + expect(result.uuid).toMatch(/^[0-9a-f-]{36}$/); + expect(result.historyId).toHaveLength(32); + expect(result.statusDetails).toBeUndefined(); + + // Steps sorted by timestamp, command suffix stripped, details summarised. + expect(result.steps.map((s) => s.name)).toEqual([ + 'applyConfiguration (config={"appId":"com.example"})', + 'inputText (text="hello")', + 'tapOnElement (selector={"idRegex":"btn"})', + ]); + expect(result.steps.every((s) => s.status === 'passed')).toBe(true); + expect(result.start).toBe(1788290360461); + expect(result.stop).toBeGreaterThanOrEqual(1788290376347); + expect(result.stop).toBe(1788290360461 + 18000); + + const label = (name: string) => + result.labels.find((l) => l.name === name)?.value; + expect(label('framework')).toBe('maestro'); + expect(label('suite')).toBe('Test Suite'); + expect(label('host')).toBe('testingbot-run-42'); + expect(label('device')).toBe('iPhone 17 Pro'); + expect(label('platform')).toBe('iOS'); + expect(label('osVersion')).toBe('26.5'); + }); + + it('marks failures with message and trace, and honours ', () => { + const [login, checkout] = junitToAllureResults(FAILING_XML, { runId: 1 }); + expect(login.status).toBe('failed'); + expect(login.statusDetails).toEqual({ + message: 'Element not found: Sign in', + trace: 'Element not found: Sign in\n at tapOn (login.yaml:4)', + }); + expect(login.steps.map((s) => s.status)).toEqual(['passed', 'failed']); + expect(checkout.status).toBe('skipped'); + expect(checkout.steps).toEqual([]); + }); + + it('uses startedAt as the start when steps carry no timestamps', () => { + const [checkout] = junitToAllureResults( + ``, + { runId: 1, startedAt: '2026-01-01T00:00:00Z' }, + ); + const expectedStart = Date.parse('2026-01-01T00:00:00Z'); + expect(checkout.start).toBe(expectedStart); + expect(checkout.stop).toBe(expectedStart + 2000); + expect(checkout.status).toBe('passed'); + }); + + it('handles testcases without a surrounding testsuite and empty input', () => { + expect( + junitToAllureResults(``, { runId: 1 }), + ).toHaveLength(1); + expect(junitToAllureResults('', { runId: 1 })).toEqual([]); + expect(junitToAllureResults('not xml at all', { runId: 1 })).toEqual([]); + }); + + it('gives each call fresh uuids but a stable historyId per flow', () => { + const a = junitToAllureResults(PASSING_XML, { runId: 1 })[0]; + const b = junitToAllureResults(PASSING_XML, { runId: 2 })[0]; + expect(a.uuid).not.toBe(b.uuid); + expect(a.historyId).toBe(b.historyId); + }); +}); + +describe('writeAllureResults', () => { + it('writes one -result.json per result under allure-results', async () => { + const mkdir = jest.spyOn(fs.promises, 'mkdir').mockResolvedValue(undefined); + const writeFile = jest + .spyOn(fs.promises, 'writeFile') + .mockResolvedValue(undefined); + const results = junitToAllureResults(FAILING_XML, { runId: 1 }); + + const dir = await writeAllureResults('/tmp/reports', results); + + expect(dir).toBe(path.join('/tmp/reports', 'allure-results')); + expect(mkdir).toHaveBeenCalledWith(dir, { recursive: true }); + expect(writeFile).toHaveBeenCalledTimes(2); + const [file, body] = writeFile.mock.calls[0]; + expect(String(file)).toBe(path.join(dir, `${results[0].uuid}-result.json`)); + expect(JSON.parse(String(body))).toMatchObject({ name: 'login' }); + jest.restoreAllMocks(); + }); +});