diff --git a/.gitignore b/.gitignore index 6197c849..117af42f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,20 @@ dist-ssr /*.ts /*.json *.tgz -example/wdio-*.json +examples/wdio/wdio-*.json +examples/wdio/wdio-*.webm +examples/nightwatch/logs/ + +# Adapter-encoded screencasts (written next to the project root by default) +selenium-video-*.webm +nightwatch-video-*.webm +packages/nightwatch-devtools/nightwatch-video-*.webm + +# trace.zip output (mode: 'trace') +trace-*.zip + +# vitest --coverage output +coverage/ # pnpm state, cache, logs, and debug files /packages/**/*.mjs diff --git a/README.md b/README.md index 62a6448f..a68f3e83 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,27 @@ Real-time capture of browser-side events through the WebDriver BiDi protocol β€” When BiDi is active in Selenium or Nightwatch, the per-command Chrome performance-log network-capture path is gated off so requests don't appear twice in the dashboard. The attach + sink logic lives in `@wdio/devtools-core`'s `bidi.ts` β€” same module both adapters consume. +### πŸ“¦ Trace mode (trace.zip) + +Headless capture path β€” no DevTools UI window opens. At session end the adapter writes a `trace-.zip` next to the user's spec / config file, suitable for offline replay, AI-agent diffing, or any consumer that prefers a portable artifact over a live UI. + +| Adapter | How to enable | +|---|---| +| **WebdriverIO** | `services: [['devtools', { mode: 'trace' }]]` | +| **Selenium** | `DevTools.configure({ mode: 'trace' })` (before importing `selenium-webdriver`) | +| **Nightwatch** | `globals: nightwatchDevtools({ mode: 'trace' })` | + +The zip contains: +- `trace.trace` β€” NDJSON `context-options` + `before`/`after` action events +- `trace.network` β€” HAR-style network entries derived from the existing capture +- `resources/page@-.jpeg` β€” screenshot per user-facing action +- `resources/elements-page@-.json` β€” flat interactable element list from `@wdio/elements` +- `resources/snapshot-page@-.txt` β€” depth-indented accessibility-tree snapshot (AI-friendly) + +What counts as a user-facing action is filtered through an allow-list in `@wdio/devtools-core/action-mapping.ts` (`url`, `click`, `setValue`, `sendKeys`, `get`, etc.). Internal commands like `findElement`/`waitUntil`/`executeScript` don't produce trace entries. + +Trace mode and live mode are **mutually exclusive** β€” `screencast` options are ignored in trace mode (live-mode feature). Live and trace serve different audiences (humans debugging vs. agents diffing), and stacking them only costs perf. + ### πŸ”οΈŽ TestLens - **Code Intelligence**: View test definitions directly in your editor - **Run/Debug Actions**: Execute individual tests or suites with inline CodeLens actions diff --git a/examples/nightwatch/nightwatch.conf.cjs b/examples/nightwatch/nightwatch.conf.cjs index a817bf57..619d1a76 100644 --- a/examples/nightwatch/nightwatch.conf.cjs +++ b/examples/nightwatch/nightwatch.conf.cjs @@ -45,6 +45,7 @@ module.exports = { // off to avoid duplicate entries. globals: nightwatchDevtools({ port: 3000, + mode: 'trace', screencast: { enabled: true, pollIntervalMs: 200 }, bidi: true }) diff --git a/examples/selenium/jest-test/test/example.js b/examples/selenium/jest-test/test/example.js index 15e9d21a..a59b252b 100644 --- a/examples/selenium/jest-test/test/example.js +++ b/examples/selenium/jest-test/test/example.js @@ -12,7 +12,7 @@ const VALID_USERNAME = 'tomsmith' const VALID_PASSWORD = 'SuperSecretPassword!' DevTools.configure({ - screencast: { enabled: true, quality: 70, maxWidth: 1280, maxHeight: 720 }, + mode: 'trace', headless: true }) diff --git a/examples/wdio/wdio.conf.ts b/examples/wdio/wdio.conf.ts index 73e88052..71830cf2 100644 --- a/examples/wdio/wdio.conf.ts +++ b/examples/wdio/wdio.conf.ts @@ -127,21 +127,7 @@ export const config: Options.Testrunner = { // Services take over a specific job you don't want to take care of. They enhance // your test setup with almost no effort. Unlike plugins, they don't add new // commands. Instead, they hook themselves up into the test process. - services: [ - 'devtools' - // [ - // 'devtools', - // { - // screencast: { - // enabled: true, - // captureFormat: 'jpeg', // 'jpeg' or 'png' β€” frame format sent by Chrome over CDP - // quality: 70, // JPEG quality 0–100 - // maxWidth: 1280, // max frame width in px - // maxHeight: 720 // max frame height in px - // } - // } - // ] - ], + services: [['devtools', { mode: 'trace' as const }]], // // Framework you want to run your specs with. // The following are supported: Mocha, Jasmine, and Cucumber diff --git a/packages/core/package.json b/packages/core/package.json index b637cd88..b7b6ad91 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -27,9 +27,11 @@ "license": "MIT", "devDependencies": { "@types/ws": "^8.18.1", + "@types/yazl": "^2.4.6", "@wdio/devtools-script": "workspace:*", "@wdio/devtools-shared": "workspace:^", "stacktrace-parser": "^0.1.11", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" } } diff --git a/packages/core/src/action-mapping.ts b/packages/core/src/action-mapping.ts new file mode 100644 index 00000000..43a39802 --- /dev/null +++ b/packages/core/src/action-mapping.ts @@ -0,0 +1,65 @@ +// Allow-list mapping from runner-native command names to trace +// vocabulary. Ported from Vince Graics' PR #209 (`@wdio/tracing-service`); the +// existing devtools UI uses its own denylist (`INTERNAL_COMMANDS`) β€” this map +// is for the trace.zip exporter to filter + rename in one step. + +export interface TraceAction { + class: string + method: string +} + +const ACTION_MAP: Record = { + // WDIO browser-level + url: { class: 'Page', method: 'navigate' }, + navigateTo: { class: 'Page', method: 'navigate' }, + back: { class: 'Page', method: 'goBack' }, + forward: { class: 'Page', method: 'goForward' }, + refresh: { class: 'Page', method: 'reload' }, + newWindow: { class: 'Page', method: 'goto' }, + // Selenium WebDriver navigation (driver.get, driver.navigate().to/back/forward/refresh) + get: { class: 'Page', method: 'navigate' }, + to: { class: 'Page', method: 'navigate' }, + // WDIO element-level + click: { class: 'Element', method: 'click' }, + doubleClick: { class: 'Element', method: 'dblclick' }, + setValue: { class: 'Element', method: 'fill' }, + selectByVisibleText: { class: 'Element', method: 'selectOption' }, + moveTo: { class: 'Element', method: 'hover' }, + scrollIntoView: { class: 'Element', method: 'scrollIntoViewIfNeeded' }, + dragAndDrop: { class: 'Element', method: 'dragTo' }, + // Selenium WebElement actions + sendKeys: { class: 'Element', method: 'fill' }, + clear: { class: 'Element', method: 'clear' }, + submit: { class: 'Element', method: 'submit' }, + // Cross-runner + keys: { class: 'Keyboard', method: 'press' }, + execute: { class: 'Page', method: 'evaluate' }, + executeAsync: { class: 'Page', method: 'evaluate' }, + switchToFrame: { class: 'Frame', method: 'goto' }, + touchAction: { class: 'Element', method: 'tap' } +} + +// Excluded by design: +// clearValue / addValue β€” WDIO fires these inside setValue (duplicate events). +// executeScript β€” Selenium's `until` polling fires it ~50ms; also recurses +// because @wdio/elements uses executeScript inside captureActionSnapshot. +// WDIO's user-facing `execute`/`executeAsync` are still captured. + +export function mapCommandToAction(command: string): TraceAction | null { + return ACTION_MAP[command] ?? null +} + +export function formatActionTitle( + action: TraceAction, + args: unknown[], + params?: Record +): string { + const firstArg = args[0] ?? params?.selector + if (firstArg === undefined) { + return `${action.class}.${action.method}()` + } + const label = ( + typeof firstArg === 'object' ? JSON.stringify(firstArg) : String(firstArg) + ).slice(0, 80) + return `${action.class}.${action.method}("${label}")` +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e34b6ac4..67879f93 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,7 +1,11 @@ // Framework-agnostic capture/reporter logic shared by @wdio/devtools-* // adapters. See ARCHITECTURE.md Β§2 and CLAUDE.md Β§2.2. +export * from './action-mapping.js' export * from './assert-patcher.js' +export * from './trace-exporter.js' +export * from './trace-har.js' +export * from './trace-zip-writer.js' export * from './bidi.js' export * from './console.js' export * from './uid.js' diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts new file mode 100644 index 00000000..cfcaef9c --- /dev/null +++ b/packages/core/src/trace-exporter.ts @@ -0,0 +1,311 @@ +// Converts a captured TraceLog into a trace.zip Buffer. +// Stays runner-agnostic so the three adapters can call this directly. + +import fs from 'node:fs/promises' +import path from 'node:path' +import type { + ActionSnapshot, + CommandLog, + ConsoleLog, + Metadata, + NetworkRequest, + TraceLog, + TraceMutation +} from '@wdio/devtools-shared' +import { formatActionTitle, mapCommandToAction } from './action-mapping.js' +import { networkRequestToHar } from './trace-har.js' +import { buildTraceZip, type TraceZipResource } from './trace-zip-writer.js' + +const TRACE_VERSION = 8 +const LIBRARY_NAME = '@wdio/devtools-core' +const LIBRARY_VERSION = '1.0.0' + +interface ContextOptionsEvent { + version: number + type: 'context-options' + origin: 'library' + libraryName: string + libraryVersion: string + browserName: string + platform: string + wallTime: number + monotonicTime: number + sdkLanguage: string + title: string + contextId: string + options: { viewport: { width: number; height: number } } +} + +interface BeforeEvent { + type: 'before' + callId: string + startTime: number + class: string + method: string + pageId: string + params: Record + title: string +} + +interface AfterEvent { + type: 'after' + callId: string + endTime: number + error?: { message: string } +} + +interface ScreencastFrameEvent { + type: 'screencast-frame' + pageId: string + sha1: string + elements?: string + width: number + height: number + timestamp: number +} + +type TraceEvent = + | ContextOptionsEvent + | BeforeEvent + | AfterEvent + | ScreencastFrameEvent + +function shortId(sessionId?: string): string { + return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) +} + +function buildContextOptions( + trace: TraceLog, + contextId: string, + wallTime: number +): ContextOptionsEvent { + const caps = trace.metadata.capabilities as + | Record + | undefined + const browserName = (caps?.browserName as string) ?? 'chromium' + const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } + return { + version: TRACE_VERSION, + type: 'context-options', + origin: 'library', + libraryName: LIBRARY_NAME, + libraryVersion: LIBRARY_VERSION, + browserName, + platform: process.platform, + wallTime, + monotonicTime: 0, + sdkLanguage: 'javascript', + title: browserName, + contextId, + options: { + viewport: { width: viewport.width, height: viewport.height } + } + } +} + +function buildActionEvents( + commands: CommandLog[], + pageId: string, + wallTime: number +): TraceEvent[] { + const events: TraceEvent[] = [] + // cmd.timestamp records command completion, so the *previous* mapped + // command's timestamp is a usable startTime for the next one. + let prevEndMs = 0 + let callCounter = 0 + for (const cmd of commands) { + const action = mapCommandToAction(cmd.command) + if (!action) { + continue + } + callCounter++ + const callId = `call@${callCounter}` + const endMs = Math.max(prevEndMs, cmd.timestamp - wallTime) + const params: Record = Object.fromEntries( + cmd.args.map((a, i) => [String(i), a]) + ) + events.push({ + type: 'before', + callId, + startTime: prevEndMs, + class: action.class, + method: action.method, + pageId, + params, + title: formatActionTitle(action, cmd.args, params) + }) + const afterEvent: AfterEvent = { + type: 'after', + callId, + endTime: endMs + } + if (cmd.error) { + const err = cmd.error as { message?: string } + afterEvent.error = { message: err.message ?? String(cmd.error) } + } + events.push(afterEvent) + prevEndMs = endMs + } + return events +} + +function buildNetworkNdjson(requests: NetworkRequest[]): Buffer { + if (!requests.length) { + return Buffer.alloc(0) + } + const lines = requests.map((r) => JSON.stringify(networkRequestToHar(r))) + return Buffer.from(lines.join('\n'), 'utf8') +} + +function buildSnapshotResources( + snapshots: ActionSnapshot[], + pageId: string +): TraceZipResource[] { + const out: TraceZipResource[] = [] + for (const snap of snapshots) { + const base = `${pageId}-${snap.timestamp}` + if (snap.screenshot) { + out.push({ + resourceName: `${base}.jpeg`, + data: Buffer.from(snap.screenshot, 'base64') + }) + } + if (snap.elements && snap.elements.length) { + out.push({ + resourceName: `elements-${base}.json`, + data: Buffer.from(JSON.stringify(snap.elements), 'utf8') + }) + } + if (snap.snapshotText) { + out.push({ + resourceName: `snapshot-${base}.txt`, + data: Buffer.from(snap.snapshotText, 'utf8') + }) + } + } + return out +} + +function buildScreencastFrames( + snapshots: ActionSnapshot[], + pageId: string, + wallTime: number, + viewport: { width: number; height: number } +): ScreencastFrameEvent[] { + return snapshots + .filter((s) => s.screenshot) + .map((s) => { + const base = `${pageId}-${s.timestamp}` + const frame: ScreencastFrameEvent = { + type: 'screencast-frame', + pageId, + sha1: `${base}.jpeg`, + width: viewport.width, + height: viewport.height, + timestamp: Math.max(0, s.timestamp - wallTime) + } + if (s.elements && s.elements.length) { + frame.elements = `elements-${base}.json` + } + return frame + }) +} + +/** + * Build a trace.zip buffer from the captured TraceLog. + * Filters commands through ACTION_MAP and renames to trace vocabulary; + * network entries become HAR resource-snapshots; per-action screenshots, + * element JSON, and snapshot text are written under `resources/`. + */ +export async function exportTraceZip( + trace: TraceLog, + opts: { sessionId?: string; wallTimeOverride?: number } = {} +): Promise { + // wallTime anchors monotonic offsets at the first captured command so + // subsequent actions render at positive deltas in the trace viewer. + const firstCommandTs = trace.commands[0]?.timestamp + const wallTime = opts.wallTimeOverride ?? firstCommandTs ?? Date.now() + const idPrefix = shortId(opts.sessionId) + const contextId = `context@${idPrefix}` + const pageId = `page@${idPrefix}` + const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } + const snapshots = trace.actionSnapshots ?? [] + const events: TraceEvent[] = [ + buildContextOptions(trace, contextId, wallTime), + ...buildScreencastFrames(snapshots, pageId, wallTime, viewport), + ...buildActionEvents(trace.commands, pageId, wallTime) + ] + const traceNdjson = events.map((e) => JSON.stringify(e)).join('\n') + const networkNdjson = buildNetworkNdjson(trace.networkRequests) + const resources = buildSnapshotResources(snapshots, pageId) + return buildTraceZip({ traceNdjson, networkNdjson, resources }) +} + +/** Minimum capturer surface needed to assemble a TraceLog. */ +export interface TraceCapturer { + mutations: TraceMutation[] + traceLogs: string[] + consoleLogs: ConsoleLog[] + networkRequests: NetworkRequest[] + commandsLog: CommandLog[] + sources: Map + metadata?: Metadata +} + +export interface WriteTraceZipOptions { + outputDir: string + sessionId: string + capabilities?: unknown + /** + * Per-action snapshots from a Phase-3-style hook. When omitted, snapshots + * are synthesized from CommandLog entries that carry a screenshot so the + * viewer still renders thumbnails for adapters without an action hook. + */ + actionSnapshots?: ActionSnapshot[] +} + +/** + * Build a TraceLog from a SessionCapturer-shaped source and write a + * Trace.zip. Returns the absolute path written. + */ +export async function writeTraceZip( + capturer: TraceCapturer, + opts: WriteTraceZipOptions +): Promise { + const baseMetadata = capturer.metadata ?? ({} as Metadata) + const actionSnapshots = + opts.actionSnapshots ?? + synthesizeSnapshotsFromCommands(capturer.commandsLog) + const traceLog: TraceLog = { + mutations: capturer.mutations, + logs: capturer.traceLogs, + consoleLogs: capturer.consoleLogs, + networkRequests: capturer.networkRequests, + metadata: { + ...baseMetadata, + ...(opts.capabilities + ? { capabilities: opts.capabilities as Metadata['capabilities'] } + : {}) + }, + commands: capturer.commandsLog, + sources: Object.fromEntries(capturer.sources), + ...(actionSnapshots.length ? { actionSnapshots } : {}) + } + const zip = await exportTraceZip(traceLog, { sessionId: opts.sessionId }) + await fs.mkdir(opts.outputDir, { recursive: true }) + const zipPath = path.join(opts.outputDir, `trace-${opts.sessionId}.zip`) + await fs.writeFile(zipPath, zip) + return zipPath +} + +function synthesizeSnapshotsFromCommands( + commands: CommandLog[] +): ActionSnapshot[] { + return commands + .filter((c) => c.screenshot && mapCommandToAction(c.command)) + .map((c) => ({ + timestamp: c.timestamp, + command: c.command, + screenshot: c.screenshot + })) +} diff --git a/packages/core/src/trace-har.ts b/packages/core/src/trace-har.ts new file mode 100644 index 00000000..565053ff --- /dev/null +++ b/packages/core/src/trace-har.ts @@ -0,0 +1,100 @@ +// Convert the existing NetworkRequest shape into trace format +// `resource-snapshot` NDJSON entries (HAR-flavoured) for trace.zip. + +import type { NetworkRequest } from '@wdio/devtools-shared' + +export interface ResourceSnapshotEntry { + type: 'resource-snapshot' + snapshot: { + startedDateTime: string + time: number + request: { + method: string + url: string + httpVersion: string + cookies: unknown[] + headers: { name: string; value: string }[] + queryString: { name: string; value: string }[] + headersSize: number + bodySize: number + } + response: { + status: number + statusText: string + httpVersion: string + cookies: unknown[] + headers: { name: string; value: string }[] + content: { size: number; mimeType: string } + redirectURL: string + headersSize: number + bodySize: number + } + cache: Record + timings: { send: number; wait: number; receive: number } + } +} + +function toHeaderArray( + h: Record | undefined +): { name: string; value: string }[] { + if (!h) { + return [] + } + return Object.entries(h).map(([name, value]) => ({ name, value })) +} + +function toQueryString(url: string): { name: string; value: string }[] { + try { + const u = new URL(url) + const out: { name: string; value: string }[] = [] + u.searchParams.forEach((value, name) => out.push({ name, value })) + return out + } catch { + return [] + } +} + +export function networkRequestToHar( + entry: NetworkRequest +): ResourceSnapshotEntry { + const startedDateTime = new Date(entry.timestamp).toISOString() + const duration = + entry.time ?? (entry.endTime ?? entry.startTime) - entry.startTime + const status = entry.response?.status ?? entry.status ?? 0 + const mimeType = entry.response?.mimeType ?? '' + const responseHeaders = entry.response?.headers ?? entry.responseHeaders + return { + type: 'resource-snapshot', + snapshot: { + startedDateTime, + time: Math.max(0, duration), + request: { + method: entry.method, + url: entry.url, + httpVersion: 'HTTP/1.1', + cookies: [], + headers: toHeaderArray(entry.requestHeaders ?? entry.headers), + queryString: toQueryString(entry.url), + headersSize: -1, + bodySize: entry.requestBody ? entry.requestBody.length : -1 + }, + response: { + status, + statusText: entry.statusText ?? '', + httpVersion: 'HTTP/1.1', + cookies: [], + headers: toHeaderArray(responseHeaders), + content: { size: entry.size ?? 0, mimeType }, + redirectURL: '', + headersSize: -1, + bodySize: entry.size ?? -1 + }, + cache: {}, + timings: { + send: -1, + wait: Math.max(0, duration), + receive: -1 + } + } + } +} diff --git a/packages/core/src/trace-zip-writer.ts b/packages/core/src/trace-zip-writer.ts new file mode 100644 index 00000000..f3dfac69 --- /dev/null +++ b/packages/core/src/trace-zip-writer.ts @@ -0,0 +1,35 @@ +// Thin yazl wrapper that packages a trace into a single Buffer. +// Ported from Vince Graics' PR #209. + +import yazl from 'yazl' + +export interface TraceZipResource { + /** Path inside the zip, e.g. `resources/page@xxx-12345.jpeg`. */ + resourceName: string + data: Buffer +} + +export interface TraceZipInputs { + /** NDJSON action events (one JSON object per line). */ + traceNdjson: string + /** NDJSON HAR resource-snapshot entries. Empty buffer when omitted. */ + networkNdjson: Buffer + /** Files written under `resources/` β€” typically screenshots + element snapshots. */ + resources: TraceZipResource[] +} + +export function buildTraceZip(inputs: TraceZipInputs): Promise { + return new Promise((resolve, reject) => { + const zipFile = new yazl.ZipFile() + zipFile.addBuffer(Buffer.from(inputs.traceNdjson, 'utf8'), 'trace.trace') + zipFile.addBuffer(inputs.networkNdjson, 'trace.network') + for (const resource of inputs.resources) { + zipFile.addBuffer(resource.data, `resources/${resource.resourceName}`) + } + zipFile.end() + const chunks: Buffer[] = [] + zipFile.outputStream.on('data', (chunk: Buffer) => chunks.push(chunk)) + zipFile.outputStream.on('end', () => resolve(Buffer.concat(chunks))) + zipFile.outputStream.on('error', reject) + }) +} diff --git a/packages/elements/.npmignore b/packages/elements/.npmignore new file mode 100644 index 00000000..344b429f --- /dev/null +++ b/packages/elements/.npmignore @@ -0,0 +1,7 @@ +src +node_modules +tests +index.html +tsconfig.json +vite.config.ts +*.tgz diff --git a/packages/elements/ROADMAP.md b/packages/elements/ROADMAP.md new file mode 100644 index 00000000..f387fb4b --- /dev/null +++ b/packages/elements/ROADMAP.md @@ -0,0 +1,81 @@ +# @wdio/elements Roadmap + +## Current state (May 2026) + +The package delivers LLM-readable element snapshots for both web and mobile: + +| Capability | Web | Mobile | +|---|---|---| +| Interactable element list | `getInteractableBrowserElements()` | `getMobileVisibleElements()` | +| Semantic tree | `getBrowserAccessibilityTree()` | *(raw `JSONElement` only)* | +| Snapshot serialization | `serializeWebSnapshot()` | `serializeMobileSnapshot()` | +| Unified API | `getElements()` returns both | `getElements()` returns both | +| Viewport filtering | `inViewportOnly` (default true) | `inViewportOnly` (default true) | +| Role classification | Computed in-browser from tag/ARIA | `ANDROID_ROLE_MAP` / `IOS_ROLE_MAP` in snapshot.ts | +| Locator generation | CSS selectors in browser script | `getSuggestedLocators()` from locator-generation.ts | +| Context disambiguation | `∈` via `inferPurpose()` | `∈` via `mobileInferPurpose()` | +| Duplicate selector indexing | N/A (selectors are unique) | `.instance(N)` suffix | + +## Architectural concerns + +### 1. Two independent mobile pipelines + +`serializeMobileSnapshot` in `snapshot.ts` has its own copies of: + +- **Role classification** β€” `ANDROID_ROLE_MAP` / `IOS_ROLE_MAP` duplicate logic from `locators/constants.ts` and `locators/element-filter.ts`. +- **Interactivity detection** β€” `isMobileInteractive()` shadows `isInteractableElement()` from `element-filter.ts`. They use different criteria (tag-based vs attribute-based) and can disagree. +- **Locator generation** β€” `getBestAndroidLocator()` / `getBestIOSLocator()` are simplified fallbacks. The full pipeline (`getSuggestedLocators()`) is now wired in when source XML is available, but the fallback still exists and the two paths can produce different selectors for the same element. + +These should be collapsed: `serializeMobileSnapshot` should consume pre-computed roles, interactivity flags, and selectors from the locator pipeline, not recompute them. + +### 2. No mobile equivalent of `getBrowserAccessibilityTree()` + +The web path returns a flat `AccessibilityNode[]` with roles, names, selectors, depths, and state. The mobile path returns a raw `JSONElement` tree β€” the snapshot does all enrichment internally via `collectMobileNodes()` β†’ `MobileFlatNode[]` (a private interface). There is no public function to get an enriched flat node list for mobile. + +**Proposal:** Extract `collectMobileNodes()` into a public `getMobileAccessibilityTree()` that returns `MobileFlatNode[]` (or a shared type). `serializeMobileSnapshot()` becomes a pure formatting pass β€” like `serializeWebSnapshot()` already is. + +### 3. Layout noise in mobile snapshots + +The Android view hierarchy includes every layout container (`FrameLayout`, `LinearLayout`, `ViewGroup`, etc.). The current noise filter (`NOISY_ROLES`) collapses anonymous containers at depth β‰₯ 2, but named containers and depth 0-1 scaffolding still appear. The web a11y tree doesn't have this problem because the browser's accessibility computation already skips layout-only `
`s. + +**Proposal:** A `collapseContainers` option on the snapshot (default `true`) that skips any container without an interactive descendant. Alternatively, the tree collection pass could flag "informative" vs "structural" containers and let the renderer decide. + +### 4. Selector format for mobile + +Mobile selectors are Appium/WDIO-specific strings (`~Accessibility`, `android=new UiSelector()...`, `id:com.example:id/foo`). The web path outputs CSS selectors (`a*=Highlights`, `#cart-icon-bubble`). An LLM/agent needs different selector parsing logic per platform. There's no common selector abstraction. + +**Proposal:** A `SelectorString` type with platform-aware parsing, or at minimum consistent prefix conventions documented for LLM consumption. + +### 5. The raw tree doesn't carry locators unless processed + +`getMobileVisibleElementsWithTree()` returns `{ elements, tree }` where `tree` is the raw `xmlToJSON()` output. Locators are only on `elements` (from `generateAllElementLocators()`). The snapshot reads locators by running `getSuggestedLocators()` again (or falling back). If a consumer wants to annotate the tree themselves, they must re-run the locator pipeline. + +**Proposal:** Enrich the tree in-place during `generateAllElementLocators()` β€” attach `_selector`, `_role`, and `_interactive` attributes to each `JSONElement` node that passes the filter. The raw tree becomes self-describing. + +## Improvement backlog + +| Priority | What | Effort | +|---|---|---| +| P0 | Merge `isMobileInteractive` + role classification into `generateAllElementLocators` β€” one source of truth | Medium | +| P1 | Extract `getMobileAccessibilityTree()` as a public API returning enriched flat nodes | Medium | +| P1 | Enrich `JSONElement` tree nodes with locators during `generateAllElementLocators()` | Small | +| P2 | `collapseContainers` option on `serializeMobileSnapshot` | Small | +| P2 | Unify web + mobile serialization into a single `serializeSnapshot()` function | Large | +| P3 | Document selector format conventions for LLM consumption | Small | +| P3 | Add `checked`/`selected`/`expanded` state rendering to mobile snapshot (parity with web) | Small | + +## Verified capabilities + +- [x] Web: viewport-only snapshot with semantic roles and unique CSS selectors +- [x] Web: `∈` disambiguation for duplicate selectors (6 "Add to Wishlist" buttons β†’ each with book title context) +- [x] Web: `statictext` role capturing visible text (book titles, promo copy, cookie text) +- [x] Web: deduplication of echoed text (child text already in parent name β†’ skipped) +- [x] Mobile: semantic role mapping (TextViewβ†’statictext, ImageViewβ†’img, Buttonβ†’button, etc.) +- [x] Mobile: full-pipeline selectors via `getSuggestedLocators()` wired into snapshot +- [x] Mobile: `~` prefix for accessibility-id, `id:` for resource-id, `android=new UiSelector()...` for compound +- [x] Mobile: `.instance(N)` indexing for duplicate selectors +- [x] Mobile: explicit tap-target promotion (clickable parent carries `β†’`, label children provide `∈` context) +- [x] Mobile: layout noise collapse for anonymous containers +- [x] Mobile: `∈` context from actual parent, not previous list-item sibling +- [x] Unified `getElements()` API returning `{ elements, tree }` for both platforms +- [x] `inViewportOnly` default `true` across all entry points with per-function toggles diff --git a/packages/elements/package.json b/packages/elements/package.json new file mode 100644 index 00000000..3c208c09 --- /dev/null +++ b/packages/elements/package.json @@ -0,0 +1,42 @@ +{ + "name": "@wdio/elements", + "version": "1.0.0", + "description": "Element detection scripts for WebdriverIO", + "author": "Vince Graics", + "license": "MIT", + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./locators": { + "types": "./dist/locators/index.d.ts", + "import": "./dist/locators/index.js" + } + }, + "types": "./dist/index.d.ts", + "repository": { + "type": "git", + "url": "git+https://github.com/webdriverio/devtools.git", + "directory": "packages/elements" + }, + "scripts": { + "build": "tsc -p ./tsconfig.json", + "lint": "eslint . --fix", + "test": "vitest run" + }, + "dependencies": { + "@xmldom/xmldom": "^0.9.8", + "xpath": "^0.0.34" + }, + "devDependencies": { + "@types/node": "25.5.2", + "@wdio/globals": "9.27.0", + "typescript": "6.0.2", + "vitest": "^4.0.16" + }, + "peerDependencies": { + "webdriverio": "^9.0.0" + } +} diff --git a/packages/elements/src/accessibility-tree.ts b/packages/elements/src/accessibility-tree.ts new file mode 100644 index 00000000..eab1aeb1 --- /dev/null +++ b/packages/elements/src/accessibility-tree.ts @@ -0,0 +1,485 @@ +/** + * Browser accessibility tree + * Single browser.execute() call: DOM walk β†’ flat accessibility node list + * + * NOTE: This script runs in browser context via browser.execute() + * It must be self-contained with no external dependencies + */ + +import { CONTAINER_ROLES, INPUT_TYPE_ROLES } from './aria-roles.js' + +export interface AccessibilityNode { + role: string + name: string + selector: string + depth: number + level: number | string + disabled: string + checked: string + expanded: string + selected: string + pressed: string + required: string + readonly: string + /** Whether the element's bounding rect intersects the viewport. */ + isInViewport?: boolean +} + +// Page-injected script: WDIO's `browser.execute` stringifies the arrow body +// and runs it in the browser. Module-level closure values are lost in +// stringification, so ARIA tables defined in aria-roles.ts are passed in +// as execute() args and re-bound here. Same constraint forces every helper +// (getRole/getAccessibleName/getSelector/...) to live inside the IIFE. +// eslint-disable-next-line max-lines-per-function +const accessibilityTreeScript = ( + inViewportOnly: boolean, + inputTypeRoles: Record, + containerRolesArr: readonly string[] +) => + // eslint-disable-next-line max-lines-per-function -- see comment above + (function () { + const INPUT_TYPE_ROLES = inputTypeRoles + const CONTAINER_ROLES = new Set(containerRolesArr) + + // ARIA role-resolution decision tree: each `if (tag === ...)` branch + // maps an HTML element to its implicit role per the WAI-ARIA spec. + // Splitting per-tag would scatter spec-equivalence groups across + // helpers without improving clarity. + // eslint-disable-next-line max-lines-per-function + function getRole(el: HTMLElement): string | null { + const explicit = el.getAttribute('role') + if (explicit) { + return explicit.split(' ')[0] + } + + const tag = el.tagName.toLowerCase() + + switch (tag) { + case 'button': + return 'button' + case 'a': + return el.hasAttribute('href') ? 'link' : null + case 'input': { + const type = (el.getAttribute('type') || 'text').toLowerCase() + if (type === 'hidden') { + return null + } + return INPUT_TYPE_ROLES[type] || 'textbox' + } + case 'select': + return 'combobox' + case 'textarea': + return 'textbox' + case 'h1': + case 'h2': + case 'h3': + case 'h4': + case 'h5': + case 'h6': + return 'heading' + case 'img': + return 'img' + case 'nav': + return 'navigation' + case 'main': + return 'main' + case 'header': + return !el.closest('article,aside,main,nav,section') ? 'banner' : null + case 'footer': + return !el.closest('article,aside,main,nav,section') + ? 'contentinfo' + : null + case 'aside': + return 'complementary' + case 'dialog': + return 'dialog' + case 'form': + return 'form' + case 'section': + return el.hasAttribute('aria-label') || + el.hasAttribute('aria-labelledby') + ? 'region' + : null + case 'summary': + return 'button' + case 'details': + return 'group' + case 'progress': + return 'progressbar' + case 'meter': + return 'meter' + case 'ul': + case 'ol': + return 'list' + case 'li': + return 'listitem' + case 'table': + return 'table' + } + + if ( + (el as HTMLElement & { contentEditable: string }).contentEditable === + 'true' + ) { + return 'textbox' + } + if ( + el.hasAttribute('tabindex') && + parseInt(el.getAttribute('tabindex') || '-1', 10) >= 0 + ) { + return 'generic' + } + + // Capture elements with visible direct text that don't match + // any semantic role β€” book titles, prices, labels, etc. + if (getDirectText(el)) { + return 'statictext' + } + + return null + } + + // Implements the WAI-ARIA Accessible Name Computation algorithm. + // The 6 sequential fallback steps (aria-label β†’ aria-labelledby β†’ + // tag-specific β†’ placeholder β†’ title β†’ childImg.alt) are spec-ordered; + // each step must run only if prior steps yielded nothing, so they don't + // factor into independent helpers without re-threading the "found" + // signal through every call. + // eslint-disable-next-line max-lines-per-function + function getAccessibleName(el: HTMLElement, role: string | null): string { + const ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { + return ariaLabel.trim() + } + + const labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + const texts = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.textContent?.trim() || '') + .filter(Boolean) + if (texts.length > 0) { + return texts.join(' ').slice(0, 200) + } + } + + const tag = el.tagName.toLowerCase() + + if ( + tag === 'img' || + (tag === 'input' && el.getAttribute('type') === 'image') + ) { + const alt = el.getAttribute('alt') + if (alt !== null) { + return alt.trim() + } + } + + if (['input', 'select', 'textarea'].includes(tag)) { + const id = el.getAttribute('id') + if (id) { + const label = document.querySelector(`label[for="${CSS.escape(id)}"]`) + if (label) { + return label.textContent?.trim() || '' + } + } + const parentLabel = el.closest('label') + if (parentLabel) { + const clone = parentLabel.cloneNode(true) as HTMLElement + clone + .querySelectorAll('input,select,textarea') + .forEach((n) => n.remove()) + const lt = clone.textContent?.trim() + if (lt) { + return lt + } + } + } + + const ph = el.getAttribute('placeholder') + if (ph) { + return ph.trim() + } + + const title = el.getAttribute('title') + if (title) { + return title.trim() + } + + // 9. Child β€” common pattern for image links and buttons + const childImg = el.querySelector('img') + if (childImg) { + const alt = childImg.getAttribute('alt') + if (alt) { + return alt.trim() + } + } + + if (role && CONTAINER_ROLES.has(role)) { + return '' + } + return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) + } + + // Selector synthesis: tries 6 strategies in priority order (text β†’ aria-label + // β†’ data-testid β†’ name attr β†’ id β†’ CSS path) and returns the first one that + // is unique on the page. Splitting per strategy would obscure the priority + // ordering and force re-uniqueness-checking across helpers. + // eslint-disable-next-line max-lines-per-function + function getSelector(element: HTMLElement): string { + const tag = element.tagName.toLowerCase() + + const text = element.textContent?.trim().replace(/\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + const sameTagElements = document.querySelectorAll(tag) + let matchCount = 0 + sameTagElements.forEach((el) => { + if (el.textContent?.includes(text)) { + matchCount++ + } + }) + if (matchCount === 1) { + return `${tag}*=${text}` + } + } + + const ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + const sel = `[aria-label="${CSS.escape(ariaLabel)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + const testId = element.getAttribute('data-testid') + if (testId) { + const sel = `[data-testid="${CSS.escape(testId)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + if (element.id) { + return `#${CSS.escape(element.id)}` + } + + const nameAttr = element.getAttribute('name') + if (nameAttr) { + const sel = `${tag}[name="${CSS.escape(nameAttr)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + if (element.className && typeof element.className === 'string') { + const classes = element.className.trim().split(/\s+/).filter(Boolean) + for (const cls of classes) { + const sel = `${tag}.${CSS.escape(cls)}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + if (classes.length >= 2) { + const sel = `${tag}${classes + .slice(0, 2) + .map((c) => `.${CSS.escape(c)}`) + .join('')}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + } + + let current: HTMLElement | null = element + const path: string[] = [] + while (current && current !== document.documentElement) { + let seg = current.tagName.toLowerCase() + if (current.id) { + path.unshift(`#${CSS.escape(current.id)}`) + break + } + const parent = current.parentElement + if (parent) { + const siblings = Array.from(parent.children).filter( + (c) => c.tagName === current!.tagName + ) + if (siblings.length > 1) { + seg += `:nth-of-type(${siblings.indexOf(current) + 1})` + } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { + break + } + } + return path.join(' > ') + } + + /** Extract text from immediate text-node children only (not nested elements). */ + function getDirectText(el: HTMLElement): string { + let text = '' + for (const child of Array.from(el.childNodes)) { + if (child.nodeType === 3 /* TEXT_NODE */) { + text += child.textContent + } + } + return text.trim().replace(/\s+/g, ' ') + } + + function isVisible(el: HTMLElement): boolean { + if (typeof el.checkVisibility === 'function') { + return el.checkVisibility({ + opacityProperty: true, + visibilityProperty: true, + contentVisibilityAuto: true + }) + } + const style = window.getComputedStyle(el) + return ( + style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + el.offsetWidth > 0 && + el.offsetHeight > 0 + ) + } + + function isInViewport(el: HTMLElement): boolean { + const rect = el.getBoundingClientRect() + return ( + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= + (window.innerHeight || document.documentElement.clientHeight) && + rect.right <= + (window.innerWidth || document.documentElement.clientWidth) + ) + } + + function getLevel(el: HTMLElement): number | undefined { + const m = el.tagName.toLowerCase().match(/^h([1-6])$/) + if (m) { + return parseInt(m[1], 10) + } + const ariaLevel = el.getAttribute('aria-level') + if (ariaLevel) { + return parseInt(ariaLevel, 10) + } + return undefined + } + + function getState(el: HTMLElement): Record { + const inputEl = el as HTMLInputElement + const isCheckable = + ['input', 'menuitemcheckbox', 'menuitemradio'].includes( + el.tagName.toLowerCase() + ) || + ['checkbox', 'radio', 'switch'].includes(el.getAttribute('role') || '') + return { + disabled: + el.getAttribute('aria-disabled') === 'true' || inputEl.disabled + ? 'true' + : '', + checked: + isCheckable && inputEl.checked + ? 'true' + : el.getAttribute('aria-checked') || '', + expanded: el.getAttribute('aria-expanded') || '', + selected: el.getAttribute('aria-selected') || '', + pressed: el.getAttribute('aria-pressed') || '', + required: + inputEl.required || el.getAttribute('aria-required') === 'true' + ? 'true' + : '', + readonly: + inputEl.readOnly || el.getAttribute('aria-readonly') === 'true' + ? 'true' + : '' + } + } + + type RawNode = Record + + const result: RawNode[] = [] + + function walk(el: HTMLElement, depth = 0): void { + if (depth > 200) { + return + } + if (!isVisible(el)) { + return + } + + const role = getRole(el) + const inViewport = isInViewport(el) + + if (!role) { + for (const child of Array.from(el.children)) { + walk(child as HTMLElement, depth + 1) + } + return + } + + // When viewport filtering is on, skip nodes outside the viewport. + // Still recurse into children β€” they may have different positioning + // (e.g. position:fixed elements inside an off-screen container). + if (inViewportOnly && !inViewport) { + for (const child of Array.from(el.children)) { + walk(child as HTMLElement, depth + 1) + } + return + } + + const name = getAccessibleName(el, role) + // Always generate a selector β€” even elements without an accessible + // name need a CSS-path fallback so the snapshot doesn't lose them. + const selector = getSelector(el) + const node: RawNode = { + role, + name, + selector, + depth, + level: getLevel(el) ?? '', + isInViewport: inViewport, + ...getState(el) + } + result.push(node) + + for (const child of Array.from(el.children)) { + walk(child as HTMLElement, depth + 1) + } + } + + for (const child of Array.from(document.body.children)) { + walk(child as HTMLElement, 0) + } + + return result + })() + +/** + * Get browser accessibility tree via a single DOM walk. + * + * @param browser WebdriverIO browser instance + * @param options {@link inViewportOnly} defaults to `true` β€” only nodes + * whose bounding rect intersects the viewport are included. + */ +// WDIO's typed Browser.execute overloads don't accept our generic injected +// script β€” narrow to a permissive execute() shape at the boundary instead. +type ExecuteLike = { + execute: (script: unknown, ...args: unknown[]) => Promise +} + +export async function getBrowserAccessibilityTree( + browser: WebdriverIO.Browser, + options: { inViewportOnly?: boolean } = {} +): Promise { + const { inViewportOnly = true } = options + return (browser as unknown as ExecuteLike).execute( + accessibilityTreeScript, + inViewportOnly, + INPUT_TYPE_ROLES, + CONTAINER_ROLES + ) as Promise +} diff --git a/packages/elements/src/aria-roles.ts b/packages/elements/src/aria-roles.ts new file mode 100644 index 00000000..b78736a9 --- /dev/null +++ b/packages/elements/src/aria-roles.ts @@ -0,0 +1,68 @@ +// WAI-ARIA role data tables shared between the accessibility-tree and +// browser-elements page-injected scripts. Defined at module level so they +// can be type-checked + reused; the consuming scripts receive them as +// arguments to `browser.execute()` since values declared at module scope +// don't survive the function-source stringification that injects the script. + +/** HTML β†’ implicit WAI-ARIA role. */ +export const INPUT_TYPE_ROLES: Record = { + text: 'textbox', + search: 'searchbox', + email: 'textbox', + url: 'textbox', + tel: 'textbox', + password: 'textbox', + number: 'spinbutton', + checkbox: 'checkbox', + radio: 'radio', + range: 'slider', + submit: 'button', + reset: 'button', + image: 'button', + file: 'button', + color: 'button' +} + +/** ARIA roles whose accessible name comes only from aria-label/labelledby, + * never from textContent (otherwise the section text leaks into the name). */ +export const CONTAINER_ROLES: readonly string[] = [ + 'navigation', + 'banner', + 'contentinfo', + 'complementary', + 'main', + 'form', + 'region', + 'group', + 'list', + 'listitem', + 'table', + 'row', + 'rowgroup', + 'generic' +] + +/** CSS selector matching all elements treated as interactable by the page-side + * element walker. Includes native form/anchor elements plus ARIA-role aliases. */ +export const INTERACTABLE_SELECTORS: readonly string[] = [ + 'a[href]', + 'button', + 'input:not([type="hidden"])', + 'select', + 'textarea', + '[role="button"]', + '[role="link"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="tab"]', + '[role="menuitem"]', + '[role="combobox"]', + '[role="option"]', + '[role="switch"]', + '[role="slider"]', + '[role="textbox"]', + '[role="searchbox"]', + '[role="spinbutton"]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])' +] diff --git a/packages/elements/src/browser-elements.ts b/packages/elements/src/browser-elements.ts new file mode 100644 index 00000000..208d7627 --- /dev/null +++ b/packages/elements/src/browser-elements.ts @@ -0,0 +1,307 @@ +/** + * Browser element detection + * Single browser.execute() call: querySelectorAll β†’ flat interactable element list + * + * NOTE: This script runs in browser context via browser.execute() + * It must be self-contained with no external dependencies + */ + +import { INTERACTABLE_SELECTORS } from './aria-roles.js' + +export interface BrowserElementInfo { + tagName: string + name: string // computed accessible name (ARIA spec) + type: string + value: string + href: string + selector: string + isInViewport: boolean + boundingBox?: { x: number; y: number; width: number; height: number } +} + +export interface GetBrowserElementsOptions { + includeBounds?: boolean + /** Only return elements whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +// Page-injected script: WDIO's `browser.execute` stringifies the arrow body +// and runs it in the browser. The selector list lives in aria-roles.ts at +// module scope so it stays type-checked, but it's passed in via execute() +// args because module-level values don't survive script stringification. +// Same constraint forces every helper below to live inside the IIFE. +// eslint-disable-next-line max-lines-per-function +const elementsScript = ( + includeBounds: boolean, + inViewportOnly: boolean, + interactableSelectorsArr: readonly string[] +) => + // eslint-disable-next-line max-lines-per-function -- see comment above + (function () { + const interactableSelectors = interactableSelectorsArr.join(',') + + function isVisible(element: HTMLElement): boolean { + if (typeof element.checkVisibility === 'function') { + return element.checkVisibility({ + opacityProperty: true, + visibilityProperty: true, + contentVisibilityAuto: true + }) + } + const style = window.getComputedStyle(element) + return ( + style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + element.offsetWidth > 0 && + element.offsetHeight > 0 + ) + } + + // WAI-ARIA Accessible Name Computation algorithm β€” see accessibility-tree.ts + // for the longer rationale. Sequential spec-ordered fallback steps. + // eslint-disable-next-line max-lines-per-function + function getAccessibleName(el: HTMLElement): string { + // 1. aria-label + const ariaLabel = el.getAttribute('aria-label') + if (ariaLabel) { + return ariaLabel.trim() + } + + // 2. aria-labelledby β€” resolve referenced elements + const labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy) { + const texts = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.textContent?.trim() || '') + .filter(Boolean) + if (texts.length > 0) { + return texts.join(' ').slice(0, 200) + } + } + + const tag = el.tagName.toLowerCase() + + // 3. alt for images and input[type=image] + if ( + tag === 'img' || + (tag === 'input' && el.getAttribute('type') === 'image') + ) { + const alt = el.getAttribute('alt') + if (alt !== null) { + return alt.trim() + } + } + + // 4. label[for=id] for form elements + if (['input', 'select', 'textarea'].includes(tag)) { + const id = el.getAttribute('id') + if (id) { + const label = document.querySelector(`label[for="${CSS.escape(id)}"]`) + if (label) { + return label.textContent?.trim() || '' + } + } + // 5. Wrapping label β€” clone, strip inputs, read text + const parentLabel = el.closest('label') + if (parentLabel) { + const clone = parentLabel.cloneNode(true) as HTMLElement + clone + .querySelectorAll('input,select,textarea') + .forEach((n) => n.remove()) + const lt = clone.textContent?.trim() + if (lt) { + return lt + } + } + } + + // 6. placeholder + const ph = el.getAttribute('placeholder') + if (ph) { + return ph.trim() + } + + // 7. title + const title = el.getAttribute('title') + if (title) { + return title.trim() + } + + // 8. text content (truncated, whitespace normalized) + return (el.textContent?.trim().replace(/\s+/g, ' ') || '').slice(0, 200) + } + + // Selector synthesis β€” see accessibility-tree.ts for the longer rationale. + // Tries 6 priority-ordered strategies; each must re-check uniqueness. + // eslint-disable-next-line max-lines-per-function + function getSelector(element: HTMLElement): string { + const tag = element.tagName.toLowerCase() + + // 1. tag*=Text β€” best per WebdriverIO docs + const text = element.textContent?.trim().replace(/\s+/g, ' ') + if (text && text.length > 0 && text.length <= 120) { + const sameTagElements = document.querySelectorAll(tag) + let matchCount = 0 + sameTagElements.forEach((el) => { + if (el.textContent?.includes(text)) { + matchCount++ + } + }) + if (matchCount === 1) { + return `${tag}*=${text}` + } + } + + // 2. aria/label + const ariaLabel = element.getAttribute('aria-label') + if (ariaLabel && ariaLabel.length <= 200) { + const sel = `[aria-label="${CSS.escape(ariaLabel)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + // 3. data-testid + const testId = element.getAttribute('data-testid') + if (testId) { + const sel = `[data-testid="${CSS.escape(testId)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + // 4. #id + if (element.id) { + return `#${CSS.escape(element.id)}` + } + + // 5. [name] β€” form elements + const nameAttr = element.getAttribute('name') + if (nameAttr) { + const sel = `${tag}[name="${CSS.escape(nameAttr)}"]` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + + // 6. tag.class β€” try each class individually, then first-two combination + if (element.className && typeof element.className === 'string') { + const classes = element.className.trim().split(/\s+/).filter(Boolean) + for (const cls of classes) { + const sel = `${tag}.${CSS.escape(cls)}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + if (classes.length >= 2) { + const sel = `${tag}${classes + .slice(0, 2) + .map((c) => `.${CSS.escape(c)}`) + .join('')}` + if (document.querySelectorAll(sel).length === 1) { + return sel + } + } + } + + // 7. CSS path fallback + let current: HTMLElement | null = element + const path: string[] = [] + while (current && current !== document.documentElement) { + let seg = current.tagName.toLowerCase() + if (current.id) { + path.unshift(`#${CSS.escape(current.id)}`) + break + } + const parent = current.parentElement + if (parent) { + const siblings = Array.from(parent.children).filter( + (c) => c.tagName === current!.tagName + ) + if (siblings.length > 1) { + seg += `:nth-of-type(${siblings.indexOf(current) + 1})` + } + } + path.unshift(seg) + current = current.parentElement + if (path.length >= 4) { + break + } + } + return path.join(' > ') + } + + const elements: Record[] = [] + const seen = new Set() + + document.querySelectorAll(interactableSelectors).forEach((el) => { + if (seen.has(el)) { + return + } + seen.add(el) + + const htmlEl = el as HTMLElement + if (!isVisible(htmlEl)) { + return + } + + const inputEl = htmlEl as HTMLInputElement + const rect = htmlEl.getBoundingClientRect() + const isInViewport = + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= + (window.innerHeight || document.documentElement.clientHeight) && + rect.right <= + (window.innerWidth || document.documentElement.clientWidth) + + if (inViewportOnly && !isInViewport) { + return + } + + const entry: Record = { + tagName: htmlEl.tagName.toLowerCase(), + name: getAccessibleName(htmlEl), + type: htmlEl.getAttribute('type') || '', + value: inputEl.value || '', + href: htmlEl.getAttribute('href') || '', + selector: getSelector(htmlEl), + isInViewport + } + + if (includeBounds) { + entry.boundingBox = { + x: rect.x + window.scrollX, + y: rect.y + window.scrollY, + width: rect.width, + height: rect.height + } + } + + elements.push(entry) + }) + + return elements + })() + +/** + * Get interactable browser elements via querySelectorAll. + */ +export async function getInteractableBrowserElements( + browser: WebdriverIO.Browser, + options: GetBrowserElementsOptions = {} +): Promise { + const { includeBounds = false, inViewportOnly = true } = options + // WDIO's typed Browser.execute overloads don't accept the injected script β€” + // narrow to a permissive execute() shape at the boundary. + type ExecuteLike = { + execute: (script: unknown, ...args: unknown[]) => Promise + } + return (browser as unknown as ExecuteLike).execute( + elementsScript, + includeBounds, + inViewportOnly, + INTERACTABLE_SELECTORS + ) as Promise +} diff --git a/packages/elements/src/get-elements.ts b/packages/elements/src/get-elements.ts new file mode 100644 index 00000000..e763a1ff --- /dev/null +++ b/packages/elements/src/get-elements.ts @@ -0,0 +1,67 @@ +import { getInteractableBrowserElements } from './browser-elements.js' +import { getMobileVisibleElementsWithTree } from './mobile-elements.js' +import type { JSONElement } from './locators/types.js' + +export type VisibleElementsResult = { + total: number + showing: number + hasMore: boolean + elements: unknown[] + /** Raw JSON element tree β€” only present for mobile (android/ios) sessions */ + tree?: JSONElement +} + +export async function getElements( + browser: WebdriverIO.Browser, + params: { + inViewportOnly?: boolean + includeContainers?: boolean + includeBounds?: boolean + limit?: number + offset?: number + } +): Promise { + const { + inViewportOnly = true, + includeContainers = false, + includeBounds = false, + limit = 0, + offset = 0 + } = params + + let elements: { isInViewport?: boolean }[] + let tree: JSONElement | undefined + + if (browser.isAndroid || browser.isIOS) { + const platform = browser.isAndroid ? 'android' : 'ios' + const result = await getMobileVisibleElementsWithTree(browser, platform, { + includeContainers, + includeBounds, + inViewportOnly + }) + elements = result.elements + tree = result.tree ?? undefined + } else { + elements = await getInteractableBrowserElements(browser, { + includeBounds, + inViewportOnly + }) + } + + const total = elements.length + + if (offset > 0) { + elements = elements.slice(offset) + } + if (limit > 0) { + elements = elements.slice(0, limit) + } + + return { + total, + showing: elements.length, + hasMore: offset + elements.length < total, + elements, + ...(tree !== undefined ? { tree } : {}) + } +} diff --git a/packages/elements/src/index.ts b/packages/elements/src/index.ts new file mode 100644 index 00000000..7aeabf78 --- /dev/null +++ b/packages/elements/src/index.ts @@ -0,0 +1,21 @@ +export { getInteractableBrowserElements } from './browser-elements.js' +export type { + BrowserElementInfo, + GetBrowserElementsOptions +} from './browser-elements.js' + +export { getBrowserAccessibilityTree } from './accessibility-tree.js' +export type { AccessibilityNode } from './accessibility-tree.js' + +export { getMobileVisibleElements } from './mobile-elements.js' +export type { + MobileElementInfo, + GetMobileElementsOptions +} from './mobile-elements.js' + +export { getElements } from './get-elements.js' +export type { VisibleElementsResult } from './get-elements.js' + +export { serializeWebSnapshot, serializeMobileSnapshot } from './snapshot.js' +export type { WebSnapshotOptions, MobileSnapshotOptions } from './snapshot.js' +export type { JSONElement } from './locators/types.js' diff --git a/packages/elements/src/locators/constants.ts b/packages/elements/src/locators/constants.ts new file mode 100644 index 00000000..540784b3 --- /dev/null +++ b/packages/elements/src/locators/constants.ts @@ -0,0 +1,169 @@ +/** + * Platform-specific element tag constants for mobile automation + */ + +export const ANDROID_INTERACTABLE_TAGS = [ + // Input elements + 'android.widget.EditText', + 'android.widget.AutoCompleteTextView', + 'android.widget.MultiAutoCompleteTextView', + 'android.widget.SearchView', + + // Button-like elements + 'android.widget.Button', + 'android.widget.ImageButton', + 'android.widget.ToggleButton', + 'android.widget.CompoundButton', + 'android.widget.RadioButton', + 'android.widget.CheckBox', + 'android.widget.Switch', + 'android.widget.FloatingActionButton', + 'com.google.android.material.button.MaterialButton', + 'com.google.android.material.floatingactionbutton.FloatingActionButton', + + // Text elements (often tappable) + 'android.widget.TextView', + 'android.widget.CheckedTextView', + + // Image elements (often tappable) + 'android.widget.ImageView', + 'android.widget.QuickContactBadge', + + // Selection elements + 'android.widget.Spinner', + 'android.widget.SeekBar', + 'android.widget.RatingBar', + 'android.widget.ProgressBar', + 'android.widget.DatePicker', + 'android.widget.TimePicker', + 'android.widget.NumberPicker', + + // List/grid items + 'android.widget.AdapterView' +] + +export const ANDROID_LAYOUT_CONTAINERS = [ + // Core ViewGroup classes + 'android.view.ViewGroup', + 'android.view.View', + 'android.widget.FrameLayout', + 'android.widget.LinearLayout', + 'android.widget.RelativeLayout', + 'android.widget.GridLayout', + 'android.widget.TableLayout', + 'android.widget.TableRow', + 'android.widget.AbsoluteLayout', + + // AndroidX layout classes + 'androidx.constraintlayout.widget.ConstraintLayout', + 'androidx.coordinatorlayout.widget.CoordinatorLayout', + 'androidx.appcompat.widget.LinearLayoutCompat', + 'androidx.cardview.widget.CardView', + 'androidx.appcompat.widget.ContentFrameLayout', + 'androidx.appcompat.widget.FitWindowsFrameLayout', + + // Scrolling containers + 'android.widget.ScrollView', + 'android.widget.HorizontalScrollView', + 'android.widget.NestedScrollView', + 'androidx.core.widget.NestedScrollView', + 'androidx.recyclerview.widget.RecyclerView', + 'android.widget.ListView', + 'android.widget.GridView', + 'android.widget.AbsListView', + + // App chrome / system elements + 'android.widget.ActionBarContainer', + 'android.widget.ActionBarOverlayLayout', + 'android.view.ViewStub', + 'androidx.appcompat.widget.ActionBarContainer', + 'androidx.appcompat.widget.ActionBarContextView', + 'androidx.appcompat.widget.ActionBarOverlayLayout', + + // Decor views + 'com.android.internal.policy.DecorView', + 'android.widget.DecorView' +] + +export const IOS_INTERACTABLE_TAGS = [ + // Input elements + 'XCUIElementTypeTextField', + 'XCUIElementTypeSecureTextField', + 'XCUIElementTypeTextView', + 'XCUIElementTypeSearchField', + + // Button-like elements + 'XCUIElementTypeButton', + 'XCUIElementTypeLink', + + // Text elements (often tappable) + 'XCUIElementTypeStaticText', + + // Image elements + 'XCUIElementTypeImage', + 'XCUIElementTypeIcon', + + // Selection elements + 'XCUIElementTypeSwitch', + 'XCUIElementTypeSlider', + 'XCUIElementTypeStepper', + 'XCUIElementTypeSegmentedControl', + 'XCUIElementTypePicker', + 'XCUIElementTypePickerWheel', + 'XCUIElementTypeDatePicker', + 'XCUIElementTypePageIndicator', + + // Table/list items + 'XCUIElementTypeCell', + 'XCUIElementTypeMenuItem', + 'XCUIElementTypeMenuBarItem', + + // Toggle elements + 'XCUIElementTypeCheckBox', + 'XCUIElementTypeRadioButton', + 'XCUIElementTypeToggle', + + // Other interactive + 'XCUIElementTypeKey', + 'XCUIElementTypeKeyboard', + 'XCUIElementTypeAlert', + 'XCUIElementTypeSheet' +] + +export const IOS_LAYOUT_CONTAINERS = [ + // Generic containers + 'XCUIElementTypeOther', + 'XCUIElementTypeGroup', + 'XCUIElementTypeLayoutItem', + + // Scroll containers + 'XCUIElementTypeScrollView', + 'XCUIElementTypeTable', + 'XCUIElementTypeCollectionView', + 'XCUIElementTypeScrollBar', + + // Navigation chrome + 'XCUIElementTypeNavigationBar', + 'XCUIElementTypeTabBar', + 'XCUIElementTypeToolbar', + 'XCUIElementTypeStatusBar', + 'XCUIElementTypeMenuBar', + + // Windows and views + 'XCUIElementTypeWindow', + 'XCUIElementTypeSheet', + 'XCUIElementTypeDrawer', + 'XCUIElementTypeDialog', + 'XCUIElementTypePopover', + 'XCUIElementTypePopUpButton', + + // Outline elements + 'XCUIElementTypeOutline', + 'XCUIElementTypeOutlineRow', + 'XCUIElementTypeBrowser', + 'XCUIElementTypeSplitGroup', + 'XCUIElementTypeSplitter', + + // Application root + 'XCUIElementTypeApplication' +] diff --git a/packages/elements/src/locators/element-filter.ts b/packages/elements/src/locators/element-filter.ts new file mode 100644 index 00000000..d249f3a2 --- /dev/null +++ b/packages/elements/src/locators/element-filter.ts @@ -0,0 +1,234 @@ +/** + * Element filtering logic for mobile automation + */ + +import type { JSONElement, FilterOptions } from './types.js' +import { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS +} from './constants.js' + +/** + * Check if element tag matches any in the list (handles partial matches) + */ +function matchesTagList(tagName: string, tagList: string[]): boolean { + if (tagList.includes(tagName)) { + return true + } + + for (const tag of tagList) { + if (tagName.endsWith(tag) || tagName.includes(tag)) { + return true + } + } + + return false +} + +/** + * Check if element matches tag name filters + */ +function matchesTagFilters( + element: JSONElement, + includeTagNames: string[], + excludeTagNames: string[] +): boolean { + if ( + includeTagNames.length > 0 && + !matchesTagList(element.tagName, includeTagNames) + ) { + return false + } + + if (matchesTagList(element.tagName, excludeTagNames)) { + return false + } + + return true +} + +/** + * Check if element matches attribute-based filters + */ +function matchesAttributeFilters( + element: JSONElement, + requireAttributes: string[], + minAttributeCount: number +): boolean { + if (requireAttributes.length > 0) { + const hasRequiredAttr = requireAttributes.some( + (attr) => element.attributes?.[attr] + ) + if (!hasRequiredAttr) { + return false + } + } + + if (element.attributes && minAttributeCount > 0) { + const attrCount = Object.values(element.attributes).filter( + (v) => v !== undefined && v !== null && v !== '' + ).length + if (attrCount < minAttributeCount) { + return false + } + } + + return true +} + +/** + * Check if element is interactable based on platform + */ +export function isInteractableElement( + element: JSONElement, + _isNative: boolean, + automationName: string +): boolean { + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const interactableTags = isAndroid + ? ANDROID_INTERACTABLE_TAGS + : IOS_INTERACTABLE_TAGS + + if (matchesTagList(element.tagName, interactableTags)) { + return true + } + + if (isAndroid) { + if ( + element.attributes?.clickable === 'true' || + element.attributes?.focusable === 'true' || + element.attributes?.checkable === 'true' || + element.attributes?.['long-clickable'] === 'true' + ) { + return true + } + } + + if (!isAndroid) { + if (element.attributes?.accessible === 'true') { + return true + } + } + + return false +} + +/** + * Check if element is a layout container + */ +export function isLayoutContainer( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const containerList = + platform === 'android' ? ANDROID_LAYOUT_CONTAINERS : IOS_LAYOUT_CONTAINERS + return matchesTagList(element.tagName, containerList) +} + +/** + * Check if element has meaningful content (text, accessibility info) + */ +export function hasMeaningfulContent( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const attrs = element.attributes + + if (attrs.text && attrs.text.trim() !== '' && attrs.text !== 'null') { + return true + } + + if (platform === 'android') { + if ( + attrs['content-desc'] && + attrs['content-desc'].trim() !== '' && + attrs['content-desc'] !== 'null' + ) { + return true + } + } else { + if (attrs.label && attrs.label.trim() !== '' && attrs.label !== 'null') { + return true + } + if (attrs.name && attrs.name.trim() !== '' && attrs.name !== 'null') { + return true + } + } + + return false +} + +/** + * Determine if an element should be included based on all filter criteria + */ +export function shouldIncludeElement( + element: JSONElement, + filters: FilterOptions, + isNative: boolean, + automationName: string +): boolean { + const { + includeTagNames = [], + excludeTagNames = ['hierarchy'], + requireAttributes = [], + minAttributeCount = 0, + fetchableOnly = false, + clickableOnly = false, + visibleOnly = true + } = filters + + if (!matchesTagFilters(element, includeTagNames, excludeTagNames)) { + if (element.attributes?.clickable !== 'true') { + return false + } + } + + if (!matchesAttributeFilters(element, requireAttributes, minAttributeCount)) { + return false + } + + if (clickableOnly && element.attributes?.clickable !== 'true') { + return false + } + + if (visibleOnly) { + const isAndroid = automationName.toLowerCase().includes('uiautomator') + if (isAndroid && element.attributes?.displayed === 'false') { + return false + } + if (!isAndroid && element.attributes?.visible === 'false') { + return false + } + } + + if ( + fetchableOnly && + !isInteractableElement(element, isNative, automationName) + ) { + return false + } + + return true +} + +/** + * Get default filter options for a platform + */ +export function getDefaultFilters( + platform: 'android' | 'ios', + includeContainers: boolean = false +): FilterOptions { + const layoutContainers = + platform === 'android' ? ANDROID_LAYOUT_CONTAINERS : IOS_LAYOUT_CONTAINERS + + return { + excludeTagNames: includeContainers + ? ['hierarchy'] + : ['hierarchy', ...layoutContainers], + fetchableOnly: !includeContainers, + visibleOnly: true, + clickableOnly: false + } +} diff --git a/packages/elements/src/locators/index.ts b/packages/elements/src/locators/index.ts new file mode 100644 index 00000000..20e2330c --- /dev/null +++ b/packages/elements/src/locators/index.ts @@ -0,0 +1,279 @@ +/** + * Mobile element locator generation + * + * Main orchestrator module that coordinates XML parsing, element filtering, + * and locator generation for mobile automation. + * + * Based on: https://github.com/appium/appium-mcp + */ + +// Types +export type { + ElementAttributes, + JSONElement, + Bounds, + FilterOptions, + UniquenessResult, + LocatorStrategy, + LocatorContext, + ElementWithLocators, + GenerateLocatorsOptions +} from './types.js' + +// Constants +export { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS, + ANDROID_LAYOUT_CONTAINERS, + IOS_LAYOUT_CONTAINERS +} from './constants.js' + +// XML Parsing +export { + xmlToJSON, + xmlToDOM, + evaluateXPath, + checkXPathUniqueness, + findDOMNodeByPath, + parseAndroidBounds, + parseIOSBounds, + flattenElementTree, + countAttributeOccurrences, + isAttributeUnique +} from './xml-parsing.js' + +// Element Filtering +export { + isInteractableElement, + isLayoutContainer, + hasMeaningfulContent, + shouldIncludeElement, + getDefaultFilters +} from './element-filter.js' + +// Locator Generation +export { + getSuggestedLocators, + getBestLocator, + locatorsToObject +} from './locator-generation.js' + +import type { + JSONElement, + FilterOptions, + LocatorStrategy, + ElementWithLocators, + GenerateLocatorsOptions, + XMLDocument +} from './types.js' + +import { + xmlToJSON, + xmlToDOM, + parseAndroidBounds, + parseIOSBounds, + findDOMNodeByPath +} from './xml-parsing.js' +import { + shouldIncludeElement, + isLayoutContainer, + hasMeaningfulContent +} from './element-filter.js' +import { getSuggestedLocators, locatorsToObject } from './locator-generation.js' + +interface ProcessingContext { + sourceXML: string + platform: 'android' | 'ios' + automationName: string + isNative: boolean + viewportSize: { width: number; height: number } + filters: FilterOptions + inViewportOnly: boolean + results: ElementWithLocators[] + parsedDOM: XMLDocument | null +} + +/** + * Parse element bounds based on platform + */ +function parseBounds( + element: JSONElement, + platform: 'android' | 'ios' +): { x: number; y: number; width: number; height: number } { + return platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) +} + +/** + * Check if bounds are within viewport + */ +function isWithinViewport( + bounds: { x: number; y: number; width: number; height: number }, + viewport: { width: number; height: number } +): boolean { + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +/** + * Transform JSONElement to ElementWithLocators + */ +function transformElement( + element: JSONElement, + locators: [LocatorStrategy, string][], + ctx: ProcessingContext +): ElementWithLocators { + const attrs = element.attributes + const bounds = parseBounds(element, ctx.platform) + + return { + tagName: element.tagName, + locators: locatorsToObject(locators), + text: attrs.text || attrs.label || '', + contentDesc: attrs['content-desc'] || '', + resourceId: attrs['resource-id'] || '', + accessibilityId: attrs.name || attrs['content-desc'] || '', + label: attrs.label || '', + value: attrs.value || '', + className: attrs.class || element.tagName, + clickable: + attrs.clickable === 'true' || + attrs.accessible === 'true' || + attrs['long-clickable'] === 'true', + enabled: attrs.enabled !== 'false', + displayed: + ctx.platform === 'android' + ? attrs.displayed !== 'false' + : attrs.visible !== 'false', + bounds, + isInViewport: isWithinViewport(bounds, ctx.viewportSize) + } +} + +/** + * Check if element should be processed + */ +function shouldProcess(element: JSONElement, ctx: ProcessingContext): boolean { + if ( + shouldIncludeElement(element, ctx.filters, ctx.isNative, ctx.automationName) + ) { + return true + } + return ( + isLayoutContainer(element, ctx.platform) && + hasMeaningfulContent(element, ctx.platform) + ) +} + +/** + * Process a single element and add to results if valid + */ +function processElement(element: JSONElement, ctx: ProcessingContext): void { + if (!shouldProcess(element, ctx)) { + return + } + + // Skip off-screen elements early when viewport filtering is on β€” + // avoids expensive locator generation for elements the caller doesn't want. + if (ctx.inViewportOnly) { + const b = parseBounds(element, ctx.platform) + if (!isWithinViewport(b, ctx.viewportSize)) { + return + } + } + + try { + const targetNode = ctx.parsedDOM + ? findDOMNodeByPath(ctx.parsedDOM, element.path) + : undefined + + const locators = getSuggestedLocators( + element, + ctx.sourceXML, + ctx.automationName, + { + sourceXML: ctx.sourceXML, + parsedDOM: ctx.parsedDOM, + isAndroid: ctx.platform === 'android' + }, + targetNode || undefined + ) + if (locators.length === 0) { + return + } + + // Stash the best locator on the tree node so serializeMobileSnapshot + // can reuse the full locator pipeline instead of recomputing. + element.attributes._selector = locators[0][1] + + const transformed = transformElement(element, locators, ctx) + if (Object.keys(transformed.locators).length === 0) { + return + } + + ctx.results.push(transformed) + } catch (error) { + console.error(`[processElement] Error at path ${element.path}:`, error) + } +} + +/** + * Recursively traverse and process element tree + */ +function traverseTree( + element: JSONElement | null, + ctx: ProcessingContext +): void { + if (!element) { + return + } + + processElement(element, ctx) + + for (const child of element.children || []) { + traverseTree(child, ctx) + } +} + +/** + * Generate locators for all elements from page source XML + */ +export function generateAllElementLocators( + sourceXML: string, + options: GenerateLocatorsOptions +): ElementWithLocators[] { + const sourceJSON = xmlToJSON(sourceXML) + + if (!sourceJSON) { + console.error( + '[generateAllElementLocators] Failed to parse page source XML' + ) + return [] + } + + const parsedDOM = xmlToDOM(sourceXML) + + const ctx: ProcessingContext = { + sourceXML, + platform: options.platform, + automationName: + options.platform === 'android' ? 'uiautomator2' : 'xcuitest', + isNative: options.isNative ?? true, + viewportSize: options.viewportSize ?? { width: 9999, height: 9999 }, + filters: options.filters ?? {}, + inViewportOnly: options.inViewportOnly ?? true, + results: [], + parsedDOM + } + + traverseTree(sourceJSON, ctx) + + return ctx.results +} diff --git a/packages/elements/src/locators/locator-generation.ts b/packages/elements/src/locators/locator-generation.ts new file mode 100644 index 00000000..5433e280 --- /dev/null +++ b/packages/elements/src/locators/locator-generation.ts @@ -0,0 +1,662 @@ +/** + * Locator strategy generation for mobile elements + */ + +import type { + JSONElement, + LocatorStrategy, + LocatorContext, + UniquenessResult, + XMLNode, + XMLDocument +} from './types.js' +import type { Element as XMLElement } from '@xmldom/xmldom' +import { + checkXPathUniqueness, + evaluateXPath, + isAttributeUnique +} from './xml-parsing.js' + +/** + * Check if a string value is valid for use in a locator + */ +function isValidValue(value: string | undefined): value is string { + return ( + value !== undefined && + value !== null && + value !== 'null' && + value.trim() !== '' + ) +} + +/** + * Escape special characters in text for use in selectors + */ +function escapeText(text: string): string { + return text.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n') +} + +/** + * Escape value for use in XPath expressions + */ +function escapeXPathValue(value: string): string { + if (!value.includes("'")) { + return `'${value}'` + } + if (!value.includes('"')) { + return `"${value}"` + } + const parts: string[] = [] + let current = '' + for (const char of value) { + if (char === "'") { + if (current) { + parts.push(`'${current}'`) + } + parts.push('"\'"') + current = '' + } else { + current += char + } + } + if (current) { + parts.push(`'${current}'`) + } + return `concat(${parts.join(',')})` +} + +/** + * Wrap non-unique XPath with index + */ +function generateIndexedXPath(baseXPath: string, index: number): string { + return `(${baseXPath})[${index}]` +} + +/** + * Add .instance(n) for UiAutomator (0-based) + */ +function generateIndexedUiAutomator( + baseSelector: string, + index: number +): string { + return `${baseSelector}.instance(${index - 1})` +} + +/** + * Check uniqueness, falling back to regex if no DOM available + */ +function checkUniqueness( + ctx: LocatorContext, + xpath: string, + targetNode?: XMLNode +): UniquenessResult { + if (ctx.parsedDOM) { + return checkXPathUniqueness(ctx.parsedDOM, xpath, targetNode) + } + + // Bounded + disjoint negated classes: attr names can never contain `=`, + // `"` or `]`; values can't contain `"`. Bounds prevent polynomial + // backtracking on adversarial input (CodeQL: js/polynomial-redos). + const match = xpath.match(/\/\/\*\[@([^="\]]{1,200})="([^"]{1,2000})"\]/) + if (match) { + const [, attr, value] = match + return { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } + } + return { isUnique: false } +} + +/** + * Get sibling index (1-based) among same-tag siblings + */ +function getSiblingIndex(element: XMLElement): number { + const parent = element.parentNode + if (!parent) { + return 1 + } + + const tagName = element.nodeName + let index = 0 + + for (let i = 0; i < parent.childNodes.length; i++) { + const child = parent.childNodes.item(i) + if (child?.nodeType === 1 && child.nodeName === tagName) { + index++ + if (child === element) { + return index + } + } + } + + return 1 +} + +/** + * Count siblings with same tag name + */ +function countSiblings(element: XMLElement): number { + const parent = element.parentNode + if (!parent) { + return 1 + } + + const tagName = element.nodeName + let count = 0 + + for (let i = 0; i < parent.childNodes.length; i++) { + const child = parent.childNodes.item(i) + if (child?.nodeType === 1 && child.nodeName === tagName) { + count++ + } + } + + return count +} + +/** + * Find unique attribute for element in XPath format + */ +function findUniqueAttribute( + element: XMLElement, + ctx: LocatorContext +): string | null { + const attrs = ctx.isAndroid + ? ['resource-id', 'content-desc', 'text'] + : ['name', 'label', 'value'] + + for (const attr of attrs) { + const value = element.getAttribute(attr) + if (value && value.trim()) { + const xpath = `//*[@${attr}=${escapeXPathValue(value)}]` + const result = ctx.parsedDOM + ? checkXPathUniqueness(ctx.parsedDOM, xpath) + : { isUnique: isAttributeUnique(ctx.sourceXML, attr, value) } + + if (result.isUnique) { + return `@${attr}=${escapeXPathValue(value)}` + } + } + } + + return null +} + +/** + * Build hierarchical XPath by traversing up the DOM tree + */ +function buildHierarchicalXPath( + ctx: LocatorContext, + element: XMLElement, + maxDepth: number = 3 +): string | null { + if (!ctx.parsedDOM) { + return null + } + + const pathParts: string[] = [] + let current: XMLElement | null = element + let depth = 0 + + while (current && depth < maxDepth) { + const tagName = current.nodeName + const uniqueAttr = findUniqueAttribute(current, ctx) + + if (uniqueAttr) { + pathParts.unshift(`//${tagName}[${uniqueAttr}]`) + break + } else { + const siblingIndex = getSiblingIndex(current) + const siblingCount = countSiblings(current) + + if (siblingCount > 1) { + pathParts.unshift(`${tagName}[${siblingIndex}]`) + } else { + pathParts.unshift(tagName) + } + } + + const parent = current.parentNode as XMLElement | null + current = parent && parent.nodeType === 1 ? parent : null + depth++ + } + + if (pathParts.length === 0) { + return null + } + + let result = pathParts[0] + for (let i = 1; i < pathParts.length; i++) { + result += '/' + pathParts[i] + } + + if (!result.startsWith('//')) { + result = '//' + result + } + + return result +} + +/** + * Add XPath locator with uniqueness checking and fallbacks + */ +function addXPathLocator( + results: [LocatorStrategy, string][], + xpath: string, + ctx: LocatorContext, + targetNode?: XMLNode +): void { + const uniqueness = checkUniqueness(ctx, xpath, targetNode) + if (uniqueness.isUnique) { + results.push(['xpath', xpath]) + } else if (uniqueness.index) { + results.push(['xpath', generateIndexedXPath(xpath, uniqueness.index)]) + } else { + if (targetNode && ctx.parsedDOM) { + // @xmldom/xmldom 0.9+ XMLNode doesn't satisfy global Node; safe at runtime + const hierarchical = buildHierarchicalXPath( + ctx, + targetNode as unknown as XMLElement + ) + if (hierarchical) { + results.push(['xpath', hierarchical]) + } + } + results.push(['xpath', xpath]) + } +} + +/** + * Check if element is within UiAutomator scope + */ +function isInUiAutomatorScope( + element: JSONElement, + doc: XMLDocument | null +): boolean { + if (!doc) { + return true + } + + const hierarchyNodes = evaluateXPath(doc, '/hierarchy/*') + if (hierarchyNodes.length === 0) { + return true + } + + const lastIndex = hierarchyNodes.length + const pathParts = element.path.split('.') + if (pathParts.length === 0 || pathParts[0] === '') { + return true + } + + const firstIndex = parseInt(pathParts[0], 10) + return firstIndex === lastIndex - 1 +} + +/** + * Build Android UiAutomator selector with multiple attributes + */ +function buildUiAutomatorSelector(element: JSONElement): string | null { + const attrs = element.attributes + const parts: string[] = [] + + if (isValidValue(attrs['resource-id'])) { + parts.push(`resourceId("${attrs['resource-id']}")`) + } + if (isValidValue(attrs.text) && attrs.text!.length < 100) { + parts.push(`text("${escapeText(attrs.text!)}")`) + } + if (isValidValue(attrs['content-desc'])) { + parts.push(`description("${attrs['content-desc']}")`) + } + if (isValidValue(attrs.class)) { + parts.push(`className("${attrs.class}")`) + } + + if (parts.length === 0) { + return null + } + + return `android=new UiSelector().${parts.join('.')}` +} + +/** + * Build iOS predicate string with multiple conditions + */ +function buildPredicateString(element: JSONElement): string | null { + const attrs = element.attributes + const conditions: string[] = [] + + if (isValidValue(attrs.name)) { + conditions.push(`name == "${escapeText(attrs.name!)}"`) + } + if (isValidValue(attrs.label)) { + conditions.push(`label == "${escapeText(attrs.label!)}"`) + } + if (isValidValue(attrs.value)) { + conditions.push(`value == "${escapeText(attrs.value!)}"`) + } + if (attrs.visible === 'true') { + conditions.push('visible == 1') + } + if (attrs.enabled === 'true') { + conditions.push('enabled == 1') + } + + if (conditions.length === 0) { + return null + } + + return `-ios predicate string:${conditions.join(' AND ')}` +} + +/** + * Build iOS class chain selector + */ +function buildClassChain(element: JSONElement): string | null { + const attrs = element.attributes + const tagName = element.tagName + + if (!tagName.startsWith('XCUI')) { + return null + } + + let selector = `**/${tagName}` + + if (isValidValue(attrs.label)) { + selector += `[\`label == "${escapeText(attrs.label!)}"\`]` + } else if (isValidValue(attrs.name)) { + selector += `[\`name == "${escapeText(attrs.name!)}"\`]` + } + + return `-ios class chain:${selector}` +} + +/** + * Build XPath for element with unique identification + */ +function buildXPath( + element: JSONElement, + _sourceXML: string, + isAndroid: boolean +): string | null { + const attrs = element.attributes + const tagName = element.tagName + const conditions: string[] = [] + + if (isAndroid) { + if (isValidValue(attrs['resource-id'])) { + conditions.push(`@resource-id="${attrs['resource-id']}"`) + } + if (isValidValue(attrs['content-desc'])) { + conditions.push(`@content-desc="${attrs['content-desc']}"`) + } + if (isValidValue(attrs.text) && attrs.text!.length < 100) { + conditions.push(`@text="${escapeText(attrs.text!)}"`) + } + } else { + if (isValidValue(attrs.name)) { + conditions.push(`@name="${attrs.name}"`) + } + if (isValidValue(attrs.label)) { + conditions.push(`@label="${attrs.label}"`) + } + if (isValidValue(attrs.value)) { + conditions.push(`@value="${attrs.value}"`) + } + } + + if (conditions.length === 0) { + return `//${tagName}` + } + + return `//${tagName}[${conditions.join(' and ')}]` +} + +/** + * Get simple locators based on single attributes + */ +function appendAndroidSimpleLocators( + attrs: JSONElement['attributes'], + ctx: LocatorContext, + inUiAutomatorScope: boolean, + results: [LocatorStrategy, string][], + targetNode?: XMLNode +): void { + const resourceId = attrs['resource-id'] + if (isValidValue(resourceId)) { + const uniqueness = checkUniqueness( + ctx, + `//*[@resource-id="${resourceId}"]`, + targetNode + ) + const base = `android=new UiSelector().resourceId("${resourceId}")` + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push(['id', base]) + } else if (uniqueness.index && inUiAutomatorScope) { + results.push(['id', generateIndexedUiAutomator(base, uniqueness.index)]) + } + } + + const contentDesc = attrs['content-desc'] + if (isValidValue(contentDesc)) { + const uniqueness = checkUniqueness( + ctx, + `//*[@content-desc="${contentDesc}"]`, + targetNode + ) + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${contentDesc}`]) + } + } + + const text = attrs.text + if (isValidValue(text) && text.length < 100) { + const uniqueness = checkUniqueness( + ctx, + `//*[@text="${escapeText(text)}"]`, + targetNode + ) + const base = `android=new UiSelector().text("${escapeText(text)}")` + if (uniqueness.isUnique && inUiAutomatorScope) { + results.push(['text', base]) + } else if (uniqueness.index && inUiAutomatorScope) { + results.push(['text', generateIndexedUiAutomator(base, uniqueness.index)]) + } + } +} + +function appendIOSSimpleLocators( + attrs: JSONElement['attributes'], + ctx: LocatorContext, + results: [LocatorStrategy, string][], + targetNode?: XMLNode +): void { + const name = attrs.name + if (isValidValue(name)) { + const uniqueness = checkUniqueness(ctx, `//*[@name="${name}"]`, targetNode) + if (uniqueness.isUnique) { + results.push(['accessibility-id', `~${name}`]) + } + } + + const label = attrs.label + if (isValidValue(label) && label !== attrs.name) { + const uniqueness = checkUniqueness( + ctx, + `//*[@label="${escapeText(label)}"]`, + targetNode + ) + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:label == "${escapeText(label)}"` + ]) + } + } + + const value = attrs.value + if (isValidValue(value)) { + const uniqueness = checkUniqueness( + ctx, + `//*[@value="${escapeText(value)}"]`, + targetNode + ) + if (uniqueness.isUnique) { + results.push([ + 'predicate-string', + `-ios predicate string:value == "${escapeText(value)}"` + ]) + } + } +} + +function getSimpleSuggestedLocators( + element: JSONElement, + ctx: LocatorContext, + automationName: string, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const results: [LocatorStrategy, string][] = [] + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const inUiAutomatorScope = isAndroid + ? isInUiAutomatorScope(element, ctx.parsedDOM) + : true + if (isAndroid) { + appendAndroidSimpleLocators( + element.attributes, + ctx, + inUiAutomatorScope, + results, + targetNode + ) + } else { + appendIOSSimpleLocators(element.attributes, ctx, results, targetNode) + } + return results +} + +/** + * Get complex locators (combinations, XPath, etc.) + */ +function getComplexSuggestedLocators( + element: JSONElement, + ctx: LocatorContext, + automationName: string, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const results: [LocatorStrategy, string][] = [] + const isAndroid = automationName.toLowerCase().includes('uiautomator') + const inUiAutomatorScope = isAndroid + ? isInUiAutomatorScope(element, ctx.parsedDOM) + : true + + if (isAndroid) { + if (inUiAutomatorScope) { + const uiAutomator = buildUiAutomatorSelector(element) + if (uiAutomator) { + results.push(['uiautomator', uiAutomator]) + } + } + + const xpath = buildXPath(element, ctx.sourceXML, true) + if (xpath) { + addXPathLocator(results, xpath, ctx, targetNode) + } + + if (inUiAutomatorScope && isValidValue(element.attributes.class)) { + results.push([ + 'class-name', + `android=new UiSelector().className("${element.attributes.class}")` + ]) + } + } else { + const predicate = buildPredicateString(element) + if (predicate) { + results.push(['predicate-string', predicate]) + } + + const classChain = buildClassChain(element) + if (classChain) { + results.push(['class-chain', classChain]) + } + + const xpath = buildXPath(element, ctx.sourceXML, false) + if (xpath) { + addXPathLocator(results, xpath, ctx, targetNode) + } + + const type = element.tagName + if (type.startsWith('XCUIElementType')) { + results.push(['class-name', `-ios class chain:**/${type}`]) + } + } + + return results +} + +/** + * Get all suggested locators for an element + */ +export function getSuggestedLocators( + element: JSONElement, + sourceXML: string, + automationName: string, + ctx?: LocatorContext, + targetNode?: XMLNode +): [LocatorStrategy, string][] { + const locatorCtx = ctx ?? { + sourceXML, + parsedDOM: null, + isAndroid: automationName.toLowerCase().includes('uiautomator') + } + + const simpleLocators = getSimpleSuggestedLocators( + element, + locatorCtx, + automationName, + targetNode + ) + const complexLocators = getComplexSuggestedLocators( + element, + locatorCtx, + automationName, + targetNode + ) + + const seen = new Set() + const results: [LocatorStrategy, string][] = [] + + for (const locator of [...simpleLocators, ...complexLocators]) { + if (!seen.has(locator[1])) { + seen.add(locator[1]) + results.push(locator) + } + } + + return results +} + +/** + * Get the best (first priority) locator for an element + */ +export function getBestLocator( + element: JSONElement, + sourceXML: string, + automationName: string +): string | null { + const locators = getSuggestedLocators(element, sourceXML, automationName) + return locators.length > 0 ? locators[0][1] : null +} + +/** + * Convert locator array to object format + */ +export function locatorsToObject( + locators: [LocatorStrategy, string][] +): Record { + const result: Record = {} + for (const [strategy, value] of locators) { + if (!result[strategy]) { + result[strategy] = value + } + } + return result +} diff --git a/packages/elements/src/locators/types.ts b/packages/elements/src/locators/types.ts new file mode 100644 index 00000000..28f5a497 --- /dev/null +++ b/packages/elements/src/locators/types.ts @@ -0,0 +1,110 @@ +/** + * Type definitions for mobile element locator generation + */ + +import type { Document as XMLDocument, Node as XMLNode } from '@xmldom/xmldom' +export type { XMLDocument, XMLNode } + +export interface ElementAttributes { + // Android attributes + 'resource-id'?: string + 'content-desc'?: string + text?: string + class?: string + package?: string + clickable?: string + 'long-clickable'?: string + focusable?: string + checkable?: string + scrollable?: string + enabled?: string + displayed?: string + bounds?: string // Format: "[x1,y1][x2,y2]" + + // iOS attributes + type?: string + name?: string + label?: string + value?: string + accessible?: string + visible?: string + x?: string + y?: string + width?: string + height?: string + + // Generic + [key: string]: string | undefined +} + +export interface JSONElement { + children: JSONElement[] + tagName: string + attributes: ElementAttributes + path: string // Dot-separated index path for tree traversal +} + +export interface Bounds { + x: number + y: number + width: number + height: number +} + +export interface FilterOptions { + includeTagNames?: string[] // Only include these tags (whitelist) + excludeTagNames?: string[] // Exclude these tags (blacklist) + requireAttributes?: string[] // Must have at least one of these attributes + minAttributeCount?: number // Minimum number of non-empty attributes + fetchableOnly?: boolean // Only interactable elements + clickableOnly?: boolean // Only elements with clickable="true" + visibleOnly?: boolean // Only visible/displayed elements +} + +export interface UniquenessResult { + isUnique: boolean + index?: number // 1-based index if not unique + totalMatches?: number +} + +export type LocatorStrategy = + | 'accessibility-id' + | 'id' + | 'class-name' + | 'xpath' + | 'predicate-string' + | 'class-chain' + | 'uiautomator' + | 'text' + +export interface LocatorContext { + sourceXML: string + parsedDOM: XMLDocument | null + isAndroid: boolean +} + +export interface ElementWithLocators { + tagName: string + locators: Record + text: string + contentDesc: string + resourceId: string + accessibilityId: string + label: string + value: string + className: string + clickable: boolean + enabled: boolean + displayed: boolean + bounds: Bounds + isInViewport: boolean +} + +export interface GenerateLocatorsOptions { + platform: 'android' | 'ios' + viewportSize?: { width: number; height: number } + filters?: FilterOptions + isNative?: boolean + /** Only return elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean +} diff --git a/packages/elements/src/locators/xml-parsing.ts b/packages/elements/src/locators/xml-parsing.ts new file mode 100644 index 00000000..09b1c13a --- /dev/null +++ b/packages/elements/src/locators/xml-parsing.ts @@ -0,0 +1,333 @@ +/** + * XML parsing utilities for mobile element source + */ + +import { DOMParser } from '@xmldom/xmldom' +import type { + Document as XMLDocument, + Element as XMLElement, + Node as XMLNode +} from '@xmldom/xmldom' +import xpath from 'xpath' +import type { + ElementAttributes, + JSONElement, + Bounds, + UniquenessResult +} from './types.js' + +/** + * Get child nodes that are elements (not text nodes, comments, etc.) + */ +function childNodesOf(node: XMLNode): XMLNode[] { + const children: XMLNode[] = [] + if (node.childNodes) { + for (let i = 0; i < node.childNodes.length; i++) { + const child = node.childNodes.item(i) + if (child?.nodeType === 1) { + children.push(child) + } + } + } + return children +} + +/** + * Recursively translate DOM node to JSONElement + */ +function translateRecursively( + domNode: XMLNode, + parentPath: string = '', + index: number | null = null +): JSONElement { + const attributes: ElementAttributes = {} + + const element = domNode as XMLElement + if (element.attributes) { + for (let attrIdx = 0; attrIdx < element.attributes.length; attrIdx++) { + const attr = element.attributes.item(attrIdx) + if (attr) { + attributes[attr.name] = attr.value.replace(/(\n)/gm, '\\n') + } + } + } + + const path = + index === null ? '' : `${parentPath ? parentPath + '.' : ''}${index}` + + return { + children: childNodesOf(domNode).map((childNode, childIndex) => + translateRecursively(childNode as XMLNode, path, childIndex) + ), + tagName: domNode.nodeName, + attributes, + path + } +} + +/** + * Compare two nodes for equality by platform-specific attributes + * (reference equality via === may fail when nodes come from different traversals) + */ +function isSameElement(node1: XMLNode, node2: XMLNode): boolean { + if (node1.nodeType !== 1 || node2.nodeType !== 1) { + return false + } + const el1 = node1 as XMLElement + const el2 = node2 as XMLElement + + if (el1.nodeName !== el2.nodeName) { + return false + } + + // For Android, compare by bounds (unique per element) + const bounds1 = el1.getAttribute('bounds') + const bounds2 = el2.getAttribute('bounds') + if (bounds1 && bounds2) { + return bounds1 === bounds2 + } + + // For iOS, compare by x, y, width, height + const x1 = el1.getAttribute('x') + const y1 = el1.getAttribute('y') + const x2 = el2.getAttribute('x') + const y2 = el2.getAttribute('y') + if (x1 && y1 && x2 && y2) { + return ( + x1 === x2 && + y1 === y2 && + el1.getAttribute('width') === el2.getAttribute('width') && + el1.getAttribute('height') === el2.getAttribute('height') + ) + } + + return false +} + +/** + * Convert XML page source to JSON tree structure + */ +export function xmlToJSON(sourceXML: string): JSONElement | null { + try { + const parser = new DOMParser() + const sourceDoc = parser.parseFromString(sourceXML, 'text/xml') + + // xmldom 0.9+ throws ParseError for fatal errors (caught below); this catches non-fatal cases + const parseErrors = sourceDoc.getElementsByTagName('parsererror') + if (parseErrors.length > 0) { + console.error( + '[xmlToJSON] XML parsing error:', + parseErrors[0].textContent + ) + return null + } + + const children = childNodesOf(sourceDoc) + const firstChild = + children[0] || + (sourceDoc.documentElement + ? childNodesOf(sourceDoc.documentElement)[0] + : null) + + return firstChild + ? translateRecursively(firstChild) + : { children: [], tagName: '', attributes: {}, path: '' } + } catch (e) { + console.error('[xmlToJSON] Failed to parse XML:', e) + return null + } +} + +/** + * Parse XML source to DOM Document for XPath evaluation + */ +export function xmlToDOM(sourceXML: string): XMLDocument | null { + try { + const parser = new DOMParser() + const doc = parser.parseFromString(sourceXML, 'text/xml') + + // xmldom 0.9+ throws ParseError for fatal errors (caught below); this catches non-fatal cases + const parseErrors = doc.getElementsByTagName('parsererror') + if (parseErrors.length > 0) { + console.error('[xmlToDOM] XML parsing error:', parseErrors[0].textContent) + return null + } + + return doc + } catch (e) { + console.error('[xmlToDOM] Failed to parse XML:', e) + return null + } +} + +/** + * Execute XPath query on DOM document + */ +export function evaluateXPath(doc: XMLDocument, xpathExpr: string): XMLNode[] { + try { + // @xmldom/xmldom 0.9+ types don't satisfy global Node; xpath still works at runtime + const nodes = xpath.select(xpathExpr, doc as unknown as Node) + if (Array.isArray(nodes)) { + return nodes as unknown as XMLNode[] + } + return [] + } catch (e) { + console.error(`[evaluateXPath] Failed to evaluate "${xpathExpr}":`, e) + return [] + } +} + +/** + * Check if an XPath selector is unique and get index if not + */ +export function checkXPathUniqueness( + doc: XMLDocument, + xpathExpr: string, + targetNode?: XMLNode +): UniquenessResult { + try { + const nodes = evaluateXPath(doc, xpathExpr) + const totalMatches = nodes.length + + if (totalMatches === 0) { + return { isUnique: false } + } + + if (totalMatches === 1) { + return { isUnique: true } + } + + // Not unique - find index of target node if provided + if (targetNode) { + for (let i = 0; i < nodes.length; i++) { + if (nodes[i] === targetNode || isSameElement(nodes[i], targetNode)) { + return { + isUnique: false, + index: i + 1, // 1-based index for XPath + totalMatches + } + } + } + } + + return { isUnique: false, totalMatches } + } catch (e) { + console.error(`[checkXPathUniqueness] Error checking "${xpathExpr}":`, e) + return { isUnique: false } + } +} + +/** + * Find DOM node by JSONElement path (e.g., "0.2.1") + */ +export function findDOMNodeByPath( + doc: XMLDocument, + path: string +): XMLNode | null { + if (!path) { + return doc.documentElement + } + + const indices = path.split('.').map(Number) + let current: XMLNode | null = doc.documentElement + + for (const index of indices) { + if (!current) { + return null + } + + const children: XMLNode[] = [] + if (current.childNodes) { + for (let i = 0; i < current.childNodes.length; i++) { + const child = current.childNodes.item(i) + if (child?.nodeType === 1) { + children.push(child) + } + } + } + + current = children[index] || null + } + + return current +} + +/** + * Parse Android bounds string "[x1,y1][x2,y2]" to coordinates + */ +export function parseAndroidBounds(bounds: string): Bounds { + const match = bounds.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/) + if (!match) { + return { x: 0, y: 0, width: 0, height: 0 } + } + + const x1 = parseInt(match[1], 10) + const y1 = parseInt(match[2], 10) + const x2 = parseInt(match[3], 10) + const y2 = parseInt(match[4], 10) + + return { + x: x1, + y: y1, + width: x2 - x1, + height: y2 - y1 + } +} + +/** + * Parse iOS element bounds from individual x, y, width, height attributes + */ +export function parseIOSBounds(attributes: ElementAttributes): Bounds { + return { + x: parseInt(attributes.x || '0', 10), + y: parseInt(attributes.y || '0', 10), + width: parseInt(attributes.width || '0', 10), + height: parseInt(attributes.height || '0', 10) + } +} + +/** + * Flatten JSON element tree to array (depth-first) + */ +export function flattenElementTree(root: JSONElement): JSONElement[] { + const result: JSONElement[] = [] + + function traverse(element: JSONElement) { + result.push(element) + for (const child of element.children) { + traverse(child) + } + } + + traverse(root) + return result +} + +/** + * Count occurrences of an attribute value in the source XML + */ +export function countAttributeOccurrences( + sourceXML: string, + attribute: string, + value: string +): number { + const escapedValue = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // `attribute` is a fixed set of XML attr names (resource-id, content-desc, + // text, name, label, value) controlled by the locator pipeline; `value` + // is regex-escaped above. No user-controlled regex source. + // eslint-disable-next-line security/detect-non-literal-regexp + const pattern = new RegExp(`${attribute}=["']${escapedValue}["']`, 'g') + const matches = sourceXML.match(pattern) + return matches ? matches.length : 0 +} + +/** + * Check if an attribute value is unique in the source (fast regex-based check) + */ +export function isAttributeUnique( + sourceXML: string, + attribute: string, + value: string +): boolean { + return countAttributeOccurrences(sourceXML, attribute, value) === 1 +} diff --git a/packages/elements/src/mobile-elements.ts b/packages/elements/src/mobile-elements.ts new file mode 100644 index 00000000..1eabbc53 --- /dev/null +++ b/packages/elements/src/mobile-elements.ts @@ -0,0 +1,199 @@ +/** + * Mobile element detection utilities for iOS and Android + * + * Uses page source parsing for optimal performance (2 HTTP calls vs 600+ for 50 elements) + */ + +import type { + ElementWithLocators, + FilterOptions, + JSONElement, + LocatorStrategy +} from './locators/index.js' +import { + generateAllElementLocators, + getDefaultFilters, + xmlToJSON +} from './locators/index.js' + +/** + * Element info returned by getMobileVisibleElements + * Uses uniform fields (all elements have same keys) to enable TOON tabular format + */ +export interface MobileElementInfo { + selector: string + tagName: string + isInViewport: boolean + text: string + resourceId: string + accessibilityId: string + isEnabled: boolean + altSelector: string // Single alternative selector (flattened for tabular format) + // Only present when includeBounds=true + bounds?: { x: number; y: number; width: number; height: number } +} + +/** + * Options for getMobileVisibleElements + */ +export interface GetMobileElementsOptions { + includeContainers?: boolean + includeBounds?: boolean + /** Only return elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean + filterOptions?: FilterOptions +} + +/** + * Locator strategy priority order for selecting best selector + * Earlier = higher priority + */ +const LOCATOR_PRIORITY: LocatorStrategy[] = [ + 'accessibility-id', // Most stable, cross-platform + 'id', // Android resource-id + 'text', // Text-based (can be fragile but readable) + 'predicate-string', // iOS predicate + 'class-chain', // iOS class chain + 'uiautomator', // Android UiAutomator compound + 'xpath' // XPath (last resort, brittle) + // 'class-name' intentionally excluded - too generic +] + +/** + * Select best locators from available strategies + * Returns [primarySelector, ...alternativeSelectors] + */ +function selectBestLocators(locators: Record): string[] { + const selected: string[] = [] + + // Find primary selector based on priority + for (const strategy of LOCATOR_PRIORITY) { + if (locators[strategy]) { + selected.push(locators[strategy]) + break + } + } + + // Add one alternative if available (different strategy) + for (const strategy of LOCATOR_PRIORITY) { + if (locators[strategy] && !selected.includes(locators[strategy])) { + selected.push(locators[strategy]) + break + } + } + + return selected +} + +/** + * Convert ElementWithLocators to MobileElementInfo + * Uses uniform fields (all elements have same keys) to enable CSV tabular format + */ +function toMobileElementInfo( + element: ElementWithLocators, + includeBounds: boolean +): MobileElementInfo { + const selectedLocators = selectBestLocators(element.locators) + + // Use contentDesc for accessibilityId on Android, or name on iOS + const accessId = element.accessibilityId || element.contentDesc + + // Build object with ALL fields for uniform schema (enables CSV tabular format) + // Empty string '' used for missing values to keep schema consistent + const info: MobileElementInfo = { + selector: selectedLocators[0] || '', + tagName: element.tagName, + isInViewport: element.isInViewport, + text: element.text || '', + resourceId: element.resourceId || '', + accessibilityId: accessId || '', + isEnabled: element.enabled !== false, + altSelector: selectedLocators[1] || '' // Single alternative (flattened for tabular) + } + + // Only include bounds if explicitly requested (adds 4 extra columns) + if (includeBounds) { + info.bounds = element.bounds + } + + return info +} + +/** + * Get viewport size from browser + */ +async function getViewportSize( + browser: WebdriverIO.Browser +): Promise<{ width: number; height: number }> { + try { + const size = await browser.getWindowSize() + return { width: size.width, height: size.height } + } catch { + return { width: 9999, height: 9999 } + } +} + +/** + * Get all visible elements from a mobile app, also returning the raw JSON element tree. + * Single parse of page source: tree and flat list share one xmlToJSON call. + * + * Performance: 2 HTTP calls (getWindowSize + getPageSource) vs 12+ per element with legacy approach + */ +export async function getMobileVisibleElementsWithTree( + browser: WebdriverIO.Browser, + platform: 'ios' | 'android', + options: GetMobileElementsOptions = {} +): Promise<{ elements: MobileElementInfo[]; tree: JSONElement | null }> { + const { + includeContainers = false, + includeBounds = false, + inViewportOnly = true, + filterOptions + } = options + + const viewportSize = await getViewportSize(browser) + const pageSource = await browser.getPageSource() + + const filters: FilterOptions = { + ...getDefaultFilters(platform, includeContainers), + ...filterOptions + } + + const tree = xmlToJSON(pageSource) + + // Stash the source XML on the root element so serializeMobileSnapshot + // can use the full locator pipeline without requiring it as a separate arg. + if (tree) { + tree.attributes._sourceXML = pageSource + } + + const elementLocators = generateAllElementLocators(pageSource, { + platform, + viewportSize, + filters, + inViewportOnly + }) + + const elements = elementLocators.map((el) => + toMobileElementInfo(el, includeBounds) + ) + return { elements, tree } +} + +/** + * Get all visible elements from a mobile app + * + * Performance: 2 HTTP calls (getWindowSize + getPageSource) vs 12+ per element with legacy approach + */ +export async function getMobileVisibleElements( + browser: WebdriverIO.Browser, + platform: 'ios' | 'android', + options: GetMobileElementsOptions = {} +): Promise { + const { elements } = await getMobileVisibleElementsWithTree( + browser, + platform, + options + ) + return elements +} diff --git a/packages/elements/src/snapshot.ts b/packages/elements/src/snapshot.ts new file mode 100644 index 00000000..b49de4c7 --- /dev/null +++ b/packages/elements/src/snapshot.ts @@ -0,0 +1,775 @@ +/** + * AI-readable snapshot serializers + * + * Converts accessibility trees and mobile element trees into depth-indented + * text files that LLMs can consume without any parsing. + */ + +import type { AccessibilityNode } from './accessibility-tree.js' +import type { JSONElement } from './locators/types.js' +import { parseAndroidBounds, parseIOSBounds } from './locators/xml-parsing.js' +import { + ANDROID_INTERACTABLE_TAGS, + IOS_INTERACTABLE_TAGS +} from './locators/constants.js' +import { getSuggestedLocators } from './locators/locator-generation.js' + +/** + * Roles that can be interacted with β€” rendered with `β†’ selector`. + * Structural roles (heading, img, form, nav, …) are intentionally excluded. + */ +const INTERACTIVE_ROLES = new Set([ + 'button', + 'link', + 'textbox', + 'checkbox', + 'radio', + 'combobox', + 'slider', + 'searchbox', + 'spinbutton', + 'switch', + 'tab', + 'menuitem', + 'option' +]) + +/** + * Walk backwards from `index` to find the nearest ancestor or preceding + * structural sibling with a non-empty name. Same-depth nodes are only + * used when they are structural (img, heading, statictext, …) β€” never + * another interactive element. + */ +function inferPurpose( + nodes: AccessibilityNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + // Same-depth sibling: only structural elements count + if (nodes[i].depth === myDepth && INTERACTIVE_ROLES.has(nodes[i].role)) { + continue + } + return nodes[i].name + } + } + return undefined +} + +export interface WebSnapshotOptions { + /** Only include nodes whose bounding rect intersects the viewport (default true). */ + inViewportOnly?: boolean +} + +/** + * Serialize a web accessibility tree into a depth-indented text snapshot. + * + * @param nodes Flat ordered node list from getBrowserAccessibilityTree() + * @param context Optional page context for the header line + * @param options {@link WebSnapshotOptions} + */ +// Single linear pass over the flat node list β€” per-node decisions (skip-by- +// viewport, role classification, statictext echo dedup, interactive vs +// structural rendering) must stay together so the inferred-purpose lookup +// can see siblings. ROADMAP P2: collapse with mobile pipeline into one +// `serializeSnapshot()`; until then this is the canonical web walker. +// eslint-disable-next-line max-lines-per-function +export function serializeWebSnapshot( + nodes: AccessibilityNode[], + context?: { url?: string; title?: string }, + options: WebSnapshotOptions = {} +): string { + const { inViewportOnly = true } = options + + let header = '[Page' + if (context?.title) { + header += `: ${context.title}` + } + if (context?.url) { + header += ` β€” ${context.url}` + } + header += ']' + + const lines: string[] = [header] + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + + // When viewport filtering is on, skip nodes that are known to be off-screen. + // Nodes from a tree captured with inViewportOnly=false will have + // isInViewport populated; nodes from a pre-filtered tree all have + // isInViewport=true (or undefined for pre-existing data). + if (inViewportOnly && node.isInViewport === false) { + continue + } + + const indent = ' '.repeat(node.depth + 1) // +1 indents everything under the header + const isInteractive = INTERACTIVE_ROLES.has(node.role) + + // Skip statictext that merely echoes the parent link/button name. + // Example: link "Highlights" β†’ a*=Highlights doesn't need + // statictext "Highlights" as a child because it adds no information. + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + const parentRole = nodes[j].role + const parentName = nodes[j].name + if ( + INTERACTIVE_ROLES.has(parentRole) && + parentName && + parentName.includes(node.name) + ) { + echoedByParent = true + } + break // only check the immediate structural parent + } + } + if (echoedByParent) { + continue + } + } + + // Heading gets level suffix: heading[2] + const roleLabel = + node.role === 'heading' && node.level + ? `heading[${node.level}]` + : node.role + + if (isInteractive) { + // No selector β†’ agent can't act on this node; skip entirely + if (!node.selector) { + continue + } + const purpose = inferPurpose(nodes, i) + if (node.name) { + // Show parent context when available β€” disambiguates + // duplicate selectors like six "Add to Wishlist" buttons. + lines.push( + purpose + ? `${indent}${roleLabel} "${node.name}" ∈ "${purpose}" β†’ ${node.selector}` + : `${indent}${roleLabel} "${node.name}" β†’ ${node.selector}` + ) + } else if (purpose) { + lines.push(`${indent}${roleLabel} ∈ "${purpose}" β†’ ${node.selector}`) + } else { + lines.push(`${indent}${roleLabel} β†’ ${node.selector}`) + } + } else { + // Container / structural: show role + name when present, no selector + lines.push( + node.name + ? `${indent}${roleLabel} "${node.name}"` + : `${indent}${roleLabel}` + ) + } + } + + return lines.join('\n') +} + +// --------------------------------------------------------------------------- +// Mobile snapshot helpers +// --------------------------------------------------------------------------- + +/** Shorten fully-qualified Android/iOS class names to the last segment. */ +function simplifyTag(tagName: string): string { + const dot = tagName.lastIndexOf('.') + if (dot !== -1) { + return tagName.slice(dot + 1) + } + return tagName.replace(/^XCUIElementType/, '') +} + +// --------------------------------------------------------------------------- +// Mobile role classification β€” maps raw Android/iOS class names to semantic +// roles so the snapshot reads like the web version (button, textbox, img, …). +// --------------------------------------------------------------------------- + +const ANDROID_ROLE_MAP: Record = { + 'android.widget.Button': 'button', + 'android.widget.ImageButton': 'button', + 'android.widget.ToggleButton': 'button', + 'android.widget.FloatingActionButton': 'button', + 'com.google.android.material.button.MaterialButton': 'button', + 'com.google.android.material.floatingactionbutton.FloatingActionButton': + 'button', + 'android.widget.EditText': 'textbox', + 'android.widget.AutoCompleteTextView': 'textbox', + 'android.widget.MultiAutoCompleteTextView': 'textbox', + 'android.widget.SearchView': 'searchbox', + 'android.widget.ImageView': 'img', + 'android.widget.QuickContactBadge': 'img', + 'android.widget.CheckBox': 'checkbox', + 'android.widget.RadioButton': 'radio', + 'android.widget.Switch': 'switch', + 'android.widget.Spinner': 'combobox', + 'android.widget.SeekBar': 'slider', + 'android.widget.RatingBar': 'slider', + 'android.widget.ProgressBar': 'progressbar', + 'android.widget.TextView': 'statictext', + 'android.widget.CheckedTextView': 'statictext', + 'android.widget.RecyclerView': 'list', + 'android.widget.ListView': 'list', + 'android.widget.GridView': 'list', + 'android.webkit.WebView': 'webview' +} + +const IOS_ROLE_MAP: Record = { + XCUIElementTypeButton: 'button', + XCUIElementTypeLink: 'link', + XCUIElementTypeTextField: 'textbox', + XCUIElementTypeSecureTextField: 'textbox', + XCUIElementTypeTextView: 'textbox', + XCUIElementTypeSearchField: 'searchbox', + XCUIElementTypeImage: 'img', + XCUIElementTypeIcon: 'img', + XCUIElementTypeSwitch: 'switch', + XCUIElementTypeSlider: 'slider', + XCUIElementTypeStepper: 'slider', + XCUIElementTypeCheckBox: 'checkbox', + XCUIElementTypeRadioButton: 'radio', + XCUIElementTypePicker: 'combobox', + XCUIElementTypePickerWheel: 'combobox', + XCUIElementTypeDatePicker: 'combobox', + XCUIElementTypeSegmentedControl: 'combobox', + XCUIElementTypeStaticText: 'statictext', + XCUIElementTypeCell: 'listitem', + XCUIElementTypeTable: 'list', + XCUIElementTypeCollectionView: 'list' +} + +function classifyMobileRole( + tagName: string, + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + return ANDROID_ROLE_MAP[tagName] || simplifyTag(tagName) + } + return IOS_ROLE_MAP[tagName] || simplifyTag(tagName) +} + +// --------------------------------------------------------------------------- +// Locator generation +// --------------------------------------------------------------------------- + +function getBestAndroidLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline (generateAllElementLocators). + // Takes priority over the simplified fallback logic below. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand in WebdriverIO ($('~foo')) + if (attrs['content-desc']) { + return `~${attrs['content-desc']}` + } + if (attrs['resource-id']) { + return `id:${attrs['resource-id']}` + } + if (attrs.text) { + return `~${attrs.text}` + } + // Fallback: class-based locator (only useful with :nth-of-type or index) + if (attrs.class) { + return `class:${simplifyTag(attrs.class)}` + } + return undefined +} + +function getBestIOSLocator( + attrs: JSONElement['attributes'] +): string | undefined { + // Pre-computed by the full locator pipeline. + if (attrs._selector) { + return attrs._selector + } + // ~ prefix = accessibility-id shorthand (maps to `name` on iOS) + if (attrs.name) { + return `~${attrs.name}` + } + if (attrs.label) { + return `~${attrs.label}` + } + if (attrs.value) { + return `~${attrs.value}` + } + // Fallback: class-based locator + if (attrs.type) { + return `class:${simplifyTag(attrs.type)}` + } + return undefined +} + +// --------------------------------------------------------------------------- +// Identity +// --------------------------------------------------------------------------- + +function getMobileNodeIdentity( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): string { + if (platform === 'android') { + const contentDesc = attrs['content-desc'] + if (contentDesc) { + return contentDesc + } + if (attrs.text) { + return attrs.text + } + // Fall back to the last segment of the resource-id (e.g. "search_action_bar") + const rid = attrs['resource-id'] + if (rid) { + const slash = rid.lastIndexOf('/') + return slash !== -1 ? rid.slice(slash + 1) : rid + } + return '' + } + return attrs.name || attrs.label || attrs.value || attrs.text || '' +} + +// --------------------------------------------------------------------------- +// Interactivity +// --------------------------------------------------------------------------- + +const ANDROID_INTERACTABLE_SET = new Set(ANDROID_INTERACTABLE_TAGS) +const IOS_INTERACTABLE_SET = new Set(IOS_INTERACTABLE_TAGS) + +/** An element is *explicitly* interactive when it carries a click/focus/check + * attribute β€” as opposed to being interactive only because its tag is in the + * interactable-tag list. Explicit parents should carry the β†’ selector, not + * their tag-interactive children. */ +function isExplicitlyInteractive( + attrs: JSONElement['attributes'], + platform: 'android' | 'ios' +): boolean { + if (platform === 'android') { + return ( + attrs.clickable === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' || + attrs['long-clickable'] === 'true' + ) + } + return attrs.accessible === 'true' +} + +function isMobileInteractive( + element: JSONElement, + platform: 'android' | 'ios' +): boolean { + const attrs = element.attributes + if (platform === 'android') { + if (ANDROID_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return ( + attrs.clickable === 'true' || + attrs['long-clickable'] === 'true' || + attrs.focusable === 'true' || + attrs.checkable === 'true' + ) + } + if (IOS_INTERACTABLE_SET.has(element.tagName)) { + return true + } + return attrs.accessible === 'true' +} + +// --------------------------------------------------------------------------- +// Viewport +// --------------------------------------------------------------------------- + +interface WalkMobileOptions { + inViewportOnly: boolean + viewport: { width: number; height: number } + /** Raw page-source XML. When provided, the full locator pipeline is used. */ + sourceXML?: string + /** 'uiautomator2' or 'xcuitest'. Required when sourceXML is set. */ + automationName?: string +} + +function isMobileInViewport( + element: JSONElement, + platform: 'android' | 'ios', + viewport: { width: number; height: number } +): boolean { + const bounds = + platform === 'android' + ? parseAndroidBounds(element.attributes.bounds || '') + : parseIOSBounds(element.attributes) + + if (bounds.width === 0 && bounds.height === 0) { + return true + } + + return ( + bounds.x >= 0 && + bounds.y >= 0 && + bounds.width > 0 && + bounds.height > 0 && + bounds.x + bounds.width <= viewport.width && + bounds.y + bounds.height <= viewport.height + ) +} + +// --------------------------------------------------------------------------- +// Flat-node representation (mirrors AccessibilityNode so both pipelines share +// inferPurpose, dedup, and rendering logic). +// --------------------------------------------------------------------------- + +interface MobileFlatNode { + role: string + name: string + selector: string + depth: number + isInteractive: boolean + /** True when the element has clickable/focusable/checkable β€” the intended tap target. */ + isExplicitInteractive: boolean + isInViewport: boolean +} + +/** + * First pass: walk the JSONElement tree, apply viewport filtering and + * collect every node into a flat array with semantic roles and selectors. + */ +// Recursive tree walker β€” splitting the viewport filter, locator pipeline, +// fallback, and node-emit branches would require threading the accumulator +// + walk options through 4 helpers. ROADMAP P0/P1: this whole pipeline gets +// merged with generateAllElementLocators; until that consolidation lands, +// keep as one walker. +// eslint-disable-next-line max-lines-per-function +function collectMobileNodes( + element: JSONElement, + platform: 'android' | 'ios', + depth: number, + nodes: MobileFlatNode[], + walkOpts: WalkMobileOptions +): void { + const attrs = element.attributes + const role = classifyMobileRole(element.tagName, platform) + const name = getMobileNodeIdentity(attrs, platform) + const explicit = isExplicitlyInteractive(attrs, platform) + const interactive = isMobileInteractive(element, platform) + const inViewport = isMobileInViewport(element, platform, walkOpts.viewport) + + // Viewport filtering + if (walkOpts.inViewportOnly) { + if (interactive && !inViewport) { + // Skip this node but still recurse (scroll children may be in view). + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } + return + } + if (!interactive && !inViewport) { + // Collapse off-screen container to a placeholder. + nodes.push({ + role: 'generic', + name: name ? `${role} "${name}"` : role, + selector: '', + depth, + isInteractive: false, + isExplicitInteractive: false, + isInViewport: false + }) + return + } + } + + // Generate a selector for every interactive element. + // Use the full locator pipeline when source XML is available; + // otherwise fall back to the simplified attribute-based heuristics. + let locator = '' + if (interactive) { + if (walkOpts.sourceXML && walkOpts.automationName) { + // Full pipeline: accessible-id, id, text, uiautomator, xpath, class-name + const suggested = getSuggestedLocators( + element, + walkOpts.sourceXML, + walkOpts.automationName, + { + sourceXML: walkOpts.sourceXML, + parsedDOM: null, + isAndroid: platform === 'android' + } + ) + if (suggested.length > 0) { + locator = suggested[0][1] // first = best priority + } + } + if (!locator) { + // Simplified fallback + locator = + (platform === 'android' + ? getBestAndroidLocator(attrs) + : getBestIOSLocator(attrs)) ?? '' + } + } + + nodes.push({ + role, + name, + selector: locator, + depth, + isInteractive: interactive, + isExplicitInteractive: explicit, + isInViewport: inViewport + }) + + for (const child of element.children || []) { + collectMobileNodes(child, platform, depth + 1, nodes, walkOpts) + } +} + +// --------------------------------------------------------------------------- +// Context inference β€” shared with the web pipeline. +// Same-depth structural siblings (img, statictext, heading, …) provide +// context for following interactive nodes. +// --------------------------------------------------------------------------- + +const MOBILE_STRUCTURAL_ROLES = new Set([ + 'img', + 'heading', + 'list', + 'listitem', + 'webview', + 'progressbar', + 'slider', + 'switch', + 'generic' +]) + +function mobileInferPurpose( + nodes: MobileFlatNode[], + index: number +): string | undefined { + const myDepth = nodes[index].depth + for (let i = index - 1; i >= 0; i--) { + if (nodes[i].depth <= myDepth && nodes[i].name) { + if ( + nodes[i].depth === myDepth && + !MOBILE_STRUCTURAL_ROLES.has(nodes[i].role) + ) { + continue + } + return nodes[i].name + } + } + return undefined +} + +// --------------------------------------------------------------------------- +// When a tag-only-interactive child (e.g. a statictext TextView) sits +// directly under an explicitly-interactive parent (e.g. a clickable +// LinearLayout row), the *parent* should carry the β†’ selector β€” the +// child is just a label. Suppress the child's interactivity so the +// parent renders as the actionable element. +// --------------------------------------------------------------------------- + +function suppressTagOnlyChildren(nodes: MobileFlatNode[]): void { + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + if (!node.isInteractive || node.isExplicitInteractive) { + continue + } + // Walk up through ALL ancestors looking for an explicitly-interactive + // parent. The immediate depth-1 parent may just be a layout wrapper; + // the real clickable row could be 2-3 levels up. + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if (nodes[j].isExplicitInteractive) { + node.isInteractive = false + break // found β€” suppress and stop + } + // keep looking upward through the ancestor chain + } + } + } +} + +// --------------------------------------------------------------------------- +// Render pass: flat nodes into lines with ∈ context, dedup, noise filter, +// and class-instance indexing. +// --------------------------------------------------------------------------- + +/** Layout roles that carry no semantic meaning by themselves. */ +const NOISY_ROLES = new Set([ + 'FrameLayout', + 'LinearLayout', + 'ViewGroup', + 'RelativeLayout', + 'View', + 'CardView', + 'ConstraintLayout', + 'ScrollView' +]) + +/** + * Pre-count selector occurrences so we can attach .instance(N) suffixes + * to duplicate selectors. + */ +function countSelectors(nodes: MobileFlatNode[]): Map { + const counts = new Map() + for (const node of nodes) { + if (node.selector) { + counts.set(node.selector, (counts.get(node.selector) ?? 0) + 1) + } + } + return counts +} + +// Render pass mirrors serializeWebSnapshot's structure (linear node-by-node +// emission with parent-context lookback, statictext echo dedup, layout-noise +// collapse, .instance(N) indexing). Splitting per-decision would lose the +// shared per-node state. ROADMAP P2: collapse with web pipeline. +// eslint-disable-next-line max-lines-per-function +function renderMobileNodes(nodes: MobileFlatNode[]): string[] { + const lines: string[] = [] + const selectorCounts = countSelectors(nodes) + const selectorIndex = new Map() + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + const indent = ' '.repeat(node.depth + 1) + + // Collapse anonymous layout containers at depth β‰₯ 2. + // Keep depth 0-1 structural chrome and any named container. + if ( + NOISY_ROLES.has(node.role) && + !node.name && + node.depth > 1 && + !node.isInteractive + ) { + continue + } + + // Off-screen containers rendered as collapsed placedersen + if (node.isInViewport === false && !node.isInteractive) { + lines.push(`${indent}β‹― ${node.name} (off-screen)`) + continue + } + + // Dedup: skip statictext whose text is echoed by the parent interactive element + if (node.role === 'statictext' && node.name) { + let echoedByParent = false + for (let j = i - 1; j >= 0; j--) { + if (nodes[j].depth < node.depth) { + if ( + nodes[j].isInteractive && + nodes[j].name && + nodes[j].name.includes(node.name) + ) { + echoedByParent = true + } + break + } + } + if (echoedByParent) { + continue + } + } + + if (node.isInteractive && node.selector) { + // Append .instance(N) when the same selector repeats + let selector = node.selector + const total = selectorCounts.get(selector) ?? 1 + if (total > 1) { + const idx = selectorIndex.get(selector) ?? 0 + selectorIndex.set(selector, idx + 1) + selector = `${selector}.instance(${idx})` + } + + const purpose = mobileInferPurpose(nodes, i) + if (node.name) { + lines.push( + purpose + ? `${indent}${node.role} "${node.name}" ∈ "${purpose}" β†’ ${selector}` + : `${indent}${node.role} "${node.name}" β†’ ${selector}` + ) + } else if (purpose) { + lines.push(`${indent}${node.role} ∈ "${purpose}" β†’ ${selector}`) + } else { + lines.push(`${indent}${node.role} β†’ ${selector}`) + } + } else { + // Container / structural / non-locatable + lines.push( + node.name + ? `${indent}${node.role} "${node.name}"` + : `${indent}${node.role}` + ) + } + } + + return lines +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export interface MobileSnapshotOptions { + /** Only include elements whose bounds intersect the viewport (default true). */ + inViewportOnly?: boolean + /** + * Raw XML page source string. When provided the full locator pipeline + * (getSuggestedLocators) runs on every interactive node, producing the same + * selectors that getElements() returns. Omit to use simplified heuristics. + */ + sourceXML?: string +} + +/** + * Serialize a mobile element tree into a depth-indented text snapshot. + * + * @param root Root JSONElement from the page source XML parse + * @param context Platform, optional device name, viewport, and source XML. + * Include `sourceXML` to use the full locator pipeline. + * @param options {@link MobileSnapshotOptions} + */ +export function serializeMobileSnapshot( + root: JSONElement, + context: { + platform: 'android' | 'ios' + deviceName?: string + viewport?: { width: number; height: number } + /** Raw page-source XML. When set, selectors match getElements() output. */ + sourceXML?: string + }, + options: MobileSnapshotOptions = {} +): string { + const { platform, deviceName, viewport, sourceXML } = context + const { inViewportOnly = true } = options + + // Auto-detect source XML stashed by getMobileVisibleElementsWithTree + const effectiveXML = sourceXML || root.attributes._sourceXML + + const effectiveViewport = viewport ?? { width: 9999, height: 9999 } + const automationName = platform === 'android' ? 'uiautomator2' : 'xcuitest' + + let header = `[${platform}` + if (deviceName) { + header += ` β€” ${deviceName}` + } + if (viewport) { + header += ` (${viewport.width}Γ—${viewport.height})` + } + header += ']' + + const nodes: MobileFlatNode[] = [] + collectMobileNodes(root, platform, 0, nodes, { + inViewportOnly, + viewport: effectiveViewport, + sourceXML: effectiveXML, + automationName: effectiveXML ? automationName : undefined + }) + + // Let explicitly-interactive parents carry the β†’ selector + suppressTagOnlyChildren(nodes) + + const lines = renderMobileNodes(nodes) + return [header, ...lines].join('\n') +} diff --git a/packages/elements/tests/accessibility-tree.test.ts b/packages/elements/tests/accessibility-tree.test.ts new file mode 100644 index 00000000..6fd7be13 --- /dev/null +++ b/packages/elements/tests/accessibility-tree.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, vi } from 'vitest' +import { getBrowserAccessibilityTree } from '../src/accessibility-tree.js' + +describe('getBrowserAccessibilityTree', () => { + it('calls browser.execute and returns result', async () => { + const nodes = [ + { + role: 'button', + name: 'Submit', + selector: 'button*=Submit', + depth: 0, + level: '', + disabled: '', + checked: '', + expanded: '', + selected: '', + pressed: '', + required: '', + readonly: '' + } + ] + const mockBrowser = { execute: vi.fn().mockResolvedValue(nodes) } as any + const result = await getBrowserAccessibilityTree(mockBrowser) + expect(mockBrowser.execute).toHaveBeenCalledTimes(1) + expect(result).toEqual(nodes) + }) +}) diff --git a/packages/elements/tests/browser-elements.test.ts b/packages/elements/tests/browser-elements.test.ts new file mode 100644 index 00000000..6fba3af5 --- /dev/null +++ b/packages/elements/tests/browser-elements.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect, vi } from 'vitest' +import { getInteractableBrowserElements } from '../src/browser-elements.js' + +describe('getInteractableBrowserElements', () => { + it('calls browser.execute with includeBounds=false by default', async () => { + const mockBrowser = { execute: vi.fn().mockResolvedValue([]) } as any + const result = await getInteractableBrowserElements(mockBrowser) + expect(mockBrowser.execute).toHaveBeenCalledTimes(1) + expect(result).toEqual([]) + }) + + it('passes includeBounds option to script', async () => { + const mockBrowser = { + execute: vi.fn().mockResolvedValue([{ tagName: 'button', name: 'OK' }]) + } as any + const result = await getInteractableBrowserElements(mockBrowser, { + includeBounds: true + }) + expect(result).toHaveLength(1) + expect(result[0].tagName).toBe('button') + }) +}) diff --git a/packages/elements/tests/locators/locator-generation.test.ts b/packages/elements/tests/locators/locator-generation.test.ts new file mode 100644 index 00000000..c5d2de4b --- /dev/null +++ b/packages/elements/tests/locators/locator-generation.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest' +import { locatorsToObject } from '@wdio/elements/locators' + +describe('locatorsToObject', () => { + it('converts locator array to object', () => { + const locators: [any, string][] = [ + ['accessibility-id', '~Submit'], + ['xpath', '//XCUIElementTypeButton[@name="Submit"]'] + ] + const result = locatorsToObject(locators) + expect(result['accessibility-id']).toBe('~Submit') + expect(result['xpath']).toBe('//XCUIElementTypeButton[@name="Submit"]') + }) + + it('returns first value for duplicate strategies', () => { + const locators: [any, string][] = [ + ['xpath', '//first'], + ['xpath', '//second'] + ] + const result = locatorsToObject(locators) + expect(result['xpath']).toBe('//first') + }) +}) diff --git a/packages/elements/tests/mobile-elements.test.ts b/packages/elements/tests/mobile-elements.test.ts new file mode 100644 index 00000000..64f19861 --- /dev/null +++ b/packages/elements/tests/mobile-elements.test.ts @@ -0,0 +1,13 @@ +import { describe, it, expect, vi } from 'vitest' +import { getMobileVisibleElements } from '../src/mobile-elements.js' + +describe('getMobileVisibleElements', () => { + it('returns empty array for unparseable XML', async () => { + const mockBrowser = { + getWindowSize: vi.fn().mockResolvedValue({ width: 375, height: 812 }), + getPageSource: vi.fn().mockResolvedValue(' & { role: string; depth: number } +): AccessibilityNode { + return { + name: '', + selector: '', + level: '', + disabled: '', + checked: '', + expanded: '', + selected: '', + pressed: '', + required: '', + readonly: '', + ...overrides + } +} + +describe('serializeWebSnapshot', () => { + it('produces a page header', () => { + const out = serializeWebSnapshot([]) + expect(out).toBe('[Page]') + }) + + it('includes title and url in header', () => { + const out = serializeWebSnapshot([], { + title: 'Login', + url: 'https://example.com/login' + }) + expect(out).toMatch('[Page: Login β€” https://example.com/login]') + }) + + it('renders interactive role with name and selector', () => { + const nodes = [ + node({ + role: 'button', + depth: 0, + name: 'Submit', + selector: 'button*=Submit' + }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('button "Submit" β†’ button*=Submit') + }) + + it('renders interactive role with ∈ ancestor name when self has no name', () => { + const nodes = [ + node({ role: 'form', depth: 0, name: 'Login form' }), + node({ role: 'checkbox', depth: 1, name: '', selector: '#remember' }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('checkbox ∈ "Login form" β†’ #remember') + }) + + it('omits interactive node with no selector regardless of name', () => { + const nodes = [ + node({ role: 'button', depth: 0, name: '', selector: '' }), + node({ + role: 'button', + depth: 0, + name: 'Named but unselector', + selector: '' + }) + ] + const out = serializeWebSnapshot(nodes) + // Only the header β€” both nodes skipped due to missing selector + expect(out.split('\n').length).toBe(1) + }) + + it('omits interactive node with ∈ context but no selector', () => { + const nodes = [ + node({ role: 'form', depth: 0, name: 'Login form' }), + node({ role: 'combobox', depth: 1, name: '', selector: '' }) + ] + const out = serializeWebSnapshot(nodes) + // combobox has ancestor context but no selector β€” must be dropped + expect(out).not.toContain('combobox') + expect(out).not.toContain('β†’') + }) + + it('renders container role without selector', () => { + const nodes = [node({ role: 'navigation', depth: 0, name: 'Main' })] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('navigation "Main"') + expect(out).not.toContain('β†’') + }) + + it('renders heading with level suffix', () => { + const nodes = [ + node({ role: 'heading', depth: 0, name: 'Sign in', level: 1 }) + ] + const out = serializeWebSnapshot(nodes) + expect(out).toContain('heading[1] "Sign in"') + }) + + it('indents nodes according to depth', () => { + const nodes = [ + node({ role: 'navigation', depth: 0, name: 'Nav' }), + node({ role: 'link', depth: 1, name: 'Home', selector: 'a*=Home' }) + ] + const lines = serializeWebSnapshot(nodes).split('\n') + // depth 0 β†’ 1 level of indent (' ' Γ— 1), depth 1 β†’ 2 levels (' ' Γ— 2) + expect(lines[1]).toMatch(/^ navigation/) + expect(lines[2]).toMatch(/^ link/) + }) + + it('renders full login page example correctly', () => { + const nodes: AccessibilityNode[] = [ + node({ role: 'navigation', depth: 0, name: 'Main' }), + node({ role: 'link', depth: 1, name: 'Home', selector: 'a*=Home' }), + node({ role: 'main', depth: 0, name: '' }), + node({ role: 'heading', depth: 1, name: 'Sign in', level: 1 }), + node({ role: 'form', depth: 1, name: 'Login' }), + node({ + role: 'textbox', + depth: 2, + name: 'Email address', + selector: '#email' + }), + node({ + role: 'button', + depth: 2, + name: 'Sign in', + selector: 'button*=Sign in' + }) + ] + const out = serializeWebSnapshot(nodes, { + title: 'Login', + url: 'https://example.com/login' + }) + expect(out).toContain('[Page: Login β€” https://example.com/login]') + expect(out).toContain('navigation "Main"') + expect(out).toContain('link "Home" ∈ "Main" β†’ a*=Home') + expect(out).toContain('heading[1] "Sign in"') + expect(out).toContain('textbox "Email address" ∈ "Login" β†’ #email') + expect(out).toContain('button "Sign in" ∈ "Login" β†’ button*=Sign in') + }) +}) + +// --------------------------------------------------------------------------- +// serializeMobileSnapshot +// --------------------------------------------------------------------------- + +function mobileEl( + tagName: string, + attrs: JSONElement['attributes'], + children: JSONElement[] = [] +): JSONElement { + return { tagName, attributes: attrs, children, path: '' } +} + +describe('serializeMobileSnapshot', () => { + it('produces a platform header with device and viewport', () => { + const root = mobileEl('hierarchy', {}) + const out = serializeMobileSnapshot(root, { + platform: 'android', + deviceName: 'Pixel 7', + viewport: { width: 412, height: 915 } + }) + expect(out).toMatch('[android β€” Pixel 7 (412Γ—915)]') + }) + + it('renders interactive Android element with accessibility-id locator', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.Button', { + clickable: 'true', + 'content-desc': 'Skip', + text: '' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('button "Skip" β†’ ~Skip') + }) + + it('falls back to resource-id when no content-desc', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.EditText', { + clickable: 'true', + 'content-desc': '', + 'resource-id': 'com.example:id/search', + text: '' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('textbox "search" β†’ id:com.example:id/search') + }) + + it('renders ∈ ancestor context when element has no identity', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl( + 'android.widget.LinearLayout', + { 'content-desc': 'Search section' }, + [ + mobileEl('android.widget.EditText', { + clickable: 'true', + 'content-desc': '', + 'resource-id': 'com.example:id/search', + text: '' + }) + ] + ) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain( + 'textbox "search" ∈ "Search section" β†’ id:com.example:id/search' + ) + }) + + it('renders iOS element with accessibility-id', () => { + const root = mobileEl('XCUIElementTypeApplication', {}, [ + mobileEl('XCUIElementTypeButton', { + accessible: 'true', + name: 'Accept All Cookies', + label: 'Accept All Cookies' + }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'ios' }) + expect(out).toContain('button "Accept All Cookies" β†’ ~Accept All Cookies') + }) + + it('simplifies iOS XCUIElementType prefix', () => { + const root = mobileEl('XCUIElementTypeApplication', {}, [ + mobileEl('XCUIElementTypeScrollView', {}) + ]) + const out = serializeMobileSnapshot(root, { platform: 'ios' }) + expect(out).toContain('ScrollView') + expect(out).not.toContain('XCUIElementType') + }) + + it('shows container without selector', () => { + const root = mobileEl('hierarchy', {}, [ + mobileEl('android.widget.FrameLayout', { 'content-desc': '' }) + ]) + const out = serializeMobileSnapshot(root, { platform: 'android' }) + expect(out).toContain('FrameLayout') + expect(out).not.toContain('β†’') + }) +}) diff --git a/packages/elements/tsconfig.json b/packages/elements/tsconfig.json new file mode 100644 index 00000000..3bcc9638 --- /dev/null +++ b/packages/elements/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "ignoreDeprecations": "6.0", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "noEmit": false, + "allowImportingTsExtensions": false, + "declaration": true, + "skipLibCheck": true, + "types": ["node", "@wdio/globals/types"] + }, + "include": ["src/**/*"] +} diff --git a/packages/nightwatch-devtools/README.md b/packages/nightwatch-devtools/README.md index 00da385d..5f404f5d 100644 --- a/packages/nightwatch-devtools/README.md +++ b/packages/nightwatch-devtools/README.md @@ -82,8 +82,9 @@ module.exports = { |--------|------|---------|-------------| | `port` | `number` | `3000` | Port for the DevTools backend server. Auto-incremented if already in use. | | `hostname` | `string` | `'localhost'` | Hostname the backend server binds to. | -| `screencast` | `ScreencastOptions` | `{ enabled: false }` | Session video recording (see [Screencast](#screencast)). | +| `screencast` | `ScreencastOptions` | `{ enabled: false }` | Session video recording β€” live mode only (see [Screencast](#screencast)). | | `bidi` | `boolean` | `false` | Opt into WebDriver BiDi capture for browser console + JS exceptions + network. Requires `webSocketUrl: true` in your capabilities and a BiDi-capable chromedriver. When attached, the per-command Chrome perf-log network path is gated off so requests don't duplicate. | +| `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-.zip` next to your `nightwatch.conf.cjs` at run end. See [Trace mode](../../README.md#-trace-mode-tracezip). | ```javascript globals: nightwatchDevtools({ diff --git a/packages/nightwatch-devtools/package.json b/packages/nightwatch-devtools/package.json index 0a92afb0..21f4f17a 100644 --- a/packages/nightwatch-devtools/package.json +++ b/packages/nightwatch-devtools/package.json @@ -43,12 +43,14 @@ "dependencies": { "@wdio/devtools-backend": "workspace:*", "@wdio/devtools-script": "workspace:*", + "@wdio/elements": "workspace:^", "@wdio/logger": "^9.18.0", "fluent-ffmpeg": "^2.1.3", "import-meta-resolve": "^4.2.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "devDependencies": { "@types/node": "25.9.1", diff --git a/packages/nightwatch-devtools/src/action-snapshot.ts b/packages/nightwatch-devtools/src/action-snapshot.ts new file mode 100644 index 00000000..171cb9d0 --- /dev/null +++ b/packages/nightwatch-devtools/src/action-snapshot.ts @@ -0,0 +1,67 @@ +// Per-action snapshot capture for Nightwatch β€” fires only in `mode: 'trace'` +// for commands in ACTION_MAP. Wraps NightwatchBrowser in a minimal +// WebdriverIO.Browser-shaped shim so @wdio/elements can run its in-page +// scripts. Returns null on failure; capture errors must not break the test. + +import { + getBrowserAccessibilityTree, + getInteractableBrowserElements, + serializeWebSnapshot +} from '@wdio/elements' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { NightwatchBrowser } from './types.js' + +interface BrowserWithUrl extends NightwatchBrowser { + getCurrentUrl?: () => Promise + getTitle?: () => Promise +} + +function shimAsWdioBrowser(browser: NightwatchBrowser): unknown { + return { + capabilities: browser.capabilities ?? {}, + isAndroid: false, + isIOS: false, + execute: (script: unknown, ...args: unknown[]) => + browser.execute( + script as string, + args.length === 1 && Array.isArray(args[0]) + ? (args[0] as unknown[]) + : args + ) + } +} + +export async function captureActionSnapshot( + browser: NightwatchBrowser, + command: string, + takeScreenshot?: () => Promise +): Promise { + try { + const timestamp = Date.now() + const b = browser as BrowserWithUrl + const browserLike = shimAsWdioBrowser(browser) as WebdriverIO.Browser + const [shot, url, title, tree, elements] = await Promise.all([ + takeScreenshot?.().catch(() => null) ?? Promise.resolve(null), + b.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), + b.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), + getBrowserAccessibilityTree(browserLike, { inViewportOnly: true }).catch( + () => [] + ), + getInteractableBrowserElements(browserLike, { + inViewportOnly: true + }).catch(() => []) + ]) + const snapshotText = serializeWebSnapshot(tree, { url, title }) + return { + timestamp, + command, + url, + title, + screenshot: shot ?? undefined, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index 395e4a77..d8033aee 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -6,7 +6,12 @@ */ import { fileURLToPath } from 'node:url' -import { errorMessage } from '@wdio/devtools-core' +import { + errorMessage, + resolveAdapterOutputDir, + writeTraceZip +} from '@wdio/devtools-core' +import { stop as stopBackend } from '@wdio/devtools-backend' import { REUSE_ENV, SCREENCAST_DEFAULTS } from '@wdio/devtools-shared' import logger from '@wdio/logger' import { @@ -98,16 +103,20 @@ class NightwatchDevToolsPlugin { #bidiAttachAttempted = false constructor(options: DevToolsOptions = {}) { + const mode = options.mode ?? 'live' + const ignore = mode === 'trace' && options.screencast?.enabled === true + if (ignore) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + } + const screencast = ignore ? {} : (options.screencast ?? {}) this.options = { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', - screencast: options.screencast ?? {}, - bidi: options.bidi ?? false - } - this.#screencastOptions = { - ...SCREENCAST_DEFAULTS, - ...(options.screencast ?? {}) + screencast, + bidi: options.bidi ?? false, + mode } + this.#screencastOptions = { ...SCREENCAST_DEFAULTS, ...screencast } this.#bidiEnabled = options.bidi === true } @@ -132,6 +141,9 @@ class NightwatchDevToolsPlugin { get port() { return self.options.port }, + get mode() { + return self.options.mode + }, get screencastOptions() { return self.#screencastOptions }, @@ -441,6 +453,12 @@ class NightwatchDevToolsPlugin { try { await this.#finalizeAllSuites(browser) this.#logRunSummary() + if (this.options.mode === 'trace') { + await this.#writeTraceZipIfNeeded() + await this.sessionCapturer?.closeWebSocket() + await stopBackend() + return + } if (!this.#devtoolsBrowser) { // Reuse mode: force one final suites broadcast so the UI reflects the // actual outcome before the process exits. @@ -464,6 +482,32 @@ class NightwatchDevToolsPlugin { logRunSummary(this.#getInternals()) } + async #writeTraceZipIfNeeded(): Promise { + if (this.options.mode !== 'trace' || !this.sessionCapturer) { + return + } + const sessionId = this.sessionCapturer.metadata?.sessionId + if (!sessionId) { + return + } + try { + if (this.sessionCapturer.snapshotCaptures.length) { + await Promise.allSettled(this.sessionCapturer.snapshotCaptures) + } + const snapshots = this.sessionCapturer.actionSnapshots + const zipPath = await writeTraceZip(this.sessionCapturer, { + outputDir: resolveAdapterOutputDir({ + configPath: this.#configPath + }), + sessionId, + actionSnapshots: snapshots.length ? snapshots : undefined + }) + log.info(`Trace.zip saved to ${zipPath}`) + } catch (err) { + log.warn(`trace.zip write failed: ${errorMessage(err)}`) + } + } + async #waitForDevtoolsBrowserClose(): Promise { await waitForDevtoolsBrowserClose(this.#getInternals()) } diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index 06c75f22..afce58b4 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -14,6 +14,7 @@ import type { TestManager } from './helpers/testManager.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { BrowserProxy } from './helpers/browserProxy.js' import type { + DevToolsMode, NightwatchBrowser, ScreencastOptions, SuiteStats, @@ -22,9 +23,10 @@ import type { export interface PluginInternals { // Config + options - options: { hostname: string; port: number } + options: { hostname: string; port: number; mode?: DevToolsMode } readonly hostname: string readonly port: number + readonly mode: DevToolsMode readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean diff --git a/packages/nightwatch-devtools/src/run-lifecycle.ts b/packages/nightwatch-devtools/src/run-lifecycle.ts index a6d7e246..4769a9b5 100644 --- a/packages/nightwatch-devtools/src/run-lifecycle.ts +++ b/packages/nightwatch-devtools/src/run-lifecycle.ts @@ -19,14 +19,18 @@ import type { SessionCapturer } from './session.js' import type { TestReporter } from './reporter.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { TestManager } from './helpers/testManager.js' -import type { NightwatchBrowser, NightwatchCurrentTest } from './types.js' +import type { + DevToolsMode, + NightwatchBrowser, + NightwatchCurrentTest +} from './types.js' import { TIMING, PLUGIN_GLOBAL_KEY } from './constants.js' import { findFreePort, resolveNightwatchConfig } from './helpers/utils.js' const log = logger('@wdio/nightwatch-devtools:run-lifecycle') export interface RunLifecycleCtx { - options: { hostname: string; port: number } + options: { hostname: string; port: number; mode?: DevToolsMode } readonly testReporter: TestReporter | undefined readonly suiteManager: SuiteManager | undefined readonly testManager: TestManager @@ -97,11 +101,15 @@ export async function runPluginBefore(ctx: PluginBeforeCtx): Promise { ctx.options.port = port const url = `http://${ctx.options.hostname}:${ctx.options.port}` log.info(`βœ“ Backend started on port ${ctx.options.port}`) - log.info(` DevTools UI: ${url}`) - await ctx.openDevtoolsBrowserAt(url) - await new Promise((resolve) => - setTimeout(resolve, TIMING.UI_CONNECTION_WAIT) - ) + if (ctx.options.mode === 'trace') { + log.info('trace mode: backend started, skipping UI window launch') + } else { + log.info(` DevTools UI: ${url}`) + await ctx.openDevtoolsBrowserAt(url) + await new Promise((resolve) => + setTimeout(resolve, TIMING.UI_CONNECTION_WAIT) + ) + } ;(globalThis as Record)[PLUGIN_GLOBAL_KEY] = ctx.plugin } catch (err) { log.error(`Failed to start backend: ${errorMessage(err)}`) diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index 5b7b29c9..438132c8 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -25,6 +25,7 @@ import { SuiteManager } from './helpers/suiteManager.js' import { BrowserProxy } from './helpers/browserProxy.js' import { ScreencastRecorder } from './screencast.js' import type { + DevToolsMode, NightwatchBrowser, ScreencastOptions, SuiteStats @@ -37,6 +38,7 @@ export interface SessionInitCtx { readonly port: number readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean + readonly mode: DevToolsMode sessionCapturer: SessionCapturer testReporter: TestReporter @@ -124,7 +126,7 @@ function broadcastSessionMetadata( ctx.srcFolders = Array.isArray(sf) ? sf : sf ? [sf] : [] } - ctx.sessionCapturer.sendUpstream('metadata', { + const metadata = { type: TraceType.Testrunner, capabilities, desiredCapabilities, @@ -133,7 +135,9 @@ function broadcastSessionMetadata( host: opts.webdriver?.host, options: ctx.buildMetadataOptions(), url: '' - }) + } + ctx.sessionCapturer.metadata = metadata + ctx.sessionCapturer.sendUpstream('metadata', metadata) const browserName = capabilities.browserName || desiredCapabilities.browserName || 'unknown' @@ -230,6 +234,7 @@ export async function ensureSessionInitialized( { port: ctx.port, hostname: ctx.hostname }, browser ) + ctx.sessionCapturer.traceMode = ctx.mode const connected = await ctx.sessionCapturer.waitForConnection(3000) if (!connected) { log.error('❌ Worker WebSocket failed to connect!') diff --git a/packages/nightwatch-devtools/src/session.ts b/packages/nightwatch-devtools/src/session.ts index d316f95e..87b3cde5 100644 --- a/packages/nightwatch-devtools/src/session.ts +++ b/packages/nightwatch-devtools/src/session.ts @@ -10,6 +10,8 @@ import { serializeError, type LogSource } from '@wdio/devtools-core' +import { mapCommandToAction } from '@wdio/devtools-core' +import { captureActionSnapshot } from './action-snapshot.js' import { NAVIGATION_COMMANDS } from './constants.js' import { parseNetworkFromPerfLogs, @@ -22,7 +24,13 @@ import { type CapturedPerformancePayload, applyPerformanceData } from '@wdio/devtools-core' -import type { CommandLog, LogLevel, NightwatchBrowser } from './types.js' +import type { + ActionSnapshot, + CommandLog, + DevToolsMode, + LogLevel, + NightwatchBrowser +} from './types.js' const log = logger('@wdio/nightwatch-devtools:SessionCapturer') @@ -105,6 +113,11 @@ export class SessionCapturer extends SessionCapturerBase { // capture path skips when set, so we don't double-emit network requests. bidiActive = false + // Populated by captureCommand when mode === 'trace' (set by the plugin). + traceMode: DevToolsMode = 'live' + readonly actionSnapshots: ActionSnapshot[] = [] + readonly snapshotCaptures: Promise[] = [] + constructor( devtoolsOptions: { hostname?: string; port?: number } = {}, browser?: NightwatchBrowser @@ -176,6 +189,24 @@ export class SessionCapturer extends SessionCapturerBase { }) } + if ( + this.traceMode === 'trace' && + !error && + this.#browser && + mapCommandToAction(command) + ) { + const browser = this.#browser + this.snapshotCaptures.push( + captureActionSnapshot(browser, command, () => + this.takeScreenshotViaHttp(browser) + ).then((snap) => { + if (snap) { + this.actionSnapshots.push(snap) + } + }) + ) + } + return true } diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index a2e2b8c0..cb519808 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -2,8 +2,10 @@ export { TraceType, + type ActionSnapshot, type CommandLog, type ConsoleLog, + type DevToolsMode, type DocumentInfo, type LogLevel, type Metadata, @@ -17,7 +19,7 @@ export { type TraceLog } from '@wdio/devtools-shared' -import type { ScreencastOptions } from '@wdio/devtools-shared' +import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' export interface CommandStackFrame { command: string @@ -93,6 +95,8 @@ export interface DevToolsOptions { * entries. Defaults to `false` β€” opt-in. */ bidi?: boolean + /** `live` (default) launches the DevTools UI; `trace` skips it. */ + mode?: DevToolsMode } export interface NightwatchBrowser { diff --git a/packages/nightwatch-devtools/tests/session.test.ts b/packages/nightwatch-devtools/tests/session.test.ts index 19822c97..4f00c948 100644 --- a/packages/nightwatch-devtools/tests/session.test.ts +++ b/packages/nightwatch-devtools/tests/session.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { SessionCapturer } from '../src/session.js' -import type { NightwatchBrowser } from '../src/types.js' +import type { CommandLog, NightwatchBrowser } from '../src/types.js' + +type CommandLogWithId = CommandLog & { _id: number } function makeMockBrowser( overrides: Partial> = {} @@ -31,8 +33,8 @@ describe('SessionCapturer.captureCommand', () => { args: ['#btn'], result: { ok: true } }) - expect((cap.commandsLog[0] as { _id: number })._id).not.toBe( - (cap.commandsLog[1] as { _id: number })._id + expect((cap.commandsLog[0] as CommandLogWithId)._id).not.toBe( + (cap.commandsLog[1] as CommandLogWithId)._id ) }) @@ -83,7 +85,7 @@ describe('SessionCapturer.replaceCommand', () => { it('splices the old entry and reissues with a new _id', async () => { const cap = makeCapturer() await cap.captureCommand('click', ['#a'], undefined, undefined) - const oldId = (cap.commandsLog[0] as { _id: number })._id + const oldId = (cap.commandsLog[0] as CommandLogWithId)._id const oldTs = cap.commandsLog[0].timestamp const { entry, oldTimestamp } = cap.replaceCommand( oldId, @@ -94,7 +96,7 @@ describe('SessionCapturer.replaceCommand', () => { ) expect(oldTimestamp).toBe(oldTs) expect(cap.commandsLog).toHaveLength(1) - expect((cap.commandsLog[0] as { _id: number })._id).not.toBe(oldId) + expect((cap.commandsLog[0] as CommandLogWithId)._id).not.toBe(oldId) expect(entry.result).toEqual({ ok: true }) }) diff --git a/packages/selenium-devtools/README.md b/packages/selenium-devtools/README.md index 7722139c..89d099a9 100644 --- a/packages/selenium-devtools/README.md +++ b/packages/selenium-devtools/README.md @@ -311,6 +311,15 @@ DevTools.configure({ captureScreenshots: false }) DevTools.configure({ rerunCommand: 'npm test -- --grep "{{testName}}"' }) ``` +#### `mode` β€” live UI vs. headless trace.zip +**Default:** `'live'` (launches the dashboard window). Set to `'trace'` to skip the dashboard entirely and write a `trace-.zip` next to your test file at session end β€” meant for CI / offline replay / agentic diffing. `screencast` is ignored in trace mode (live-mode-only feature). + +```javascript +DevTools.configure({ mode: 'trace' }) +``` + +See [Trace mode](../../README.md#-trace-mode-tracezip) in the root README for the trace.zip layout and consumer notes. + --- ## Common recipes diff --git a/packages/selenium-devtools/package.json b/packages/selenium-devtools/package.json index 0b118b7f..93ec7223 100644 --- a/packages/selenium-devtools/package.json +++ b/packages/selenium-devtools/package.json @@ -44,10 +44,12 @@ "dependencies": { "@wdio/devtools-backend": "workspace:*", "@wdio/devtools-script": "workspace:*", + "@wdio/elements": "workspace:^", "@wdio/logger": "^9.18.0", "stacktrace-parser": "^0.1.11", "webdriverio": "^9.27.2", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "optionalDependencies": { "fluent-ffmpeg": "^2.1.3" diff --git a/packages/selenium-devtools/src/action-snapshot.ts b/packages/selenium-devtools/src/action-snapshot.ts new file mode 100644 index 00000000..84a081a4 --- /dev/null +++ b/packages/selenium-devtools/src/action-snapshot.ts @@ -0,0 +1,62 @@ +// Per-action snapshot capture for Selenium β€” fires only in `mode: 'trace'` +// for commands in ACTION_MAP. Wraps the SeleniumDriverLike in a minimal +// WebdriverIO.Browser-shaped shim so @wdio/elements can run its in-page +// scripts via driver.executeScript. Returns null on failure; capture errors +// must not break the user's test. + +import { + getBrowserAccessibilityTree, + getInteractableBrowserElements, + serializeWebSnapshot +} from '@wdio/elements' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { SeleniumDriverLike } from './types.js' + +interface DriverWithUrl extends SeleniumDriverLike { + getCurrentUrl?: () => Promise + getTitle?: () => Promise +} + +function shimAsWdioBrowser(driver: SeleniumDriverLike): unknown { + return { + capabilities: {}, + isAndroid: false, + isIOS: false, + execute: (script: unknown, ...args: unknown[]) => + driver.executeScript(script as string, ...args) + } +} + +export async function captureActionSnapshot( + driver: SeleniumDriverLike, + command: string +): Promise { + try { + const timestamp = Date.now() + const d = driver as DriverWithUrl + const browserLike = shimAsWdioBrowser(driver) as WebdriverIO.Browser + const [screenshot, url, title, tree, elements] = await Promise.all([ + d.takeScreenshot?.().catch(() => undefined) ?? Promise.resolve(undefined), + d.getCurrentUrl?.().catch(() => undefined) ?? Promise.resolve(undefined), + d.getTitle?.().catch(() => undefined) ?? Promise.resolve(undefined), + getBrowserAccessibilityTree(browserLike, { inViewportOnly: true }).catch( + () => [] + ), + getInteractableBrowserElements(browserLike, { + inViewportOnly: true + }).catch(() => []) + ]) + const snapshotText = serializeWebSnapshot(tree, { url, title }) + return { + timestamp, + command, + url, + title, + screenshot, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/selenium-devtools/src/helpers/commandPostActions.ts b/packages/selenium-devtools/src/helpers/commandPostActions.ts index d90b9880..ad90666c 100644 --- a/packages/selenium-devtools/src/helpers/commandPostActions.ts +++ b/packages/selenium-devtools/src/helpers/commandPostActions.ts @@ -3,17 +3,21 @@ import { CAPTURE_PERFORMANCE_SCRIPT, applyPerformanceData, errorMessage, + mapCommandToAction, toError, type CapturedPerformancePayload, type RetryTracker } from '@wdio/devtools-core' import { getDriverOriginals, getElementOriginals } from '../driverPatcher.js' import { captureOrReplaceCommand } from './captureOrReplaceCommand.js' +import { captureActionSnapshot } from '../action-snapshot.js' import type { SessionCapturer } from '../session.js' import type { TestManager } from './testManager.js' import type { + ActionSnapshot, CapturedCommand, CommandLog, + DevToolsMode, SeleniumDriverLike } from '../types.js' @@ -144,10 +148,12 @@ export interface OnCommandCtx { readonly sessionCapturer: SessionCapturer | undefined readonly testManager: TestManager | undefined readonly retryTracker: RetryTracker - readonly options: { captureScreenshots: boolean } + readonly options: { captureScreenshots: boolean; mode?: DevToolsMode } readonly scriptInjected: boolean readonly finalized: boolean readonly driver: SeleniumDriverLike | undefined + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] setScriptInjected(v: boolean): void } @@ -214,4 +220,19 @@ export async function handleOnCommand( ctx.driver ) } + if ( + ctx.options.mode === 'trace' && + !error && + ctx.driver && + mapCommandToAction(cmd.command) + ) { + const driver = ctx.driver + ctx.snapshotCaptures.push( + captureActionSnapshot(driver, cmd.command).then((snap) => { + if (snap) { + ctx.actionSnapshots.push(snap) + } + }) + ) + } } diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 0e95b7de..4a673549 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -1,7 +1,6 @@ // @wdio/selenium-devtools β€” runner-agnostic Selenium WebDriver adapter. // Side-effect import that patches selenium-webdriver and starts the backend. -// MUST be the first import β€” see setupConsole.ts. import './setupConsole.js' import logger from '@wdio/logger' import { startDetachedBackend } from './helpers/detachedBackend.js' @@ -49,7 +48,9 @@ import { NAVIGATION_COMMANDS } from './constants.js' import { + type ActionSnapshot, type CapturedCommand, + type DevToolsMode, type DevToolsOptions, type ScreencastOptions, type SeleniumDriverLike, @@ -86,6 +87,8 @@ class SeleniumDevToolsPlugin { #retryTracker = new RetryTracker() #screencast?: ScreencastRecorder #screencastOptions: ScreencastOptions + #actionSnapshots: ActionSnapshot[] = [] + #snapshotCaptures: Promise[] = [] #sessionId?: string #uiUrlOpened = false #testFilePath?: string @@ -116,11 +119,11 @@ class SeleniumDevToolsPlugin { this.#options = { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', - // Default true to match @wdio/devtools-service and @wdio/nightwatch-devtools. openUi: options.openUi ?? true, captureScreenshots: options.captureScreenshots ?? true, rerunCommand: options.rerunCommand, - headless: options.headless ?? false + headless: options.headless ?? false, + mode: options.mode ?? 'live' } this.#rerunManager = new RerunManager(RUNNER) if (options.rerunCommand) { @@ -187,8 +190,8 @@ class SeleniumDevToolsPlugin { ) } this.#backendStarted = true - // Skip when in REUSE mode β€” the rerun child reuses the parent's window. - if (this.#options.openUi && !this.#isReuse) { + const { mode, openUi } = this.#options + if (mode !== 'trace' && openUi && !this.#isReuse) { this.#openUiWindow() } } catch (err) { @@ -229,24 +232,32 @@ class SeleniumDevToolsPlugin { screencast?: ScreencastOptions headless?: boolean openUi?: boolean + mode?: DevToolsMode } = {} ) { if ('rerunCommand' in opts) { this.#rerunManager.configure(opts.rerunCommand) this.#options.rerunCommand = opts.rerunCommand } - if (opts.screencast) { - this.#screencastOptions = { - ...this.#screencastOptions, - ...opts.screencast - } - } if (typeof opts.headless === 'boolean') { this.#options.headless = opts.headless } if (typeof opts.openUi === 'boolean') { this.#options.openUi = opts.openUi } + if (opts.mode) { + this.#options.mode = opts.mode + } + if (opts.screencast) { + if (this.#options.mode === 'trace' && opts.screencast.enabled) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + } else { + this.#screencastOptions = { + ...this.#screencastOptions, + ...opts.screencast + } + } + } } get options() { @@ -367,6 +378,12 @@ class SeleniumDevToolsPlugin { setScriptInjected: (v) => { self.#scriptInjected = v }, + get actionSnapshots() { + return self.#actionSnapshots + }, + get snapshotCaptures() { + return self.#snapshotCaptures + }, ensureBackendStarted: () => self.ensureBackendStarted(), flushPendingTestActions: () => self.#flushPendingTestActions(), resetRetryTracker: () => self.#retryTracker.reset(), @@ -565,7 +582,13 @@ if (!registerHooks()) { registerProcessHooks(plugin) export const DevTools = { - configure: (opts: { rerunCommand?: string }) => plugin.configure(opts), + configure: (opts: { + rerunCommand?: string + screencast?: ScreencastOptions + headless?: boolean + openUi?: boolean + mode?: DevToolsMode + }) => plugin.configure(opts), startTest: (name: string, meta?: { file?: string }) => plugin.startTest(name, meta), endTest: (state: TestStats['state'] = 'passed') => plugin.endTest(state) diff --git a/packages/selenium-devtools/src/plugin-internals.ts b/packages/selenium-devtools/src/plugin-internals.ts index 9484d07b..6bd73e99 100644 --- a/packages/selenium-devtools/src/plugin-internals.ts +++ b/packages/selenium-devtools/src/plugin-internals.ts @@ -12,7 +12,12 @@ import type { TestReporter } from './reporter.js' import type { SuiteManager } from './helpers/suiteManager.js' import type { TestManager } from './helpers/testManager.js' import type { ScreencastRecorder } from './screencast.js' -import type { ScreencastOptions, SeleniumDriverLike } from './types.js' +import type { + ActionSnapshot, + DevToolsMode, + ScreencastOptions, + SeleniumDriverLike +} from './types.js' import type { RetryTracker } from '@wdio/devtools-core' import type { PendingTestAction, PendingScenario } from './test-management.js' @@ -24,6 +29,7 @@ export interface PluginInternals { openUi: boolean captureScreenshots: boolean rerunCommand?: string + mode?: DevToolsMode } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -49,6 +55,10 @@ export interface PluginInternals { pendingTestActions: PendingTestAction[] pendingScenario: PendingScenario | null + // trace-mode snapshot accumulators (mode === 'trace' only) + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] + // Plugin-side delegates setFinalized(v: boolean): void setScriptInjected(v: boolean): void diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 59c823e5..13934512 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -12,7 +12,8 @@ import logger from '@wdio/logger' import { errorMessage, finalizeScreencast, - resolveAdapterOutputDir + resolveAdapterOutputDir, + writeTraceZip } from '@wdio/devtools-core' import { TIMING } from './constants.js' import { SessionCapturer } from './session.js' @@ -22,7 +23,13 @@ import { ScreencastRecorder } from './screencast.js' import { buildDriverMetadata } from './helpers/driverMetadata.js' import { attachBidiHandlers, buildBidiSinks } from './bidi.js' import { gracefulShutdown } from './helpers/processHooks.js' -import type { ScreencastOptions, SeleniumDriverLike } from './types.js' +import type { + ActionSnapshot, + DevToolsMode, + Metadata, + ScreencastOptions, + SeleniumDriverLike +} from './types.js' import type { TestManager } from './helpers/testManager.js' const log = logger('@wdio/selenium-devtools:session-lifecycle') @@ -34,6 +41,7 @@ export interface SessionLifecycleCtx { openUi: boolean captureScreenshots: boolean rerunCommand?: string + mode?: DevToolsMode } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -53,6 +61,10 @@ export interface SessionLifecycleCtx { testFilePath: string | undefined keepAliveTimer: ReturnType | undefined + // Populated by handleOnCommand when mode === 'trace'. + readonly actionSnapshots: ActionSnapshot[] + readonly snapshotCaptures: Promise[] + setFinalized(v: boolean): void ensureBackendStarted(): Promise flushPendingTestActions(): void @@ -136,6 +148,10 @@ async function initPerDriverCapture( }) ctx.sessionId = sessionId if (metadata) { + // buildDriverMetadata returns a Record-shaped payload; the relevant + // Metadata fields (sessionId, capabilities, viewport, ...) are present + // at runtime but TS can't prove the discriminant `type`. + ctx.sessionCapturer.metadata = metadata as unknown as Metadata ctx.sessionCapturer.sendUpstream('metadata', metadata) } @@ -198,6 +214,9 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { } ctx.setFinalized(true) const shutdownStart = Date.now() + // Capture for the trace.zip write before onDriverEnd clears ctx state. + const capturerAtStart = ctx.sessionCapturer + const testFilePathAtStart = ctx.testFilePath try { await onDriverEnd(ctx).catch(() => {}) @@ -211,14 +230,48 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { ctx.testManager?.finalizeSession() ctx.testReporter?.updateSuites() + const sessionId = capturerAtStart?.metadata?.sessionId + if (ctx.options.mode === 'trace' && capturerAtStart && sessionId) { + try { + if (ctx.snapshotCaptures.length) { + await Promise.allSettled(ctx.snapshotCaptures) + } + const zipPath = await writeTraceZip(capturerAtStart, { + outputDir: resolveAdapterOutputDir({ + testFilePath: testFilePathAtStart + }), + sessionId, + actionSnapshots: ctx.actionSnapshots.length + ? ctx.actionSnapshots + : undefined + }) + log.info(`Trace.zip saved to ${zipPath}`) + } catch (err) { + log.warn(`trace.zip write failed: ${errorMessage(err)}`) + } + } + logSessionSummary(ctx) ctx.sessionCapturer?.cleanup() - if (ctx.options.openUi && !ctx.isReuse) { + if (ctx.options.openUi && ctx.options.mode !== 'trace' && !ctx.isReuse) { handleInteractivePath(ctx, shutdownStart) return } + // trace mode: no UI to wait for; close the WS so the backend can wind + // down naturally. process.exit is avoided β€” Jest/runners may treat + // forced exits as failures. + if (ctx.options.mode === 'trace' && !ctx.isReuse) { + try { + await ctx.sessionCapturer?.closeWebSocket() + } catch { + /* best-effort */ + } + log.info(`πŸ›‘ Shutdown complete (${Date.now() - shutdownStart}ms)`) + return + } + // Non-interactive path (no dashboard or rerun child). Don't close the // WS yet: this `onSessionEnd` is reached via the patched `driver.quit()` // (cucumber's per-scenario `After` hook), but the runner's diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index d239a308..dc451c45 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -2,8 +2,10 @@ export { TraceType, + type ActionSnapshot, type CommandLog, type ConsoleLog, + type DevToolsMode, type DocumentInfo, type LogLevel, type Metadata, @@ -19,6 +21,8 @@ export interface DevToolsOptions { hostname?: string /** Open a Chrome window pointing at the UI. Default true. */ openUi?: boolean + /** `live` (default) launches the DevTools UI; `trace` skips it. Overrides `openUi`. */ + mode?: DevToolsMode /** Capture screenshots after each command. Default true. */ captureScreenshots?: boolean /** Command template for per-test rerun. {{testName}} is substituted. */ @@ -35,7 +39,7 @@ export interface DevToolsOptions { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing selenium-internal imports. -import type { ScreencastOptions } from '@wdio/devtools-shared' +import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' /** diff --git a/packages/service/README.md b/packages/service/README.md index 887f885d..d5532798 100644 --- a/packages/service/README.md +++ b/packages/service/README.md @@ -46,7 +46,8 @@ services: [['devtools', options]] | `port` | `number` | random | Port the DevTools UI server listens on | | `hostname` | `string` | `'localhost'` | Hostname the DevTools UI server binds to | | `devtoolsCapabilities` | `Capabilities` | Chrome 1600Γ—1200 | Capabilities used to open the DevTools UI window | -| `screencast` | `ScreencastOptions` | β€” | Session video recording (see below) | +| `screencast` | `ScreencastOptions` | β€” | Session video recording (live mode only β€” see below) | +| `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-.zip` at session end. See [Trace mode](../../README.md#-trace-mode-tracezip) | ## Screencast Recording diff --git a/packages/service/package.json b/packages/service/package.json index 66461d70..b00eb468 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -40,6 +40,7 @@ "@babel/types": "^7.29.7", "@wdio/devtools-backend": "workspace:^", "@wdio/devtools-script": "workspace:^", + "@wdio/elements": "workspace:^", "@wdio/logger": "9.18.0", "@wdio/reporter": "9.27.2", "@wdio/types": "9.27.2", @@ -47,7 +48,8 @@ "import-meta-resolve": "^4.2.0", "stack-trace": "^1.0.0", "stacktrace-parser": "^0.1.11", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yazl": "^2.5.1" }, "license": "MIT", "devDependencies": { diff --git a/packages/service/src/action-snapshot.ts b/packages/service/src/action-snapshot.ts new file mode 100644 index 00000000..8763f393 --- /dev/null +++ b/packages/service/src/action-snapshot.ts @@ -0,0 +1,54 @@ +// Per-action snapshot capture β€” fires only in `mode: 'trace'` for commands +// in the action allow-list (see @wdio/devtools-core/action-mapping). Returns +// null on failure; snapshot errors must not break the user's test. + +import { + getBrowserAccessibilityTree, + getElements, + serializeMobileSnapshot, + serializeWebSnapshot +} from '@wdio/elements' +import type { ActionSnapshot } from '@wdio/devtools-shared' + +export async function captureActionSnapshot( + browser: WebdriverIO.Browser, + command: string +): Promise { + try { + const timestamp = Date.now() + const isMobile = !!(browser.isAndroid || browser.isIOS) + const [screenshot, url, title] = await Promise.all([ + browser.takeScreenshot().catch(() => undefined), + browser.getUrl().catch(() => undefined), + browser.getTitle().catch(() => undefined) + ]) + let elements: unknown[] = [] + let snapshotText: string | undefined + if (isMobile) { + const result = await getElements(browser, { inViewportOnly: true }) + elements = result.elements + if (result.tree) { + const platform = browser.isAndroid ? 'android' : 'ios' + snapshotText = serializeMobileSnapshot(result.tree, { platform }) + } + } else { + const [tree, flatResult] = await Promise.all([ + getBrowserAccessibilityTree(browser, { inViewportOnly: true }), + getElements(browser, { inViewportOnly: true }) + ]) + elements = flatResult.elements + snapshotText = serializeWebSnapshot(tree, { url, title }) + } + return { + timestamp, + command, + url, + title, + screenshot, + elements, + snapshotText + } + } catch { + return null + } +} diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 1f54eba1..558f0755 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -3,7 +3,13 @@ import fs from 'node:fs/promises' import path from 'node:path' import logger from '@wdio/logger' -import { errorMessage } from '@wdio/devtools-core' +import { + errorMessage, + mapCommandToAction, + writeTraceZip +} from '@wdio/devtools-core' +import { captureActionSnapshot } from './action-snapshot.js' +import type { ActionSnapshot } from '@wdio/devtools-shared' import { SevereServiceError } from 'webdriverio' import type { Services, Reporters, Capabilities, Options } from '@wdio/types' import type { WebDriverCommands } from '@wdio/protocols' @@ -47,9 +53,18 @@ export default class DevToolsHookService implements Services.ServiceInstance { #bidiListenersSetup = false #screencastRecorder?: ScreencastRecorder #screencastOptions?: ScreencastOptions + #options: ServiceOptions + #actionSnapshots: ActionSnapshot[] = [] + #snapshotCaptures: Promise[] = [] constructor(serviceOptions: ServiceOptions = {}) { - this.#screencastOptions = serviceOptions.screencast + this.#options = serviceOptions + if (serviceOptions.mode === 'trace' && serviceOptions.screencast?.enabled) { + log.warn('trace mode: ignoring screencast option (live-mode feature)') + this.#screencastOptions = undefined + } else { + this.#screencastOptions = serviceOptions.screencast + } } /** @@ -275,7 +290,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { if (frame?.command === command) { this.#commandStack.pop() if (this.#browser) { - return this.#sessionCapturer.afterCommand( + const captured = this.#sessionCapturer.afterCommand( this.#browser, command, args, @@ -283,6 +298,21 @@ export default class DevToolsHookService implements Services.ServiceInstance { error, frame.callSource ) + if ( + this.#options.mode === 'trace' && + !error && + mapCommandToAction(command) + ) { + const browser = this.#browser + this.#snapshotCaptures.push( + captureActionSnapshot(browser, command).then((snap) => { + if (snap) { + this.#actionSnapshots.push(snap) + } + }) + ) + } + return captured } } @@ -304,6 +334,11 @@ export default class DevToolsHookService implements Services.ServiceInstance { // Stop and encode the screencast for the current session. await this.#finalizeScreencast(this.#browser.sessionId) + // Drain in-flight per-action snapshots before writing the trace. + if (this.#snapshotCaptures.length) { + await Promise.allSettled(this.#snapshotCaptures) + } + const outputDir = this.#outputDir const { ...options } = this.#browser.options const traceLog: TraceLog = { @@ -319,7 +354,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { }, commands: this.#sessionCapturer.commandsLog, sources: Object.fromEntries(this.#sessionCapturer.sources), - suites: this.#testReporters.map((reporter) => reporter.report) + suites: this.#testReporters.map((reporter) => reporter.report), + ...(this.#actionSnapshots.length + ? { actionSnapshots: this.#actionSnapshots } + : {}) } const traceFilePath = path.join( @@ -329,6 +367,18 @@ export default class DevToolsHookService implements Services.ServiceInstance { await fs.writeFile(traceFilePath, JSON.stringify(traceLog)) log.info(`DevTools trace saved to ${traceFilePath}`) + if (this.#options.mode === 'trace') { + const zipPath = await writeTraceZip(this.#sessionCapturer, { + outputDir, + sessionId: this.#browser.sessionId, + capabilities: this.#browser.capabilities, + actionSnapshots: this.#actionSnapshots.length + ? this.#actionSnapshots + : undefined + }) + log.info(`Trace.zip saved to ${zipPath}`) + } + // Clean up console patching this.#sessionCapturer.cleanup() } diff --git a/packages/service/src/launcher.ts b/packages/service/src/launcher.ts index b8fd252d..d874513a 100644 --- a/packages/service/src/launcher.ts +++ b/packages/service/src/launcher.ts @@ -136,6 +136,10 @@ export class DevToolsAppLauncher { port, hostname: this.#options.hostname || 'localhost' }) + if (this.#options.mode === 'trace') { + log.info('trace mode: backend started, skipping UI window launch') + return + } this.#browser = await remote({ automationProtocol: 'devtools', capabilities: { diff --git a/packages/service/src/types.ts b/packages/service/src/types.ts index a4dce985..b20de413 100644 --- a/packages/service/src/types.ts +++ b/packages/service/src/types.ts @@ -23,8 +23,12 @@ export { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing service-internal imports. -import type { ScreencastOptions } from '@wdio/devtools-shared' -export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' +import type { DevToolsMode, ScreencastOptions } from '@wdio/devtools-shared' +export type { + DevToolsMode, + ScreencastFrame, + ScreencastOptions +} from '@wdio/devtools-shared' export interface ExtendedCapabilities extends WebdriverIO.Capabilities { 'wdio:devtoolsOptions'?: ServiceOptions @@ -58,6 +62,8 @@ export interface ServiceOptions { * uses CDP push mode; all other browsers fall back to screenshot polling. */ screencast?: ScreencastOptions + /** `live` (default) launches the DevTools UI; `trace` skips it. */ + mode?: DevToolsMode } declare namespace WebdriverIO { diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index f7b7fdfd..ca7bc2d8 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -16,6 +16,9 @@ export enum TraceType { export type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'running' +/** `live` opens the DevTools UI window; `trace` skips it and lets a downstream exporter consume captured state. */ +export type DevToolsMode = 'live' | 'trace' + /** * Enum-style accessor for the canonical TestStatus values. Adapter code uses * this for readable comparisons (`state === TEST_STATE.PASSED`). The app's @@ -244,6 +247,20 @@ export interface TraceMutation { url?: string } +/** + * Captured at each user-facing action boundary in `trace` mode. Feeds the + * downstream trace.zip exporter (Phase 4). `screenshot` is base64-encoded JPEG. + */ +export interface ActionSnapshot { + timestamp: number + command: string + url?: string + title?: string + screenshot?: string + elements?: unknown[] + snapshotText?: string +} + export interface TraceLog { mutations: TraceMutation[] logs: string[] @@ -255,6 +272,8 @@ export interface TraceLog { suites?: Record[] screencast?: ScreencastInfo config?: { configFile?: string } + /** Per-action snapshots captured in `mode: 'trace'` for the trace.zip exporter. */ + actionSnapshots?: ActionSnapshot[] } // ─── Preserve-and-rerun ───────────────────────────────────────────────────── diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 55b17f19..cd5c1104 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -299,6 +299,9 @@ importers: '@types/ws': specifier: ^8.18.1 version: 8.18.1 + '@types/yazl': + specifier: ^2.4.6 + version: 2.4.6 '@wdio/devtools-script': specifier: workspace:* version: link:../script @@ -311,6 +314,34 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 + + packages/elements: + dependencies: + '@xmldom/xmldom': + specifier: ^0.9.8 + version: 0.9.10 + webdriverio: + specifier: ^9.0.0 + version: 9.27.2(puppeteer-core@21.11.0) + xpath: + specifier: ^0.0.34 + version: 0.0.34 + devDependencies: + '@types/node': + specifier: 25.9.1 + version: 25.9.1 + '@wdio/globals': + specifier: 9.27.0 + version: 9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + typescript: + specifier: 6.0.2 + version: 6.0.2 + vitest: + specifier: ^4.0.16 + version: 4.1.8(@types/node@25.9.1)(@vitest/coverage-v8@4.1.8)(happy-dom@20.9.0)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) packages/nightwatch-devtools: dependencies: @@ -320,6 +351,9 @@ importers: '@wdio/devtools-script': specifier: workspace:* version: link:../script + '@wdio/elements': + specifier: workspace:^ + version: link:../elements '@wdio/logger': specifier: ^9.18.0 version: 9.18.0 @@ -341,6 +375,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@types/node': specifier: 25.9.1 @@ -394,6 +431,9 @@ importers: '@wdio/devtools-script': specifier: workspace:* version: link:../script + '@wdio/elements': + specifier: workspace:^ + version: link:../elements '@wdio/logger': specifier: ^9.18.0 version: 9.18.0 @@ -406,6 +446,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@cucumber/cucumber': specifier: ^13.0.0 @@ -465,6 +508,9 @@ importers: '@wdio/devtools-script': specifier: workspace:^ version: link:../script + '@wdio/elements': + specifier: workspace:^ + version: link:../elements '@wdio/logger': specifier: 9.18.0 version: 9.18.0 @@ -495,6 +541,9 @@ importers: ws: specifier: ^8.21.0 version: 8.21.0 + yazl: + specifier: ^2.5.1 + version: 2.5.1 devDependencies: '@types/babel__core': specifier: ^7.20.5 @@ -2319,6 +2368,9 @@ packages: '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@types/yazl@2.4.6': + resolution: {integrity: sha512-/ifFjQtcKaoZOjl5NNCQRR0fAKafB3Foxd7J/WvFPTMea46zekapcR30uzkwIkKAAuq5T6d0dkwz754RFH27hg==} + '@typescript-eslint/eslint-plugin@8.60.1': resolution: {integrity: sha512-JQ4S5GB0tfjO8BuJ4fcX+HodkzJjYBV+7OJ+wLygaX7OGQ7FudyHL4NSCA6ob+w3Yn+5MkKIozOwQhXeM7opVg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2570,6 +2622,13 @@ packages: resolution: {integrity: sha512-xoBgmACafV4L7e7e3DUN8UM1N+I225oms38JtxtfgrMfvHm8QtcmZWXfycxEGM28Gm2M3NmeV3oso7hZeBk6Ww==} engines: {node: '>=18.20.0'} + '@wdio/globals@9.27.0': + resolution: {integrity: sha512-yT6EAyvEqm+wFD11fg89BMxvFkYLgnIVCihfJx+k73Gm3utL/DfZQpSheQdwrlQzu5p7jHi/JwOD76740F5Peg==} + engines: {node: '>=18.20.0'} + peerDependencies: + expect-webdriverio: ^5.6.5 + webdriverio: ^9.0.0 + '@wdio/globals@9.27.2': resolution: {integrity: sha512-Rx9bqD4/8iR3CNPMWYxywQSCqsR/WGwIYT2Q0uUmrvPxOdYFridDEhVRGO32kQ55UM5+JXzXppxgwGLRQ60fJg==} engines: {node: '>=18.20.0'} @@ -2634,6 +2693,10 @@ packages: resolution: {integrity: sha512-Rj8AP/VYVd5clZFKy+P7zzoXCKshjrog6lcV65nnUzATbUYT/PpUCy6OhEWHTSmLQY2Oc5ztY/IetLSg4nmB3w==} engines: {node: '>=18'} + '@xmldom/xmldom@0.9.10': + resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} + engines: {node: '>=14.6'} + '@zip.js/zip.js@2.8.26': resolution: {integrity: sha512-RQ4h9F6DOiHxpdocUDrOl6xBM+yOtz+LkUol47AVWcfebGBDpZ7w7Xvz9PS24JgXvLGiXXzSAfdCdVy1tPlaFA==} engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=18.0.0'} @@ -6739,6 +6802,11 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@6.0.2: + resolution: {integrity: sha512-bGdAIrZ0wiGDo5l8c++HWtbaNCWTS4UTv7RaTH/ThVIgjkveJt83m74bBHMJkuCbslY8ixgLBVZJIOiQlQTjfQ==} + engines: {node: '>=14.17'} + hasBin: true + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -7197,6 +7265,10 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + xpath@0.0.34: + resolution: {integrity: sha512-FxF6+rkr1rNSQrhUNYrAFJpRXNzlDoMxeXN5qI84939ylEv3qqPFKa85Oxr6tDaJKqwW6KKyo2v26TSv3k6LeA==} + engines: {node: '>=0.6.0'} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -7235,6 +7307,9 @@ packages: yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yazl@2.5.1: + resolution: {integrity: sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==} + yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} @@ -9070,6 +9145,10 @@ snapshots: '@types/node': 25.9.1 optional: true + '@types/yazl@2.4.6': + dependencies: + '@types/node': 25.9.1 + '@typescript-eslint/eslint-plugin@8.60.1(@typescript-eslint/parser@8.60.1(eslint@10.4.1(jiti@2.7.0))(typescript@6.0.3))(eslint@10.4.1(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -9416,6 +9495,11 @@ snapshots: '@wdio/types': 9.27.2 chalk: 5.6.2 + '@wdio/globals@9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0))': + dependencies: + expect-webdriverio: 5.6.7(@wdio/globals@9.27.0)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + webdriverio: 9.27.2(puppeteer-core@21.11.0) + '@wdio/globals@9.27.2(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0))': dependencies: expect-webdriverio: 5.6.7(@wdio/globals@9.27.2)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)) @@ -9558,6 +9642,8 @@ snapshots: dependencies: '@wdio/logger': 9.18.0 + '@xmldom/xmldom@0.9.10': {} + '@zip.js/zip.js@2.8.26': {} abort-controller@3.0.0: @@ -11082,6 +11168,16 @@ snapshots: expect-type@1.3.0: {} + expect-webdriverio@5.6.7(@wdio/globals@9.27.0)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)): + dependencies: + '@vitest/snapshot': 4.1.8 + '@wdio/globals': 9.27.0(expect-webdriverio@5.6.7)(webdriverio@9.27.2(puppeteer-core@21.11.0)) + '@wdio/logger': 9.18.0 + deep-eql: 5.0.2 + expect: 30.4.1 + jest-matcher-utils: 30.4.1 + webdriverio: 9.27.2(puppeteer-core@21.11.0) + expect-webdriverio@5.6.7(@wdio/globals@9.27.2)(@wdio/logger@9.18.0)(webdriverio@9.27.2(puppeteer-core@21.11.0)): dependencies: '@vitest/snapshot': 4.1.8 @@ -14384,6 +14480,8 @@ snapshots: typescript@5.9.3: optional: true + typescript@6.0.2: {} + typescript@6.0.3: {} ua-parser-js@1.0.41: {} @@ -14833,6 +14931,8 @@ snapshots: xmlchars@2.2.0: {} + xpath@0.0.34: {} + y18n@5.0.8: {} yallist@3.1.1: {} @@ -14877,6 +14977,10 @@ snapshots: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 + yazl@2.5.1: + dependencies: + buffer-crc32: 0.2.13 + yn@3.1.1: {} yocto-queue@0.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c577713a..038df802 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,6 +2,7 @@ packages: - 'packages/shared' - 'packages/core' + - 'packages/elements' - 'packages/backend' - 'packages/script' - 'packages/service' diff --git a/tsconfig.json b/tsconfig.json index 97c599d0..6cd2e8c9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,8 @@ "@wdio/devtools-shared/*": ["packages/shared/src/*"], "@wdio/devtools-core": ["packages/core/src/index.ts"], "@wdio/devtools-core/*": ["packages/core/src/*"], + "@wdio/elements": ["packages/elements/src/index.ts"], + "@wdio/elements/*": ["packages/elements/src/*"], "@wdio/devtools-backend": ["packages/backend/src/index.ts"], "@wdio/devtools-backend/*": ["packages/backend/src/*"], "@wdio/devtools-script": ["packages/script/src/index.ts"],