From f94b48d05054bb57d82f82df1ebbfb31b38f16f3 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Mon, 14 Sep 2026 03:09:40 +0800 Subject: [PATCH] Add isolated browser smoke tests with supervised cleanup Load real Chromium and Firefox extensions with isolated profiles and loopback mock requests. Bound cleanup and propagate failure and cancellation through preflight and artifact reads. Use isolated Node supervisors and private control channels for group termination. Never signal a remembered numeric process group from the runner. Preserve profiles on detected ownership loss or when group termination cannot be confirmed. Distinguish target results from supervisor termination. Document that descendants escaping these groups are outside the cleanup guarantee and may outlive profile removal. Install Firefox from a verified per-run build snapshot. Cancel active snapshot transfers and wait for stream closure before rejecting them. Recheck Chromium artifacts before launch and use unambiguous directory hashes that include empty directories. Verify popup identity, streaming, complete outbound history, error reporting, and conversation records returned by the background. Cover process isolation, control-channel failures, signal exit codes, artifact consistency, startup failures, and corrupted conversation context. Keep compatibility entry points and document prerequisites, path resolution, and cleanup limitations. --- package.json | 1 + scripts/run-smoke.sh | 4 + scripts/smoke/README.md | 162 ++++ scripts/smoke/artifacts.mjs | 72 ++ scripts/smoke/cdp.mjs | 112 +++ scripts/smoke/chromium.mjs | 240 ++++++ scripts/smoke/firefox.mjs | 279 +++++++ scripts/smoke/lifecycle.mjs | 455 +++++++++++ scripts/smoke/mock-server.mjs | 120 +++ scripts/smoke/runner.mjs | 357 +++++++++ scripts/smoke/scenarios.mjs | 309 ++++++++ scripts/smoke/supervisor.mjs | 102 +++ scripts/xvfb-smoke.mjs | 5 + tests/fixtures/smoke/fake-browser.fixture | 11 + tests/fixtures/smoke/process.fixture | 41 + tests/unit/scripts/smoke-artifacts.test.mjs | 141 ++++ tests/unit/scripts/smoke-chromium.test.mjs | 746 ++++++++++++++++++ tests/unit/scripts/smoke-firefox.test.mjs | 675 ++++++++++++++++ .../unit/scripts/smoke-interruption.test.mjs | 554 +++++++++++++ tests/unit/scripts/smoke-lifecycle.test.mjs | 645 +++++++++++++++ tests/unit/scripts/smoke-mock-server.test.mjs | 123 +++ tests/unit/scripts/smoke-runner.test.mjs | 404 ++++++++++ tests/unit/scripts/smoke-scenarios.test.mjs | 438 ++++++++++ 23 files changed, 5996 insertions(+) create mode 100644 scripts/run-smoke.sh create mode 100644 scripts/smoke/README.md create mode 100644 scripts/smoke/artifacts.mjs create mode 100644 scripts/smoke/cdp.mjs create mode 100644 scripts/smoke/chromium.mjs create mode 100644 scripts/smoke/firefox.mjs create mode 100644 scripts/smoke/lifecycle.mjs create mode 100644 scripts/smoke/mock-server.mjs create mode 100644 scripts/smoke/runner.mjs create mode 100644 scripts/smoke/scenarios.mjs create mode 100644 scripts/smoke/supervisor.mjs create mode 100644 scripts/xvfb-smoke.mjs create mode 100644 tests/fixtures/smoke/fake-browser.fixture create mode 100644 tests/fixtures/smoke/process.fixture create mode 100644 tests/unit/scripts/smoke-artifacts.test.mjs create mode 100644 tests/unit/scripts/smoke-chromium.test.mjs create mode 100644 tests/unit/scripts/smoke-firefox.test.mjs create mode 100644 tests/unit/scripts/smoke-interruption.test.mjs create mode 100644 tests/unit/scripts/smoke-lifecycle.test.mjs create mode 100644 tests/unit/scripts/smoke-mock-server.test.mjs create mode 100644 tests/unit/scripts/smoke-runner.test.mjs create mode 100644 tests/unit/scripts/smoke-scenarios.test.mjs diff --git a/package.json b/package.json index a75f48586..3bef6cfaa 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dev": "node build.mjs --development", "analyze": "node build.mjs --analyze", "lint": "eslint --ext .js,.mjs,.jsx .", + "smoke": "node scripts/xvfb-smoke.mjs", "test": "node --import ./tests/setup/browser-shim.mjs --test", "test:coverage": "c8 --all --reporter=text --reporter=lcov --reporter=json-summary --include=\"src/**/*.{mjs,jsx,js}\" node --import ./tests/setup/browser-shim.mjs --test", "lint:fix": "eslint --ext .js,.mjs,.jsx . --fix", diff --git a/scripts/run-smoke.sh b/scripts/run-smoke.sh new file mode 100644 index 000000000..6df5a11f9 --- /dev/null +++ b/scripts/run-smoke.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Compatibility entry point; Node owns resources and the exit status. +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) || exit 2 +exec node "$script_dir/xvfb-smoke.mjs" "$@" diff --git a/scripts/smoke/README.md b/scripts/smoke/README.md new file mode 100644 index 000000000..71b99a91c --- /dev/null +++ b/scripts/smoke/README.md @@ -0,0 +1,162 @@ +# Browser smoke tests + +This maintained Linux smoke tool loads the built extension into real, isolated +headless Chromium and Firefox sessions. It does not use Xvfb, real API credentials, +external test websites, or the user's browser profile. It does not install software, +download browsers, build the extension, or integrate with CI automatically. + +## Prerequisites and usage + +Install Node 22+, project dependencies (`npm ci`), full Chromium or Chrome for +Testing, Firefox, and a native Linux geckodriver binary. Browser installation is an +explicit prerequisite, not part of the runner. An npm geckodriver launcher is +rejected because it may download a driver. Use the actual extracted executable. + +Build in the same worktree first: + +```sh +npm run build +npm run smoke -- --browser all \ + --chromium-path /path/to/chromium \ + --firefox-path /path/to/firefox \ + --geckodriver-path /path/to/geckodriver +``` + +Firefox installs a temporary add-on from a per-run copy of `build/firefox`, not +`build/firefox.zip`. The copy must match the directory hash recorded during +preflight; a changed or incomplete build fails before Firefox starts. Popup paths +and expected identity come from that verified copy's manifest, including its query +string. This validates the unpacked build, not the distribution ZIP. A missing or +stale distribution ZIP does not select different JavaScript for the smoke run. + +Use `--browser chromium` or `--browser firefox` for a single browser. The default +is `all`, with Chromium followed by Firefox. All selected prerequisites are checked +before any browser starts; missing browsers are failures, not skipped passes. +Paths supplied on the command line override PATH discovery. Discovery searches +explicit PATH directories; empty components do not imply the current directory. +Use an explicit `--*-path ./executable` or PATH entry `.` to select a local binary. +Relative executable and artifact-parent paths use the runner's working directory. +Direct Node and shell invocations preserve the caller's directory. `npm run smoke` +runs from the package root, including when selected with `npm --prefix PATH`. +The runner does not use `INIT_CWD` for path resolution. Use absolute paths when +invoking the npm entry from another directory to avoid ambiguity. +Build entry paths are fixed at `build/` in the runner's own worktree. +Chromium accepts a stable build-root symlink, whose canonical target may be outside +the worktree; Firefox requires a non-symlink build root. Paths containing spaces work. + +`node scripts/xvfb-smoke.mjs` and `sh scripts/run-smoke.sh` are compatibility +entry points to the same runner. Despite the historical filename, Xvfb is not +required. `--help` lists options. There is no automatic sandbox-disabling flag; +the browser must be able to start with its normal sandbox. + +Use full Chromium or Chrome for Testing, not `chrome-headless-shell` or branded +Google Chrome. Chrome 137+ removed the command-line extension-loading flag from +branded builds. See the [Chromium announcement](https://groups.google.com/a/chromium.org/g/chromium-extensions/c/1-g8EFx2BBY/m/S0ET5wPjCAAJ). +Firefox requires geckodriver's `--allow-system-access` for privileged popup +navigation, and dynamic port flags; see [geckodriver flags](https://firefox-source-docs.mozilla.org/testing/geckodriver/Flags.html). + +## What is verified + +- The installed extension's identity, version, and rendered popup. +- A popup extension port reaches the real background implementation, which sends + a Chat Completions request to the run's loopback-only mock server. +- A first answer arrives while the server deliberately holds the rest of the + response; after release, the complete Unicode answer, one completion message, + and one conversation record are asserted. +- An HTTP failure reports an error without a successful completion or new record. + +This tests the committed Chat Completions route and does not require the separate +Responses API feature. Exact byte splitting, UTF-8 boundaries, and CRLF parsing +remain deterministic unit tests: separate server writes do not guarantee separate +browser network reads. These checks are a focused runtime smoke test, not a +replacement for all site-adapter and keyboard interaction testing in AGENTS.md. + +## Isolation, results, and cleanup + +Every invocation gets unique browser profiles, a unique artifact directory, and +OS-assigned loopback ports. Concurrent invocations do not share browser state. +Chromium's debugging endpoint is read from its own profile's `DevToolsActivePort`; +geckodriver's endpoint comes from its own startup log. + +`--artifacts-dir PATH` selects a parent directory. The tool creates a unique +`chatgptbox-smoke-*` child, retaining `report.json`, browser logs, and available +failure screenshots/DOM. Reports include selected browsers, versions, build +hashes, checks, failure stage, cleanup errors, and exit status. Retained artifacts +must stay outside the selected build directories; parents inside a build, +including symlink aliases, are rejected before writing. Ancestor directories +such as the worktree root or `/tmp` are allowed because each run gets a new child. +Retained artifacts include Firefox's `extension/` snapshot and its `snapshotDir` +in the report. +For Firefox, `preflightManifestVersion` identifies the initial manifest, and +`manifestVersion` is set from the verified snapshot after successful startup. +Chromium's `manifestVersion` starts with the preflight value and is updated from +the rechecked build on successful startup. `artifactSha256` is the preflight +directory hash; Firefox verifies its snapshot against it before startup. +Chromium rechecks the live directory against that hash before launch and derives +startup identity from the on-disk manifest, not stale caller metadata. Directory +hashes include entry types, length-framed paths and file bytes, and empty directories. +If startup fails, the retained snapshot may be absent or incomplete. +Do not modify a build or the retained snapshot during a run. +Use trusted builds and keep their paths, the artifact parent, and all ancestors +stable during the run. Path-based validation and hashing are not an atomic security +boundary against concurrent directory or symlink substitution and cannot prevent +outside bytes entering retained artifacts if this precondition is violated. +Temporary browser profiles are removed after confirming that their managed process +groups have no live members. If group termination cannot be confirmed within the +cleanup budget, the run fails and preserves the profile at the path recorded in the +report. The tool never removes the artifact parent or user profiles. Browser logs +and failure DOM may contain test data; review artifacts before sharing. + +Each managed command runs inside an isolated group with a small Node supervisor. +Shutdown first closes browser protocols, then asks the supervisor through a private +IPC channel to signal its own group with SIGTERM and finally SIGKILL (including +itself). The parent never sends termination signals to remembered numeric PIDs or +PGIDs, which can be reused by the OS. The target does not inherit the control channel. +Target exit reports are separate from supervisor exit: if a final group SIGKILL +prevents a target report, its exit status is explicitly unavailable, not inferred. + +Browser and driver executables must keep profile-using descendants in these managed +groups. Do not use wrappers that daemonize or move those descendants to another +process group or session. This is process-group cleanup, not containment of every +descendant: a process that leaves its group (for example, using `setsid()`) is not +reliably detected or terminated. In that case, the run may report success and remove +its temporary profile while the escaped process remains alive. A PASS does not prove +that no descendants escaped their groups. + +Cleanup is idempotent and bounded, and attempts remaining resources even when one +cleanup fails. SIGINT and SIGTERM trigger cleanup; a second signal asks supervisors +to accelerate termination. Control-channel loss triggers local supervisor cleanup, +but unexpected loss of the supervisor remains an error and preserves its profile. +An unresponsive or forcibly killed supervisor, machine failure, or uninterruptible +processes can prevent cleanup. Detected loss of supervisor ownership or failure to +confirm group termination is reported as an error; the parent does not retry +termination using an unverified group ID. This does not detect the escaped processes +described above. + +Each browser startup is limited to 60 seconds. Request timeouts default to 10 seconds +for Chromium CDP and 30 seconds for Firefox WebDriver; Firefox session creation uses +60 seconds, still subject to the startup limit. The selected-browser loop has a shared +five-minute limit; each lifecycle cleanup has a ten-second waiting budget. Readiness checks +may retry, but session creation, addon installation, and test requests never do. + +| Exit status | Meaning | +| ----------- | --------------------------------------------------- | +| 0 | Every selected browser passed and cleanup succeeded | +| 1 | Test, runtime, reporting, or cleanup failure | +| 2 | Invalid arguments or missing prerequisites/builds | +| 130 | SIGINT | +| 143 | SIGTERM | + +The shell and npm entry points preserve failure status. No fixed `/tmp/smoke.done` +or `/tmp/smoke.out` files are used. A Firefox-only pass is not an all-browser pass. +The console prints the artifact location, not a provisional PASS or exit code: +signals can still arrive while that output is pending. Use the process exit status +and `report.json` for the final result. + +## Development checks + +Run the targeted formatter on changed files, `npm run lint`, `shellcheck +scripts/run-smoke.sh`, `npm test`, `npm run build`, then the actual browser smoke. +`npm test` includes fake-protocol and real child-process lifecycle tests, but never +launches a real browser automatically. It needs loopback socket access for mock +servers. No new browser automation package or indirect `ws` dependency is used. diff --git a/scripts/smoke/artifacts.mjs b/scripts/smoke/artifacts.mjs new file mode 100644 index 000000000..20dbc005f --- /dev/null +++ b/scripts/smoke/artifacts.mjs @@ -0,0 +1,72 @@ +import { createHash } from 'node:crypto' +import { Buffer } from 'node:buffer' +import { createReadStream, createWriteStream } from 'node:fs' +import { lstat, mkdir, readFile, readdir } from 'node:fs/promises' +import { join } from 'node:path' +import { pipeline } from 'node:stream/promises' + +export async function hashArtifact(directory, signal) { + const hash = createHash('sha256').update('chatgptbox-smoke-artifact-v2\0') + function frame(bytes) { + const length = Buffer.alloc(8) + length.writeBigUInt64BE(BigInt(bytes.length)) + hash.update(length).update(bytes) + } + async function visit(relative = '') { + signal?.throwIfAborted() + const entries = await readdir(join(directory, relative), { withFileTypes: true }) + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + for (const entry of entries) { + signal?.throwIfAborted() + const path = join(relative, entry.name) + if (entry.isDirectory()) { + hash.update('d') + frame(Buffer.from(path)) + await visit(path) + } else if (entry.isFile()) { + hash.update('f') + frame(Buffer.from(path)) + frame(await readFile(join(directory, path), { signal })) + } else throw new Error(`Unexpected non-regular build artifact: ${path}`) + } + } + await visit() + signal?.throwIfAborted() + return hash.digest('hex') +} + +export async function snapshotArtifact(source, destination, expectedHash, signal) { + signal?.throwIfAborted() + if (typeof expectedHash !== 'string' || !/^[a-f0-9]{64}$/.test(expectedHash)) + throw new Error('Missing preflight build hash') + if (!(await lstat(source)).isDirectory()) throw new Error('Build source must be a directory') + // The destination belongs to this run's retained artifacts, never a browser profile. + await mkdir(destination) + async function copyDirectory(from, to) { + signal?.throwIfAborted() + for (const entry of await readdir(from, { withFileTypes: true })) { + signal?.throwIfAborted() + const input = join(from, entry.name) + const output = join(to, entry.name) + if (entry.isDirectory()) { + await mkdir(output) + await copyDirectory(input, output) + } else if (entry.isFile()) { + try { + // pipeline destroys both streams on abort and waits for their closure. + await pipeline(createReadStream(input), createWriteStream(output, { flags: 'wx' }), { + signal, + }) + } catch (error) { + signal?.throwIfAborted() + throw error + } + } else throw new Error(`Unexpected non-regular build artifact: ${input}`) + } + } + await copyDirectory(source, destination) + signal?.throwIfAborted() + const actualHash = await hashArtifact(destination, signal) + if (actualHash !== expectedHash) throw new Error('Build changed since preflight') + return actualHash +} diff --git a/scripts/smoke/cdp.mjs b/scripts/smoke/cdp.mjs new file mode 100644 index 000000000..85aeb2ff5 --- /dev/null +++ b/scripts/smoke/cdp.mjs @@ -0,0 +1,112 @@ +import { bounded } from './lifecycle.mjs' + +// Register the transport before awaiting the handshake or issuing any RPCs. +export async function connectCDP( + url, + { lifecycle, signal, timeoutMs = 10000, WebSocketImpl = globalThis.WebSocket }, +) { + signal?.throwIfAborted() + let socket + let nextId = 0 + let failure + let closed = false + let closing + let closeFailure + const pending = new Map() + let resolveOpen, rejectOpen, resolveClosed + const opened = new Promise((resolve, reject) => { + resolveOpen = resolve + rejectOpen = reject + }) + const finished = new Promise((resolve) => { + resolveClosed = resolve + }) + const fail = (error) => { + failure ??= error + rejectOpen(failure) + for (const request of pending.values()) request.reject(failure) + pending.clear() + } + const onAbort = () => { + const error = signal.reason ?? new Error('CDP connection aborted') + rejectOpen(error) + for (const [id, request] of pending) { + if (request.signal !== signal) continue + request.reject(error) + pending.delete(id) + } + } + const cleanup = () => { + if (closing) return closing + if (!socket) return Promise.resolve() + closeFailure = failure + signal?.removeEventListener('abort', onAbort) + fail(new Error('CDP connection closed')) + closing = (async () => { + if (closed) return + socket.close() + await bounded(finished, 2000, 'Close CDP connection') + })() + return closing + } + const close = async () => { + await cleanup() + if (closeFailure) throw closeFailure + } + lifecycle.defer('Close Chromium CDP connection', cleanup) + socket = new WebSocketImpl(url) + socket.addEventListener('open', resolveOpen, { once: true }) + socket.addEventListener('error', () => fail(new Error('CDP WebSocket transport error'))) + socket.addEventListener('close', () => { + closed = true + signal?.removeEventListener('abort', onAbort) + fail(new Error('CDP WebSocket disconnected')) + resolveClosed() + }) + socket.addEventListener('message', ({ data }) => { + let message + try { + message = JSON.parse(data) + if (!message || typeof message !== 'object' || Array.isArray(message)) { + throw new Error('Expected a CDP message object') + } + } catch (error) { + fail(new Error('Invalid CDP WebSocket message', { cause: error })) + return + } + const request = pending.get(message.id) + if (!request) return + pending.delete(message.id) + if (message.error) { + const error = new Error(`${request.method}: ${message.error.message}`) + error.code = message.error.code + request.reject(error) + } else if (!Object.hasOwn(message, 'result')) { + request.reject(new Error(`${request.method}: missing CDP result`)) + } else { + request.resolve(message.result) + } + }) + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) onAbort() + await bounded(opened, timeoutMs, 'Connect Chromium CDP', signal) + + return { + close, + async request(method, params = {}, sessionId, options = {}) { + const requestSignal = Object.hasOwn(options, 'signal') ? options.signal : signal + requestSignal?.throwIfAborted() + if (failure) throw failure + const id = ++nextId + const response = new Promise((resolve, reject) => { + pending.set(id, { resolve, reject, method, signal: requestSignal }) + }) + try { + socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) })) + return await bounded(response, options.timeoutMs ?? timeoutMs, method, requestSignal) + } finally { + pending.delete(id) + } + }, + } +} diff --git a/scripts/smoke/chromium.mjs b/scripts/smoke/chromium.mjs new file mode 100644 index 000000000..77955cc30 --- /dev/null +++ b/scripts/smoke/chromium.mjs @@ -0,0 +1,240 @@ +import { Buffer } from 'node:buffer' +import { createHash } from 'node:crypto' +import { readFile, realpath, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { URL } from 'node:url' +import { connectCDP } from './cdp.mjs' +import { bounded, waitFor } from './lifecycle.mjs' +import { hashArtifact } from './artifacts.mjs' + +export async function startChromium({ + executable, + extensionDir, + profileDir, + artifactsDir, + lifecycle, + signal, + artifactSha256, +}) { + signal?.throwIfAborted() + extensionDir = await realpath(path.resolve(extensionDir)) + profileDir = path.resolve(profileDir) + artifactsDir = path.resolve(artifactsDir) + if (extensionDir.includes(',')) throw new Error('Chromium extension path must not contain commas') + if (typeof artifactSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(artifactSha256)) + throw new Error('Missing preflight build hash') + if ((await hashArtifact(extensionDir, signal)) !== artifactSha256) + throw new Error('Build changed since preflight') + const manifest = JSON.parse( + await readFile(path.join(extensionDir, 'manifest.json'), { encoding: 'utf8', signal }), + ) + const workerPath = manifest.background?.service_worker + const popupPath = manifest.action?.default_popup + if (!workerPath || !popupPath) { + throw new Error('Chromium build must declare a service worker and an action popup') + } + // Linux unpacked IDs use the manifest public key, or the canonical load path. + // Chromium's components/crx_file/id_util.cc maps the first 128 SHA-256 bits to a-p. + const extensionId = createHash('sha256') + .update(manifest.key ? Buffer.from(manifest.key, 'base64') : extensionDir) + .digest('hex') + .slice(0, 32) + .replace(/[0-9a-f]/g, (digit) => String.fromCharCode(97 + Number.parseInt(digit, 16))) + const extensionOrigin = `chrome-extension://${extensionId}/` + const portFile = path.join(profileDir, 'DevToolsActivePort') + try { + await readFile(portFile) + throw new Error('Chromium profile already contains DevToolsActivePort; use a fresh profile') + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + signal?.throwIfAborted() + const browser = lifecycle.spawn( + executable, + [ + '--headless=new', + '--no-first-run', + '--no-default-browser-check', + '--remote-debugging-address=127.0.0.1', + '--remote-debugging-port=0', + `--user-data-dir=${profileDir}`, + `--load-extension=${extensionDir}`, + `--disable-extensions-except=${extensionDir}`, + 'about:blank', + ], + { logPath: path.join(artifactsDir, 'chromium.log') }, + ) + let cdp, sessionId, popupUrl + const evaluateExpression = async (expression, options) => { + if (!options?.diagnostic && !options?.cleanup) browser.assertRunning() + const response = await cdp.request( + 'Runtime.evaluate', + { expression, awaitPromise: true, returnByValue: true }, + sessionId, + options, + ) + if (response.exceptionDetails) { + const detail = response.exceptionDetails + throw new Error(`Chromium evaluation failed: ${detail.exception?.description ?? detail.text}`) + } + if (!response.result) throw new Error('Chromium evaluation returned no result') + if (response.result.subtype === 'error') { + throw new Error(`Chromium evaluation returned an Error: ${response.result.description}`) + } + return response.result.value + } + const evaluate = (fn, ...args) => { + if (typeof fn !== 'function') throw new TypeError('evaluate requires a function') + return evaluateExpression(`(${fn.toString()})(...${JSON.stringify(args)})`) + } + const evaluateCleanup = async (fn, ...args) => { + if (typeof fn !== 'function') throw new TypeError('evaluate requires a function') + const controller = new AbortController() + try { + return await evaluateExpression(`(${fn.toString()})(...${JSON.stringify(args)})`, { + cleanup: true, + signal: controller.signal, + timeoutMs: 5000, + }) + } finally { + controller.abort() + } + } + const capture = async (prefix) => { + if (!sessionId) throw new Error('Chromium popup is not attached; capture unavailable') + // Diagnostic requests have their own short deadlines. + const options = { signal: undefined, timeoutMs: 2000, diagnostic: true } + const destination = path.resolve(artifactsDir, prefix) + if (path.dirname(destination) !== artifactsDir) { + throw new Error('Capture prefix must name a file inside artifactsDir') + } + const results = await Promise.allSettled([ + (async () => { + const { data } = await cdp.request( + 'Page.captureScreenshot', + { format: 'png' }, + sessionId, + options, + ) + await bounded( + writeFile(`${destination}.png`, Buffer.from(data, 'base64')), + 2000, + 'Write Chromium screenshot', + ) + })(), + (async () => { + const dom = await evaluateExpression('document.documentElement.outerHTML', options) + await bounded(writeFile(`${destination}.html`, dom), 2000, 'Write Chromium DOM') + })(), + ]) + const errors = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason) + if (errors.length) throw new AggregateError(errors, 'Chromium capture failed') + } + try { + const endpoint = await waitFor( + async () => { + browser.assertRunning() + let content + try { + content = await readFile(portFile, 'utf8') + } catch (error) { + if (error.code === 'ENOENT') return false + throw error + } + const [port, websocketPath] = content.trim().split(/\r?\n/) + // Chromium may still be writing this file; wait for the complete port and UUID. + if (!port || !websocketPath) return false + if ( + !/^\d+$/.test(port) || + Number(port) < 1 || + Number(port) > 65535 || + !/^\/devtools\/browser\/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/i.test( + websocketPath, + ) + ) { + return false + } + return `ws://127.0.0.1:${port}${websocketPath}` + }, + { signal, label: 'Complete Chromium DevToolsActivePort record' }, + ) + cdp = await connectCDP(endpoint, { lifecycle, signal }) + const { product: browserVersion } = await cdp.request('Browser.getVersion') + await waitFor( + async () => { + browser.assertRunning() + const { targetInfos } = await cdp.request('Target.getTargets') + const workers = targetInfos.filter((target) => { + if (target.type !== 'service_worker' || !target.url.startsWith('chrome-extension://')) { + return false + } + return target.url === new URL(workerPath, extensionOrigin).href + }) + if (workers.length > 1) + throw new Error('Multiple matching Chromium extension service workers') + return workers[0] + }, + { signal, label: 'Chromium extension service worker' }, + ) + popupUrl = new URL(popupPath, extensionOrigin).href + if (!popupUrl.startsWith(extensionOrigin)) throw new Error('Popup must belong to the extension') + const { targetId } = await cdp.request('Target.createTarget', { url: popupUrl }) + const attached = await cdp.request('Target.attachToTarget', { targetId, flatten: true }) + sessionId = attached.sessionId + if (!sessionId) throw new Error('Chromium popup attachment returned no session ID') + await cdp.request('Runtime.enable', {}, sessionId) + await cdp.request('Page.enable', {}, sessionId) + await waitFor( + async () => { + try { + return await evaluateExpression( + `location.href === ${JSON.stringify(popupUrl)} && document.readyState === 'complete'`, + ) + } catch (error) { + // Only a read-only readiness probe may retry a navigation context transition. + if ( + error.code === -32000 && + /^Runtime\.evaluate: (Execution context was destroyed|Cannot find context with specified id|Cannot find default execution context)\.?$/.test( + error.message, + ) + ) { + return false + } + throw error + } + }, + { signal, label: 'Chromium extension popup' }, + ) + return { + evaluate, + evaluateCleanup, + capture, + // Process-group termination belongs to lifecycle.spawn's registered cleanup. + async close() { + browser.assertRunning() + await cdp.close() + }, + metadata: { + browserVersion, + extensionId, + popupUrl, + expectedVersion: manifest.version, + expectedName: manifest.name, + manifestVersion: manifest.version, + }, + } + } catch (error) { + if (sessionId) { + try { + await capture('chromium-startup-failure') + } catch (captureError) { + error.captureError = captureError + } + } + throw new Error(`Chromium startup failed: ${error.message}\n${browser.output().slice(-4000)}`, { + cause: error, + }) + } +} diff --git a/scripts/smoke/firefox.mjs b/scripts/smoke/firefox.mjs new file mode 100644 index 000000000..e6a378c06 --- /dev/null +++ b/scripts/smoke/firefox.mjs @@ -0,0 +1,279 @@ +import { Buffer } from 'node:buffer' +import { readFile, writeFile } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import process from 'node:process' +import { bounded, waitFor } from './lifecycle.mjs' +import { snapshotArtifact } from './artifacts.mjs' + +export async function startFirefox({ + executable, + geckodriver, + extensionDir, + artifactSha256, + profileDir, + artifactsDir, + lifecycle, + signal, +}) { + signal?.throwIfAborted() + const snapshotDir = resolve(artifactsDir, 'extension') + await snapshotArtifact(extensionDir, snapshotDir, artifactSha256, signal) + const manifest = JSON.parse(await readFile(join(snapshotDir, 'manifest.json'), { signal })) + if (!manifest.name || !manifest.version) throw new Error('Invalid Firefox snapshot manifest') + const popup = manifest.action?.default_popup || manifest.browser_action?.default_popup + if ( + typeof popup !== 'string' || + !popup.trim() || + popup !== popup.trim() || + /^[a-z][a-z\d+.-]*:|^\/\//i.test(popup) + ) + throw new Error('Invalid Firefox manifest popup path') + signal?.throwIfAborted() + const managed = lifecycle.spawn( + geckodriver, + [ + '--host', + '127.0.0.1', + '--port', + '0', + '--websocket-port', + '0', + '--profile-root', + resolve(profileDir), + '--allow-system-access', + ], + { logPath: join(artifactsDir, 'geckodriver.log'), env: { ...process.env } }, + ) + let origin + let sessionId + let closing + + async function request(method, path, body, { cleanup = false, timeoutMs = 30000 } = {}) { + if (!cleanup) { + signal?.throwIfAborted() + managed.assertRunning() + } + const controller = new AbortController() + const requestSignal = + signal && !cleanup ? AbortSignal.any([signal, controller.signal]) : controller.signal + const label = `Firefox WebDriver ${method} ${path}` + try { + return await bounded( + (async () => { + const response = await fetch(`${origin}${path}`, { + method, + headers: { 'Content-Type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: requestSignal, + redirect: 'error', + }) + const text = await response.text() + let payload + try { + payload = JSON.parse(text) + } catch { + throw new Error(`${label}: HTTP ${response.status}, invalid JSON response`) + } + if (!response.ok || payload?.value?.error) { + throw new Error( + `${label}: HTTP ${response.status}: ${payload?.value?.error || 'request failed'}: ${ + payload?.value?.message || '' + }`, + ) + } + if (!payload || !Object.hasOwn(payload, 'value')) { + throw new Error(`${label}: missing WebDriver value`) + } + return payload.value + })(), + timeoutMs, + label, + requestSignal, + ) + } finally { + controller.abort() + } + } + + function command(method, path, body, options) { + return request(method, `/session/${encodeURIComponent(sessionId)}${path}`, body, options) + } + + function close() { + if (!closing) { + closing = sessionId + ? command('DELETE', '', undefined, { cleanup: true, timeoutMs: 5000 }) + : Promise.resolve() + } + return closing + } + lifecycle.defer('Firefox WebDriver session', close) + + origin = await waitFor( + async () => { + managed.assertRunning() + const match = managed.output().match(/Listening on 127\.0\.0\.1:(\d+)(?=\r?\n)/) + if (!match) return false + const port = Number(match[1]) + if (port < 1 || port > 65535) throw new Error('Invalid geckodriver listening port') + return `http://127.0.0.1:${port}` + }, + { timeoutMs: 15000, signal, label: 'geckodriver listening port' }, + ) + await waitFor( + async () => { + const status = await request('GET', '/status') + if (typeof status?.ready !== 'boolean') + throw new Error('Invalid geckodriver readiness status') + return status.ready + }, + { timeoutMs: 15000, signal, label: 'geckodriver ready' }, + ) + + const session = await request( + 'POST', + '/session', + { + capabilities: { + alwaysMatch: { + browserName: 'firefox', + 'moz:firefoxOptions': { + binary: executable, + args: ['-headless'], + prefs: { + 'intl.accept_languages': 'en-US,en', + 'browser.shell.checkDefaultBrowser': false, + }, + }, + }, + }, + }, + { timeoutMs: 60000 }, + ) + sessionId = session?.sessionId + if (typeof sessionId !== 'string' || !sessionId.trim()) { + sessionId = undefined + throw new Error('Firefox WebDriver returned an empty session ID') + } + await command('POST', '/timeouts', { script: 30000, pageLoad: 30000, implicit: 0 }) + const extensionId = await command('POST', '/moz/addon/install', { + path: snapshotDir, + temporary: true, + }) + if (typeof extensionId !== 'string' || !extensionId.trim()) { + throw new Error('Firefox returned an empty addon ID') + } + const previousHandles = await command('GET', '/window/handles') + await command('POST', '/moz/context', { context: 'chrome' }) + const extensionBase = await command('POST', '/execute/sync', { + script: `const policy = WebExtensionPolicy.getByID(arguments[0]); + if (!policy) throw new Error('Installed extension policy not found'); + return policy.getURL('');`, + args: [extensionId], + }) + if (typeof extensionBase !== 'string' || !extensionBase.startsWith('moz-extension://')) { + throw new Error('Firefox returned an invalid extension URL') + } + const base = new URL(extensionBase) + const target = new URL(popup, base) + if ( + target.protocol !== 'moz-extension:' || + target.host !== base.host || + target.username || + target.password + ) + throw new Error('Firefox popup must belong to the installed extension') + const popupUrl = target.href + await command('POST', '/execute/sync', { + script: `const tab = gBrowser.addTab(arguments[0], { + triggeringPrincipal: Services.scriptSecurityManager.getSystemPrincipal() + }); + gBrowser.selectedTab = tab;`, + args: [popupUrl], + }) + await command('POST', '/moz/context', { context: 'content' }) + const handle = await waitFor( + async () => { + const handles = await command('GET', '/window/handles') + return handles.find((candidate) => !previousHandles.includes(candidate)) + }, + { timeoutMs: 10000, signal, label: 'Firefox popup window' }, + ) + await command('POST', '/window', { handle }) + + async function evaluateWithOptions(options, fn, ...args) { + if (typeof fn !== 'function') throw new TypeError('evaluate requires a function') + const result = await command( + 'POST', + '/execute/async', + { + script: `const done = arguments[arguments.length - 1]; + const args = Array.prototype.slice.call(arguments, 0, -1); + Promise.resolve().then(() => (${fn.toString()})(...args)).then( + value => done({ ok: true, value: value === undefined ? null : value }), + error => done({ ok: false, smokeError: { + message: String(error), stack: String(error && error.stack || '') + } }) + );`, + args, + }, + options, + ) + if (result?.ok !== true) { + const detail = result?.smokeError + throw new Error( + `Firefox evaluation failed: ${detail?.message || 'invalid script result'}${ + detail?.stack ? `\n${detail.stack}` : '' + }`, + ) + } + return result.value + } + const evaluate = (fn, ...args) => evaluateWithOptions(undefined, fn, ...args) + const evaluateCleanup = (fn, ...args) => + evaluateWithOptions({ cleanup: true, timeoutMs: 5000 }, fn, ...args) + + await waitFor( + () => evaluate((url) => location.href === url && document.readyState === 'complete', popupUrl), + { timeoutMs: 15000, signal, label: 'Firefox popup document' }, + ) + + async function capture(prefix) { + const destination = resolve(artifactsDir, prefix) + if (dirname(destination) !== resolve(artifactsDir)) { + throw new Error('Capture prefix must name a file inside artifactsDir') + } + const results = await Promise.allSettled([ + (async () => { + const screenshot = await command('GET', '/screenshot') + await writeFile(`${destination}.png`, Buffer.from(screenshot, 'base64')) + })(), + (async () => { + const source = await command('GET', '/source') + await writeFile(`${destination}.html`, source) + })(), + ]) + const errors = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason) + if (errors.length) + throw new AggregateError(errors, `Firefox capture failed: ${errors.map(String).join('; ')}`) + } + + return { + evaluate, + evaluateCleanup, + capture, + close, + metadata: { + browserVersion: session.capabilities?.browserVersion, + extensionId, + popupUrl, + expectedVersion: manifest.version, + expectedName: manifest.name, + artifactSha256, + snapshotDir, + manifestVersion: manifest.version, + }, + } +} diff --git a/scripts/smoke/lifecycle.mjs b/scripts/smoke/lifecycle.mjs new file mode 100644 index 000000000..eabdf8345 --- /dev/null +++ b/scripts/smoke/lifecycle.mjs @@ -0,0 +1,455 @@ +import { spawn as spawnChild } from 'node:child_process' +import { createWriteStream } from 'node:fs' +import { readdir, readFile } from 'node:fs/promises' +import { performance } from 'node:perf_hooks' +import process from 'node:process' +import { clearTimeout, setTimeout } from 'node:timers' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' + +function checkTimeout(ms) { + if (!Number.isFinite(ms) || ms <= 0 || ms > 2147483647) { + throw new RangeError('Timeout must be positive and at most 2147483647 ms') + } +} + +function cancellation(signal) { + return signal.reason ?? new Error('Operation aborted') +} + +// This bounds waiting, not the underlying operation. Callers must arrange its cleanup. +export function bounded(promise, ms, label = 'Operation', signal) { + return new Promise((resolve, reject) => { + let timer + const finish = (callback, value) => { + clearTimeout(timer) + signal?.removeEventListener('abort', abort) + callback(value) + } + const abort = () => finish(reject, cancellation(signal)) + // Observe even an already rejected promise when cancellation wins the race. + Promise.resolve(promise).then( + (value) => finish(resolve, value), + (error) => finish(reject, error), + ) + try { + checkTimeout(ms) + } catch (error) { + reject(error) + return + } + if (signal?.aborted) { + abort() + return + } + signal?.addEventListener('abort', abort, { once: true }) + timer = setTimeout(() => finish(reject, new Error(`${label} timed out after ${ms} ms`)), ms) + }) +} + +// Only an explicit falsy result means "not ready"; probe errors are never retried. +export async function waitFor(asyncProbe, { timeoutMs = 60000, signal, label = 'Readiness' } = {}) { + checkTimeout(timeoutMs) + const deadline = performance.now() + timeoutMs + for (;;) { + if (signal?.aborted) throw cancellation(signal) + const remaining = deadline - performance.now() + if (remaining <= 0) throw new Error(`${label} timed out after ${timeoutMs} ms`) + const result = await bounded( + Promise.resolve().then(() => { + if (signal?.aborted) throw cancellation(signal) + return asyncProbe() + }), + remaining, + label, + signal, + ) + if (result) return result + const pause = Math.min(50, deadline - performance.now()) + if (pause > 0) await delay(pause, undefined, { signal }) + } +} + +// Linux may retain orphan zombies until PID 1 reaps them. They cannot run or hold +// resources, and Node can only reap its own direct children. Other POSIX hosts +// use the process-group existence check without relying on /proc. +async function hasLiveMembers(pgid) { + if (process.platform !== 'linux') return true + const entries = (await readdir('/proc')).filter((entry) => /^\d+$/.test(entry)) + // Bound concurrent descriptor use without paying one async round trip per PID. + for (let offset = 0; offset < entries.length; offset += 64) { + const live = await Promise.all( + entries.slice(offset, offset + 64).map(async (entry) => { + try { + const stat = await readFile(`/proc/${entry}/stat`, 'utf8') + const [state, , group] = stat.slice(stat.lastIndexOf(')') + 2).split(' ') + return Number(group) === pgid && state !== 'Z' && state !== 'X' + } catch (error) { + if (error.code !== 'ENOENT' && error.code !== 'ESRCH') throw error + return false + } + }), + ) + if (live.some(Boolean)) return true + } + return false +} + +export function createLifecycle({ signal, cleanupTimeoutMs = 10000 } = {}) { + checkTimeout(cleanupTimeoutMs) + const resources = [] + const groups = new Set() + const errors = [] + let cleaning = false + let forced = false + let cleanupPromise + + const report = (label, cause) => { + const error = new Error(`${label}: ${cause?.message ?? String(cause)}`, { cause }) + errors.push(error) + return error + } + + function assertOpen() { + if (signal?.aborted) throw cancellation(signal) + if (cleaning || forced) throw new Error('Lifecycle is closing; new resources are not allowed') + } + + function defer(label, asyncCleanup) { + assertOpen() + if (typeof asyncCleanup !== 'function') throw new TypeError('Cleanup must be a function') + resources.push({ label, run: asyncCleanup }) + } + + function groupExists(record, retire = true) { + if (record.retired || !record.child.pid) return false + try { + process.kill(-record.child.pid, 0) + return true + } catch (error) { + if (error.code !== 'ESRCH') throw error + if (retire) { + record.retired = true + groups.delete(record) + } + return false + } + } + + // Read-only guard for dependent resources such as browser profiles. Unknown + // process state (including /proc permission failures) must preserve the resource. + async function assertProcessesStopped() { + for (const record of groups) { + if (record.uncertain) throw new Error('Supervisor ownership was lost; preserve its profile') + if (groupExists(record, false) && (await hasLiveMembers(record.child.pid))) { + throw new Error(`Process group ${record.child.pid} still has live members`) + } + } + } + + function killGroup(record, signalName) { + record.request(signalName === 'SIGKILL' ? 'force' : 'stop') + } + + function spawn(command, args = [], { logPath, cwd, env } = {}) { + assertOpen() + if (process.platform === 'win32') throw new Error('Smoke process groups require a POSIX host') + const child = spawnChild( + process.execPath, + [fileURLToPath(new URL('./supervisor.mjs', import.meta.url))], + { + detached: true, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }, + ) + const record = { child, retired: false, uncertain: false } + // Register ownership synchronously, before opening logs or awaiting readiness. + groups.add(record) + let ended = false + let failure + let tail = '' + let log + let logClosed = Promise.resolve() + let supervisorEnded = false + let ready = false + let requested + let startSent = false + let targetStarted = false + let terminating = false + let graceMs = 100 + let targetResult + let resolveTarget + let rejectTarget + const fail = (label, error) => { + const reported = report(label, error) + failure ??= reported + } + const exited = new Promise((resolve, reject) => { + resolveTarget = resolve + rejectTarget = reject + }) + // Some callers only use assertRunning; never leave a spawn rejection unhandled. + exited.catch(() => {}) + function lost(error) { + record.uncertain = true + fail('Supervisor control', error) + rejectTarget(failure) + } + function send(message) { + if (!child.connected) { + if (!supervisorEnded) lost(new Error('Control channel unavailable')) + return + } + try { + child.send({ version: 1, ...message }, (error) => { + if (error && !supervisorEnded) { + lost(error) + if (child.connected) child.disconnect() + } + }) + } catch (error) { + lost(error) + if (child.connected) child.disconnect() + } + } + record.request = (type) => { + if (supervisorEnded || record.retired) return + if (requested === 'force' || (requested === 'stop' && type === 'stop')) return + requested = type + if (ready) send({ type, graceMs }) + } + child.on('message', (message) => { + const invalid = () => { + lost(new Error('Invalid supervisor message')) + record.request('force') + } + if (!message || message.version !== 1) { + invalid() + return + } + if (message.type === 'ready') { + if (ready) return invalid() + ready = true + if (!requested && (cleaning || forced || signal?.aborted || record.uncertain)) { + record.request(forced ? 'force' : 'stop') + return + } + if (requested) send({ type: requested, graceMs }) + else { + startSent = true + send({ type: 'start', command, args, cwd, env }) + } + } else if (message.type === 'started') { + if ( + !startSent || + targetStarted || + ended || + !Number.isSafeInteger(message.pid) || + message.pid <= 0 + ) { + return invalid() + } + targetStarted = true + } else if (message.type === 'target-exit') { + if ( + !targetStarted || + ended || + !( + (Number.isInteger(message.code) && message.signal === null) || + (message.code === null && typeof message.signal === 'string') + ) + ) + return invalid() + ended = true + targetResult = { code: message.code, signal: message.signal } + resolveTarget(targetResult) + } else if (message.type === 'start-error') { + if (!startSent || ended || typeof message.message !== 'string') return invalid() + ended = true + fail( + `Process ${command}`, + Object.assign(new Error(message.message), { code: message.code }), + ) + rejectTarget(failure) + } else if (message.type === 'terminating') { + if (!ready || terminating) return invalid() + terminating = true + } else invalid() + }) + let resolveIpcClosed + const ipcClosed = new Promise((resolve) => { + resolveIpcClosed = resolve + }) + const supervisorExit = new Promise((resolve) => { + child.once('exit', (code, signalName) => { + supervisorEnded = true + resolve({ code, signalName }) + }) + child.once('error', (error) => { + lost(error) + if (!child.pid) { + supervisorEnded = true + resolveIpcClosed() + resolve({ code: null, signalName: null }) + } + }) + }) + child.on('disconnect', () => { + if (!requested && !supervisorEnded) lost(new Error('Control channel disconnected')) + resolveIpcClosed() + }) + // OS exit and IPC reads are independent. Process queued final messages + // before deciding that an acknowledgement or target result was lost. + const supervisorSettled = Promise.all([supervisorExit, ipcClosed]).then(([result]) => { + if (!terminating || result.signalName !== 'SIGKILL') { + lost(new Error(`Supervisor exited unexpectedly (${result.code ?? result.signalName})`)) + } + rejectTarget(new Error('Target exit status unavailable after supervisor termination')) + }) + // IPC disconnect does not reliably produce ChildProcess.close on all Node + // versions. Reaping and output draining are independent completion facts. + const drained = Promise.all( + [child.stdout, child.stderr].map( + (stream) => new Promise((resolve) => stream.once('close', resolve)), + ), + ) + + async function stop(budgetMs) { + const deadline = performance.now() + budgetMs + try { + graceMs = Math.min(1000, budgetMs / 2) + killGroup(record, forced ? 'SIGKILL' : 'SIGTERM') + // Reap the direct child and drain both pipes before flushing the log. + await bounded( + supervisorSettled, + Math.max(1, deadline - performance.now()), + `Reap ${command}`, + ) + await bounded(drained, Math.max(1, deadline - performance.now()), `Drain ${command}`) + if (record.uncertain) throw new Error('Supervisor ownership was lost') + while (groupExists(record)) { + if (!(await hasLiveMembers(child.pid))) { + record.retired = true + groups.delete(record) + break + } + if (performance.now() >= deadline) throw new Error('Process group did not terminate') + await delay(10) + } + log?.end() + await bounded(logClosed, Math.max(1, deadline - performance.now()), `Flush ${command} log`) + } finally { + // A timeout must not leave our pipes or file descriptors keeping Node alive. + child.stdout.destroy() + child.stderr.destroy() + log?.destroy() + if (!supervisorEnded) record.uncertain = true + if (child.connected) child.disconnect() + child.unref() + } + } + resources.push({ label: `Stop ${command}`, run: stop, record }) + + if (logPath !== undefined) { + try { + log = createWriteStream(logPath, { flags: 'a' }) + logClosed = new Promise((resolve) => log.once('close', resolve)) + log.on('error', (error) => { + fail(`Log ${logPath}`, error) + child.stdout.resume() + child.stderr.resume() + }) + log.on('drain', () => { + child.stdout.resume() + child.stderr.resume() + }) + } catch (error) { + fail(`Log ${logPath}`, error) + } + } + for (const stream of [child.stdout, child.stderr]) { + stream.setEncoding('utf8') + stream.on('error', (error) => fail(`Output ${command}`, error)) + stream.on('data', (chunk) => { + // Keep diagnostics bounded while preserving complete output on disk. + tail = (tail + chunk).slice(-1024 * 1024) + if (log && !log.destroyed && !log.writableEnded && !log.write(chunk)) { + child.stdout.pause() + child.stderr.pause() + } + }) + } + return { + child, + exited, + output: () => tail, + assertRunning() { + assertOpen() + if (failure) throw failure + if (ended || supervisorEnded) { + throw new Error( + `Process ${command} exited unexpectedly (${ + targetResult?.code ?? targetResult?.signal ?? 'target status unavailable' + })\n${tail}`, + ) + } + }, + } + } + + function cleanup() { + if (cleanupPromise) return cleanupPromise + cleaning = true + signal?.removeEventListener('abort', onAbort) + // Defer invocation until the shared promise is installed, including reentrant cleanup. + cleanupPromise = Promise.resolve().then(async () => { + const deadline = performance.now() + cleanupTimeoutMs + while (resources.length) { + // Share one deadline fairly, so a stalled resource cannot starve later cleanup. + const remaining = deadline - performance.now() + const budgetMs = Math.max(1, remaining / resources.length) + const { label, run, record } = resources.pop() + try { + const operation = Promise.resolve().then(() => (record ? run(budgetMs) : run())) + if (remaining <= 0) { + // Invoke independent cleanup, but do not add a timer per resource + // after the overall deadline. The timeout remains an explicit failure. + operation.catch(() => {}) + throw new Error('Overall cleanup deadline exceeded') + } + await bounded(operation, budgetMs, label) + } catch (error) { + report(label, error) + if (record) { + try { + killGroup(record, 'SIGKILL') + } catch (killError) { + report(label, killError) + } + } + } + } + return errors + }) + return cleanupPromise + } + + function force() { + forced = true + for (const record of groups) { + try { + killGroup(record, 'SIGKILL') + } catch (error) { + report('Force process group', error) + } + } + return cleanup() + } + + const onAbort = () => { + cleanup() + } + if (signal?.aborted) cleanup() + else signal?.addEventListener('abort', onAbort, { once: true }) + return { defer, spawn, cleanup, force, assertProcessesStopped } +} diff --git a/scripts/smoke/mock-server.mjs b/scripts/smoke/mock-server.mjs new file mode 100644 index 000000000..76104fcb8 --- /dev/null +++ b/scripts/smoke/mock-server.mjs @@ -0,0 +1,120 @@ +import { createServer } from 'node:http' +import { Buffer } from 'node:buffer' +import { bounded } from './lifecycle.mjs' + +export async function startMockServer({ lifecycle, signal }) { + signal?.throwIfAborted() + const requests = [] + const sockets = new Set() + const pending = new Set() + let released = false + let closing + const event = (content, finishReason = null) => + `data: ${JSON.stringify({ + choices: [{ delta: { content }, finish_reason: finishReason }], + })}\r\n\r\n` + const finish = (response) => { + pending.delete(response) + if (response.destroyed) return + const bytes = Buffer.from(event('δΈ–η•ŒπŸ™‚', 'stop') + 'data: [DONE]\r\n\r\n') + // Exercise split UTF-8 and CRLF writes without assuming network read boundaries. + for (const byte of bytes) response.write(Buffer.from([byte])) + response.end() + } + const server = createServer((request, response) => { + response.on('error', () => response.destroy()) + response.setHeader('Access-Control-Allow-Origin', '*') + response.setHeader('Access-Control-Allow-Headers', 'content-type, authorization') + response.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS') + if (request.method === 'OPTIONS') { + response.writeHead(204).end() + return + } + const entry = { method: request.method, path: request.url, body: null, status: null } + requests.push(entry) + let body = '' + request.setEncoding('utf8') + request.on('error', () => response.destroy()) + request.on('data', (chunk) => { + body += chunk + if (Buffer.byteLength(body) > 65536) request.destroy() + }) + request.on('end', () => { + const json = (status, value) => { + entry.status = status + response.writeHead(status, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(value)) + } + if ( + request.method !== 'POST' || + !['/success/v1/chat/completions', '/error/v1/chat/completions'].includes(request.url) + ) { + json(404, { error: { message: 'Unknown smoke endpoint' } }) + return + } + try { + entry.body = JSON.parse(body) + } catch { + json(400, { error: { message: 'Invalid JSON' } }) + return + } + if ( + request.headers.authorization !== 'Bearer smoke-local-only' || + entry.body?.model !== 'smoke-model' || + entry.body?.stream !== true || + !Array.isArray(entry.body?.messages) + ) { + json(400, { error: { message: 'Invalid Chat Completions request' } }) + return + } + if (request.url === '/error/v1/chat/completions') { + json(503, { error: { message: 'Smoke upstream unavailable', code: 'smoke_503' } }) + return + } + entry.status = 200 + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + }) + pending.add(response) + response.once('close', () => pending.delete(response)) + response.write(event('Hello ')) + if (released) finish(response) + }) + }) + server.on('connection', (socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + }) + let listening + const close = () => { + closing ??= (async () => { + // A cancelled startup may still finish listening; close it before returning. + await listening?.catch(() => {}) + const closed = new Promise((resolve, reject) => { + server.close((error) => { + if (error && error.code !== 'ERR_SERVER_NOT_RUNNING') reject(error) + else resolve() + }) + }) + for (const socket of sockets) socket.destroy() + await closed + })() + return closing + } + lifecycle.defer('mock HTTP server', close) + listening = new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + await bounded(listening, 10000, 'mock HTTP listen', signal) + return { + baseUrl: `http://127.0.0.1:${server.address().port}`, + requests, + release() { + released = true + for (const response of pending) finish(response) + }, + close, + } +} diff --git a/scripts/smoke/runner.mjs b/scripts/smoke/runner.mjs new file mode 100644 index 000000000..762902cb4 --- /dev/null +++ b/scripts/smoke/runner.mjs @@ -0,0 +1,357 @@ +import process from 'node:process' +import { constants, mkdtempSync, writeFileSync } from 'node:fs' +import { access, lstat, mkdir, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { fileURLToPath } from 'node:url' +import { bounded, createLifecycle } from './lifecycle.mjs' +import { startChromium } from './chromium.mjs' +import { startFirefox } from './firefox.mjs' +import { runScenarios } from './scenarios.mjs' +import { hashArtifact } from './artifacts.mjs' + +export { hashArtifact } + +export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +export const HELP = `Usage: npm run smoke -- [options] + --browser all|chromium|firefox Selected browsers (default: all, run serially) + --chromium-path PATH Full Chromium or Chrome for Testing executable + --firefox-path PATH Firefox executable + --geckodriver-path PATH Native geckodriver executable (not an npm wrapper) + --artifacts-dir PATH Parent directory for a unique retained run directory + --help Show this help + +Requires Linux, Node 22+, and production builds in this worktree. +No downloads, dependency installation, or builds are performed automatically. +Exit: 0 pass; 1 test/runtime/cleanup failure; 2 preflight; 130 SIGINT; 143 SIGTERM. +` + +export function parseArgs(args) { + const options = { browser: 'all' } + const allowed = new Set([ + 'browser', + 'chromium-path', + 'firefox-path', + 'geckodriver-path', + 'artifacts-dir', + ]) + const seen = new Set() + for (let index = 0; index < args.length; index++) { + const key = args[index].slice(2) + if (args[index] === '--help') { + options.help = true + continue + } + if (!args[index].startsWith('--') || !allowed.has(key) || seen.has(key)) { + throw new Error(`Unknown or repeated option: ${args[index]}`) + } + const value = args[++index] + if (!value || value.startsWith('--')) throw new Error(`Missing value for --${key}`) + seen.add(key) + options[key] = value + } + if (!['all', 'chromium', 'firefox'].includes(options.browser)) + throw new Error(`Invalid browser: ${options.browser}`) + return options +} + +export async function findExecutable(explicit, candidates, pathValue = process.env.PATH || '') { + const paths = explicit + ? [resolve(explicit)] + : pathValue + .split(delimiter) + .filter(Boolean) + .flatMap((dir) => candidates.map((name) => join(dir, name))) + for (const path of paths) { + try { + await access(path, constants.X_OK) + if ((await stat(path)).isFile()) return resolve(path) + } catch (error) { + if (!['ENOENT', 'EACCES', 'ENOTDIR'].includes(error.code)) throw error + } + } + throw new Error( + `Executable not found: ${explicit || candidates.join(' / ')}; specify its --*-path`, + ) +} + +export async function preflight(options, root = ROOT, signal) { + signal?.throwIfAborted() + if (process.platform !== 'linux') throw new Error('Smoke currently supports Linux only') + if (Number(process.versions.node.split('.')[0]) < 22) throw new Error('Node 22+ is required') + const browsers = options.browser === 'all' ? ['chromium', 'firefox'] : [options.browser] + const jobs = [] + for (const browser of browsers) { + signal?.throwIfAborted() + const executable = await findExecutable( + options[`${browser}-path`], + browser === 'chromium' ? ['chromium', 'chromium-browser', 'chrome-for-testing'] : ['firefox'], + ) + const extensionDir = join(root, 'build', browser) + if (browser === 'firefox' && !(await lstat(extensionDir)).isDirectory()) + throw new Error('Firefox build source must be a directory') + for (const name of ['manifest.json', 'background.js', 'popup.html', 'popup.js']) { + if (!(await stat(join(extensionDir, name))).isFile()) + throw new Error(`Missing build file: ${name}`) + } + const manifest = JSON.parse( + await readFile(join(extensionDir, 'manifest.json'), { encoding: 'utf8', signal }), + ) + if (!manifest.version || !manifest.name) throw new Error(`Invalid ${browser} manifest`) + const job = { + browser, + executable, + extensionDir, + manifest, + artifactSha256: await hashArtifact(extensionDir, signal), + } + if (browser === 'firefox') { + job.geckodriver = await findExecutable(options['geckodriver-path'], ['geckodriver']) + // npm launchers may download binaries; require an already installed native driver. + const header = (await readFile(job.geckodriver, { signal })).subarray(0, 4) + if (header.toString('hex') !== '7f454c46') + throw new Error('Specify a native Linux geckodriver binary, not a downloading wrapper') + } + signal?.throwIfAborted() + jobs.push(job) + } + return jobs +} + +// Resolve existing symlinks even when the requested directory has not been created yet. +async function resolveFuturePath(path) { + try { + return await realpath(path) + } catch (error) { + if (error.code !== 'ENOENT') throw error + // A dangling symlink is not a missing directory that mkdir can safely create. + const entry = await lstat(path).catch((failure) => { + if (failure.code !== 'ENOENT') throw failure + }) + if (entry || dirname(path) === path) throw error + return join(await resolveFuturePath(dirname(path)), basename(path)) + } +} + +export async function validateArtifactParent(parent, browsers, root = ROOT) { + const resolvedParent = await resolveFuturePath(resolve(parent)) + for (const browser of browsers) { + const build = await resolveFuturePath(join(root, 'build', browser)) + const nesting = relative(build, resolvedParent) + if (!nesting || (!isAbsolute(nesting) && nesting !== '..' && !nesting.startsWith(`..${sep}`))) + throw new Error(`Artifact parent must be outside the selected ${browser} build`) + } + // Ancestors are safe: mkdtemp creates a new, exclusive sibling of any existing build. + return resolvedParent +} + +export async function run(args) { + let options + try { + options = parseArgs(args) + } catch (error) { + console.error(error.message) + return 2 + } + if (options.help) { + console.log(HELP) + return 0 + } + const controller = new AbortController() + let lifecycle = createLifecycle({ signal: controller.signal }) + let signalCode + let activeAdapter + let artifactsDir + const report = { + result: 'FAIL', + selectedBrowsers: options.browser === 'all' ? ['chromium', 'firefox'] : [options.browser], + nodeVersion: process.version, + root: ROOT, + browsers: [], + failedStage: 'preflight', + cleanup: [], + } + function abort(error, code) { + signalCode ||= code + if (controller.signal.aborted) { + lifecycle.force() + return + } + controller.abort(error) + } + const onInterrupt = () => abort(new Error('Interrupted by SIGINT'), 130) + const onTerminate = () => abort(new Error('Interrupted by SIGTERM'), 143) + const onFatal = (error) => abort(error instanceof Error ? error : new Error(String(error)), 1) + process.on('SIGINT', onInterrupt) + process.on('SIGTERM', onTerminate) + process.on('uncaughtException', onFatal) + process.on('unhandledRejection', onFatal) + process.stdout.on('error', onFatal) + process.stderr.on('error', onFatal) + let exitCode = 1 + async function cleanupBrowser() { + const errors = await lifecycle.cleanup() + report.cleanup = [ + ...new Set([...report.cleanup, ...errors.map((error) => String(error.stack || error))]), + ] + return errors + } + function applyLateSignal() { + if (!signalCode || exitCode === signalCode) return + exitCode = signalCode + report.exitCode = exitCode + report.result = 'FAIL' + report.failedStage ||= 'reporting' + // No await between this final signal snapshot, report update, and returning the status. + if (artifactsDir) + writeFileSync(join(artifactsDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`) + } + try { + const parent = await validateArtifactParent( + options['artifacts-dir'] || tmpdir(), + report.selectedBrowsers, + ) + controller.signal.throwIfAborted() + await mkdir(parent, { recursive: true }) + controller.signal.throwIfAborted() + artifactsDir = mkdtempSync(join(parent, 'chatgptbox-smoke-')) + report.artifactsDir = artifactsDir + const jobs = await bounded( + preflight(options, ROOT, controller.signal), + 60000, + 'Preflight', + controller.signal, + ) + report.failedStage = 'startup' + await bounded( + (async () => { + for (const job of jobs) { + controller.signal.throwIfAborted() + if (report.browsers.length) lifecycle = createLifecycle({ signal: controller.signal }) + const browserDir = join(artifactsDir, job.browser) + await mkdir(browserDir) + controller.signal.throwIfAborted() + // Keep acquisition and registration synchronous so cancellation cannot leak a profile. + const profileDir = mkdtempSync(join(tmpdir(), `chatgptbox-${job.browser}-`)) + const profileLifecycle = lifecycle + lifecycle.defer(`${job.browser} profile`, async () => { + await profileLifecycle.assertProcessesStopped() + await rm(profileDir, { recursive: true, force: true }) + }) + const browserReport = { + browser: job.browser, + executable: job.executable, + artifactSha256: job.artifactSha256, + ...(job.browser === 'firefox' + ? { + preflightManifestVersion: job.manifest.version, + snapshotDir: join(browserDir, 'extension'), + } + : { manifestVersion: job.manifest.version }), + profileDir, + checks: [], + } + report.browsers.push(browserReport) + report.failedStage = `${job.browser}:startup` + activeAdapter = await bounded( + (job.browser === 'chromium' ? startChromium : startFirefox)({ + ...job, + profileDir, + artifactsDir: browserDir, + lifecycle, + signal: controller.signal, + }), + 60000, + `${job.browser} startup`, + controller.signal, + ) + Object.assign(browserReport, activeAdapter.metadata) + report.failedStage = `${job.browser}:scenarios` + Object.assign( + browserReport, + await runScenarios(activeAdapter, { + lifecycle, + signal: controller.signal, + checks: browserReport.checks, + }), + ) + report.failedStage = `${job.browser}:shutdown` + await activeAdapter.close() + activeAdapter = undefined + report.failedStage = `${job.browser}:cleanup` + const cleanupErrors = await cleanupBrowser() + if (cleanupErrors.length) + throw new AggregateError(cleanupErrors, `${job.browser} cleanup failed`) + } + })(), + 300000, + 'Smoke run', + controller.signal, + ) + exitCode = 0 + report.failedStage = null + } catch (error) { + exitCode = signalCode || (report.failedStage === 'preflight' ? 2 : 1) + report.error = String(error.stack || error) + if (!artifactsDir && !process.stderr.destroyed) console.error(String(error.message || error)) + report.cleanup.push( + ...(error.cleanupErrors || []).map((failure) => String(failure.stack || failure)), + ) + if (activeAdapter && !controller.signal.aborted) { + try { + await bounded(activeAdapter.capture('failure'), 3000, 'Failure capture') + } catch (captureError) { + report.captureError = String(captureError) + } + } + } finally { + controller.abort(new Error('Smoke run finished')) + await cleanupBrowser() + if (report.cleanup.length && exitCode === 0) { + exitCode = 1 + report.failedStage = 'cleanup' + } + exitCode = signalCode || exitCode + report.exitCode = exitCode + report.result = exitCode === 0 ? 'PASS' : 'FAIL' + try { + if (artifactsDir) + await writeFile(join(artifactsDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`) + applyLateSignal() + await new Promise((resolve, reject) => { + process.stdout.write( + `Smoke report; artifacts: ${artifactsDir || 'unavailable'}\n`, + (error) => (error ? reject(error) : resolve()), + ) + }) + } catch (error) { + exitCode = signalCode || 1 + report.exitCode = exitCode + report.result = 'FAIL' + report.failedStage ||= 'reporting' + report.reportingError = String(error.stack || error) + if (artifactsDir) { + try { + await writeFile(join(artifactsDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`) + } catch (writeError) { + if (!process.stderr.destroyed) + console.error(`Cannot write smoke report: ${writeError.message}`) + } + } + if (!process.stderr.destroyed) console.error(`Smoke reporting failed: ${error.message}`) + } + try { + applyLateSignal() + } catch (error) { + exitCode = signalCode || 1 + if (!process.stderr.destroyed) console.error(`Cannot finalize smoke report: ${error.message}`) + } + process.removeListener('SIGINT', onInterrupt) + process.removeListener('SIGTERM', onTerminate) + process.removeListener('uncaughtException', onFatal) + process.removeListener('unhandledRejection', onFatal) + process.stdout.removeListener('error', onFatal) + process.stderr.removeListener('error', onFatal) + } + return exitCode +} diff --git a/scripts/smoke/scenarios.mjs b/scripts/smoke/scenarios.mjs new file mode 100644 index 000000000..bd5a0cf88 --- /dev/null +++ b/scripts/smoke/scenarios.mjs @@ -0,0 +1,309 @@ +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import { bounded, waitFor } from './lifecycle.mjs' +import { startMockServer } from './mock-server.mjs' + +export function assertPopupIdentity(identity, metadata) { + assert.ok(metadata.expectedName, 'Adapter metadata must include expectedName') + assert.ok(metadata.expectedVersion, 'Adapter metadata must include expectedVersion') + assert.ok(metadata.extensionId, 'Adapter metadata must include the installed extensionId') + assert.ok(metadata.popupUrl, 'Adapter metadata must include the installed popupUrl') + assert.equal(identity.visible, true, 'Installed popup tabs must be visible') + assert.equal( + identity.extensionId, + metadata.extensionId, + 'Runtime identity must match installation', + ) + assert.equal( + identity.name, + metadata.expectedName, + 'Runtime name must match the installed manifest', + ) + assert.equal( + identity.version, + metadata.expectedVersion, + 'Runtime version must match the installed manifest', + ) + assert.equal( + identity.location, + identity.popupUrl, + 'The evaluated page must be the manifest popup', + ) + assert.equal(identity.popupUrl, metadata.popupUrl) +} + +export function assertScenarioMessages(messages, { phase, question, history = [] }) { + const errors = messages.filter((message) => message.error) + const done = messages.filter((message) => message.done === true) + const answers = messages.filter((message) => message.answer != null) + const acknowledgements = messages.filter((message) => message.session && message.done !== true) + assert.equal( + acknowledgements.length, + 1, + 'The background must acknowledge the initial session once', + ) + const acknowledgement = acknowledgements[0] + assert.equal(acknowledgement.answer == null && !acknowledgement.error, true) + assert.deepEqual( + acknowledgement.session.conversationRecords, + history, + 'The initial acknowledgement must preserve history', + ) + assert.ok( + messages.indexOf(acknowledgement) < + messages.findIndex((message) => message.answer != null || message.error), + 'The initial acknowledgement must precede the first answer or error', + ) + if (phase === 'error') { + assert.equal(errors.length, 1, 'HTTP failure must emit one error') + assert.match(String(errors[0].error), /smoke_503|Smoke upstream unavailable/) + assert.equal(done.length, 0, 'HTTP failure must not complete successfully') + assert.equal(answers.length, 0, 'HTTP failure must not emit an answer') + return + } + assert.equal(errors.length, 0, 'Successful stream must not emit errors') + if (phase === 'partial') { + assert.equal(done.length, 0, 'The gated stream must still be live') + assert.deepEqual( + answers.map(({ answer }) => answer), + ['Hello '], + 'The browser must observe exactly the first delta', + ) + return + } + assert.equal(phase, 'final') + assert.equal(done.length, 1, 'The stream must complete exactly once') + // Mock content events, not network chunks, determine the cumulative answers. + assert.deepEqual( + answers.map(({ answer }) => answer), + ['Hello ', 'Hello δΈ–η•ŒπŸ™‚'], + 'Every cumulative answer must match the mock stream exactly once and in order', + ) + assert.equal(done[0].answer, null, 'Completion must not carry answer content') + assert.ok( + messages.indexOf(done[0]) > messages.indexOf(answers.at(-1)), + 'Completion must follow every answer', + ) + assert.deepEqual( + done[0].session?.conversationRecords, + [...history, { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }], + 'The stream must add exactly one complete history record', + ) +} + +// These functions are serialized into the installed popup and must remain self-contained. +function popupIdentity() { + const api = globalThis.browser || globalThis.chrome + if (!api?.runtime?.id) throw new Error('Missing extension runtime in popup') + const manifest = api.runtime.getManifest() + const visible = [...document.querySelectorAll('#app [role="tab"]')].some((tab) => { + const style = globalThis.getComputedStyle(tab) + return ( + [...tab.getClientRects()].some((rect) => rect.width > 0 && rect.height > 0) && + style.visibility !== 'hidden' && + style.display !== 'none' && + style.opacity !== '0' + ) + }) + return { + visible, + extensionId: api.runtime.id, + name: manifest.name, + version: manifest.version, + popupUrl: api.runtime.getURL( + manifest.action?.default_popup || manifest.browser_action?.default_popup, + ), + location: globalThis.location.href, + } +} + +async function configure(baseUrl) { + const api = globalThis.browser || globalThis.chrome + await api.storage.local.set({ + preferredLanguage: 'en', + customOpenAIProviders: ['success', 'error'].map((name) => ({ + id: `smoke-${name}`, + name: `Smoke ${name}`, + baseUrl: `${baseUrl}/${name}/v1`, + chatCompletionsPath: '/chat/completions', + enabled: true, + })), + providerSecrets: { + 'smoke-success': 'smoke-local-only', + 'smoke-error': 'smoke-local-only', + }, + maxConversationContextLength: 4, + maxResponseTokenLength: 64, + temperatureOverrideEnabled: false, + }) +} + +export function beginRequest(key, question, history) { + const api = globalThis.browser || globalThis.chrome + const state = (window.__chatGPTBoxSmoke ??= {}) + if (state[key]) throw new Error(`Smoke request already started: ${key}`) + const port = api.runtime.connect({ name: `smoke-${key}` }) + const request = (state[key] = { port, messages: [], disconnected: false }) + port.onMessage.addListener((message) => request.messages.push(message)) + port.onDisconnect.addListener(() => { + request.disconnected = true + request.disconnectError = api.runtime.lastError?.message || 'Unexpected port disconnect' + }) + port.postMessage({ + session: { + question, + modelName: 'customModel', + conversationRecords: history, + isRetry: false, + apiMode: { + groupName: 'customApiModelKeys', + itemName: `Smoke ${key}`, + isCustom: true, + providerId: `smoke-${key}`, + customName: 'smoke-model', + customUrl: '', + apiKey: '', + active: true, + }, + }, + }) +} + +export function snapshot(key) { + const request = window.__chatGPTBoxSmoke?.[key] + if (!request) throw new Error(`Missing smoke request: ${key}`) + if (request.disconnected) throw new Error(request.disconnectError) + return request.messages +} + +export function disconnectPorts() { + const state = window.__chatGPTBoxSmoke + if (!state) return + const errors = [] + try { + for (const request of Object.values(state)) { + try { + request.port.disconnect() + } catch (error) { + errors.push(error) + } + } + } finally { + delete window.__chatGPTBoxSmoke + } + if (errors.length) throw new AggregateError(errors, 'Failed to disconnect smoke ports') +} + +export async function runScenarios(adapter, { lifecycle, signal, checks = [] }) { + const evaluate = (fn, ...args) => { + signal?.throwIfAborted() + return bounded(adapter.evaluate(fn, ...args), 10000, `popup ${fn.name}`, signal) + } + const identity = await waitFor( + async () => { + const value = await evaluate(popupIdentity) + return value.visible && value + }, + { signal, label: 'visible installed popup tabs' }, + ) + assertPopupIdentity(identity, adapter.metadata) + checks.push('Installed popup visible with matching runtime identity and version') + + const mock = await startMockServer({ lifecycle, signal }) + let disconnected = false + let disconnecting + const disconnect = async () => { + if (disconnected) return + if (disconnecting) return disconnecting + disconnecting = bounded( + Promise.resolve().then(() => adapter.evaluateCleanup(disconnectPorts)), + 5000, + 'disconnect smoke ports', + ) + try { + await disconnecting + disconnected = true + } finally { + disconnecting = undefined + } + } + lifecycle.defer('smoke runtime ports', disconnect) + let primaryError + try { + await evaluate(configure, mock.baseUrl) + const question = 'Smoke success question' + await evaluate(beginRequest, 'success', question, []) + const partial = await waitFor( + async () => { + const messages = await evaluate(snapshot, 'success') + if (messages.some((message) => message.error || message.done)) { + assertScenarioMessages(messages, { phase: 'partial', question }) + } + return messages.some((message) => typeof message.answer === 'string') && messages + }, + { signal, label: 'first streamed answer before release' }, + ) + assertScenarioMessages(partial, { phase: 'partial', question }) + checks.push('Partial Hello answer observed while the SSE response remains gated') + mock.release() + await waitFor( + async () => { + const messages = await evaluate(snapshot, 'success') + assert.equal(messages.filter((message) => message.error).length, 0) + return messages.some((message) => message.done) + }, + { signal, label: 'completed streamed answer' }, + ) + const history = [{ question, answer: 'Hello δΈ–η•ŒπŸ™‚' }] + await evaluate(beginRequest, 'error', 'Smoke error question', history) + await waitFor( + async () => { + const messages = await evaluate(snapshot, 'error') + assert.equal(messages.filter((message) => message.done).length, 0) + return messages.some((message) => message.error) + }, + { signal, label: 'HTTP 503 error from background' }, + ) + // Keep both ports open through the error round and a short drain window to catch late done messages. + await delay(150, undefined, { signal }) + assertScenarioMessages(await evaluate(snapshot, 'success'), { phase: 'final', question }) + checks.push('Exact Unicode answer, one completion, and one conversation record') + assertScenarioMessages(await evaluate(snapshot, 'error'), { phase: 'error', history }) + checks.push('HTTP 503 reported without completion or a new conversation record') + assert.equal(mock.requests.length, 2, 'Each scenario must issue exactly one HTTP request') + assert.deepEqual( + mock.requests.map(({ status }) => status), + [200, 503], + ) + assert.deepEqual(mock.requests[0].body.messages, [{ role: 'user', content: question }]) + assert.deepEqual(mock.requests[1].body.messages, [ + { role: 'user', content: question }, + { role: 'assistant', content: 'Hello δΈ–η•ŒπŸ™‚' }, + { role: 'user', content: 'Smoke error question' }, + ]) + } catch (error) { + primaryError = error instanceof Error ? error : new Error(String(error), { cause: error }) + } + const cleanupErrors = [] + for (const cleanup of [disconnect, () => mock.close()]) { + try { + await cleanup() + } catch (error) { + cleanupErrors.push( + error instanceof Error ? error : new Error(String(error), { cause: error }), + ) + } + } + if (primaryError) { + if (cleanupErrors.length) { + primaryError.cleanupErrors = [...(primaryError.cleanupErrors ?? []), ...cleanupErrors] + } + throw primaryError + } + if (cleanupErrors.length) { + const error = new AggregateError(cleanupErrors, 'Smoke scenario cleanup failed') + error.cleanupErrors = cleanupErrors + throw error + } + return { checks, requests: mock.requests } +} diff --git a/scripts/smoke/supervisor.mjs b/scripts/smoke/supervisor.mjs new file mode 100644 index 000000000..74d45a27d --- /dev/null +++ b/scripts/smoke/supervisor.mjs @@ -0,0 +1,102 @@ +// Internal entry point: lifecycle starts this process in a new POSIX session. +// Never use a remembered PID to signal a group, including on error paths. +import { spawn } from 'node:child_process' +import process from 'node:process' +import { clearTimeout, setTimeout } from 'node:timers' + +if (process.platform === 'win32' || typeof process.send !== 'function') { + throw new Error('The smoke supervisor requires an isolated POSIX IPC child') +} + +let started = false +let stopping = false +let forcing = false +let timer + +function force() { + if (forcing) return + forcing = true + stopping = true + clearTimeout(timer) + // The sender is itself a member, so this cannot select a recycled group. + // SIGKILL includes us: do not promise a subsequent target-exit message. + const kill = () => process.kill(0, 'SIGKILL') + timer = setTimeout(kill, 100) + if (process.connected) { + try { + process.send({ version: 1, type: 'terminating' }, kill) + } catch { + kill() + } + } else kill() +} + +function stop(graceMs = 100) { + if (stopping) return + stopping = true + timer = setTimeout(force, graceMs) + process.kill(0, 'SIGTERM') +} + +function send(message) { + if (!process.connected) { + stop() + return + } + process.send({ version: 1, ...message }, (error) => { + if (error) stop() + }) +} + +// Remain alive through the graceful group signal, including after target exit. +process.on('SIGTERM', () => {}) +process.on('SIGINT', () => stop()) +process.on('disconnect', () => stop()) +process.on('error', () => stop()) +process.on('message', (message) => { + if (!message || message.version !== 1) { + stop() + return + } + if (message.type === 'force') { + force() + return + } + if (message.type === 'stop') { + if (!Number.isFinite(message.graceMs) || message.graceMs < 0 || message.graceMs > 1000) { + stop() + return + } + stop(message.graceMs) + return + } + if (stopping) return + if ( + message.type !== 'start' || + started || + typeof message.command !== 'string' || + !Array.isArray(message.args) || + !message.args.every((arg) => typeof arg === 'string') + ) { + stop() + return + } + started = true + try { + const target = spawn(message.command, message.args, { + cwd: message.cwd, + env: message.env, + detached: false, + // The target must not inherit the private control channel. + stdio: ['ignore', 'inherit', 'inherit'], + }) + target.once('spawn', () => send({ type: 'started', pid: target.pid })) + target.once('error', (error) => + send({ type: 'start-error', message: error.message, code: error.code }), + ) + target.once('exit', (code, signal) => send({ type: 'target-exit', code, signal })) + } catch (error) { + send({ type: 'start-error', message: error.message, code: error.code }) + } +}) +send({ type: 'ready' }) diff --git a/scripts/xvfb-smoke.mjs b/scripts/xvfb-smoke.mjs new file mode 100644 index 000000000..d41abc153 --- /dev/null +++ b/scripts/xvfb-smoke.mjs @@ -0,0 +1,5 @@ +// Compatibility name: native headless browsers no longer require Xvfb. +import process from 'node:process' +import { run } from './smoke/runner.mjs' + +process.exitCode = await run(process.argv.slice(2)) diff --git a/tests/fixtures/smoke/fake-browser.fixture b/tests/fixtures/smoke/fake-browser.fixture new file mode 100644 index 000000000..cd320e60a --- /dev/null +++ b/tests/fixtures/smoke/fake-browser.fixture @@ -0,0 +1,11 @@ +#!/usr/bin/env node +// A deliberately unready browser used to exercise the production runner's cleanup. +const process = require('node:process') +const profile = process.argv + .find((arg) => arg.startsWith('--user-data-dir=')) + ?.slice('--user-data-dir='.length) +process.on('SIGTERM', () => {}) +// Failure-only safety net; successful tests must observe runner cleanup first. +setTimeout(() => process.exit(99), 30000).unref() +process.stdout.write(`${JSON.stringify({ pid: process.pid, profile })}\n`) +setInterval(() => {}, 1000) diff --git a/tests/fixtures/smoke/process.fixture b/tests/fixtures/smoke/process.fixture new file mode 100644 index 000000000..567b69b2c --- /dev/null +++ b/tests/fixtures/smoke/process.fixture @@ -0,0 +1,41 @@ +const { spawn } = require('node:child_process') +const process = require('node:process') +const { clearInterval, setInterval, setTimeout } = require('node:timers') + +const mode = process.argv[2] +if (mode === 'early-exit') { + process.stdout.write('stdout before exit\n') + process.stderr.write('stderr before exit\n') + process.exitCode = 7 +} else if (mode === 'descendant') { + const child = spawn(process.execPath, [__filename, 'ignore-term'], { + stdio: ['ignore', 'inherit', 'inherit', 'ipc'], + }) + child.on('message', (message) => { + if (message === 'ready') { + process.stdout.write(`DESCENDANT ${child.pid}\n`, () => process.exit(0)) + } + }) + child.on('error', () => process.exit(1)) +} else { + if (mode === 'ignore-term') { + process.on('SIGTERM', () => process.stdout.write('IGNORED TERM\n')) + } + function announce() { + process.stdout.write(`READY ${process.pid}\n`) + process.stderr.write('stderr ready\n') + if (process.send) process.send('ready') + } + if (mode === 'flood') { + Promise.all([ + new Promise((resolve) => process.stdout.write('O'.repeat(2 * 1024 * 1024), resolve)), + new Promise((resolve) => process.stderr.write('E'.repeat(2 * 1024 * 1024), resolve)), + ]).then(announce) + } else announce() + const timer = setInterval(() => {}, 1000) + // Fixture safety net: a failed test cannot leave a live process indefinitely. + setTimeout(() => { + clearInterval(timer) + process.exit(99) + }, 15000).unref() +} diff --git a/tests/unit/scripts/smoke-artifacts.test.mjs b/tests/unit/scripts/smoke-artifacts.test.mjs new file mode 100644 index 000000000..b5f1893cc --- /dev/null +++ b/tests/unit/scripts/smoke-artifacts.test.mjs @@ -0,0 +1,141 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import streams from 'node:fs' +import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { PassThrough } from 'node:stream' +import { setImmediate } from 'node:timers/promises' +import test from 'node:test' +import { hashArtifact, snapshotArtifact } from '../../../scripts/smoke/artifacts.mjs' +import { bounded } from '../../../scripts/smoke/lifecycle.mjs' + +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), 'smoke-artifacts-')) + t.after(() => rm(root, { recursive: true, force: true })) + const source = join(root, 'source') + const destination = join(root, 'snapshot') + await mkdir(source) + await mkdir(join(source, 'nested')) + await writeFile(join(source, 'nested/file'), 'original') + const hash = await hashArtifact(source) + return { root, source, destination, hash } +} + +test('snapshot includes nested files and is independent from later build writes', async (t) => { + const { source, destination, hash } = await fixture(t) + assert.equal(await snapshotArtifact(source, destination, hash), hash) + await writeFile(join(source, 'nested/file'), 'new build') + assert.equal(await readFile(join(destination, 'nested/file'), 'utf8'), 'original') + assert.equal(await hashArtifact(destination), hash) +}) + +for (const operation of ['add', 'remove']) { + test(`snapshot rejects an empty directory ${operation} after preflight`, async (t) => { + const { source, destination } = await fixture(t) + const empty = join(source, 'empty') + if (operation === 'remove') await mkdir(empty) + const hash = await hashArtifact(source) + if (operation === 'add') await mkdir(empty) + else await rm(empty, { recursive: true }) + assert.notEqual(await hashArtifact(source), hash) + await assert.rejects( + snapshotArtifact(source, destination, hash), + /Build changed since preflight/, + ) + }) +} + +test('binary contents cannot impersonate a second artifact entry', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'smoke-hash-framing-')) + t.after(() => rm(root, { recursive: true, force: true })) + const first = join(root, 'first') + const second = join(root, 'second') + await mkdir(first) + await mkdir(second) + await writeFile(join(first, 'a'), Buffer.from('first\0b\0second')) + await writeFile(join(second, 'a'), 'first') + await writeFile(join(second, 'b'), 'second') + const hash = await hashArtifact(first) + assert.notEqual(await hashArtifact(second), hash) + await assert.rejects( + snapshotArtifact(second, join(root, 'snapshot'), hash), + /Build changed since preflight/, + ) +}) + +test( + 'snapshot rejects symlinks instead of reading outside the build', + { skip: process.platform !== 'linux' }, + async (t) => { + const { root, source, destination, hash } = await fixture(t) + await writeFile(join(root, 'outside'), 'not a build artifact') + await symlink(join(root, 'outside'), join(source, 'link')) + await assert.rejects(snapshotArtifact(source, destination, hash), /non-regular build artifact/) + }, +) + +test('snapshot never overwrites an existing destination', async (t) => { + const { source, destination, hash } = await fixture(t) + await mkdir(destination) + await writeFile(join(destination, 'existing'), 'retain') + await assert.rejects(snapshotArtifact(source, destination, hash), { code: 'EEXIST' }) + assert.equal(await readFile(join(destination, 'existing'), 'utf8'), 'retain') +}) + +test('snapshot checks preflight hash and cancellation before acquiring a destination', async (t) => { + const { source, destination, hash } = await fixture(t) + await assert.rejects(snapshotArtifact(source, destination), /Missing preflight build hash/) + const controller = new AbortController() + const reason = new Error('Cancelled before snapshot') + controller.abort(reason) + await assert.rejects( + snapshotArtifact(source, destination, hash, controller.signal), + (error) => error === reason, + ) + await assert.rejects(stat(destination), { code: 'ENOENT' }) +}) + +test('snapshot cancellation waits for the transfer to close', { timeout: 5000 }, async (t) => { + const { source, destination, hash } = await fixture(t) + const started = Promise.withResolvers() + const destroying = Promise.withResolvers() + const release = Promise.withResolvers() + const input = new PassThrough({ + destroy(error, callback) { + destroying.resolve() + release.promise.then(() => callback(error)) + }, + }) + const mockRead = t.mock.method(streams, 'createReadStream', () => { + input.write('partial') + started.resolve() + return input + }) + syncBuiltinESMExports() + t.after(() => { + release.resolve() + input.destroy() + mockRead.mock.restore() + syncBuiltinESMExports() + }) + const controller = new AbortController() + const reason = new Error('Cancel the active transfer') + let settled = false + const rejected = assert.rejects( + snapshotArtifact(source, destination, hash, controller.signal).finally(() => { + settled = true + }), + (error) => error === reason, + ) + await bounded(started.promise, 1000, 'Snapshot transfer') + controller.abort(reason) + await bounded(destroying.promise, 1000, 'Transfer destruction') + await setImmediate() + assert.equal(settled, false, 'A pending close must not be abandoned') + release.resolve() + await bounded(rejected, 1000, 'Snapshot cancellation') + assert.equal(input.closed, true) +}) diff --git a/tests/unit/scripts/smoke-chromium.test.mjs b/tests/unit/scripts/smoke-chromium.test.mjs new file mode 100644 index 000000000..5aaf57655 --- /dev/null +++ b/tests/unit/scripts/smoke-chromium.test.mjs @@ -0,0 +1,746 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import { createHash } from 'node:crypto' +import { once } from 'node:events' +import fs, { mkdtemp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { writeFileSync } from 'node:fs' +import { createServer } from 'node:http' +import { syncBuiltinESMExports } from 'node:module' +import os from 'node:os' +import path from 'node:path' +import { test } from 'node:test' +import { clearTimeout, setTimeout } from 'node:timers' +import { runInNewContext } from 'node:vm' +import { connectCDP } from '../../../scripts/smoke/cdp.mjs' +import { startChromium } from '../../../scripts/smoke/chromium.mjs' +import { createLifecycle } from '../../../scripts/smoke/lifecycle.mjs' +import { hashArtifact } from '../../../scripts/smoke/artifacts.mjs' + +function fakeSocket(onSend, { open = true } = {}) { + const sockets = [] + class FakeWebSocket { + constructor(url) { + this.url = url + this.listeners = new Map() + this.closeCount = 0 + this.sent = [] + sockets.push(this) + if (open) globalThis.queueMicrotask(() => this.emit('open')) + } + addEventListener(type, fn) { + const listeners = this.listeners.get(type) ?? [] + listeners.push(fn) + this.listeners.set(type, listeners) + } + emit(type, event = {}) { + for (const listener of this.listeners.get(type) ?? []) listener(event) + } + reply(message) { + this.emit('message', { data: JSON.stringify(message) }) + } + send(data) { + const message = JSON.parse(data) + this.sent.push(message) + onSend?.(message, this) + } + close() { + this.closeCount++ + this.emit('close') + } + } + return { FakeWebSocket, sockets } +} + +function cleanupRegistry(t) { + const callbacks = [] + t.after(async () => { + for (const callback of callbacks.reverse()) await callback() + }) + return { + callbacks, + defer(label, cleanup) { + callbacks.push(cleanup) + }, + } +} + +test('CDP registers cleanup before handshake and closes a stalled connection', async (t) => { + const lifecycle = cleanupRegistry(t) + const { FakeWebSocket, sockets } = fakeSocket(undefined, { open: false }) + const connecting = connectCDP('ws://127.0.0.1:1234', { + lifecycle, + timeoutMs: 10, + WebSocketImpl: FakeWebSocket, + }) + assert.equal(lifecycle.callbacks.length, 1) + await assert.rejects(connecting, /Connect Chromium CDP timed out/) + await lifecycle.callbacks[0]() + await lifecycle.callbacks[0]() + assert.equal(sockets[0].closeCount, 1) +}) + +test('CDP does not open a transport when cleanup registration is rejected', async () => { + const { FakeWebSocket, sockets } = fakeSocket() + await assert.rejects( + connectCDP('ws://127.0.0.1:1234', { + lifecycle: { + defer() { + throw new Error('Lifecycle is already closing') + }, + }, + WebSocketImpl: FakeWebSocket, + }), + /Lifecycle is already closing/, + ) + assert.equal(sockets.length, 0) +}) + +test('CDP correlates concurrent responses and surfaces protocol errors without retry', async (t) => { + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + WebSocketImpl: FakeWebSocket, + }) + const socket = sockets[0] + const first = client.request('Target.createTarget', { url: 'about:blank' }) + const second = client.request('Runtime.evaluate', {}, 'popup-session') + socket.reply({ method: 'Runtime.consoleAPICalled', params: {} }) + socket.reply({ id: socket.sent[1].id, result: { value: 42 } }) + socket.reply({ id: socket.sent[0].id, error: { code: -1, message: 'Target denied' } }) + assert.deepEqual(await second, { value: 42 }) + await assert.rejects(first, /Target.createTarget: Target denied/) + assert.equal(socket.sent[1].sessionId, 'popup-session') + assert.equal(socket.sent.length, 2) +}) + +test('CDP request timeout ignores late responses and rejects pending requests on disconnect', async (t) => { + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + timeoutMs: 10, + WebSocketImpl: FakeWebSocket, + }) + await assert.rejects(client.request('Target.createTarget'), /Target.createTarget timed out/) + const next = client.request('Browser.getVersion') + sockets[0].reply({ id: sockets[0].sent[0].id, result: { wrong: true } }) + sockets[0].emit('close') + await assert.rejects(next, /disconnected/) + await assert.rejects(client.request('Browser.getVersion'), /disconnected/) + assert.equal(sockets[0].sent.length, 2) +}) + +test('CDP cancellation prevents new RPCs and permits bounded diagnostic requests', async (t) => { + const controller = new AbortController() + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + signal: controller.signal, + WebSocketImpl: FakeWebSocket, + }) + const pending = client.request('Runtime.evaluate') + controller.abort(new Error('Cancelled by test')) + await assert.rejects(pending, /Cancelled by test/) + await assert.rejects(client.request('Target.createTarget'), /Cancelled by test/) + const capture = client.request('Page.captureScreenshot', {}, 'popup', { signal: undefined }) + sockets[0].reply({ id: sockets[0].sent.at(-1).id, result: { data: 'png' } }) + assert.deepEqual(await capture, { data: 'png' }) + assert.equal(sockets[0].sent.length, 2) +}) + +test('CDP rejects malformed messages and missing results', async (t) => { + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + WebSocketImpl: FakeWebSocket, + }) + const missing = client.request('Browser.getVersion') + sockets[0].reply({ id: sockets[0].sent.at(-1).id }) + await assert.rejects(missing, /missing CDP result/) + const malformed = client.request('Runtime.evaluate') + sockets[0].emit('message', { data: 'not JSON' }) + await assert.rejects(malformed, /Invalid CDP WebSocket message/) +}) + +test('CDP run cancellation preserves an already pending independent cleanup RPC', async (t) => { + const controller = new AbortController() + const cleanup = new AbortController() + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + signal: controller.signal, + WebSocketImpl: FakeWebSocket, + }) + const normal = client.request('Runtime.evaluate') + const independent = client.request('Runtime.evaluate', {}, 'popup', { + signal: cleanup.signal, + timeoutMs: 1000, + }) + controller.abort(new Error('Run cancelled')) + await assert.rejects(normal, /Run cancelled/) + sockets[0].reply({ id: sockets[0].sent[0].id, result: { late: true } }) + sockets[0].reply({ id: sockets[0].sent[1].id, result: { cleaned: true } }) + assert.deepEqual(await independent, { cleaned: true }) + await assert.rejects( + client.request('Runtime.evaluate', {}, 'popup', { + signal: cleanup.signal, + timeoutMs: 10, + }), + /timed out after 10 ms/, + ) + await assert.rejects(client.request('Runtime.evaluate'), /Run cancelled/) + assert.equal(sockets[0].sent.length, 3) +}) + +test('Chromium cleanup evaluation survives cancellation and retains a five-second deadline', async (t) => { + const controller = new AbortController() + const fixture = await chromiumFixture(t, { signal: controller.signal }) + const adapter = await startChromium(fixture.options) + const normal = adapter.evaluate(() => new Promise(() => {})) + controller.abort(new Error('Run cancelled')) + await assert.rejects(normal, /Run cancelled/) + await assert.rejects( + Promise.resolve().then(() => adapter.evaluate(() => 1)), + /Run cancelled/, + ) + assert.equal(await adapter.evaluateCleanup((a, b) => a + b, 2, 3), 5) + await assert.rejects( + adapter.evaluateCleanup(async () => { + throw new Error('Cleanup failed') + }), + /Cleanup failed/, + ) + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + const stalled = adapter.evaluateCleanup(() => new Promise(() => {})) + const rejected = assert.rejects(stalled, /timed out after 5000 ms/) + t.mock.timers.tick(5001) + await rejected +}) + +test('CDP uses the Node builtin WebSocket against a local protocol peer', async (t) => { + const server = createServer() + const peers = new Set() + t.after(async () => { + for (const peer of peers) peer.destroy() + await new Promise((resolve) => server.close(resolve)) + }) + server.on('upgrade', (request, peer) => { + peers.add(peer) + peer.on('close', () => peers.delete(peer)) + const accept = createHash('sha1') + .update(`${request.headers['sec-websocket-key']}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`) + .digest('base64') + peer.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`, + ) + let input = Buffer.alloc(0) + peer.on('data', (chunk) => { + input = Buffer.concat([input, chunk]) + while (input.length >= 2) { + const opcode = input[0] & 15 + const length = input[1] & 127 + // The test sends only small, single-frame CDP commands and a close frame. + assert.ok(length < 126) + assert.ok(input[1] & 128) + if (input.length < 6 + length) return + const mask = input.subarray(2, 6) + const payload = Buffer.from(input.subarray(6, 6 + length)) + for (let i = 0; i < payload.length; i++) payload[i] ^= mask[i % 4] + input = input.subarray(6 + length) + if (opcode === 8) { + peer.end(Buffer.from([0x88, 0])) + return + } + assert.equal(opcode, 1) + const message = JSON.parse(payload.toString()) + assert.equal(message.method, 'Browser.getVersion') + const response = Buffer.from( + JSON.stringify({ id: message.id, result: { product: 'Chromium/native-transport' } }), + ) + peer.write(Buffer.concat([Buffer.from([0x81, response.length]), response])) + } + }) + }) + const listening = once(server, 'listening') + server.listen(0, '127.0.0.1') + await listening + const client = await connectCDP(`ws://127.0.0.1:${server.address().port}/devtools/browser/test`, { + lifecycle: cleanupRegistry(t), + }) + assert.deepEqual(await client.request('Browser.getVersion'), { + product: 'Chromium/native-transport', + }) + await client.close() +}) + +async function chromiumFixture( + t, + { + protocolError, + evaluationError, + screenshotError, + contextTransition, + partialPort, + signal, + key, + } = {}, +) { + const directory = await fs.realpath(await mkdtemp(path.join(os.tmpdir(), 'smoke-chromium-test-'))) + t.after(() => rm(directory, { recursive: true, force: true })) + const extensionDir = path.join(directory, 'extension') + const profileDir = path.join(directory, 'profile') + const artifactsDir = path.join(directory, 'artifacts') + await mkdir(extensionDir) + await mkdir(profileDir) + await mkdir(artifactsDir) + await writeFile( + path.join(extensionDir, 'manifest.json'), + JSON.stringify({ + version: '1.2.3', + name: 'ChatGPTBox', + ...(key ? { key } : {}), + background: { service_worker: 'background.js' }, + action: { default_popup: 'popup.html?popup=true' }, + }), + ) + const extensionId = createHash('sha256') + .update(key ? Buffer.from(key, 'base64') : extensionDir) + .digest('hex') + .slice(0, 32) + .replace(/[0-9a-f]/g, (digit) => String.fromCharCode(97 + Number.parseInt(digit, 16))) + const popupUrl = `chrome-extension://${extensionId}/popup.html?popup=true` + const lifecycle = cleanupRegistry(t) + const launches = [] + const portRecord = '43210\n/devtools/browser/12345678-1234-1234-1234-123456789abc\n' + lifecycle.spawn = (executable, args, options) => { + launches.push({ executable, args, options }) + const portFile = path.join(profileDir, 'DevToolsActivePort') + writeFileSync(portFile, partialPort ? '43210\n/devtools/browser/1234' : portRecord) + if (partialPort) { + const timer = setTimeout(() => writeFileSync(portFile, portRecord), 10) + t.after(() => clearTimeout(timer)) + } + return { + assertRunning() { + signal?.throwIfAborted() + }, + output: () => 'Fake browser output', + } + } + const { FakeWebSocket, sockets } = fakeSocket((message, socket) => { + if (message.method === protocolError) { + socket.reply({ id: message.id, error: { message: 'Deliberate protocol failure' } }) + return + } + let result = {} + switch (message.method) { + case 'Browser.getVersion': + result = { product: 'Chromium/150.0.0.0' } + break + case 'Target.getTargets': + result = { + targetInfos: [ + { type: 'page', targetId: 'unrelated', url: 'https://example.test/' }, + { + type: 'service_worker', + targetId: 'built-in-component', + url: `chrome-extension://${'b'.repeat(32)}/background.js`, + }, + { + type: 'service_worker', + targetId: 'worker', + url: `chrome-extension://${extensionId}/background.js`, + }, + ], + } + break + case 'Target.createTarget': + assert.equal(message.params.url, popupUrl) + result = { targetId: 'popup' } + break + case 'Target.attachToTarget': + assert.equal(message.params.targetId, 'popup') + result = { sessionId: 'popup-session' } + break + case 'Runtime.evaluate': { + assert.equal(message.sessionId, 'popup-session') + assert.equal(message.params.awaitPromise, true) + assert.equal(message.params.returnByValue, true) + if (contextTransition && message.params.expression.includes('readyState')) { + contextTransition = false + socket.reply({ + id: message.id, + error: { code: -32000, message: 'Execution context was destroyed.' }, + }) + return + } + if (evaluationError && message.params.expression.includes('readyState')) { + result = { + exceptionDetails: { exception: { description: 'Popup initialization failed' } }, + } + break + } + const value = runInNewContext(message.params.expression, { + location: { href: popupUrl }, + document: { + readyState: 'complete', + documentElement: { outerHTML: 'Extension popup' }, + }, + chrome: { runtime: { id: extensionId } }, + }) + Promise.resolve(value).then( + (value) => socket.reply({ id: message.id, result: { result: { value } } }), + (error) => + socket.reply({ + id: message.id, + result: { exceptionDetails: { exception: { description: error.message } } }, + }), + ) + return + } + case 'Page.captureScreenshot': + if (screenshotError) { + socket.reply({ id: message.id, error: { message: 'Screenshot failed' } }) + return + } + result = { data: Buffer.from('fake-png').toString('base64') } + break + } + socket.reply({ id: message.id, result }) + }) + t.mock.method(globalThis, 'WebSocket', function (url) { + return new FakeWebSocket(url) + }) + return { + options: { + executable: '/fake/chromium', + extensionDir, + profileDir, + artifactsDir, + lifecycle, + signal, + artifactSha256: await hashArtifact(extensionDir), + }, + launches, + sockets, + extensionId, + popupUrl, + } +} + +test('Chromium launches isolated native headless and evaluates async code in its extension popup', async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium(fixture.options) + const launch = fixture.launches[0] + assert.equal(launch.executable, '/fake/chromium') + assert.ok(launch.args.includes('--headless=new')) + assert.ok(launch.args.includes('--remote-debugging-port=0')) + assert.ok(launch.args.includes(`--load-extension=${fixture.options.extensionDir}`)) + assert.ok(launch.args.includes(`--disable-extensions-except=${fixture.options.extensionDir}`)) + assert.ok(launch.args.includes(`--user-data-dir=${fixture.options.profileDir}`)) + assert.ok(!launch.args.includes('--no-sandbox')) + assert.equal( + fixture.sockets[0].url, + 'ws://127.0.0.1:43210/devtools/browser/12345678-1234-1234-1234-123456789abc', + ) + assert.deepEqual(adapter.metadata, { + browserVersion: 'Chromium/150.0.0.0', + extensionId: fixture.extensionId, + popupUrl: fixture.popupUrl, + expectedVersion: '1.2.3', + expectedName: 'ChatGPTBox', + manifestVersion: '1.2.3', + }) + const identity = await adapter.evaluate(async (arg) => { + await Promise.resolve() + return { id: globalThis.chrome.runtime.id, arg } + }, 'quotes " and newlines\n') + assert.equal(identity.id, fixture.extensionId) + assert.equal(identity.arg, 'quotes " and newlines\n') + await assert.rejects( + adapter.evaluate(async () => { + throw new Error('Scenario failed') + }), + /Scenario failed/, + ) + await adapter.capture('failure') + assert.equal( + await readFile(path.join(fixture.options.artifactsDir, 'failure.png'), 'utf8'), + 'fake-png', + ) + assert.match( + await readFile(path.join(fixture.options.artifactsDir, 'failure.html'), 'utf8'), + /Extension popup/, + ) + await assert.rejects(adapter.capture('../outside'), /inside artifactsDir/) + await adapter.close() + await adapter.close() + assert.equal(fixture.sockets[0].closeCount, 1) +}) + +for (const event of ['close', 'error']) { + test(`Chromium close rejects an idle transport ${event} while the browser is alive`, async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium(fixture.options) + fixture.sockets[0].emit(event) + const expected = event === 'close' ? /disconnected/ : /transport error/ + await assert.rejects(adapter.close(), expected) + await assert.rejects(adapter.close(), expected) + await fixture.options.lifecycle.callbacks[0]() + assert.equal(fixture.sockets[0].closeCount, event === 'close' ? 0 : 1) + }) +} + +test('Chromium retries only the popup readiness context transition', async (t) => { + const fixture = await chromiumFixture(t, { contextTransition: true }) + await startChromium(fixture.options) + assert.equal( + fixture.sockets[0].sent.filter( + (message) => + message.method === 'Runtime.evaluate' && message.params.expression.includes('readyState'), + ).length, + 2, + ) + assert.equal( + fixture.sockets[0].sent.filter((message) => message.method === 'Target.createTarget').length, + 1, + ) +}) + +test('Chromium waits for a complete DevToolsActivePort record before connecting', async (t) => { + const fixture = await chromiumFixture(t, { partialPort: true }) + await startChromium(fixture.options) + assert.equal(fixture.sockets.length, 1) + assert.equal( + fixture.sockets[0].url, + 'ws://127.0.0.1:43210/devtools/browser/12345678-1234-1234-1234-123456789abc', + ) +}) + +test('Chromium diagnostics bypass cancelled liveness checks while the transport remains open', async (t) => { + const controller = new AbortController() + const fixture = await chromiumFixture(t, { signal: controller.signal }) + const adapter = await startChromium(fixture.options) + controller.abort(new Error('Scenario cancelled')) + await assert.rejects( + adapter.evaluate(() => true), + /Scenario cancelled/, + ) + await adapter.capture('cancelled') + assert.equal( + await readFile(path.join(fixture.options.artifactsDir, 'cancelled.png'), 'utf8'), + 'fake-png', + ) + assert.match( + await readFile(path.join(fixture.options.artifactsDir, 'cancelled.html'), 'utf8'), + /Extension popup/, + ) +}) + +test('Chromium ignores component workers with the same script path', async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium(fixture.options) + assert.equal(adapter.metadata.extensionId, fixture.extensionId) + assert.notEqual(adapter.metadata.extensionId, 'b'.repeat(32)) +}) + +test('Chromium uses the manifest public key instead of the load path for identity', async (t) => { + const fixture = await chromiumFixture(t, { + key: Buffer.from('Fixture public key').toString('base64'), + }) + const adapter = await startChromium(fixture.options) + assert.equal(adapter.metadata.extensionId, fixture.extensionId) +}) + +test( + 'Chromium resolves a symlink before selecting its worker and loading the extension', + { skip: os.platform() === 'win32' }, + async (t) => { + const fixture = await chromiumFixture(t) + const alias = path.join(path.dirname(fixture.options.extensionDir), 'extension alias') + await fs.symlink(fixture.options.extensionDir, alias, 'dir') + const adapter = await startChromium({ ...fixture.options, extensionDir: alias }) + assert.equal(adapter.metadata.extensionId, fixture.extensionId) + assert.ok(fixture.launches[0].args.includes(`--load-extension=${fixture.options.extensionDir}`)) + }, +) + +test('Chromium uses the verified build manifest instead of stale caller metadata', async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium({ + ...fixture.options, + manifest: { + name: 'Runner build', + version: '2.3.4', + background: { service_worker: 'background.js' }, + action: { default_popup: 'popup.html?popup=true' }, + }, + }) + assert.equal(adapter.metadata.expectedName, 'ChatGPTBox') + assert.equal(adapter.metadata.expectedVersion, '1.2.3') + assert.equal(adapter.metadata.manifestVersion, '1.2.3') +}) + +for (const change of [ + 'manifest', + 'invalid manifest', + 'missing manifest', + 'file', + 'empty directory', +]) { + test(`Chromium rejects a changed ${change} after preflight before spawning`, async (t) => { + const fixture = await chromiumFixture(t) + const { extensionDir } = fixture.options + const manifestPath = path.join(extensionDir, 'manifest.json') + if (change === 'manifest') { + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) + manifest.version = '9.9.9' + await writeFile(manifestPath, JSON.stringify(manifest)) + } else if (change === 'invalid manifest') { + await writeFile(manifestPath, '{') + } else if (change === 'missing manifest') { + await rm(manifestPath) + } else if (change === 'file') { + await writeFile(path.join(extensionDir, 'new.js'), 'changed') + } else { + await mkdir(path.join(extensionDir, 'new-directory')) + } + await assert.rejects(startChromium(fixture.options), /Build changed since preflight/) + assert.equal(fixture.launches.length, 0) + }) +} + +test('Chromium requires a valid preflight hash before spawning', async (t) => { + const fixture = await chromiumFixture(t) + for (const artifactSha256 of [undefined, '', 'invalid']) { + await assert.rejects( + startChromium({ ...fixture.options, artifactSha256 }), + /Missing preflight build hash/, + ) + } + assert.equal(fixture.launches.length, 0) +}) + +test('Chromium captures PNG and DOM when popup startup fails', async (t) => { + const fixture = await chromiumFixture(t, { evaluationError: true }) + await assert.rejects( + startChromium(fixture.options), + /Popup initialization failed.*\nFake browser output/, + ) + assert.equal( + await readFile(path.join(fixture.options.artifactsDir, 'chromium-startup-failure.png'), 'utf8'), + 'fake-png', + ) + assert.match( + await readFile( + path.join(fixture.options.artifactsDir, 'chromium-startup-failure.html'), + 'utf8', + ), + /Extension popup/, + ) +}) + +test('Chromium captures DOM even if screenshot fails', async (t) => { + const fixture = await chromiumFixture(t, { screenshotError: true }) + const adapter = await startChromium(fixture.options) + await assert.rejects(adapter.capture('failure'), /Chromium capture failed/) + assert.match( + await readFile(path.join(fixture.options.artifactsDir, 'failure.html'), 'utf8'), + /Extension popup/, + ) +}) + +test('Chromium does not retry target creation failures and retains cleanup', async (t) => { + const fixture = await chromiumFixture(t, { protocolError: 'Target.createTarget' }) + await assert.rejects( + startChromium(fixture.options), + /Target.createTarget: Deliberate protocol failure/, + ) + assert.equal( + fixture.sockets[0].sent.filter((message) => message.method === 'Target.createTarget').length, + 1, + ) + assert.equal(fixture.options.lifecycle.callbacks.length, 1) +}) + +test('Chromium rejects stale profile port files without launching', async (t) => { + const fixture = await chromiumFixture(t) + await writeFile( + path.join(fixture.options.profileDir, 'DevToolsActivePort'), + '1234\n/devtools/browser/stale', + ) + await assert.rejects(startChromium(fixture.options), /use a fresh profile/) + assert.equal(fixture.launches.length, 0) +}) + +test('Chromium cancellation before startup does not spawn a process', async (t) => { + const fixture = await chromiumFixture(t) + const controller = new AbortController() + controller.abort(new Error('Stop now')) + await assert.rejects(startChromium({ ...fixture.options, signal: controller.signal }), /Stop now/) + assert.equal(fixture.launches.length, 0) +}) + +test( + 'Chromium does not recreate an owned profile after cancellation during manifest reading', + { timeout: 5000 }, + async (t) => { + const controller = new AbortController() + const fixture = await chromiumFixture(t, { signal: controller.signal }) + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.force()) + lifecycle.defer('Chromium profile', () => + rm(fixture.options.profileDir, { recursive: true, force: true }), + ) + const started = Promise.withResolvers() + const release = Promise.withResolvers() + const originalReadFile = readFile + const manifestPath = path.join(fixture.options.extensionDir, 'manifest.json') + const mockRead = t.mock.method(fs, 'readFile', async (...args) => { + const bytes = await originalReadFile(...args) + if (args[0] === manifestPath) { + started.resolve() + await release.promise + } + return bytes + }) + syncBuiltinESMExports() + t.after(() => { + release.resolve() + mockRead.mock.restore() + syncBuiltinESMExports() + }) + const reason = new Error('Cancelled while reading manifest') + const rejected = assert.rejects( + startChromium({ ...fixture.options, lifecycle }), + (error) => error === reason, + ) + await started.promise + controller.abort(reason) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(stat(fixture.options.profileDir), { code: 'ENOENT' }) + release.resolve() + await rejected + await assert.rejects(stat(fixture.options.profileDir), { code: 'ENOENT' }) + assert.equal(fixture.sockets.length, 0) + }, +) + +test('Chromium normal close rejects a late browser exit while lifecycle cleanup remains available', async (t) => { + const fixture = await chromiumFixture(t) + const originalSpawn = fixture.options.lifecycle.spawn + let browser + fixture.options.lifecycle.spawn = (...args) => (browser = originalSpawn(...args)) + const adapter = await startChromium(fixture.options) + const failure = new Error('Process Chromium exited unexpectedly (7)') + t.mock.method(browser, 'assertRunning', () => { + throw failure + }) + fixture.sockets[0].emit('close') + await assert.rejects(adapter.close(), (error) => error === failure) + await fixture.options.lifecycle.callbacks[0]() +}) diff --git a/tests/unit/scripts/smoke-firefox.test.mjs b/tests/unit/scripts/smoke-firefox.test.mjs new file mode 100644 index 000000000..9640cace0 --- /dev/null +++ b/tests/unit/scripts/smoke-firefox.test.mjs @@ -0,0 +1,675 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import streams from 'node:fs' +import fs, { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import { setTimeout } from 'node:timers/promises' +import { URL } from 'node:url' +import { runInNewContext } from 'node:vm' +import test from 'node:test' +import { startFirefox } from '../../../scripts/smoke/firefox.mjs' +import { bounded, createLifecycle, waitFor } from '../../../scripts/smoke/lifecycle.mjs' +import { hashArtifact } from '../../../scripts/smoke/runner.mjs' + +const popupUrl = 'moz-extension://owned-extension/popup.html?popup=true' + +async function fixture(t, override = () => {}, popup = 'popup.html?popup=true') { + const root = await mkdtemp(join(tmpdir(), 'smoke-firefox-test-')) + t.after(() => rm(root, { recursive: true, force: true })) + const calls = [] + const cleanups = [] + const spawns = [] + let handlesReads = 0 + const controller = new AbortController() + const options = { + executable: '/usr/lib/firefox/firefox', + geckodriver: '/owned/geckodriver', + archive: join(root, 'extension.zip'), + extensionDir: join(root, 'build'), + manifest: { + version: '2.6.1', + name: 'ChatGPTBox', + browser_action: { default_popup: popup }, + }, + profileDir: join(root, 'profile'), + artifactsDir: join(root, 'artifacts'), + signal: controller.signal, + lifecycle: { + spawn(...args) { + spawns.push(args) + return { + child: {}, + exited: new Promise(() => {}), + output: () => '1720000000000\tgeckodriver\tINFO\tListening on 127.0.0.1:41239\n', + assertRunning() {}, + } + }, + defer(label, cleanup) { + cleanups.push({ label, cleanup }) + }, + }, + } + await mkdir(options.profileDir) + await mkdir(options.artifactsDir) + await writeFile(options.archive, 'owned archive bytes') + await mkdir(options.extensionDir) + await writeFile(join(options.extensionDir, 'manifest.json'), JSON.stringify(options.manifest)) + await writeFile(join(options.extensionDir, 'background.js'), 'current build') + options.artifactSha256 = await hashArtifact(options.extensionDir) + t.mock.method(globalThis, 'fetch', async (url, init) => { + assert.equal(new URL(url).origin, 'http://127.0.0.1:41239') + const path = new URL(url).pathname + const call = { path, method: init.method, body: init.body && JSON.parse(init.body), init } + calls.push(call) + const overridden = await override(call, { controller, cleanups }) + if (overridden) return overridden + let value = null + if (path === '/status') value = { ready: true } + else if (path === '/session') { + assert.equal(cleanups.length, 1, 'cleanup must be registered before creating a session') + value = { sessionId: 'owned-session', capabilities: { browserVersion: '154.0' } } + } else if (path.endsWith('/moz/addon/install')) value = 'addon@example.test' + else if (path.endsWith('/window/handles')) { + value = handlesReads++ ? ['original', 'popup'] : ['original'] + } else if (path.endsWith('/execute/sync') && call.body.script.includes('getByID')) { + value = 'moz-extension://owned-extension/' + } else if (path.endsWith('/execute/async')) { + value = await new Promise((resolve, reject) => { + try { + runInNewContext(`(function () { ${call.body.script} }).apply(null, args)`, { + args: [...call.body.args, resolve], + location: { + href: calls.find( + (entry) => + typeof entry.body?.script === 'string' && + entry.body.script.includes('gBrowser.addTab'), + )?.body.args[0], + }, + document: { readyState: 'complete' }, + browser: { storage: { local: { get: async () => ({ stored: true }) } } }, + }) + } catch (error) { + reject(error) + } + }) + } else if (path.endsWith('/screenshot')) value = Buffer.from('image bytes').toString('base64') + else if (path.endsWith('/source')) value = 'popup' + return Response.json({ value }) + }) + return { options, calls, cleanups, spawns, controller } +} + +test('Firefox opens the popup declared by the installed manifest including its query', async (t) => { + const declared = 'custom/declared-popup.html?from=manifest' + const { options, calls } = await fixture(t, undefined, declared) + const adapter = await startFirefox(options) + assert.equal(adapter.metadata.popupUrl, `moz-extension://owned-extension/${declared}`) + assert.deepEqual( + calls.find( + (call) => + typeof call.body?.script === 'string' && call.body.script.includes('gBrowser.addTab'), + ).body.args, + [`moz-extension://owned-extension/${declared}`], + ) + await adapter.close() +}) + +test('Firefox installs the verified directory snapshot instead of a stale archive', async (t) => { + const { options, calls } = await fixture(t) + const adapter = await startFirefox(options) + const installed = calls.find((call) => call.path.endsWith('/moz/addon/install')).body.path + assert.equal(installed, join(options.artifactsDir, 'extension')) + assert.equal(await hashArtifact(installed), options.artifactSha256) + await writeFile(join(options.extensionDir, 'background.js'), 'later build') + assert.equal(await readFile(join(installed, 'background.js'), 'utf8'), 'current build') + assert.equal(adapter.metadata.artifactSha256, options.artifactSha256) + assert.equal(Object.hasOwn(adapter.metadata, 'archiveSha256'), false) + await adapter.close() +}) + +test('Firefox rejects a build changed since preflight before launching processes', async (t) => { + const { options, calls, spawns } = await fixture(t) + await writeFile(join(options.extensionDir, 'background.js'), 'changed build') + await assert.rejects(startFirefox(options), /Build changed since preflight/) + assert.equal(spawns.length, 0) + assert.equal(calls.length, 0) +}) + +for (const popup of [ + null, + '', + ' ', + ' popup.html', + 'https://example.test/popup.html', + 'javascript:alert(1)', + '//other-extension/popup.html', + 'moz-extension://other-extension/popup.html', + 'moz-extension://owned-extension/popup.html', +]) { + test(`Firefox rejects invalid manifest popup ${JSON.stringify( + popup, + )} before startup`, async (t) => { + const { options, calls, spawns } = await fixture(t, undefined, popup) + await assert.rejects(startFirefox(options), /Invalid Firefox manifest popup path/) + assert.equal(calls.length, 0) + assert.equal(spawns.length, 0) + }) +} + +test('Firefox uses snapshot manifest identity and action popup precedence', async (t) => { + const { options } = await fixture(t) + const manifest = { + ...options.manifest, + name: 'Current snapshot', + version: '3.0', + action: { default_popup: 'action-popup.html?source=action#tab' }, + } + await writeFile(join(options.extensionDir, 'manifest.json'), JSON.stringify(manifest)) + options.artifactSha256 = await hashArtifact(options.extensionDir) + // The earlier metadata may predate the directory hash; use the snapshot instead. + const adapter = await startFirefox(options) + assert.equal(adapter.metadata.expectedName, manifest.name) + assert.equal(adapter.metadata.expectedVersion, manifest.version) + assert.equal(adapter.metadata.manifestVersion, manifest.version) + assert.equal( + adapter.metadata.popupUrl, + 'moz-extension://owned-extension/action-popup.html?source=action#tab', + ) + await adapter.close() +}) + +test( + 'Firefox does not recreate an owned profile after cancellation during snapshot validation', + { timeout: 5000 }, + async (t) => { + const { options, controller, calls } = await fixture(t) + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.force()) + lifecycle.defer('Firefox profile', () => + rm(options.profileDir, { recursive: true, force: true }), + ) + const started = Promise.withResolvers() + const release = Promise.withResolvers() + const originalReadFile = readFile + const mockRead = t.mock.method(fs, 'readFile', async (...args) => { + const bytes = await originalReadFile(...args) + if (args[0] === join(options.artifactsDir, 'extension/manifest.json')) { + started.resolve() + await release.promise + } + return bytes + }) + syncBuiltinESMExports() + t.after(() => { + release.resolve() + mockRead.mock.restore() + syncBuiltinESMExports() + }) + const reason = new Error('Cancelled while validating snapshot') + const rejected = assert.rejects( + startFirefox({ ...options, lifecycle }), + (error) => error === reason, + ) + await started.promise + controller.abort(reason) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + release.resolve() + await rejected + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + assert.equal(calls.length, 0) + }, +) + +test( + 'Firefox cancellation closes an in-flight snapshot transfer before rejecting startup', + { timeout: 5000 }, + async (t) => { + const { options, controller, calls } = await fixture(t) + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.force()) + lifecycle.defer('Firefox profile', () => + rm(options.profileDir, { recursive: true, force: true }), + ) + const spawn = t.mock.method(lifecycle, 'spawn', () => { + throw new Error('Unexpected browser startup') + }) + const started = Promise.withResolvers() + const originalRead = streams.createReadStream + const originalWrite = streams.createWriteStream + const input = new PassThrough() + let output + const mockRead = t.mock.method(streams, 'createReadStream', (path, ...args) => { + if (path === join(options.extensionDir, 'background.js')) { + input.write('partial') + started.resolve() + return input + } + return originalRead(path, ...args) + }) + const mockWrite = t.mock.method(streams, 'createWriteStream', (path, ...args) => { + const result = originalWrite(path, ...args) + if (path === join(options.artifactsDir, 'extension/background.js')) output = result + return result + }) + syncBuiltinESMExports() + t.after(() => { + input.destroy() + output?.destroy() + mockRead.mock.restore() + mockWrite.mock.restore() + syncBuiltinESMExports() + }) + const reason = new Error('Cancelled during snapshot copy') + const rejected = assert.rejects( + startFirefox({ ...options, lifecycle }), + (error) => error === reason, + ) + await bounded(started.promise, 1000, 'Snapshot transfer start') + await waitFor(() => output?.bytesWritten > 0, { timeoutMs: 1000 }) + controller.abort(reason) + await bounded(rejected, 1000, 'Snapshot cancellation') + assert.equal(input.closed, true) + assert.equal(output.closed, true) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + assert.equal( + await readFile(join(options.artifactsDir, 'extension/background.js'), 'utf8'), + 'partial', + ) + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + assert.equal(spawn.mock.callCount(), 0) + assert.equal(calls.length, 0) + }, +) + +test('Firefox uses an owned headless session and privileged extension popup context', async (t) => { + const { options, calls, cleanups, spawns } = await fixture(t) + const adapter = await startFirefox(options) + assert.deepEqual(spawns[0].slice(0, 2), [ + options.geckodriver, + [ + '--host', + '127.0.0.1', + '--port', + '0', + '--websocket-port', + '0', + '--profile-root', + options.profileDir, + '--allow-system-access', + ], + ]) + assert.equal(spawns[0][2].logPath, join(options.artifactsDir, 'geckodriver.log')) + const session = calls.find((call) => call.path === '/session') + assert.equal( + session.body.capabilities.alwaysMatch['moz:firefoxOptions'].binary, + options.executable, + ) + assert.deepEqual(session.body.capabilities.alwaysMatch['moz:firefoxOptions'].args, ['-headless']) + assert.deepEqual(calls.find((call) => call.path.endsWith('/moz/addon/install')).body, { + path: join(options.artifactsDir, 'extension'), + temporary: true, + }) + assert.deepEqual( + calls.filter((call) => call.path.endsWith('/moz/context')).map((call) => call.body.context), + ['chrome', 'content'], + ) + const tab = calls.find( + (call) => call.path.endsWith('/execute/sync') && call.body.script.includes('gBrowser.addTab'), + ) + assert.match(tab.body.script, /Services\.scriptSecurityManager\.getSystemPrincipal\(\)/) + assert.deepEqual(tab.body.args, [popupUrl]) + assert.deepEqual(calls.find((call) => call.path.endsWith('/window')).body, { handle: 'popup' }) + assert.deepEqual(adapter.metadata, { + browserVersion: '154.0', + extensionId: 'addon@example.test', + popupUrl, + expectedVersion: '2.6.1', + expectedName: 'ChatGPTBox', + artifactSha256: options.artifactSha256, + snapshotDir: join(options.artifactsDir, 'extension'), + manifestVersion: '2.6.1', + }) + assert.equal(await adapter.evaluate(async (a, b) => Promise.resolve(a + b), 3, 4), 7) + assert.equal( + await adapter.evaluate(async () => (await globalThis.browser.storage.local.get()).stored), + true, + ) + await assert.rejects( + adapter.evaluate(async () => { + throw new Error('page failure') + }), + /page failure/, + ) + await adapter.capture('popup-initial') + await assert.rejects(adapter.capture('../escape'), /inside artifactsDir/) + assert.equal( + await readFile(join(options.artifactsDir, 'popup-initial.png'), 'utf8'), + 'image bytes', + ) + assert.equal( + await readFile(join(options.artifactsDir, 'popup-initial.html'), 'utf8'), + 'popup', + ) + await adapter.close() + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) +}) + +test('Firefox capture retains DOM when screenshot fails', async (t) => { + const { options } = await fixture(t, ({ path }) => { + if (path.endsWith('/screenshot')) { + return Response.json({ value: { error: 'unknown error', message: 'Screenshot failed' } }) + } + }) + const adapter = await startFirefox(options) + await assert.rejects(adapter.capture('failure'), (error) => { + assert.ok(error instanceof AggregateError) + assert.equal(error.errors.length, 1) + assert.match(error.errors[0].message, /Screenshot failed/) + assert.match(String(error), /Screenshot failed/) + return true + }) + assert.equal( + await readFile(join(options.artifactsDir, 'failure.html'), 'utf8'), + 'popup', + ) + await adapter.close() +}) + +test('Firefox capture preserves both diagnostic errors', async (t) => { + const { options } = await fixture(t, ({ path }) => { + if (path.endsWith('/screenshot') || path.endsWith('/source')) { + return Response.json({ value: { error: 'unknown error', message: `Failed ${path}` } }) + } + }) + const adapter = await startFirefox(options) + await assert.rejects(adapter.capture('failure'), (error) => { + assert.ok(error instanceof AggregateError) + assert.equal(error.errors.length, 2) + assert.match(error.errors[0].message, /screenshot/) + assert.match(error.errors[1].message, /source/) + assert.match(String(error), /Failed \/session\/owned-session\/screenshot/) + assert.match(String(error), /Failed \/session\/owned-session\/source/) + return true + }) + await adapter.close() +}) + +test('Firefox capture cancels both pending diagnostics with the run signal', async (t) => { + const { options, controller, calls } = await fixture(t, ({ path, init }) => { + if (path.endsWith('/screenshot') || path.endsWith('/source')) { + return new Promise((resolve, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason), { once: true }) + }) + } + }) + const adapter = await startFirefox(options) + const reason = new Error('Run cancelled during capture') + const captured = adapter.capture('failure') + const rejected = assert.rejects(captured, (error) => { + assert.ok(error instanceof AggregateError) + assert.deepEqual(error.errors, [reason, reason]) + return true + }) + const diagnostics = calls.filter( + ({ path }) => path.endsWith('/screenshot') || path.endsWith('/source'), + ) + assert.equal(diagnostics.length, 2) + controller.abort(reason) + await rejected + assert.ok(diagnostics.every(({ init }) => init.signal.aborted)) + await adapter.close() +}) + +for (const path of ['/session', '/session/owned-session/moz/addon/install']) { + for (const status of [200, 500]) { + test(`Firefox rejects WebDriver error at ${path} with HTTP ${status} without retry`, async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path === path) { + return Response.json( + { value: { error: 'unknown error', message: 'broken operation' } }, + { status }, + ) + } + }) + await assert.rejects(startFirefox(options), /broken operation/) + assert.equal(calls.filter((call) => call.path === path).length, 1) + await cleanups[0].cleanup() + assert.equal( + calls.filter((call) => call.method === 'DELETE').length, + path === '/session' ? 0 : 1, + ) + }) + } +} + +test('Firefox rejects failed HTTP status even with a successful-looking value', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/session') + return Response.json({ value: { sessionId: 'bad' } }, { status: 502 }) + }) + await assert.rejects(startFirefox(options), /HTTP 502/) + assert.equal(calls.filter((call) => call.path === '/session').length, 1) +}) + +test('Firefox rejects malformed HTTP JSON without retrying session creation', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/session') return new Response('error', { status: 503 }) + }) + await assert.rejects(startFirefox(options), /HTTP 503, invalid JSON/) + assert.equal(calls.filter((call) => call.path === '/session').length, 1) +}) + +for (const value of ['', ' ', null]) { + test(`Firefox rejects empty addon ID ${JSON.stringify(value)} and retains cleanup`, async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path.endsWith('/moz/addon/install')) return Response.json({ value }) + }) + await assert.rejects(startFirefox(options), /empty addon ID/) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/addon/install')).length, 1) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/context')).length, 0) + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) + }) +} + +test('Firefox retries explicit read-only not-ready status', async (t) => { + let statusReads = 0 + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/status') return Response.json({ value: { ready: ++statusReads > 1 } }) + }) + const adapter = await startFirefox(options) + assert.equal(statusReads, 2) + assert.equal(calls.filter((call) => call.path === '/session').length, 1) + await adapter.close() +}) + +test('Firefox preserves evaluation messages when the stack contains only locations', async (t) => { + const { options } = await fixture(t) + const adapter = await startFirefox(options) + await assert.rejects( + adapter.evaluate(() => { + const error = new Error('Popup snapshot is missing') + error.stack = 'pageFunction@moz-extension://owned-extension/popup.html:4:2' + throw error + }), + (error) => { + assert.match(error.message, /^Firefox evaluation failed: Error: Popup snapshot is missing/) + assert.match(error.message, /pageFunction@moz-extension:/) + assert.doesNotMatch(error.message, /WebDriver POST/) + return true + }, + ) + await adapter.close() +}) + +test('Firefox waits for a complete listening-port log record', async (t) => { + const { options, calls } = await fixture(t) + const originalSpawn = options.lifecycle.spawn + let reads = 0 + options.lifecycle.spawn = (...args) => ({ + ...originalSpawn(...args), + output: () => + ++reads === 1 + ? 'geckodriver INFO Listening on 127.0.0.1:4' + : 'geckodriver INFO Listening on 127.0.0.1:41239\n', + }) + const adapter = await startFirefox(options) + assert.equal(reads, 2) + assert.equal(calls.filter((call) => call.path === '/status').length, 1) + await adapter.close() +}) + +test('Firefox rejects malformed readiness instead of retrying', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/status') return Response.json({ value: {} }) + }) + await assert.rejects(startFirefox(options), /Invalid geckodriver readiness status/) + assert.equal(calls.length, 1) +}) + +test('Firefox refuses an empty session ID before installing the addon', async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path === '/session') return Response.json({ value: { sessionId: '' } }) + }) + await assert.rejects(startFirefox(options), /empty session ID/) + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'POST').length, 1) + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 0) +}) + +test('Firefox surfaces failed session cleanup and does not retry it', async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.method === 'DELETE') { + return Response.json({ value: { error: 'unknown error', message: 'cannot close' } }) + } + }) + const adapter = await startFirefox(options) + await assert.rejects(adapter.close(), /cannot close/) + await assert.rejects(cleanups[0].cleanup(), /cannot close/) + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) +}) + +test('Firefox never retries a network failure during addon installation', async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path.endsWith('/moz/addon/install')) throw new TypeError('fetch failed') + }) + await assert.rejects(startFirefox(options), /fetch failed/) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/addon/install')).length, 1) + await cleanups[0].cleanup() +}) + +test('Firefox aborts a stalled installation and still deletes its session once', async (t) => { + const { options, calls, cleanups, controller } = await fixture(t, async (call) => { + if (call.path.endsWith('/moz/addon/install')) { + await setTimeout(10) + controller.abort(new Error('test operation timeout')) + return new Promise(() => {}) + } + }) + await assert.rejects(startFirefox(options), /test operation timeout/) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/addon/install')).length, 1) + assert.equal( + calls.find((call) => call.path.endsWith('/moz/addon/install')).init.signal.aborted, + true, + ) + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) + assert.equal( + calls.find((call) => call.method === 'DELETE').init.signal.reason?.message === + 'test operation timeout', + false, + ) +}) + +for (const path of ['/session', '/session/owned-session/moz/addon/install']) { + test(`Firefox bounds a stalled ${path} request without retrying`, async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path === path) { + queueMicrotask(() => t.mock.timers.tick(path === '/session' ? 60001 : 30001)) + return new Promise(() => {}) + } + }) + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + await assert.rejects(startFirefox(options), path === '/session' ? /60000 ms/ : /30000 ms/) + assert.equal(calls.filter((call) => call.path === path).length, 1) + assert.equal(calls.find((call) => call.path === path).init.signal.aborted, true) + await cleanups[0].cleanup() + }) +} + +test('Firefox bounds session shutdown to five seconds without retry', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.method === 'DELETE') { + queueMicrotask(() => t.mock.timers.tick(5001)) + return new Promise(() => {}) + } + }) + const adapter = await startFirefox(options) + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + await assert.rejects(adapter.close(), /5000 ms/) + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) + assert.equal(calls.find((call) => call.method === 'DELETE').init.signal.aborted, true) +}) + +test('Firefox cleanup evaluation survives cancellation and retains a five-second deadline', async (t) => { + let stall = false + const started = Promise.withResolvers() + const { options, calls, controller } = await fixture(t, (call) => { + if (stall && call.path.endsWith('/execute/async')) { + started.resolve() + return new Promise(() => {}) + } + }) + const adapter = await startFirefox(options) + stall = true + const normal = adapter.evaluate(() => 1) + await started.promise + controller.abort(new Error('Run cancelled')) + await assert.rejects(normal, /Run cancelled/) + const count = calls.length + await assert.rejects( + adapter.evaluate(() => 1), + /Run cancelled/, + ) + assert.equal(calls.length, count) + stall = false + assert.equal(await adapter.evaluateCleanup((a, b) => a + b, 2, 3), 5) + await assert.rejects( + adapter.evaluateCleanup(async () => { + throw new Error('Cleanup failed') + }), + /Cleanup failed/, + ) + stall = true + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + const stalled = adapter.evaluateCleanup(() => 1) + const cleanupSignal = calls.at(-1).init.signal + assert.equal(cleanupSignal.aborted, false) + assert.notEqual(cleanupSignal, controller.signal) + const rejected = assert.rejects(stalled, /timed out after 5000 ms/) + t.mock.timers.tick(5001) + await rejected + assert.equal(cleanupSignal.aborted, true) + await adapter.close() +}) diff --git a/tests/unit/scripts/smoke-interruption.test.mjs b/tests/unit/scripts/smoke-interruption.test.mjs new file mode 100644 index 000000000..de7b51a76 --- /dev/null +++ b/tests/unit/scripts/smoke-interruption.test.mjs @@ -0,0 +1,554 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import process from 'node:process' +import { spawn } from 'node:child_process' +import { + chmod, + cp, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + stat, + writeFile, +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { ROOT } from '../../../scripts/smoke/runner.mjs' +import { bounded, waitFor } from '../../../scripts/smoke/lifecycle.mjs' + +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), 'smoke checkout = with spaces ')) + const cleanups = [] + t.after(async () => { + const results = await Promise.allSettled(cleanups.map(async (cleanup) => cleanup())) + const errors = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason) + if (errors.length) { + t.diagnostic(`Preserved fixture after cleanup failure: ${root}`) + throw new AggregateError(errors, 'Fixture cleanup failed') + } + await rm(root, { recursive: true, force: true }) + }) + await cp(join(ROOT, 'scripts/smoke'), join(root, 'scripts/smoke'), { recursive: true }) + for (const name of ['xvfb-smoke.mjs', 'run-smoke.sh']) { + await cp(join(ROOT, 'scripts', name), join(root, 'scripts', name)) + } + const build = join(root, 'build/chromium') + await mkdir(build, { recursive: true }) + await writeFile( + join(build, 'manifest.json'), + JSON.stringify({ + name: 'Smoke fixture', + version: '1.0', + background: { service_worker: 'background.js' }, + action: { default_popup: 'popup.html' }, + }), + ) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), '') + const executable = join(root, 'fake browser') + await cp(join(ROOT, 'tests/fixtures/smoke/fake-browser.fixture'), executable) + await chmod(executable, 0o700) + return { root, executable, cleanups } +} + +async function isLive(pid) { + try { + const contents = await readFile(`/proc/${pid}/stat`, 'utf8') + const state = contents.slice(contents.lastIndexOf(')') + 2).split(' ')[0] + if (!/^[A-Z]$/.test(state)) throw new Error('Invalid process state') + return !['Z', 'X'].includes(state) + } catch (error) { + if (error.code === 'ENOENT' || error.code === 'ESRCH') return false + throw error + } +} + +async function stopRunner(child, exited) { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM') + try { + await bounded(exited, 15000, 'Fixture runner exit') + } catch { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + await bounded(exited, 2000, 'Reap fixture runner') + } +} + +async function launch(t, fixture, tag) { + const parent = join(fixture.root, tag) + const child = spawn( + 'sh', + [ + join(fixture.root, 'scripts/run-smoke.sh'), + '--browser', + 'chromium', + '--chromium-path', + fixture.executable, + '--artifacts-dir', + parent, + ], + { + cwd: tmpdir(), + env: { ...process.env, TMPDIR: fixture.root }, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let output = '' + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.on('data', (chunk) => { + output += chunk + }) + const exited = new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (code, signal) => resolve({ code, signal })) + }) + exited.catch(() => {}) + let browser + fixture.cleanups.push(async () => { + await stopRunner(child, exited) + if (!browser) throw new Error('Cannot verify fixture browser cleanup') + // Never signal a remembered target PID as a process group. The fixture's + // watchdog is only a final safety net after a failed test, not a pass condition. + await waitFor(async () => !(await isLive(browser.pid)), { timeoutMs: 35000 }) + }) + const artifacts = await waitFor( + async () => { + if (child.exitCode !== null) throw new Error(`Runner exited early: ${output}`) + try { + const entries = await readdir(parent) + if (!entries.length) return false + const directory = join(parent, entries[0]) + const log = await readFile(join(directory, 'chromium/chromium.log'), 'utf8') + if (!log.endsWith('\n')) return false + browser = JSON.parse(log.trim()) + return directory + } catch (error) { + if (error.code === 'ENOENT') return false + throw error + } + }, + { timeoutMs: 10000, label: 'Fixture browser startup' }, + ) + return { child, browser, artifacts, exited, output: () => output } +} + +for (const [signal, expectedCode] of [ + ['SIGINT', 130], + ['SIGTERM', 143], +]) { + test( + `runner forwards ${signal} to the active preflight artifact walk`, + { skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + const ready = join(setup.root, 'hash-ready') + const stopped = join(setup.root, 'hash-stopped') + await writeFile( + join(setup.root, 'scripts/smoke/artifacts.mjs'), + ` + import { writeFileSync } from 'node:fs' + export async function hashArtifact(directory, signal) { + return new Promise((resolve, reject) => { + signal?.addEventListener('abort', () => { + writeFileSync(${JSON.stringify(stopped)}, 'cancelled') + reject(signal.reason) + }, { once: true }) + writeFileSync(${JSON.stringify(ready)}, 'ready') + }) + } + export async function snapshotArtifact() { throw new Error('Must not start a browser') } + `, + ) + const child = spawn( + process.execPath, + [ + join(setup.root, 'scripts/xvfb-smoke.mjs'), + '--browser', + 'chromium', + '--chromium-path', + setup.executable, + '--artifacts-dir', + join(setup.root, 'artifacts'), + ], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ) + let output = '' + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.on('data', (chunk) => { + output += chunk + }) + const exited = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', (code) => resolve(code)) + }) + exited.catch(() => {}) + setup.cleanups.push(() => stopRunner(child, exited)) + await waitFor( + async () => { + try { + return await readFile(ready, 'utf8') + } catch (error) { + if (error.code === 'ENOENT') return false + throw error + } + }, + { timeoutMs: 10000, label: 'Preflight hash entry' }, + ) + child.kill(signal) + assert.equal(await bounded(exited, 10000, 'Cancelled preflight exit'), expectedCode, output) + assert.equal(await readFile(stopped, 'utf8'), 'cancelled') + const [runDirectory] = await readdir(join(setup.root, 'artifacts')) + const report = JSON.parse( + await readFile(join(setup.root, 'artifacts', runDirectory, 'report.json'), 'utf8'), + ) + assert.equal(report.failedStage, 'preflight') + assert.equal(report.exitCode, expectedCode) + assert.deepEqual(report.browsers, []) + assert.deepEqual(report.cleanup, []) + }, + ) +} + +test( + 'shell runner preserves SIGINT/SIGTERM, cleans owned profiles and isolates simultaneous runs', + { timeout: 60000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + const first = await launch(t, setup, 'first') + const second = await launch(t, setup, 'second') + assert.notEqual(first.browser.profile, second.browser.profile) + // Foreground Ctrl+C reaches the runner group; detached browser groups are owned separately. + process.kill(-first.child.pid, 'SIGINT') + await delay(50) + first.child.kill('SIGINT') + assert.deepEqual(await bounded(first.exited, 15000, 'SIGINT exit'), { code: 130, signal: null }) + assert.equal(await isLive(second.browser.pid), true) + assert.doesNotThrow(() => process.kill(second.child.pid, 0)) + assert.equal((await stat(second.browser.profile)).isDirectory(), true) + second.child.kill('SIGTERM') + assert.deepEqual(await bounded(second.exited, 15000, 'SIGTERM exit'), { + code: 143, + signal: null, + }) + for (const [run, code] of [ + [first, 130], + [second, 143], + ]) { + await assert.rejects(stat(run.browser.profile), { code: 'ENOENT' }) + assert.equal(await isLive(run.browser.pid), false) + const report = JSON.parse(await readFile(join(run.artifacts, 'report.json'), 'utf8')) + assert.equal(report.exitCode, code, run.output()) + assert.equal(report.result, 'FAIL') + assert.deepEqual(report.cleanup, []) + assert.equal(report.root, await realpath(setup.root)) + } + }, +) + +test( + 'all-browser runner reaps the previous browser and removes its profile before starting the next', + { timeout: 75000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + await cp(join(setup.root, 'build/chromium'), join(setup.root, 'build/firefox'), { + recursive: true, + }) + await writeFile(join(setup.root, 'build/firefox.zip'), 'Fixture archive') + const marker = join(setup.root, 'previous-browser.json') + setup.cleanups.push(async () => { + const previous = JSON.parse(await readFile(marker, 'utf8')) + await waitFor(async () => !(await isLive(previous.targetPid)), { timeoutMs: 35000 }) + }) + await writeFile( + join(setup.root, 'scripts/smoke/chromium.mjs'), + ` + import process from 'node:process' + import { writeFile } from 'node:fs/promises' + import { waitFor } from './lifecycle.mjs' + export async function startChromium({ lifecycle, profileDir }) { + const managed = lifecycle.spawn(process.execPath, ['-e', + 'setTimeout(() => process.exit(99), 30000); console.log(process.pid)']) + const targetPid = await waitFor(() => { + managed.assertRunning() + const output = managed.output() + return /^\\d+\\n$/.test(output) && Number(output.trim()) + }, { timeoutMs: 3000 }) + await writeFile(${JSON.stringify( + marker, + )}, JSON.stringify({ targetPid, supervisorPid: managed.child.pid, profileDir })) + return { metadata: {}, close: async () => {}, capture: async () => {} } + } + `, + ) + await writeFile( + join(setup.root, 'scripts/smoke/firefox.mjs'), + ` + import process from 'node:process' + import assert from 'node:assert/strict' + import { readFile, stat } from 'node:fs/promises' + const isLive = ${isLive.toString()} + export async function startFirefox() { + const previous = JSON.parse(await readFile(${JSON.stringify(marker)}, 'utf8')) + assert.throws(() => process.kill(previous.supervisorPid, 0), { code: 'ESRCH' }) + assert.equal(await isLive(previous.targetPid), false) + await assert.rejects(stat(previous.profileDir), { code: 'ENOENT' }) + return { metadata: {}, close: async () => {}, capture: async () => {} } + } + `, + ) + await writeFile( + join(setup.root, 'scripts/smoke/scenarios.mjs'), + ` + export async function runScenarios() { return { checks: ['Fixture check'], requests: [] } } + `, + ) + const child = spawn( + process.execPath, + [ + join(setup.root, 'scripts/xvfb-smoke.mjs'), + '--browser', + 'all', + '--chromium-path', + setup.executable, + '--firefox-path', + process.execPath, + '--geckodriver-path', + process.execPath, + '--artifacts-dir', + join(setup.root, 'artifacts'), + ], + { + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let output = '' + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.on('data', (chunk) => { + output += chunk + }) + const exit = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', resolve) + }) + setup.cleanups.push(() => stopRunner(child, exit)) + assert.equal(await bounded(exit, 10000, 'Sequential browsers'), 0, output) + }, +) + +test( + 'runner reports success, failures, EPIPE and late signals truthfully', + { timeout: 60000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + // Substitute protocol boundaries only; exercise the real CLI, report and lifecycle. + await writeFile( + join(setup.root, 'scripts/smoke/chromium.mjs'), + ` + import process from 'node:process' + export async function startChromium({ lifecycle }) { + lifecycle.defer('fixture cleanup', async () => { + if (process.env.SMOKE_FIXTURE_MODE === 'cleanup') throw new Error('Fixture cleanup failure') + }) + return { metadata: {}, close: async () => {}, capture: async () => {} } + } + `, + ) + await writeFile( + join(setup.root, 'scripts/smoke/scenarios.mjs'), + ` + import process from 'node:process' + import { setTimeout } from 'node:timers/promises' + export async function runScenarios() { + await setTimeout(50) + if (process.env.SMOKE_FIXTURE_MODE === 'scenario-cleanup') { + throw Object.assign(new Error('Fixture primary failure'), { + cleanupErrors: [new Error('Fixture secondary cleanup failure')] + }) + } + if (process.env.SMOKE_FIXTURE_MODE === 'scenario') throw new Error('Fixture scenario failure') + return { checks: ['Fixture check'], requests: [] } + } + `, + ) + const interruptImport = join(setup.root, 'interrupt-report.mjs') + await writeFile( + interruptImport, + ` + import fs from 'node:fs/promises' + import process from 'node:process' + import { syncBuiltinESMExports } from 'node:module' + import { setTimeout } from 'node:timers/promises' + const original = fs.writeFile + let sent = false + fs.writeFile = async (...args) => { + const result = await original(...args) + if (!sent && String(args[0]).endsWith('report.json')) { + sent = true + process.kill(process.pid, 'SIGTERM') + await setTimeout(20) + } + return result + } + syncBuiltinESMExports() + `, + ) + const stdoutInterruptImport = join(setup.root, 'interrupt-stdout.mjs') + await writeFile( + stdoutInterruptImport, + ` + import process from 'node:process' + import { setInterval, clearInterval } from 'node:timers' + const original = process.stdout.write + let sent = false + process.stdout.write = function (chunk, callback) { + return original.call(this, chunk, (error) => { + if (error || sent) return callback(error) + sent = true + const signal = process.env.SMOKE_FIXTURE_MODE === 'stdout-int' ? 'SIGINT' : 'SIGTERM' + // Release the write only after the runner has observed the signal. + const keepAlive = setInterval(() => {}, 1000) + process.once(signal, () => { + clearInterval(keepAlive) + callback() + }) + process.kill(process.pid, signal) + }) + } + `, + ) + for (const [mode, expected] of [ + ['success', 0], + ['scenario', 1], + ['scenario-cleanup', 1], + ['cleanup', 1], + ['epipe', 1], + ['late-signal', 143], + ['stdout-int', 130], + ['stdout-term', 143], + ]) { + const parent = join(setup.root, mode) + const preload = + mode === 'late-signal' + ? interruptImport + : mode.startsWith('stdout-') + ? stdoutInterruptImport + : undefined + const child = spawn( + preload ? process.execPath : 'sh', + [ + ...(preload + ? ['--import', preload, join(setup.root, 'scripts/xvfb-smoke.mjs')] + : [join(setup.root, 'scripts/run-smoke.sh')]), + '--browser', + 'chromium', + '--chromium-path', + setup.executable, + '--artifacts-dir', + parent, + ], + { + env: { ...process.env, SMOKE_FIXTURE_MODE: mode }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let output = '' + if (mode === 'epipe') child.stdout.destroy() + else + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.resume() + const exit = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', (code) => resolve(code)) + }) + setup.cleanups.push(() => stopRunner(child, exit)) + assert.equal(await bounded(exit, 5000, `${mode} exit`), expected, mode) + const directories = await readdir(parent) + const report = JSON.parse(await readFile(join(parent, directories[0], 'report.json'), 'utf8')) + assert.equal(report.exitCode, expected, mode) + assert.equal(report.result, expected === 0 ? 'PASS' : 'FAIL', mode) + if (mode.startsWith('stdout-')) { + assert.doesNotMatch(output, /PASS|exit 0/, mode) + assert.match(output, /artifacts:/, mode) + } + await assert.rejects(stat(report.browsers[0].profileDir), { code: 'ENOENT' }) + if (mode === 'epipe') assert.equal(report.failedStage, 'reporting') + if (mode === 'cleanup') assert.match(report.cleanup[0], /Fixture cleanup failure/) + if (mode === 'scenario-cleanup') { + assert.match(report.error, /Fixture primary failure/) + assert.match(report.cleanup[0], /Fixture secondary cleanup failure/) + } + } + }, +) + +test( + 'Firefox startup failures report preflight identity separately from the snapshot', + { timeout: 10000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + await cp(join(setup.root, 'build/chromium'), join(setup.root, 'build/firefox'), { + recursive: true, + }) + await writeFile( + join(setup.root, 'scripts/smoke/firefox.mjs'), + ` + import { join } from 'node:path' + import { snapshotArtifact } from './artifacts.mjs' + export async function startFirefox(options) { + await snapshotArtifact(options.extensionDir, join(options.artifactsDir, 'extension'), + options.artifactSha256, options.signal) + throw new Error('Fixture Firefox startup failure') + } + `, + ) + const parent = join(setup.root, 'startup-failure') + const child = spawn( + process.execPath, + [ + join(setup.root, 'scripts/xvfb-smoke.mjs'), + '--browser', + 'firefox', + '--firefox-path', + setup.executable, + '--geckodriver-path', + process.execPath, + '--artifacts-dir', + parent, + ], + { stdio: 'ignore' }, + ) + const exit = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', resolve) + }) + setup.cleanups.push(() => stopRunner(child, exit)) + assert.equal(await bounded(exit, 5000, 'Firefox startup failure exit'), 1) + const directories = await readdir(parent) + const report = JSON.parse(await readFile(join(parent, directories[0], 'report.json'), 'utf8')) + assert.equal(report.failedStage, 'firefox:startup') + assert.match(report.error, /Fixture Firefox startup failure/) + assert.equal(report.browsers[0].preflightManifestVersion, '1.0') + assert.equal(Object.hasOwn(report.browsers[0], 'manifestVersion'), false) + assert.equal( + report.browsers[0].snapshotDir, + join(await realpath(parent), directories[0], 'firefox/extension'), + ) + assert.equal((await stat(join(report.browsers[0].snapshotDir, 'manifest.json'))).isFile(), true) + await assert.rejects(stat(report.browsers[0].profileDir), { code: 'ENOENT' }) + assert.deepEqual(report.cleanup, []) + }, +) diff --git a/tests/unit/scripts/smoke-lifecycle.test.mjs b/tests/unit/scripts/smoke-lifecycle.test.mjs new file mode 100644 index 000000000..f9ab3442f --- /dev/null +++ b/tests/unit/scripts/smoke-lifecycle.test.mjs @@ -0,0 +1,645 @@ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { performance } from 'node:perf_hooks' +import process from 'node:process' +import test from 'node:test' +import timers from 'node:timers' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' +import { bounded, createLifecycle, waitFor } from '../../../scripts/smoke/lifecycle.mjs' + +const fixture = fileURLToPath(new URL('../../fixtures/smoke/process.fixture', import.meta.url)) +const options = { timeout: 10000, skip: process.platform === 'win32' } + +function owned(t, options = {}) { + const lifecycle = createLifecycle({ cleanupTimeoutMs: 1500, ...options }) + t.after(() => lifecycle.force()) + return lifecycle +} + +function stalledCleanupClock(t) { + const clock = { elapsed: 0, delays: [] } + const setTimeout = timers.setTimeout + const time = t.mock.method(performance, 'now', () => clock.elapsed) + const timeout = t.mock.method(timers, 'setTimeout', (callback, ms, ...args) => { + clock.delays.push(ms) + // Only the two stalled-callback tests use this clock. Advance by the + // requested budget when its timer fires, independently of worker pauses. + return setTimeout(() => { + clock.elapsed += ms + callback(...args) + }, 0) + }) + syncBuiltinESMExports() + t.after(() => { + time.mock.restore() + timeout.mock.restore() + syncBuiltinESMExports() + }) + return clock +} + +async function ready(managed) { + await waitFor( + () => { + managed.assertRunning() + return managed.output().includes('READY') && managed.output().includes('stderr ready') + }, + { timeoutMs: 3000, label: 'Fixture startup' }, + ) +} + +async function isLive(pid) { + try { + process.kill(pid, 0) + if (process.platform === 'linux') { + const stat = await readFile(`/proc/${pid}/stat`, 'utf8') + return !['Z', 'X'].includes(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0]) + } + return true + } catch (error) { + if (error.code === 'ESRCH' || error.code === 'ENOENT') return false + throw error + } +} + +test( + 'bounded settles success, failure, timeout, and cancellation without late rejections', + options, + async () => { + assert.equal(await bounded(Promise.resolve(42), 100, 'Success'), 42) + const failure = new Error('Original failure') + await assert.rejects(bounded(Promise.reject(failure), 100, 'Failure'), (e) => e === failure) + await assert.rejects(bounded(new Promise(() => {}), 20, 'Stalled'), /Stalled timed out/) + const controller = new AbortController() + const pending = bounded( + delay(50).then(() => Promise.reject(failure)), + 500, + 'Abort', + controller.signal, + ) + controller.abort(failure) + await assert.rejects(pending, (e) => e === failure) + await assert.rejects( + bounded( + Promise.reject(new Error('Late rejection')), + 500, + 'Already aborted', + controller.signal, + ), + (e) => e === failure, + ) + await delay(75) + for (const ms of [0, -1, Infinity, NaN, 2147483648]) { + await assert.rejects(bounded(Promise.resolve(), ms, 'Invalid'), RangeError) + } + }, +) + +test('waitFor retries only falsy readiness and bounds a hung probe', options, async () => { + let attempts = 0 + assert.equal( + await waitFor(() => (++attempts === 3 ? 'ready' : false), { timeoutMs: 1000 }), + 'ready', + ) + assert.equal(attempts, 3) + attempts = 0 + await assert.rejects( + waitFor(() => { + attempts++ + throw new Error('Fatal probe') + }), + /Fatal probe/, + ) + assert.equal(attempts, 1) + await assert.rejects( + waitFor(() => new Promise(() => {}), { timeoutMs: 20, label: 'Hung probe' }), + /Hung probe timed out/, + ) + await assert.rejects( + waitFor(() => false, { timeoutMs: 20 }), + /timed out/, + ) + const controller = new AbortController() + controller.abort() + await assert.rejects( + waitFor(() => assert.fail('Aborted probe ran'), { signal: controller.signal }), + { name: 'AbortError' }, + ) +}) + +test( + 'cleanup is LIFO and idempotent, and continues after failed or hung callbacks', + options, + async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 180 }) + const seen = [] + lifecycle.defer('First', (...args) => { + assert.deepEqual(args, []) + seen.push('first') + }) + lifecycle.defer('Failed', () => { + seen.push('failed') + throw new Error('Cleanup failed') + }) + lifecycle.defer('Hung', () => { + seen.push('hung') + return new Promise(() => {}) + }) + const started = performance.now() + const cleanup = lifecycle.cleanup() + assert.equal(lifecycle.cleanup(), cleanup) + assert.throws(() => lifecycle.defer('Late', () => {}), /closing/) + assert.throws(() => lifecycle.spawn(process.execPath, [fixture, 'hang']), /closing/) + const errors = await cleanup + assert.equal(errors.length, 2) + assert.match(errors[0].message, /Hung.*timed out/) + assert.match(errors[1].message, /Failed.*Cleanup failed/) + assert.deepEqual(seen, ['hung', 'failed', 'first']) + assert.ok(performance.now() - started < 1000) + assert.equal(await lifecycle.cleanup(), errors) + }, +) + +test('spawn failure is observable and does not prevent independent cleanup', options, async (t) => { + const lifecycle = owned(t) + let cleaned = false + lifecycle.defer('Independent', () => { + cleaned = true + }) + const managed = lifecycle.spawn('/nonexistent/chatgptbox-smoke-executable', []) + await assert.rejects(managed.exited, /ENOENT/) + assert.throws(() => managed.assertRunning(), /ENOENT/) + const errors = await lifecycle.cleanup() + assert.equal(cleaned, true) + assert.ok(errors.some((error) => /ENOENT/.test(error.message))) +}) + +test('multiple stalled resources share one aggregate cleanup budget', options, async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 200 }) + const clock = stalledCleanupClock(t) + const seen = [] + for (let index = 0; index < 4; index++) { + lifecycle.defer(`Stalled ${index}`, () => { + seen.push(index) + return new Promise(() => {}) + }) + } + assert.equal((await lifecycle.cleanup()).length, 4) + assert.deepEqual(seen, [3, 2, 1, 0]) + assert.equal(clock.elapsed, 200, 'Cleanup multiplied the timeout by resource count') +}) + +test( + 'an exhausted aggregate deadline does not add a timer for every remaining resource', + options, + async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 20 }) + const clock = stalledCleanupClock(t) + let invoked = 0 + for (let index = 0; index < 250; index++) { + lifecycle.defer(`Stalled ${index}`, () => { + invoked++ + return new Promise(() => {}) + }) + } + const errors = await lifecycle.cleanup() + assert.equal(errors.length, 250) + assert.equal(invoked, 250) + assert.ok(errors.some((error) => /Overall cleanup deadline exceeded/.test(error.message))) + assert.equal(clock.elapsed, 20, 'Cleanup added per-resource delays after its deadline') + assert.ok(clock.delays.length <= 20, 'Expired resources must not allocate more timers') + }, +) + +test( + 'a failed callback cannot strand processes or remove their profile before reaping', + options, + async (t) => { + const lifecycle = owned(t) + let managed + let profileCleaned = false + lifecycle.defer('Profile', async () => { + assert.equal(await isLive(managed.child.pid), false) + assert.notEqual(managed.child.signalCode, null) + profileCleaned = true + }) + managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + lifecycle.defer('Broken protocol close', () => { + throw new Error('Protocol unavailable') + }) + const errors = await lifecycle.cleanup() + assert.equal(errors.length, 1) + assert.match(errors[0].message, /Protocol unavailable/) + assert.equal(profileCleaned, true) + }, +) + +test( + 'early exit preserves status and both output streams in diagnostics and log', + options, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-lifecycle-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const lifecycle = owned(t) + const logPath = join(directory, 'process.log') + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit'], { logPath }) + assert.deepEqual(await bounded(managed.exited, 3000, 'Early exit'), { code: 7, signal: null }) + assert.throws(() => managed.assertRunning(), /exited unexpectedly/) + assert.deepEqual(await lifecycle.cleanup(), []) + for (const output of [managed.output(), await readFile(logPath, 'utf8')]) { + assert.match(output, /stdout before exit/) + assert.match(output, /stderr before exit/) + } + }, +) + +test('cleanup terminates and reaps a hanging owned child', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.deepEqual(await managed.exited, { code: null, signal: 'SIGTERM' }) + assert.equal(await isLive(managed.child.pid), false) + await lifecycle.assertProcessesStopped() +}) + +test('parent cleanup never signals a remembered numeric process group', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + const originalKill = process.kill + const destructiveSignals = [] + try { + process.kill = (pid, signal) => { + if (signal !== 0) destructiveSignals.push({ pid, signal }) + return originalKill.call(process, pid, signal) + } + assert.deepEqual(await lifecycle.cleanup(), []) + assert.deepEqual(destructiveSignals, []) + } finally { + process.kill = originalKill + } +}) + +test('force before supervisor readiness prevents target startup', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit']) + assert.deepEqual(await lifecycle.force(), []) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.equal(managed.output(), '') + assert.equal(await isLive(managed.child.pid), false) +}) + +test('cleanup waiting on another resource cannot start a late-ready target', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit']) + lifecycle.defer('Protocol close', () => delay(300)) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.equal(managed.output(), '') +}) + +test('the target does not inherit the supervisor control channel', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, ['-e', 'console.log(typeof process.send)']) + assert.deepEqual(await bounded(managed.exited, 3000), { code: 0, signal: null }) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.equal(managed.output().trim(), 'undefined') +}) + +test( + 'control EOF invokes supervisor cleanup without waiting for ChildProcess close', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + managed.child.disconnect() + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /disconnected|ownership was lost/.test(error.message))) + await waitFor(async () => !(await isLive(managed.child.pid)), { timeoutMs: 2000 }) + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) + }, +) + +test('invalid supervisor results cannot resolve the target successfully', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + managed.child.emit('message', { version: 1, type: 'target-exit' }) + await assert.rejects(managed.exited, /Invalid supervisor message/) + assert.ok((await lifecycle.cleanup()).length > 0) + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) +}) + +test( + 'supervisor exit before final IPC delivery preserves the target result', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + const originalEmit = managed.child.emit + const pending = [] + let exited = false + let disconnected = false + let sawTerminating = false + managed.child.emit = function (event, ...args) { + if (event === 'message' && args[0]?.type === 'terminating') sawTerminating = true + if ( + !exited && + event === 'message' && + ['target-exit', 'terminating'].includes(args[0]?.type) + ) { + pending.push(args) + return true + } + if (event === 'disconnect' && !exited) { + disconnected = true + return true + } + const result = originalEmit.call(this, event, ...args) + if (event === 'exit') { + exited = true + for (const message of pending) originalEmit.call(this, 'message', ...message) + if (disconnected) originalEmit.call(this, 'disconnect') + } + return result + } + try { + assert.deepEqual(await lifecycle.cleanup(), []) + assert.equal(sawTerminating, true) + assert.deepEqual(await managed.exited, { code: null, signal: 'SIGTERM' }) + } finally { + managed.child.emit = originalEmit + } + }, +) + +test( + 'unexpected supervisor exit remains uncertain even after its group disappears', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit']) + assert.deepEqual(await bounded(managed.exited, 3000), { code: 7, signal: null }) + // The fixture has already ended; only our directly owned supervisor remains. + const exited = new Promise((resolve) => managed.child.once('exit', resolve)) + managed.child.kill('SIGKILL') + await bounded(exited, 2000) + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /Supervisor/.test(error.message))) + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) + }, +) + +test( + 'a timed-out process cleanup preserves its guarded profile until the process stops', + options, + async (t) => { + const profileDir = await mkdtemp(join(tmpdir(), 'smoke-lifecycle-preserved-profile-')) + const lifecycle = owned(t, { cleanupTimeoutMs: 1000 }) + lifecycle.defer('Profile', async () => { + await lifecycle.assertProcessesStopped() + await rm(profileDir, { recursive: true }) + }) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + const originalSend = managed.child.send + const originalDisconnect = managed.child.disconnect + let suppressedCommands = 0 + try { + await ready(managed) + managed.child.send = () => { + suppressedCommands++ + return true + } + managed.child.disconnect = () => {} + const errors = await lifecycle.cleanup() + assert.ok(suppressedCommands > 0) + assert.ok(errors.length > 0) + await assert.rejects( + bounded(lifecycle.assertProcessesStopped(), 2000, 'Check surviving process'), + /ownership was lost/, + ) + assert.equal(await isLive(managed.child.pid), true) + assert.equal((await stat(profileDir)).isDirectory(), true) + } finally { + managed.child.send = originalSend + managed.child.disconnect = originalDisconnect + if (managed.child.connected) managed.child.disconnect() + await lifecycle.force() + await waitFor(async () => !(await isLive(managed.child.pid)), { timeoutMs: 2000 }) + await rm(profileDir, { recursive: true, force: true }) + } + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) + }, +) + +test( + 'the stopped-process guard preserves ownership on missing or unknown process state', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + const originalKill = process.kill + let inspectionError + const checks = [] + try { + await ready(managed) + process.kill = (pid, signal) => { + if (pid === -managed.child.pid) { + checks.push(signal) + if (inspectionError) throw inspectionError + } + return originalKill.call(process, pid, signal) + } + inspectionError = Object.assign(new Error('Process not visible'), { code: 'ESRCH' }) + await lifecycle.assertProcessesStopped() + inspectionError = Object.assign(new Error('Process inspection denied'), { code: 'EPERM' }) + await assert.rejects(lifecycle.assertProcessesStopped(), (error) => error === inspectionError) + inspectionError = undefined + await assert.rejects(lifecycle.assertProcessesStopped(), /still has live members/) + assert.deepEqual(checks, [0, 0, 0]) + } finally { + process.kill = originalKill + await lifecycle.force() + await assert.rejects( + bounded(managed.exited, 2000, 'Reap inspection fixture'), + /status unavailable/, + ) + } + await lifecycle.assertProcessesStopped() + }, +) + +test('cleanup escalates an owned group ignoring TERM to KILL', options, async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 800 }) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.match(managed.output(), /IGNORED TERM/) + assert.equal(await isLive(managed.child.pid), false) +}) + +test('descendants remain owned after their group leader exits', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'descendant']) + assert.deepEqual(await bounded(managed.exited, 3000, 'Leader exit'), { code: 0, signal: null }) + const descendant = Number(managed.output().match(/DESCENDANT (\d+)/)?.[1]) + assert.ok(Number.isInteger(descendant)) + assert.equal(await isLive(descendant), true) + assert.deepEqual(await lifecycle.cleanup(), []) + await waitFor(async () => !(await isLive(descendant)), { timeoutMs: 1000 }) +}) + +test( + 'independent lifecycles and an unrelated sentinel survive another lifecycle cleanup', + options, + async (t) => { + const sentinel = spawn(process.execPath, [fixture, 'hang'], { stdio: 'ignore' }) + const sentinelExit = new Promise((resolve) => sentinel.once('exit', resolve)) + t.after(async () => { + sentinel.kill('SIGKILL') + await bounded(sentinelExit, 2000, 'Sentinel exit') + }) + const first = owned(t) + const second = owned(t) + const one = first.spawn(process.execPath, [fixture, 'hang']) + const two = second.spawn(process.execPath, [fixture, 'hang']) + await Promise.all([ready(one), ready(two)]) + assert.deepEqual(await first.cleanup(), []) + two.assertRunning() + assert.equal(await isLive(two.child.pid), true) + assert.equal(await isLive(sentinel.pid), true) + assert.deepEqual(await second.cleanup(), []) + assert.equal(await isLive(sentinel.pid), true) + }, +) + +test( + 'abort starts cleanup and force acts as the second signal without global handlers', + options, + async (t) => { + const termListeners = process.listenerCount('SIGTERM') + const intListeners = process.listenerCount('SIGINT') + const controller = new AbortController() + const lifecycle = owned(t, { signal: controller.signal, cleanupTimeoutMs: 5000 }) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + controller.abort() + assert.throws(() => lifecycle.spawn(process.execPath, [fixture, 'hang']), { + name: 'AbortError', + }) + await waitFor(() => managed.output().includes('IGNORED TERM'), { timeoutMs: 1000 }) + const started = performance.now() + assert.equal(lifecycle.force(), lifecycle.cleanup()) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.ok(performance.now() - started < 750) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.equal(process.listenerCount('SIGTERM'), termListeners) + assert.equal(process.listenerCount('SIGINT'), intListeners) + }, +) + +test( + 'pre-aborted lifecycle and force before spawn forbid acquiring resources', + options, + async (t) => { + const controller = new AbortController() + controller.abort() + const aborted = owned(t, { signal: controller.signal }) + assert.throws(() => aborted.spawn(process.execPath, [fixture, 'hang']), { name: 'AbortError' }) + assert.deepEqual(await aborted.cleanup(), []) + const forced = owned(t) + assert.deepEqual(await forced.force(), []) + assert.throws(() => forced.spawn(process.execPath, [fixture, 'hang']), /closing/) + }, +) + +test('log errors are handled, visible, and do not strand a running process', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang'], { + logPath: '/nonexistent/chatgptbox-smoke-directory/output.log', + }) + await waitFor(() => managed.output().includes('READY'), { timeoutMs: 3000 }) + await waitFor( + () => { + try { + managed.assertRunning() + return false + } catch (error) { + assert.match(error.message, /Log.*ENOENT/) + return true + } + }, + { timeoutMs: 3000, label: 'Log stream failure' }, + ) + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /Log.*ENOENT/.test(error.message))) + assert.equal(await isLive(managed.child.pid), false) +}) + +test('output stream errors are handled and retained during cleanup', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + managed.child.stdout.destroy(new Error('Read stream failure')) + await waitFor( + () => { + try { + managed.assertRunning() + return false + } catch (error) { + assert.match(error.message, /Read stream failure/) + return true + } + }, + { timeoutMs: 3000, label: 'Output stream failure' }, + ) + const errors = await lifecycle.cleanup() + assert.equal(errors.length, 1) + assert.match(errors[0].message, /Read stream failure/) + assert.equal(await isLive(managed.child.pid), false) +}) + +test( + 'a real log write error releases pipe backpressure and still allows cleanup', + { + ...options, + skip: process.platform !== 'linux', + }, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'flood'], { logPath: '/dev/full' }) + await waitFor(() => managed.output().includes('stderr ready'), { timeoutMs: 3000 }) + assert.throws(() => managed.assertRunning(), /ENOSPC/) + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /ENOSPC/.test(error.message))) + assert.equal(await isLive(managed.child.pid), false) + }, +) + +test( + 'large output is drained to disk while the diagnostic tail stays bounded', + options, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-lifecycle-flood-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const lifecycle = owned(t) + const logPath = join(directory, 'flood.log') + const managed = lifecycle.spawn(process.execPath, [fixture, 'flood'], { logPath }) + await ready(managed) + assert.ok(managed.output().length <= 1024 * 1024) + assert.deepEqual(await lifecycle.cleanup(), []) + const log = await readFile(logPath, 'utf8') + assert.equal(log.match(/O/g).length, 2 * 1024 * 1024) + assert.equal(log.match(/E/g).length, 2 * 1024 * 1024 + 1) + }, +) diff --git a/tests/unit/scripts/smoke-mock-server.test.mjs b/tests/unit/scripts/smoke-mock-server.test.mjs new file mode 100644 index 000000000..1201dbe10 --- /dev/null +++ b/tests/unit/scripts/smoke-mock-server.test.mjs @@ -0,0 +1,123 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import { TextDecoder } from 'node:util' +import { startMockServer } from '../../../scripts/smoke/mock-server.mjs' + +async function setup(t) { + const cleanups = [] + const mock = await startMockServer({ + lifecycle: { defer: (label, cleanup) => cleanups.push({ label, cleanup }) }, + }) + assert.equal(cleanups.length, 1) + t.after(() => cleanups[0].cleanup()) + return mock +} + +function post(mock, path, body = { model: 'smoke-model', stream: true, messages: [] }) { + return globalThis.fetch(mock.baseUrl + path, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: 'Bearer smoke-local-only' }, + body: JSON.stringify(body), + signal: globalThis.AbortSignal.timeout(5000), + }) +} + +test( + 'mock gates the remainder until a client has observed the complete first event', + { timeout: 10000 }, + async (t) => { + const mock = await setup(t) + assert.match(mock.baseUrl, /^http:\/\/127\.0\.0\.1:\d+$/) + const response = await post(mock, '/success/v1/chat/completions') + assert.equal(response.status, 200) + const reader = response.body.getReader() + const decoder = new TextDecoder() + let text = '' + while (!text.includes('\r\n\r\n')) { + const part = await reader.read() + assert.equal(part.done, false) + text += decoder.decode(part.value, { stream: true }) + } + assert.match(text, /Hello /) + assert.doesNotMatch(text, /δΈ–η•Œ|DONE/) + const waiting = reader.read() + assert.equal(await Promise.race([waiting, delay(30, 'still gated')]), 'still gated') + mock.release() + let part = await waiting + while (!part.done) { + text += decoder.decode(part.value, { stream: true }) + part = await reader.read() + } + text += decoder.decode() + const events = text + .split('\r\n\r\n') + .filter(Boolean) + .map((line) => line.slice(6)) + assert.equal(events.length, 3) + assert.equal(JSON.parse(events[0]).choices[0].delta.content, 'Hello ') + assert.equal(JSON.parse(events[1]).choices[0].delta.content, 'δΈ–η•ŒπŸ™‚') + assert.equal(events[2], '[DONE]') + assert.equal(mock.requests.length, 1) + assert.equal(mock.requests[0].status, 200) + await mock.close() + await mock.close() + }, +) + +test('mock reports HTTP 503 and records the Chat Completions body', async (t) => { + const mock = await setup(t) + const response = await post(mock, '/error/v1/chat/completions') + assert.equal(response.status, 503) + assert.deepEqual(await response.json(), { + error: { message: 'Smoke upstream unavailable', code: 'smoke_503' }, + }) + assert.deepEqual(mock.requests, [ + { + method: 'POST', + path: '/error/v1/chat/completions', + body: { model: 'smoke-model', stream: true, messages: [] }, + status: 503, + }, + ]) +}) + +test('mock rejects a Responses endpoint and invalid Chat Completions payload', async (t) => { + const mock = await setup(t) + for (const [path, body, status] of [ + ['/v1/responses', {}, 404], + ['/success/v1/chat/completions', { input: 'wrong API' }, 400], + ]) { + const response = await post(mock, path, body) + assert.equal(response.status, status) + await response.text() + } +}) + +test( + 'cleanup closes an unreleased response and its listening socket', + { timeout: 10000 }, + async (t) => { + const mock = await setup(t) + const response = await post(mock, '/success/v1/chat/completions') + const consumed = response.text().then( + () => 'ended', + () => 'closed', + ) + await mock.close() + assert.equal(await consumed, 'closed') + await assert.rejects(post(mock, '/error/v1/chat/completions')) + }, +) + +test('an already aborted startup allocates no server', async () => { + let registrations = 0 + await assert.rejects( + startMockServer({ + lifecycle: { defer: () => registrations++ }, + signal: globalThis.AbortSignal.abort(new Error('cancelled')), + }), + /cancelled/, + ) + assert.equal(registrations, 0) +}) diff --git a/tests/unit/scripts/smoke-runner.test.mjs b/tests/unit/scripts/smoke-runner.test.mjs new file mode 100644 index 000000000..b4dcac48f --- /dev/null +++ b/tests/unit/scripts/smoke-runner.test.mjs @@ -0,0 +1,404 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import process from 'node:process' +import { mkdtemp, mkdir, writeFile, rm, chmod, symlink, rename } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import fs from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { bounded, createLifecycle, waitFor } from '../../../scripts/smoke/lifecycle.mjs' +import { + parseArgs, + findExecutable, + hashArtifact, + preflight, + ROOT, + validateArtifactParent, +} from '../../../scripts/smoke/runner.mjs' + +test('artifact parent validation resolves aliases and missing descendants without writing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'smoke-parent-')) + t.after(() => rm(root, { recursive: true, force: true })) + const build = join(root, 'build/chromium') + await mkdir(build, { recursive: true }) + await writeFile(join(build, 'keep'), 'unchanged') + const before = await hashArtifact(build) + await symlink(build, join(root, 'alias')) + await symlink(join(root, 'build'), join(root, 'build-alias')) + for (const parent of [ + build, + join(build, 'new/deep'), + join(root, 'alias'), + join(root, 'alias/new/deep'), + join(root, 'build-alias/chromium/new'), + ]) { + await assert.rejects(validateArtifactParent(parent, ['chromium'], root), /outside.*build/) + } + assert.equal(await hashArtifact(build), before) + const realRoot = await fs.realpath(root) + assert.equal(await validateArtifactParent(root, ['chromium'], root), realRoot) + const sibling = join(root, 'build/chromium-other/new') + assert.equal( + await validateArtifactParent(sibling, ['chromium'], root), + join(realRoot, 'build/chromium-other/new'), + ) + await assert.rejects(fs.stat(sibling), { code: 'ENOENT' }) + await symlink(build, join(root, 'build/firefox')) + await assert.rejects( + validateArtifactParent(join(build, 'new'), ['firefox'], root), + /outside.*build/, + ) +}) + +test( + 'real runner rejects unsafe artifact parents before writes and accepts build ancestors', + { skip: process.platform !== 'linux' }, + async (t) => { + const root = await mkdtemp(join(tmpdir(), 'smoke-parent-runner-')) + let retainFixture = false + t.after(async () => { + if (retainFixture) { + t.diagnostic(`Retaining CLI fixture after incomplete cleanup: ${root}`) + return + } + await rm(root, { recursive: true, force: true }) + }) + await fs.cp(join(ROOT, 'scripts/smoke'), join(root, 'scripts/smoke'), { recursive: true }) + await fs.cp(join(ROOT, 'scripts/xvfb-smoke.mjs'), join(root, 'scripts/xvfb-smoke.mjs')) + await writeFile( + join(root, 'scripts/smoke/scenarios.mjs'), + 'export async function runScenarios() { return { checks: [], requests: [] } }\n', + ) + for (const browser of ['chromium', 'firefox']) { + const build = join(root, 'build', browser) + await mkdir(build, { recursive: true }) + await writeFile( + join(build, 'manifest.json'), + JSON.stringify({ name: 'Fixture', version: '1' }), + ) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'fixture') + await writeFile( + join(root, 'scripts/smoke', `${browser}.mjs`), + `export async function start${browser === 'chromium' ? 'Chromium' : 'Firefox'}() { + return { metadata: {}, close: async () => {}, capture: async () => {} } + }`, + ) + const before = await hashArtifact(build) + const alias = join(root, `${browser}-alias`) + await symlink(build, alias) + for (const parent of [build, join(build, 'new/deep'), alias, join(alias, 'new/deep'), root]) { + const result = await invoke( + process.execPath, + [ + join(root, 'scripts/xvfb-smoke.mjs'), + '--browser', + browser, + `--${browser}-path`, + process.execPath, + '--geckodriver-path', + process.execPath, + '--artifacts-dir', + parent, + ], + root, + ).catch((error) => { + if (error.cleanupErrors?.length) retainFixture = true + throw error + }) + assert.equal(result.code, parent === root ? 0 : 2, result.output) + if (parent !== root) { + assert.match(result.output, /artifacts: unavailable/) + assert.match(result.output, /Artifact parent must be outside the selected/) + } + assert.equal(await hashArtifact(build), before) + } + } + }, +) + +test('smoke CLI accepts explicit selections and rejects typos, duplicates and missing values', () => { + assert.deepEqual(parseArgs([]), { browser: 'all' }) + assert.deepEqual(parseArgs(['--browser', 'firefox', '--firefox-path', '/a path/firefox']), { + browser: 'firefox', + 'firefox-path': '/a path/firefox', + }) + for (const args of [ + ['--browser'], + ['--browser', 'chrome'], + ['--no-sandbox'], + ['x'], + ['--browser', 'all', '--browser', 'all'], + ]) { + assert.throws(() => parseArgs(args)) + } +}) + +test('explicit executable wins PATH; invalid explicit paths never fall back', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-path-test-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const executable = join(directory, 'a browser') + await writeFile(executable, '#!/bin/sh\nexit 0\n') + await chmod(executable, 0o700) + assert.equal(await findExecutable(executable, ['node'], process.env.PATH), executable) + assert.equal(await findExecutable(undefined, ['a browser'], directory), executable) + await assert.rejects( + findExecutable(join(directory, 'missing'), ['node'], process.env.PATH), + /not found/, + ) + await assert.rejects(findExecutable(directory, [], ''), /not found/) +}) + +test('build hash is stable and covers nested content and file names', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-hash-test-')) + t.after(() => rm(directory, { recursive: true, force: true })) + await mkdir(join(directory, 'nested')) + await writeFile(join(directory, 'nested/a'), 'one') + const first = await hashArtifact(directory) + assert.equal(await hashArtifact(directory), first) + await writeFile(join(directory, 'nested/a'), 'two') + const changedContent = await hashArtifact(directory) + assert.notEqual(changedContent, first) + await rename(join(directory, 'nested/a'), join(directory, 'nested/renamed')) + const changedName = await hashArtifact(directory) + assert.notEqual(changedName, changedContent) + await writeFile(join(directory, 'nested/b'), 'two') + assert.notEqual(await hashArtifact(directory), changedName) +}) + +test('preflight rejects missing artifacts', { skip: process.platform !== 'linux' }, async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-preflight-test-')) + t.after(() => rm(directory, { recursive: true, force: true })) + await assert.rejects( + preflight({ browser: 'chromium', 'chromium-path': process.execPath }, directory), + /ENOENT/, + ) +}) + +test('preflight preserves cancellation before checking executables or artifacts', async () => { + const controller = new AbortController() + const reason = new Error('Cancelled before preflight') + controller.abort(reason) + await assert.rejects( + preflight({ browser: 'chromium', 'chromium-path': '/missing' }, '/missing', controller.signal), + (error) => error === reason, + ) +}) + +test( + 'preflight passes cancellation into artifact reads and stops the walk', + { skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-preflight-abort-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const build = join(directory, 'build/chromium') + await mkdir(build, { recursive: true }) + await writeFile(join(build, 'manifest.json'), JSON.stringify({ name: 'Fixture', version: '1' })) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'current build') + const controller = new AbortController() + const reason = new Error('Cancelled during artifact read') + const originalReadFile = fs.readFile + let cancelledRead = false + let readsAfterAbort = 0 + t.mock.method(fs, 'readFile', async (path, options) => { + if (controller.signal.aborted) readsAfterAbort++ + if (path === join(build, 'background.js')) { + cancelledRead = true + assert.equal(options?.signal, controller.signal) + controller.abort(reason) + } + return originalReadFile(path, options) + }) + syncBuiltinESMExports() + t.after(() => { + t.mock.restoreAll() + syncBuiltinESMExports() + }) + await assert.rejects( + preflight( + { browser: 'chromium', 'chromium-path': process.execPath }, + directory, + controller.signal, + ), + (error) => error.name === 'AbortError' && error.cause === reason, + ) + assert.equal(cancelledRead, true) + assert.equal(readsAfterAbort, 0) + }, +) + +test( + 'Firefox preflight selects the directory independently of distribution ZIPs', + { skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-firefox-preflight-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const build = join(directory, 'build/firefox') + await mkdir(build, { recursive: true }) + await writeFile(join(build, 'manifest.json'), JSON.stringify({ name: 'Fixture', version: '1' })) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'current build') + const options = { + browser: 'firefox', + 'firefox-path': process.execPath, + 'geckodriver-path': process.execPath, + } + const [withoutZip] = await preflight(options, directory) + await writeFile(join(directory, 'build/firefox.zip'), 'stale archive') + const [withStaleZip] = await preflight(options, directory) + assert.equal(withoutZip.artifactSha256, await hashArtifact(build)) + assert.deepEqual(withStaleZip, withoutZip) + assert.equal(Object.hasOwn(withStaleZip, 'archive'), false) + assert.equal(Object.hasOwn(withStaleZip, 'archiveSha256'), false) + }, +) + +test( + 'Firefox preflight rejects a symlink build root before snapshot startup', + { skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-firefox-symlink-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const build = join(directory, 'actual-build') + await mkdir(build) + await mkdir(join(directory, 'build')) + await writeFile(join(build, 'manifest.json'), JSON.stringify({ name: 'Fixture', version: '1' })) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'current build') + await symlink(build, join(directory, 'build/firefox')) + await assert.rejects( + preflight( + { + browser: 'firefox', + 'firefox-path': process.execPath, + 'geckodriver-path': process.execPath, + }, + directory, + ), + /Firefox build source must be a directory/, + ) + }, +) + +async function invoke(command, args, cwd) { + const lifecycle = createLifecycle() + let managed + let result + let failure + try { + managed = lifecycle.spawn(command, args, { cwd }) + result = await bounded(managed.exited, 10000, 'CLI timeout') + } catch (error) { + failure = error + } + // Reuse owned-group termination, child reaping and pipe draining before settling. + const cleanupErrors = await lifecycle.cleanup() + if (cleanupErrors.length) + throw Object.assign( + new AggregateError( + failure ? [failure, ...cleanupErrors] : cleanupErrors, + failure?.message || 'CLI cleanup failed', + failure ? { cause: failure } : undefined, + ), + { cleanupErrors }, + ) + if (failure) throw failure + return { code: result.code, output: managed.output() } +} + +test( + 'CLI invocation preserves exit status and output and reports spawn failure', + { skip: process.platform !== 'linux' }, + async () => { + const result = await invoke( + process.execPath, + [ + '-e', + 'process.stdout.write("stdout"); process.stderr.write("stderr"); process.exitCode = 7', + ], + tmpdir(), + ) + assert.equal(result.code, 7) + assert.match(result.output, /stdout/) + assert.match(result.output, /stderr/) + await assert.rejects( + invoke(join(import.meta.dirname, 'missing-cli-executable'), [], tmpdir()), + /ENOENT/, + ) + }, +) + +test( + 'CLI timeouts stop owned descendants before rejecting', + { timeout: 30000, skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-cli-timeout-')) + const marker = join(directory, 'pids.json') + async function running(pid) { + try { + const stat = await fs.readFile(`/proc/${pid}/stat`, 'utf8') + return !['Z', 'X'].includes(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0]) + } catch (error) { + if (error.code === 'ENOENT' || error.code === 'ESRCH') return false + throw error + } + } + t.after(async () => { + let pids = [] + try { + pids = JSON.parse(await fs.readFile(marker, 'utf8')) + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + for (const pid of pids) { + assert.ok(Number.isInteger(pid) && pid > 0) + if (await running(pid)) { + try { + process.kill(pid, 'SIGKILL') + } catch (error) { + if (error.code !== 'ESRCH') throw error + } + } + } + await waitFor(async () => (await Promise.all(pids.map(running))).every((value) => !value), { + timeoutMs: 5000, + label: 'owned CLI fixture processes stopped', + }) + await rm(directory, { recursive: true, force: true }) + }) + const source = ` + const { spawn } = require('node:child_process') + const { writeFileSync } = require('node:fs') + const child = spawn(process.execPath, ['-e', + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)" + ], { stdio: ['ignore', 'inherit', 'inherit'] }) + writeFileSync(process.argv[1], JSON.stringify([process.pid, child.pid])) + process.on('SIGTERM', () => {}) + setInterval(() => {}, 1000) + ` + await assert.rejects(invoke(process.execPath, ['-e', source, marker], directory), /CLI timeout/) + const pids = JSON.parse(await fs.readFile(marker, 'utf8')) + assert.equal(pids.length, 2) + for (const pid of pids) assert.equal(await running(pid), false, `Owned process ${pid} survived`) + }, +) + +test( + 'Node, shell and npm entries work outside the repository and propagate argument failures', + { skip: process.platform !== 'linux' }, + async () => { + const entries = [ + [process.execPath, [join(ROOT, 'scripts/xvfb-smoke.mjs')]], + ['sh', [join(ROOT, 'scripts/run-smoke.sh')]], + ['npm', ['--prefix', ROOT, 'run', 'smoke', '--']], + ] + for (const [command, args] of entries) { + const help = await invoke(command, [...args, '--help'], tmpdir()) + assert.equal(help.code, 0, help.output) + assert.match(help.output, /Usage: npm run smoke/) + const invalid = await invoke(command, [...args, '--invalid'], tmpdir()) + assert.equal(invalid.code, 2, invalid.output) + } + }, +) diff --git a/tests/unit/scripts/smoke-scenarios.test.mjs b/tests/unit/scripts/smoke-scenarios.test.mjs new file mode 100644 index 000000000..dfcd536d9 --- /dev/null +++ b/tests/unit/scripts/smoke-scenarios.test.mjs @@ -0,0 +1,438 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { runInNewContext } from 'node:vm' +import { Server } from 'node:http' +import { + assertPopupIdentity, + assertScenarioMessages, + beginRequest, + disconnectPorts, + runScenarios, + snapshot, +} from '../../../scripts/smoke/scenarios.mjs' +import { createLifecycle } from '../../../scripts/smoke/lifecycle.mjs' + +for (const { name, primaryFailure = false, corruptContext } of [ + { name: 'scenario cleanup retains all failures with a primary error', primaryFailure: true }, + { name: 'scenario cleanup retains all failures without a primary error' }, + ...['missing', 'reordered', 'corrupted', 'extra success'].map((corruptContext) => ({ + name: `runScenarios rejects ${corruptContext} outbound context and cleans up`, + corruptContext, + })), +]) { + test(name, async (t) => { + const lifecycle = createLifecycle() + t.after(() => lifecycle.cleanup()) + const primary = new Error('PRIMARY_CONFIGURATION_FAILURE') + const disconnectFailure = new Error('SECONDARY_DISCONNECT_FAILURE') + const closeFailure = new Error('SECONDARY_SERVER_CLOSE_FAILURE') + const cleanupCalls = [] + const close = Server.prototype.close + t.mock.method(Server.prototype, 'close', function (callback) { + cleanupCalls.push('close') + return close.call(this, (error) => callback(error || closeFailure)) + }) + const metadata = { + expectedName: 'ChatGPTBox', + expectedVersion: '2.7.0', + extensionId: 'smoke-extension', + popupUrl: 'moz-extension://smoke-uuid/popup.html', + } + let baseUrl + let completed = false + let streamed + let disconnectAttempts = 0 + const adapter = { + metadata, + evaluateCleanup(fn, ...args) { + return this.evaluate(fn, ...args) + }, + async evaluate(fn, ...args) { + if (fn.name === 'popupIdentity') + return { + visible: true, + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + } + if (fn.name === 'configure') { + if (primaryFailure) throw primary + baseUrl = args[0] + return + } + if (fn.name === 'disconnectPorts') { + cleanupCalls.push('disconnect') + if (++disconnectAttempts === 1) throw disconnectFailure + return + } + if (fn.name === 'beginRequest') { + const [key, question, history] = args + let messages = [ + ...history.flatMap(({ question, answer }) => [ + { role: 'user', content: question }, + { role: 'assistant', content: answer }, + ]), + { role: 'user', content: question }, + ] + if (key === 'error') { + if (corruptContext === 'missing') messages = messages.slice(-1) + if (corruptContext === 'reordered') { + messages = [messages[1], messages[0], messages[2]] + } + if (corruptContext === 'corrupted') messages[1].content = 'Hello δΈ–η•ŒοΏ½' + } else if (corruptContext === 'extra success') { + messages.unshift({ role: 'user', content: 'Unexpected previous question' }) + } + const response = await globalThis.fetch(`${baseUrl}/${key}/v1/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer smoke-local-only', + }, + body: JSON.stringify({ + model: 'smoke-model', + stream: true, + messages, + }), + signal: globalThis.AbortSignal.timeout(5000), + }) + if (key === 'success') { + streamed = response.text().then(() => { + completed = true + }) + streamed.catch(() => {}) + } else await response.text() + return + } + if (fn.name === 'snapshot') { + const history = [{ question: 'Smoke success question', answer: 'Hello δΈ–η•ŒπŸ™‚' }] + if (args[0] === 'error') return [session(history), { error: 'smoke_503' }] + return completed + ? [ + session([]), + { answer: 'Hello ', done: false }, + { answer: 'Hello δΈ–η•ŒπŸ™‚', done: false }, + { ...session(history), answer: null, done: true }, + ] + : [session([]), { answer: 'Hello ', done: false }] + } + throw new Error(`Unexpected evaluation: ${fn.name}`) + }, + } + const checks = [] + let reported + await assert.rejects(runScenarios(adapter, { lifecycle, checks }), (error) => { + reported = error + if (primaryFailure) assert.equal(error, primary) + else if (corruptContext) { + assert.ok(error instanceof assert.AssertionError) + assert.equal(error.code, 'ERR_ASSERTION') + assert.equal(error.operator, 'deepStrictEqual') + const expected = [{ role: 'user', content: 'Smoke success question' }] + if (corruptContext !== 'extra success') { + expected.push( + { role: 'assistant', content: 'Hello δΈ–η•ŒπŸ™‚' }, + { role: 'user', content: 'Smoke error question' }, + ) + } + assert.deepEqual(error.expected, expected) + assert.notDeepEqual(error.actual, expected) + assert.equal(checks.length, 4) + } else { + assert.ok(error instanceof AggregateError) + assert.deepEqual(error.errors, [disconnectFailure, closeFailure]) + assert.equal(checks.length, 4) + } + assert.deepEqual(error.cleanupErrors, [disconnectFailure, closeFailure]) + assert.ok(error.cleanupErrors.every((failure) => failure instanceof Error)) + return true + }) + assert.deepEqual(cleanupCalls, ['disconnect', 'close']) + await streamed + await lifecycle.cleanup() + assert.equal( + disconnectAttempts, + 2, + 'A later successful disconnect retry must not erase its first failure', + ) + assert.deepEqual(reported.cleanupErrors, [disconnectFailure, closeFailure]) + }) +} + +test('request state survives fresh evaluation sandboxes sharing the popup window', () => { + const window = {} + let onMessage + let disconnected = 0 + const sent = [] + const port = { + onMessage: { + addListener: (listener) => { + onMessage = listener + }, + }, + onDisconnect: { addListener() {} }, + postMessage: (message) => sent.push(message), + disconnect: () => disconnected++, + } + const browser = { runtime: { connect: () => port } } + const evaluate = (fn, ...args) => + runInNewContext(`(${fn.toString()})(...args)`, { + window, + browser, + args, + }) + evaluate(beginRequest, 'success', 'Question', []) + assert.equal(sent.length, 1) + onMessage({ answer: 'Hello ', done: false }) + assert.equal(evaluate(snapshot, 'success')[0].answer, 'Hello ') + onMessage({ answer: 'Hello δΈ–η•ŒπŸ™‚', done: false }) + assert.equal(evaluate(snapshot, 'success')[1].answer, 'Hello δΈ–η•ŒπŸ™‚') + assert.throws(() => evaluate(beginRequest, 'success', 'Duplicate', []), /already started/) + assert.equal(sent.length, 1) + evaluate(disconnectPorts) + assert.equal(disconnected, 1) + assert.equal(window.__chatGPTBoxSmoke, undefined) + evaluate(disconnectPorts) + assert.equal(disconnected, 1) +}) + +test('a failed scenario preserves completed checks in the caller array and disconnects ports', async (t) => { + const lifecycle = createLifecycle() + t.after(async () => assert.deepEqual(await lifecycle.cleanup(), [])) + const checks = ['Existing runner check'] + const failure = new Error('Configuration write failed') + const metadata = { + expectedName: 'ChatGPTBox', + expectedVersion: '2.7.0', + extensionId: 'smoke-extension', + popupUrl: 'moz-extension://smoke-uuid/popup.html', + } + const calls = [] + const adapter = { + metadata, + evaluateCleanup(fn, ...args) { + return this.evaluate(fn, ...args) + }, + async evaluate(fn) { + calls.push(fn.name) + if (fn.name === 'popupIdentity') { + return { + visible: true, + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + } + } + if (fn.name === 'disconnectPorts') return + throw failure + }, + } + await assert.rejects(runScenarios(adapter, { lifecycle, checks }), (error) => error === failure) + assert.deepEqual(checks, [ + 'Existing runner check', + 'Installed popup visible with matching runtime identity and version', + ]) + assert.deepEqual(calls, ['popupIdentity', 'configure', 'disconnectPorts']) +}) + +test('popup identity uses the adapter installation manifest and rejects mismatches or missing expectations', () => { + const metadata = { + expectedName: 'ChatGPTBox', + expectedVersion: '2.7.0', + extensionId: 'smoke-extension', + popupUrl: 'moz-extension://smoke-uuid/popup.html', + } + const identity = { + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + visible: true, + } + assertPopupIdentity(identity, metadata) + for (const override of [ + { name: 'Wrong extension' }, + { version: '0.0.0' }, + { extensionId: 'other-extension' }, + { location: 'https://example.com/' }, + { visible: false }, + ]) { + assert.throws(() => assertPopupIdentity({ ...identity, ...override }, metadata)) + } + for (const key of Object.keys(metadata)) { + assert.throws(() => assertPopupIdentity(identity, { ...metadata, [key]: undefined })) + } +}) + +const question = 'Smoke success question' +const history = [{ question: 'Previous question', answer: 'Previous answer' }] +const session = (records) => ({ session: { conversationRecords: records } }) +const partial = [session(history), { answer: 'Hello ', done: false }] +const completed = [ + ...partial, + { answer: 'Hello δΈ–η•ŒπŸ™‚', done: false }, + { ...session([...history, { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }]), answer: null, done: true }, +] +const failure = [session(history), { error: '{"error":{"code":"smoke_503"}}' }] + +test('scenario assertions accept partial, final, and HTTP failure messages', () => { + assertScenarioMessages(partial, { phase: 'partial', question, history }) + assertScenarioMessages(completed, { phase: 'final', question, history }) + assertScenarioMessages(failure, { phase: 'error', question, history }) +}) + +test('partial proof rejects completion before release and incorrect first answers', () => { + for (const messages of [ + completed, + [session(history), { answer: 'Hello δΈ–η•ŒπŸ™‚' }], + [session(history)], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'partial', question, history })) + } +}) + +test('partial proof requires one unchanged session acknowledgement before the answer', () => { + for (const messages of [ + [partial[1]], + [partial[1], partial[0]], + [partial[0], partial[0], partial[1]], + [{ ...partial[0], ...partial[1] }], + [session([]), partial[1]], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'partial', question, history })) + } +}) + +test('final proof rejects duplicate completion, corrupted Unicode, and duplicate history', () => { + const duplicateHistory = { + ...session([ + ...history, + { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }, + { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }, + ]), + done: true, + } + for (const messages of [ + [...completed, completed.at(-1)], + [...partial, { answer: 'Hello δΈ–η•ŒοΏ½' }, completed.at(-1)], + [...completed.slice(0, -1), duplicateHistory], + partial, + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'final', question, history })) + } +}) + +test('final proof preserves the initial acknowledgement and completion ordering', () => { + for (const messages of [ + [...completed.slice(0, -1), session(history), completed.at(-1)], + [...completed.slice(0, -1), session([]), completed.at(-1)], + [session([]), ...completed.slice(1)], + [...partial, completed.at(-1), completed[2]], + [...partial, { ...completed.at(-1), answer: 'Hello δΈ–η•ŒπŸ™‚' }], + [...completed.slice(0, -1), { ...completed.at(-1), answer: undefined }], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'final', question, history })) + } +}) + +test('HTTP failure requires one standalone acknowledgement before the error', () => { + for (const messages of [ + [failure[1]], + [failure[1], failure[0]], + [failure[0], failure[0], failure[1]], + [{ ...failure[0], ...failure[1] }], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'error', question, history })) + } +}) + +test('HTTP failure proof rejects false success, unrelated errors, and modified history', () => { + for (const messages of [ + [...failure, { done: true }], + [...failure, { answer: 'unexpected' }], + [...failure, session([...history, { question, answer: '' }])], + [session(history), { error: 'Unexpected provider failure' }], + [session([]), failure[1]], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'error', question, history })) + } +}) + +test('answer sequences reject corruption, duplication, omission and reordering before a valid final answer', () => { + for (const answers of [ + ['wrong', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 'corrupt', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 42, 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 'Hello ', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 'Hello δΈ–η•ŒπŸ™‚', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello δΈ–η•ŒπŸ™‚', 'Hello ', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello δΈ–η•ŒπŸ™‚'], + ]) { + assert.throws(() => + assertScenarioMessages( + [session(history), ...answers.map((answer) => ({ answer, done: false })), completed.at(-1)], + { phase: 'final', question, history }, + ), + ) + } + for (const answers of [ + ['bad', 'Hello '], + ['Hello ', 'Hello '], + ]) { + assert.throws(() => + assertScenarioMessages( + answers.map((answer) => ({ answer })), + { phase: 'partial', question, history }, + ), + ) + } +}) + +test('scenario cancellation uses independent cleanup and successful disconnect is idempotent', async (t) => { + const controller = new AbortController() + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.cleanup()) + const reason = new Error('Cancelled during configuration') + let disconnects = 0 + const metadata = { + expectedName: 'Fixture', + expectedVersion: '1', + extensionId: 'fixture', + popupUrl: 'moz-extension://fixture/popup.html', + } + const adapter = { + metadata, + async evaluate(fn) { + controller.signal.throwIfAborted() + if (fn.name === 'popupIdentity') + return { + visible: true, + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + } + assert.equal(fn.name, 'configure') + controller.abort(reason) + throw reason + }, + async evaluateCleanup(fn) { + assert.equal(controller.signal.aborted, true) + assert.equal(fn, disconnectPorts) + disconnects++ + }, + } + await assert.rejects(runScenarios(adapter, { lifecycle, signal: controller.signal }), (error) => { + assert.equal(error, reason) + assert.equal(error.cleanupErrors, undefined) + return true + }) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.equal(disconnects, 1) +})