diff --git a/package.json b/package.json index a75f48586..3bef6cfaa 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dev": "node build.mjs --development", "analyze": "node build.mjs --analyze", "lint": "eslint --ext .js,.mjs,.jsx .", + "smoke": "node scripts/xvfb-smoke.mjs", "test": "node --import ./tests/setup/browser-shim.mjs --test", "test:coverage": "c8 --all --reporter=text --reporter=lcov --reporter=json-summary --include=\"src/**/*.{mjs,jsx,js}\" node --import ./tests/setup/browser-shim.mjs --test", "lint:fix": "eslint --ext .js,.mjs,.jsx . --fix", diff --git a/scripts/run-smoke.sh b/scripts/run-smoke.sh new file mode 100644 index 000000000..6df5a11f9 --- /dev/null +++ b/scripts/run-smoke.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Compatibility entry point; Node owns resources and the exit status. +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) || exit 2 +exec node "$script_dir/xvfb-smoke.mjs" "$@" diff --git a/scripts/smoke/README.md b/scripts/smoke/README.md new file mode 100644 index 000000000..71b99a91c --- /dev/null +++ b/scripts/smoke/README.md @@ -0,0 +1,162 @@ +# Browser smoke tests + +This maintained Linux smoke tool loads the built extension into real, isolated +headless Chromium and Firefox sessions. It does not use Xvfb, real API credentials, +external test websites, or the user's browser profile. It does not install software, +download browsers, build the extension, or integrate with CI automatically. + +## Prerequisites and usage + +Install Node 22+, project dependencies (`npm ci`), full Chromium or Chrome for +Testing, Firefox, and a native Linux geckodriver binary. Browser installation is an +explicit prerequisite, not part of the runner. An npm geckodriver launcher is +rejected because it may download a driver. Use the actual extracted executable. + +Build in the same worktree first: + +```sh +npm run build +npm run smoke -- --browser all \ + --chromium-path /path/to/chromium \ + --firefox-path /path/to/firefox \ + --geckodriver-path /path/to/geckodriver +``` + +Firefox installs a temporary add-on from a per-run copy of `build/firefox`, not +`build/firefox.zip`. The copy must match the directory hash recorded during +preflight; a changed or incomplete build fails before Firefox starts. Popup paths +and expected identity come from that verified copy's manifest, including its query +string. This validates the unpacked build, not the distribution ZIP. A missing or +stale distribution ZIP does not select different JavaScript for the smoke run. + +Use `--browser chromium` or `--browser firefox` for a single browser. The default +is `all`, with Chromium followed by Firefox. All selected prerequisites are checked +before any browser starts; missing browsers are failures, not skipped passes. +Paths supplied on the command line override PATH discovery. Discovery searches +explicit PATH directories; empty components do not imply the current directory. +Use an explicit `--*-path ./executable` or PATH entry `.` to select a local binary. +Relative executable and artifact-parent paths use the runner's working directory. +Direct Node and shell invocations preserve the caller's directory. `npm run smoke` +runs from the package root, including when selected with `npm --prefix PATH`. +The runner does not use `INIT_CWD` for path resolution. Use absolute paths when +invoking the npm entry from another directory to avoid ambiguity. +Build entry paths are fixed at `build/` in the runner's own worktree. +Chromium accepts a stable build-root symlink, whose canonical target may be outside +the worktree; Firefox requires a non-symlink build root. Paths containing spaces work. + +`node scripts/xvfb-smoke.mjs` and `sh scripts/run-smoke.sh` are compatibility +entry points to the same runner. Despite the historical filename, Xvfb is not +required. `--help` lists options. There is no automatic sandbox-disabling flag; +the browser must be able to start with its normal sandbox. + +Use full Chromium or Chrome for Testing, not `chrome-headless-shell` or branded +Google Chrome. Chrome 137+ removed the command-line extension-loading flag from +branded builds. See the [Chromium announcement](https://groups.google.com/a/chromium.org/g/chromium-extensions/c/1-g8EFx2BBY/m/S0ET5wPjCAAJ). +Firefox requires geckodriver's `--allow-system-access` for privileged popup +navigation, and dynamic port flags; see [geckodriver flags](https://firefox-source-docs.mozilla.org/testing/geckodriver/Flags.html). + +## What is verified + +- The installed extension's identity, version, and rendered popup. +- A popup extension port reaches the real background implementation, which sends + a Chat Completions request to the run's loopback-only mock server. +- A first answer arrives while the server deliberately holds the rest of the + response; after release, the complete Unicode answer, one completion message, + and one conversation record are asserted. +- An HTTP failure reports an error without a successful completion or new record. + +This tests the committed Chat Completions route and does not require the separate +Responses API feature. Exact byte splitting, UTF-8 boundaries, and CRLF parsing +remain deterministic unit tests: separate server writes do not guarantee separate +browser network reads. These checks are a focused runtime smoke test, not a +replacement for all site-adapter and keyboard interaction testing in AGENTS.md. + +## Isolation, results, and cleanup + +Every invocation gets unique browser profiles, a unique artifact directory, and +OS-assigned loopback ports. Concurrent invocations do not share browser state. +Chromium's debugging endpoint is read from its own profile's `DevToolsActivePort`; +geckodriver's endpoint comes from its own startup log. + +`--artifacts-dir PATH` selects a parent directory. The tool creates a unique +`chatgptbox-smoke-*` child, retaining `report.json`, browser logs, and available +failure screenshots/DOM. Reports include selected browsers, versions, build +hashes, checks, failure stage, cleanup errors, and exit status. Retained artifacts +must stay outside the selected build directories; parents inside a build, +including symlink aliases, are rejected before writing. Ancestor directories +such as the worktree root or `/tmp` are allowed because each run gets a new child. +Retained artifacts include Firefox's `extension/` snapshot and its `snapshotDir` +in the report. +For Firefox, `preflightManifestVersion` identifies the initial manifest, and +`manifestVersion` is set from the verified snapshot after successful startup. +Chromium's `manifestVersion` starts with the preflight value and is updated from +the rechecked build on successful startup. `artifactSha256` is the preflight +directory hash; Firefox verifies its snapshot against it before startup. +Chromium rechecks the live directory against that hash before launch and derives +startup identity from the on-disk manifest, not stale caller metadata. Directory +hashes include entry types, length-framed paths and file bytes, and empty directories. +If startup fails, the retained snapshot may be absent or incomplete. +Do not modify a build or the retained snapshot during a run. +Use trusted builds and keep their paths, the artifact parent, and all ancestors +stable during the run. Path-based validation and hashing are not an atomic security +boundary against concurrent directory or symlink substitution and cannot prevent +outside bytes entering retained artifacts if this precondition is violated. +Temporary browser profiles are removed after confirming that their managed process +groups have no live members. If group termination cannot be confirmed within the +cleanup budget, the run fails and preserves the profile at the path recorded in the +report. The tool never removes the artifact parent or user profiles. Browser logs +and failure DOM may contain test data; review artifacts before sharing. + +Each managed command runs inside an isolated group with a small Node supervisor. +Shutdown first closes browser protocols, then asks the supervisor through a private +IPC channel to signal its own group with SIGTERM and finally SIGKILL (including +itself). The parent never sends termination signals to remembered numeric PIDs or +PGIDs, which can be reused by the OS. The target does not inherit the control channel. +Target exit reports are separate from supervisor exit: if a final group SIGKILL +prevents a target report, its exit status is explicitly unavailable, not inferred. + +Browser and driver executables must keep profile-using descendants in these managed +groups. Do not use wrappers that daemonize or move those descendants to another +process group or session. This is process-group cleanup, not containment of every +descendant: a process that leaves its group (for example, using `setsid()`) is not +reliably detected or terminated. In that case, the run may report success and remove +its temporary profile while the escaped process remains alive. A PASS does not prove +that no descendants escaped their groups. + +Cleanup is idempotent and bounded, and attempts remaining resources even when one +cleanup fails. SIGINT and SIGTERM trigger cleanup; a second signal asks supervisors +to accelerate termination. Control-channel loss triggers local supervisor cleanup, +but unexpected loss of the supervisor remains an error and preserves its profile. +An unresponsive or forcibly killed supervisor, machine failure, or uninterruptible +processes can prevent cleanup. Detected loss of supervisor ownership or failure to +confirm group termination is reported as an error; the parent does not retry +termination using an unverified group ID. This does not detect the escaped processes +described above. + +Each browser startup is limited to 60 seconds. Request timeouts default to 10 seconds +for Chromium CDP and 30 seconds for Firefox WebDriver; Firefox session creation uses +60 seconds, still subject to the startup limit. The selected-browser loop has a shared +five-minute limit; each lifecycle cleanup has a ten-second waiting budget. Readiness checks +may retry, but session creation, addon installation, and test requests never do. + +| Exit status | Meaning | +| ----------- | --------------------------------------------------- | +| 0 | Every selected browser passed and cleanup succeeded | +| 1 | Test, runtime, reporting, or cleanup failure | +| 2 | Invalid arguments or missing prerequisites/builds | +| 130 | SIGINT | +| 143 | SIGTERM | + +The shell and npm entry points preserve failure status. No fixed `/tmp/smoke.done` +or `/tmp/smoke.out` files are used. A Firefox-only pass is not an all-browser pass. +The console prints the artifact location, not a provisional PASS or exit code: +signals can still arrive while that output is pending. Use the process exit status +and `report.json` for the final result. + +## Development checks + +Run the targeted formatter on changed files, `npm run lint`, `shellcheck +scripts/run-smoke.sh`, `npm test`, `npm run build`, then the actual browser smoke. +`npm test` includes fake-protocol and real child-process lifecycle tests, but never +launches a real browser automatically. It needs loopback socket access for mock +servers. No new browser automation package or indirect `ws` dependency is used. diff --git a/scripts/smoke/artifacts.mjs b/scripts/smoke/artifacts.mjs new file mode 100644 index 000000000..20dbc005f --- /dev/null +++ b/scripts/smoke/artifacts.mjs @@ -0,0 +1,72 @@ +import { createHash } from 'node:crypto' +import { Buffer } from 'node:buffer' +import { createReadStream, createWriteStream } from 'node:fs' +import { lstat, mkdir, readFile, readdir } from 'node:fs/promises' +import { join } from 'node:path' +import { pipeline } from 'node:stream/promises' + +export async function hashArtifact(directory, signal) { + const hash = createHash('sha256').update('chatgptbox-smoke-artifact-v2\0') + function frame(bytes) { + const length = Buffer.alloc(8) + length.writeBigUInt64BE(BigInt(bytes.length)) + hash.update(length).update(bytes) + } + async function visit(relative = '') { + signal?.throwIfAborted() + const entries = await readdir(join(directory, relative), { withFileTypes: true }) + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + for (const entry of entries) { + signal?.throwIfAborted() + const path = join(relative, entry.name) + if (entry.isDirectory()) { + hash.update('d') + frame(Buffer.from(path)) + await visit(path) + } else if (entry.isFile()) { + hash.update('f') + frame(Buffer.from(path)) + frame(await readFile(join(directory, path), { signal })) + } else throw new Error(`Unexpected non-regular build artifact: ${path}`) + } + } + await visit() + signal?.throwIfAborted() + return hash.digest('hex') +} + +export async function snapshotArtifact(source, destination, expectedHash, signal) { + signal?.throwIfAborted() + if (typeof expectedHash !== 'string' || !/^[a-f0-9]{64}$/.test(expectedHash)) + throw new Error('Missing preflight build hash') + if (!(await lstat(source)).isDirectory()) throw new Error('Build source must be a directory') + // The destination belongs to this run's retained artifacts, never a browser profile. + await mkdir(destination) + async function copyDirectory(from, to) { + signal?.throwIfAborted() + for (const entry of await readdir(from, { withFileTypes: true })) { + signal?.throwIfAborted() + const input = join(from, entry.name) + const output = join(to, entry.name) + if (entry.isDirectory()) { + await mkdir(output) + await copyDirectory(input, output) + } else if (entry.isFile()) { + try { + // pipeline destroys both streams on abort and waits for their closure. + await pipeline(createReadStream(input), createWriteStream(output, { flags: 'wx' }), { + signal, + }) + } catch (error) { + signal?.throwIfAborted() + throw error + } + } else throw new Error(`Unexpected non-regular build artifact: ${input}`) + } + } + await copyDirectory(source, destination) + signal?.throwIfAborted() + const actualHash = await hashArtifact(destination, signal) + if (actualHash !== expectedHash) throw new Error('Build changed since preflight') + return actualHash +} diff --git a/scripts/smoke/cdp.mjs b/scripts/smoke/cdp.mjs new file mode 100644 index 000000000..85aeb2ff5 --- /dev/null +++ b/scripts/smoke/cdp.mjs @@ -0,0 +1,112 @@ +import { bounded } from './lifecycle.mjs' + +// Register the transport before awaiting the handshake or issuing any RPCs. +export async function connectCDP( + url, + { lifecycle, signal, timeoutMs = 10000, WebSocketImpl = globalThis.WebSocket }, +) { + signal?.throwIfAborted() + let socket + let nextId = 0 + let failure + let closed = false + let closing + let closeFailure + const pending = new Map() + let resolveOpen, rejectOpen, resolveClosed + const opened = new Promise((resolve, reject) => { + resolveOpen = resolve + rejectOpen = reject + }) + const finished = new Promise((resolve) => { + resolveClosed = resolve + }) + const fail = (error) => { + failure ??= error + rejectOpen(failure) + for (const request of pending.values()) request.reject(failure) + pending.clear() + } + const onAbort = () => { + const error = signal.reason ?? new Error('CDP connection aborted') + rejectOpen(error) + for (const [id, request] of pending) { + if (request.signal !== signal) continue + request.reject(error) + pending.delete(id) + } + } + const cleanup = () => { + if (closing) return closing + if (!socket) return Promise.resolve() + closeFailure = failure + signal?.removeEventListener('abort', onAbort) + fail(new Error('CDP connection closed')) + closing = (async () => { + if (closed) return + socket.close() + await bounded(finished, 2000, 'Close CDP connection') + })() + return closing + } + const close = async () => { + await cleanup() + if (closeFailure) throw closeFailure + } + lifecycle.defer('Close Chromium CDP connection', cleanup) + socket = new WebSocketImpl(url) + socket.addEventListener('open', resolveOpen, { once: true }) + socket.addEventListener('error', () => fail(new Error('CDP WebSocket transport error'))) + socket.addEventListener('close', () => { + closed = true + signal?.removeEventListener('abort', onAbort) + fail(new Error('CDP WebSocket disconnected')) + resolveClosed() + }) + socket.addEventListener('message', ({ data }) => { + let message + try { + message = JSON.parse(data) + if (!message || typeof message !== 'object' || Array.isArray(message)) { + throw new Error('Expected a CDP message object') + } + } catch (error) { + fail(new Error('Invalid CDP WebSocket message', { cause: error })) + return + } + const request = pending.get(message.id) + if (!request) return + pending.delete(message.id) + if (message.error) { + const error = new Error(`${request.method}: ${message.error.message}`) + error.code = message.error.code + request.reject(error) + } else if (!Object.hasOwn(message, 'result')) { + request.reject(new Error(`${request.method}: missing CDP result`)) + } else { + request.resolve(message.result) + } + }) + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) onAbort() + await bounded(opened, timeoutMs, 'Connect Chromium CDP', signal) + + return { + close, + async request(method, params = {}, sessionId, options = {}) { + const requestSignal = Object.hasOwn(options, 'signal') ? options.signal : signal + requestSignal?.throwIfAborted() + if (failure) throw failure + const id = ++nextId + const response = new Promise((resolve, reject) => { + pending.set(id, { resolve, reject, method, signal: requestSignal }) + }) + try { + socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) })) + return await bounded(response, options.timeoutMs ?? timeoutMs, method, requestSignal) + } finally { + pending.delete(id) + } + }, + } +} diff --git a/scripts/smoke/chromium.mjs b/scripts/smoke/chromium.mjs new file mode 100644 index 000000000..77955cc30 --- /dev/null +++ b/scripts/smoke/chromium.mjs @@ -0,0 +1,240 @@ +import { Buffer } from 'node:buffer' +import { createHash } from 'node:crypto' +import { readFile, realpath, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { URL } from 'node:url' +import { connectCDP } from './cdp.mjs' +import { bounded, waitFor } from './lifecycle.mjs' +import { hashArtifact } from './artifacts.mjs' + +export async function startChromium({ + executable, + extensionDir, + profileDir, + artifactsDir, + lifecycle, + signal, + artifactSha256, +}) { + signal?.throwIfAborted() + extensionDir = await realpath(path.resolve(extensionDir)) + profileDir = path.resolve(profileDir) + artifactsDir = path.resolve(artifactsDir) + if (extensionDir.includes(',')) throw new Error('Chromium extension path must not contain commas') + if (typeof artifactSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(artifactSha256)) + throw new Error('Missing preflight build hash') + if ((await hashArtifact(extensionDir, signal)) !== artifactSha256) + throw new Error('Build changed since preflight') + const manifest = JSON.parse( + await readFile(path.join(extensionDir, 'manifest.json'), { encoding: 'utf8', signal }), + ) + const workerPath = manifest.background?.service_worker + const popupPath = manifest.action?.default_popup + if (!workerPath || !popupPath) { + throw new Error('Chromium build must declare a service worker and an action popup') + } + // Linux unpacked IDs use the manifest public key, or the canonical load path. + // Chromium's components/crx_file/id_util.cc maps the first 128 SHA-256 bits to a-p. + const extensionId = createHash('sha256') + .update(manifest.key ? Buffer.from(manifest.key, 'base64') : extensionDir) + .digest('hex') + .slice(0, 32) + .replace(/[0-9a-f]/g, (digit) => String.fromCharCode(97 + Number.parseInt(digit, 16))) + const extensionOrigin = `chrome-extension://${extensionId}/` + const portFile = path.join(profileDir, 'DevToolsActivePort') + try { + await readFile(portFile) + throw new Error('Chromium profile already contains DevToolsActivePort; use a fresh profile') + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + signal?.throwIfAborted() + const browser = lifecycle.spawn( + executable, + [ + '--headless=new', + '--no-first-run', + '--no-default-browser-check', + '--remote-debugging-address=127.0.0.1', + '--remote-debugging-port=0', + `--user-data-dir=${profileDir}`, + `--load-extension=${extensionDir}`, + `--disable-extensions-except=${extensionDir}`, + 'about:blank', + ], + { logPath: path.join(artifactsDir, 'chromium.log') }, + ) + let cdp, sessionId, popupUrl + const evaluateExpression = async (expression, options) => { + if (!options?.diagnostic && !options?.cleanup) browser.assertRunning() + const response = await cdp.request( + 'Runtime.evaluate', + { expression, awaitPromise: true, returnByValue: true }, + sessionId, + options, + ) + if (response.exceptionDetails) { + const detail = response.exceptionDetails + throw new Error(`Chromium evaluation failed: ${detail.exception?.description ?? detail.text}`) + } + if (!response.result) throw new Error('Chromium evaluation returned no result') + if (response.result.subtype === 'error') { + throw new Error(`Chromium evaluation returned an Error: ${response.result.description}`) + } + return response.result.value + } + const evaluate = (fn, ...args) => { + if (typeof fn !== 'function') throw new TypeError('evaluate requires a function') + return evaluateExpression(`(${fn.toString()})(...${JSON.stringify(args)})`) + } + const evaluateCleanup = async (fn, ...args) => { + if (typeof fn !== 'function') throw new TypeError('evaluate requires a function') + const controller = new AbortController() + try { + return await evaluateExpression(`(${fn.toString()})(...${JSON.stringify(args)})`, { + cleanup: true, + signal: controller.signal, + timeoutMs: 5000, + }) + } finally { + controller.abort() + } + } + const capture = async (prefix) => { + if (!sessionId) throw new Error('Chromium popup is not attached; capture unavailable') + // Diagnostic requests have their own short deadlines. + const options = { signal: undefined, timeoutMs: 2000, diagnostic: true } + const destination = path.resolve(artifactsDir, prefix) + if (path.dirname(destination) !== artifactsDir) { + throw new Error('Capture prefix must name a file inside artifactsDir') + } + const results = await Promise.allSettled([ + (async () => { + const { data } = await cdp.request( + 'Page.captureScreenshot', + { format: 'png' }, + sessionId, + options, + ) + await bounded( + writeFile(`${destination}.png`, Buffer.from(data, 'base64')), + 2000, + 'Write Chromium screenshot', + ) + })(), + (async () => { + const dom = await evaluateExpression('document.documentElement.outerHTML', options) + await bounded(writeFile(`${destination}.html`, dom), 2000, 'Write Chromium DOM') + })(), + ]) + const errors = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason) + if (errors.length) throw new AggregateError(errors, 'Chromium capture failed') + } + try { + const endpoint = await waitFor( + async () => { + browser.assertRunning() + let content + try { + content = await readFile(portFile, 'utf8') + } catch (error) { + if (error.code === 'ENOENT') return false + throw error + } + const [port, websocketPath] = content.trim().split(/\r?\n/) + // Chromium may still be writing this file; wait for the complete port and UUID. + if (!port || !websocketPath) return false + if ( + !/^\d+$/.test(port) || + Number(port) < 1 || + Number(port) > 65535 || + !/^\/devtools\/browser\/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/i.test( + websocketPath, + ) + ) { + return false + } + return `ws://127.0.0.1:${port}${websocketPath}` + }, + { signal, label: 'Complete Chromium DevToolsActivePort record' }, + ) + cdp = await connectCDP(endpoint, { lifecycle, signal }) + const { product: browserVersion } = await cdp.request('Browser.getVersion') + await waitFor( + async () => { + browser.assertRunning() + const { targetInfos } = await cdp.request('Target.getTargets') + const workers = targetInfos.filter((target) => { + if (target.type !== 'service_worker' || !target.url.startsWith('chrome-extension://')) { + return false + } + return target.url === new URL(workerPath, extensionOrigin).href + }) + if (workers.length > 1) + throw new Error('Multiple matching Chromium extension service workers') + return workers[0] + }, + { signal, label: 'Chromium extension service worker' }, + ) + popupUrl = new URL(popupPath, extensionOrigin).href + if (!popupUrl.startsWith(extensionOrigin)) throw new Error('Popup must belong to the extension') + const { targetId } = await cdp.request('Target.createTarget', { url: popupUrl }) + const attached = await cdp.request('Target.attachToTarget', { targetId, flatten: true }) + sessionId = attached.sessionId + if (!sessionId) throw new Error('Chromium popup attachment returned no session ID') + await cdp.request('Runtime.enable', {}, sessionId) + await cdp.request('Page.enable', {}, sessionId) + await waitFor( + async () => { + try { + return await evaluateExpression( + `location.href === ${JSON.stringify(popupUrl)} && document.readyState === 'complete'`, + ) + } catch (error) { + // Only a read-only readiness probe may retry a navigation context transition. + if ( + error.code === -32000 && + /^Runtime\.evaluate: (Execution context was destroyed|Cannot find context with specified id|Cannot find default execution context)\.?$/.test( + error.message, + ) + ) { + return false + } + throw error + } + }, + { signal, label: 'Chromium extension popup' }, + ) + return { + evaluate, + evaluateCleanup, + capture, + // Process-group termination belongs to lifecycle.spawn's registered cleanup. + async close() { + browser.assertRunning() + await cdp.close() + }, + metadata: { + browserVersion, + extensionId, + popupUrl, + expectedVersion: manifest.version, + expectedName: manifest.name, + manifestVersion: manifest.version, + }, + } + } catch (error) { + if (sessionId) { + try { + await capture('chromium-startup-failure') + } catch (captureError) { + error.captureError = captureError + } + } + throw new Error(`Chromium startup failed: ${error.message}\n${browser.output().slice(-4000)}`, { + cause: error, + }) + } +} diff --git a/scripts/smoke/firefox.mjs b/scripts/smoke/firefox.mjs new file mode 100644 index 000000000..e6a378c06 --- /dev/null +++ b/scripts/smoke/firefox.mjs @@ -0,0 +1,279 @@ +import { Buffer } from 'node:buffer' +import { readFile, writeFile } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import process from 'node:process' +import { bounded, waitFor } from './lifecycle.mjs' +import { snapshotArtifact } from './artifacts.mjs' + +export async function startFirefox({ + executable, + geckodriver, + extensionDir, + artifactSha256, + profileDir, + artifactsDir, + lifecycle, + signal, +}) { + signal?.throwIfAborted() + const snapshotDir = resolve(artifactsDir, 'extension') + await snapshotArtifact(extensionDir, snapshotDir, artifactSha256, signal) + const manifest = JSON.parse(await readFile(join(snapshotDir, 'manifest.json'), { signal })) + if (!manifest.name || !manifest.version) throw new Error('Invalid Firefox snapshot manifest') + const popup = manifest.action?.default_popup || manifest.browser_action?.default_popup + if ( + typeof popup !== 'string' || + !popup.trim() || + popup !== popup.trim() || + /^[a-z][a-z\d+.-]*:|^\/\//i.test(popup) + ) + throw new Error('Invalid Firefox manifest popup path') + signal?.throwIfAborted() + const managed = lifecycle.spawn( + geckodriver, + [ + '--host', + '127.0.0.1', + '--port', + '0', + '--websocket-port', + '0', + '--profile-root', + resolve(profileDir), + '--allow-system-access', + ], + { logPath: join(artifactsDir, 'geckodriver.log'), env: { ...process.env } }, + ) + let origin + let sessionId + let closing + + async function request(method, path, body, { cleanup = false, timeoutMs = 30000 } = {}) { + if (!cleanup) { + signal?.throwIfAborted() + managed.assertRunning() + } + const controller = new AbortController() + const requestSignal = + signal && !cleanup ? AbortSignal.any([signal, controller.signal]) : controller.signal + const label = `Firefox WebDriver ${method} ${path}` + try { + return await bounded( + (async () => { + const response = await fetch(`${origin}${path}`, { + method, + headers: { 'Content-Type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: requestSignal, + redirect: 'error', + }) + const text = await response.text() + let payload + try { + payload = JSON.parse(text) + } catch { + throw new Error(`${label}: HTTP ${response.status}, invalid JSON response`) + } + if (!response.ok || payload?.value?.error) { + throw new Error( + `${label}: HTTP ${response.status}: ${payload?.value?.error || 'request failed'}: ${ + payload?.value?.message || '' + }`, + ) + } + if (!payload || !Object.hasOwn(payload, 'value')) { + throw new Error(`${label}: missing WebDriver value`) + } + return payload.value + })(), + timeoutMs, + label, + requestSignal, + ) + } finally { + controller.abort() + } + } + + function command(method, path, body, options) { + return request(method, `/session/${encodeURIComponent(sessionId)}${path}`, body, options) + } + + function close() { + if (!closing) { + closing = sessionId + ? command('DELETE', '', undefined, { cleanup: true, timeoutMs: 5000 }) + : Promise.resolve() + } + return closing + } + lifecycle.defer('Firefox WebDriver session', close) + + origin = await waitFor( + async () => { + managed.assertRunning() + const match = managed.output().match(/Listening on 127\.0\.0\.1:(\d+)(?=\r?\n)/) + if (!match) return false + const port = Number(match[1]) + if (port < 1 || port > 65535) throw new Error('Invalid geckodriver listening port') + return `http://127.0.0.1:${port}` + }, + { timeoutMs: 15000, signal, label: 'geckodriver listening port' }, + ) + await waitFor( + async () => { + const status = await request('GET', '/status') + if (typeof status?.ready !== 'boolean') + throw new Error('Invalid geckodriver readiness status') + return status.ready + }, + { timeoutMs: 15000, signal, label: 'geckodriver ready' }, + ) + + const session = await request( + 'POST', + '/session', + { + capabilities: { + alwaysMatch: { + browserName: 'firefox', + 'moz:firefoxOptions': { + binary: executable, + args: ['-headless'], + prefs: { + 'intl.accept_languages': 'en-US,en', + 'browser.shell.checkDefaultBrowser': false, + }, + }, + }, + }, + }, + { timeoutMs: 60000 }, + ) + sessionId = session?.sessionId + if (typeof sessionId !== 'string' || !sessionId.trim()) { + sessionId = undefined + throw new Error('Firefox WebDriver returned an empty session ID') + } + await command('POST', '/timeouts', { script: 30000, pageLoad: 30000, implicit: 0 }) + const extensionId = await command('POST', '/moz/addon/install', { + path: snapshotDir, + temporary: true, + }) + if (typeof extensionId !== 'string' || !extensionId.trim()) { + throw new Error('Firefox returned an empty addon ID') + } + const previousHandles = await command('GET', '/window/handles') + await command('POST', '/moz/context', { context: 'chrome' }) + const extensionBase = await command('POST', '/execute/sync', { + script: `const policy = WebExtensionPolicy.getByID(arguments[0]); + if (!policy) throw new Error('Installed extension policy not found'); + return policy.getURL('');`, + args: [extensionId], + }) + if (typeof extensionBase !== 'string' || !extensionBase.startsWith('moz-extension://')) { + throw new Error('Firefox returned an invalid extension URL') + } + const base = new URL(extensionBase) + const target = new URL(popup, base) + if ( + target.protocol !== 'moz-extension:' || + target.host !== base.host || + target.username || + target.password + ) + throw new Error('Firefox popup must belong to the installed extension') + const popupUrl = target.href + await command('POST', '/execute/sync', { + script: `const tab = gBrowser.addTab(arguments[0], { + triggeringPrincipal: Services.scriptSecurityManager.getSystemPrincipal() + }); + gBrowser.selectedTab = tab;`, + args: [popupUrl], + }) + await command('POST', '/moz/context', { context: 'content' }) + const handle = await waitFor( + async () => { + const handles = await command('GET', '/window/handles') + return handles.find((candidate) => !previousHandles.includes(candidate)) + }, + { timeoutMs: 10000, signal, label: 'Firefox popup window' }, + ) + await command('POST', '/window', { handle }) + + async function evaluateWithOptions(options, fn, ...args) { + if (typeof fn !== 'function') throw new TypeError('evaluate requires a function') + const result = await command( + 'POST', + '/execute/async', + { + script: `const done = arguments[arguments.length - 1]; + const args = Array.prototype.slice.call(arguments, 0, -1); + Promise.resolve().then(() => (${fn.toString()})(...args)).then( + value => done({ ok: true, value: value === undefined ? null : value }), + error => done({ ok: false, smokeError: { + message: String(error), stack: String(error && error.stack || '') + } }) + );`, + args, + }, + options, + ) + if (result?.ok !== true) { + const detail = result?.smokeError + throw new Error( + `Firefox evaluation failed: ${detail?.message || 'invalid script result'}${ + detail?.stack ? `\n${detail.stack}` : '' + }`, + ) + } + return result.value + } + const evaluate = (fn, ...args) => evaluateWithOptions(undefined, fn, ...args) + const evaluateCleanup = (fn, ...args) => + evaluateWithOptions({ cleanup: true, timeoutMs: 5000 }, fn, ...args) + + await waitFor( + () => evaluate((url) => location.href === url && document.readyState === 'complete', popupUrl), + { timeoutMs: 15000, signal, label: 'Firefox popup document' }, + ) + + async function capture(prefix) { + const destination = resolve(artifactsDir, prefix) + if (dirname(destination) !== resolve(artifactsDir)) { + throw new Error('Capture prefix must name a file inside artifactsDir') + } + const results = await Promise.allSettled([ + (async () => { + const screenshot = await command('GET', '/screenshot') + await writeFile(`${destination}.png`, Buffer.from(screenshot, 'base64')) + })(), + (async () => { + const source = await command('GET', '/source') + await writeFile(`${destination}.html`, source) + })(), + ]) + const errors = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason) + if (errors.length) + throw new AggregateError(errors, `Firefox capture failed: ${errors.map(String).join('; ')}`) + } + + return { + evaluate, + evaluateCleanup, + capture, + close, + metadata: { + browserVersion: session.capabilities?.browserVersion, + extensionId, + popupUrl, + expectedVersion: manifest.version, + expectedName: manifest.name, + artifactSha256, + snapshotDir, + manifestVersion: manifest.version, + }, + } +} diff --git a/scripts/smoke/lifecycle.mjs b/scripts/smoke/lifecycle.mjs new file mode 100644 index 000000000..eabdf8345 --- /dev/null +++ b/scripts/smoke/lifecycle.mjs @@ -0,0 +1,455 @@ +import { spawn as spawnChild } from 'node:child_process' +import { createWriteStream } from 'node:fs' +import { readdir, readFile } from 'node:fs/promises' +import { performance } from 'node:perf_hooks' +import process from 'node:process' +import { clearTimeout, setTimeout } from 'node:timers' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' + +function checkTimeout(ms) { + if (!Number.isFinite(ms) || ms <= 0 || ms > 2147483647) { + throw new RangeError('Timeout must be positive and at most 2147483647 ms') + } +} + +function cancellation(signal) { + return signal.reason ?? new Error('Operation aborted') +} + +// This bounds waiting, not the underlying operation. Callers must arrange its cleanup. +export function bounded(promise, ms, label = 'Operation', signal) { + return new Promise((resolve, reject) => { + let timer + const finish = (callback, value) => { + clearTimeout(timer) + signal?.removeEventListener('abort', abort) + callback(value) + } + const abort = () => finish(reject, cancellation(signal)) + // Observe even an already rejected promise when cancellation wins the race. + Promise.resolve(promise).then( + (value) => finish(resolve, value), + (error) => finish(reject, error), + ) + try { + checkTimeout(ms) + } catch (error) { + reject(error) + return + } + if (signal?.aborted) { + abort() + return + } + signal?.addEventListener('abort', abort, { once: true }) + timer = setTimeout(() => finish(reject, new Error(`${label} timed out after ${ms} ms`)), ms) + }) +} + +// Only an explicit falsy result means "not ready"; probe errors are never retried. +export async function waitFor(asyncProbe, { timeoutMs = 60000, signal, label = 'Readiness' } = {}) { + checkTimeout(timeoutMs) + const deadline = performance.now() + timeoutMs + for (;;) { + if (signal?.aborted) throw cancellation(signal) + const remaining = deadline - performance.now() + if (remaining <= 0) throw new Error(`${label} timed out after ${timeoutMs} ms`) + const result = await bounded( + Promise.resolve().then(() => { + if (signal?.aborted) throw cancellation(signal) + return asyncProbe() + }), + remaining, + label, + signal, + ) + if (result) return result + const pause = Math.min(50, deadline - performance.now()) + if (pause > 0) await delay(pause, undefined, { signal }) + } +} + +// Linux may retain orphan zombies until PID 1 reaps them. They cannot run or hold +// resources, and Node can only reap its own direct children. Other POSIX hosts +// use the process-group existence check without relying on /proc. +async function hasLiveMembers(pgid) { + if (process.platform !== 'linux') return true + const entries = (await readdir('/proc')).filter((entry) => /^\d+$/.test(entry)) + // Bound concurrent descriptor use without paying one async round trip per PID. + for (let offset = 0; offset < entries.length; offset += 64) { + const live = await Promise.all( + entries.slice(offset, offset + 64).map(async (entry) => { + try { + const stat = await readFile(`/proc/${entry}/stat`, 'utf8') + const [state, , group] = stat.slice(stat.lastIndexOf(')') + 2).split(' ') + return Number(group) === pgid && state !== 'Z' && state !== 'X' + } catch (error) { + if (error.code !== 'ENOENT' && error.code !== 'ESRCH') throw error + return false + } + }), + ) + if (live.some(Boolean)) return true + } + return false +} + +export function createLifecycle({ signal, cleanupTimeoutMs = 10000 } = {}) { + checkTimeout(cleanupTimeoutMs) + const resources = [] + const groups = new Set() + const errors = [] + let cleaning = false + let forced = false + let cleanupPromise + + const report = (label, cause) => { + const error = new Error(`${label}: ${cause?.message ?? String(cause)}`, { cause }) + errors.push(error) + return error + } + + function assertOpen() { + if (signal?.aborted) throw cancellation(signal) + if (cleaning || forced) throw new Error('Lifecycle is closing; new resources are not allowed') + } + + function defer(label, asyncCleanup) { + assertOpen() + if (typeof asyncCleanup !== 'function') throw new TypeError('Cleanup must be a function') + resources.push({ label, run: asyncCleanup }) + } + + function groupExists(record, retire = true) { + if (record.retired || !record.child.pid) return false + try { + process.kill(-record.child.pid, 0) + return true + } catch (error) { + if (error.code !== 'ESRCH') throw error + if (retire) { + record.retired = true + groups.delete(record) + } + return false + } + } + + // Read-only guard for dependent resources such as browser profiles. Unknown + // process state (including /proc permission failures) must preserve the resource. + async function assertProcessesStopped() { + for (const record of groups) { + if (record.uncertain) throw new Error('Supervisor ownership was lost; preserve its profile') + if (groupExists(record, false) && (await hasLiveMembers(record.child.pid))) { + throw new Error(`Process group ${record.child.pid} still has live members`) + } + } + } + + function killGroup(record, signalName) { + record.request(signalName === 'SIGKILL' ? 'force' : 'stop') + } + + function spawn(command, args = [], { logPath, cwd, env } = {}) { + assertOpen() + if (process.platform === 'win32') throw new Error('Smoke process groups require a POSIX host') + const child = spawnChild( + process.execPath, + [fileURLToPath(new URL('./supervisor.mjs', import.meta.url))], + { + detached: true, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }, + ) + const record = { child, retired: false, uncertain: false } + // Register ownership synchronously, before opening logs or awaiting readiness. + groups.add(record) + let ended = false + let failure + let tail = '' + let log + let logClosed = Promise.resolve() + let supervisorEnded = false + let ready = false + let requested + let startSent = false + let targetStarted = false + let terminating = false + let graceMs = 100 + let targetResult + let resolveTarget + let rejectTarget + const fail = (label, error) => { + const reported = report(label, error) + failure ??= reported + } + const exited = new Promise((resolve, reject) => { + resolveTarget = resolve + rejectTarget = reject + }) + // Some callers only use assertRunning; never leave a spawn rejection unhandled. + exited.catch(() => {}) + function lost(error) { + record.uncertain = true + fail('Supervisor control', error) + rejectTarget(failure) + } + function send(message) { + if (!child.connected) { + if (!supervisorEnded) lost(new Error('Control channel unavailable')) + return + } + try { + child.send({ version: 1, ...message }, (error) => { + if (error && !supervisorEnded) { + lost(error) + if (child.connected) child.disconnect() + } + }) + } catch (error) { + lost(error) + if (child.connected) child.disconnect() + } + } + record.request = (type) => { + if (supervisorEnded || record.retired) return + if (requested === 'force' || (requested === 'stop' && type === 'stop')) return + requested = type + if (ready) send({ type, graceMs }) + } + child.on('message', (message) => { + const invalid = () => { + lost(new Error('Invalid supervisor message')) + record.request('force') + } + if (!message || message.version !== 1) { + invalid() + return + } + if (message.type === 'ready') { + if (ready) return invalid() + ready = true + if (!requested && (cleaning || forced || signal?.aborted || record.uncertain)) { + record.request(forced ? 'force' : 'stop') + return + } + if (requested) send({ type: requested, graceMs }) + else { + startSent = true + send({ type: 'start', command, args, cwd, env }) + } + } else if (message.type === 'started') { + if ( + !startSent || + targetStarted || + ended || + !Number.isSafeInteger(message.pid) || + message.pid <= 0 + ) { + return invalid() + } + targetStarted = true + } else if (message.type === 'target-exit') { + if ( + !targetStarted || + ended || + !( + (Number.isInteger(message.code) && message.signal === null) || + (message.code === null && typeof message.signal === 'string') + ) + ) + return invalid() + ended = true + targetResult = { code: message.code, signal: message.signal } + resolveTarget(targetResult) + } else if (message.type === 'start-error') { + if (!startSent || ended || typeof message.message !== 'string') return invalid() + ended = true + fail( + `Process ${command}`, + Object.assign(new Error(message.message), { code: message.code }), + ) + rejectTarget(failure) + } else if (message.type === 'terminating') { + if (!ready || terminating) return invalid() + terminating = true + } else invalid() + }) + let resolveIpcClosed + const ipcClosed = new Promise((resolve) => { + resolveIpcClosed = resolve + }) + const supervisorExit = new Promise((resolve) => { + child.once('exit', (code, signalName) => { + supervisorEnded = true + resolve({ code, signalName }) + }) + child.once('error', (error) => { + lost(error) + if (!child.pid) { + supervisorEnded = true + resolveIpcClosed() + resolve({ code: null, signalName: null }) + } + }) + }) + child.on('disconnect', () => { + if (!requested && !supervisorEnded) lost(new Error('Control channel disconnected')) + resolveIpcClosed() + }) + // OS exit and IPC reads are independent. Process queued final messages + // before deciding that an acknowledgement or target result was lost. + const supervisorSettled = Promise.all([supervisorExit, ipcClosed]).then(([result]) => { + if (!terminating || result.signalName !== 'SIGKILL') { + lost(new Error(`Supervisor exited unexpectedly (${result.code ?? result.signalName})`)) + } + rejectTarget(new Error('Target exit status unavailable after supervisor termination')) + }) + // IPC disconnect does not reliably produce ChildProcess.close on all Node + // versions. Reaping and output draining are independent completion facts. + const drained = Promise.all( + [child.stdout, child.stderr].map( + (stream) => new Promise((resolve) => stream.once('close', resolve)), + ), + ) + + async function stop(budgetMs) { + const deadline = performance.now() + budgetMs + try { + graceMs = Math.min(1000, budgetMs / 2) + killGroup(record, forced ? 'SIGKILL' : 'SIGTERM') + // Reap the direct child and drain both pipes before flushing the log. + await bounded( + supervisorSettled, + Math.max(1, deadline - performance.now()), + `Reap ${command}`, + ) + await bounded(drained, Math.max(1, deadline - performance.now()), `Drain ${command}`) + if (record.uncertain) throw new Error('Supervisor ownership was lost') + while (groupExists(record)) { + if (!(await hasLiveMembers(child.pid))) { + record.retired = true + groups.delete(record) + break + } + if (performance.now() >= deadline) throw new Error('Process group did not terminate') + await delay(10) + } + log?.end() + await bounded(logClosed, Math.max(1, deadline - performance.now()), `Flush ${command} log`) + } finally { + // A timeout must not leave our pipes or file descriptors keeping Node alive. + child.stdout.destroy() + child.stderr.destroy() + log?.destroy() + if (!supervisorEnded) record.uncertain = true + if (child.connected) child.disconnect() + child.unref() + } + } + resources.push({ label: `Stop ${command}`, run: stop, record }) + + if (logPath !== undefined) { + try { + log = createWriteStream(logPath, { flags: 'a' }) + logClosed = new Promise((resolve) => log.once('close', resolve)) + log.on('error', (error) => { + fail(`Log ${logPath}`, error) + child.stdout.resume() + child.stderr.resume() + }) + log.on('drain', () => { + child.stdout.resume() + child.stderr.resume() + }) + } catch (error) { + fail(`Log ${logPath}`, error) + } + } + for (const stream of [child.stdout, child.stderr]) { + stream.setEncoding('utf8') + stream.on('error', (error) => fail(`Output ${command}`, error)) + stream.on('data', (chunk) => { + // Keep diagnostics bounded while preserving complete output on disk. + tail = (tail + chunk).slice(-1024 * 1024) + if (log && !log.destroyed && !log.writableEnded && !log.write(chunk)) { + child.stdout.pause() + child.stderr.pause() + } + }) + } + return { + child, + exited, + output: () => tail, + assertRunning() { + assertOpen() + if (failure) throw failure + if (ended || supervisorEnded) { + throw new Error( + `Process ${command} exited unexpectedly (${ + targetResult?.code ?? targetResult?.signal ?? 'target status unavailable' + })\n${tail}`, + ) + } + }, + } + } + + function cleanup() { + if (cleanupPromise) return cleanupPromise + cleaning = true + signal?.removeEventListener('abort', onAbort) + // Defer invocation until the shared promise is installed, including reentrant cleanup. + cleanupPromise = Promise.resolve().then(async () => { + const deadline = performance.now() + cleanupTimeoutMs + while (resources.length) { + // Share one deadline fairly, so a stalled resource cannot starve later cleanup. + const remaining = deadline - performance.now() + const budgetMs = Math.max(1, remaining / resources.length) + const { label, run, record } = resources.pop() + try { + const operation = Promise.resolve().then(() => (record ? run(budgetMs) : run())) + if (remaining <= 0) { + // Invoke independent cleanup, but do not add a timer per resource + // after the overall deadline. The timeout remains an explicit failure. + operation.catch(() => {}) + throw new Error('Overall cleanup deadline exceeded') + } + await bounded(operation, budgetMs, label) + } catch (error) { + report(label, error) + if (record) { + try { + killGroup(record, 'SIGKILL') + } catch (killError) { + report(label, killError) + } + } + } + } + return errors + }) + return cleanupPromise + } + + function force() { + forced = true + for (const record of groups) { + try { + killGroup(record, 'SIGKILL') + } catch (error) { + report('Force process group', error) + } + } + return cleanup() + } + + const onAbort = () => { + cleanup() + } + if (signal?.aborted) cleanup() + else signal?.addEventListener('abort', onAbort, { once: true }) + return { defer, spawn, cleanup, force, assertProcessesStopped } +} diff --git a/scripts/smoke/mock-server.mjs b/scripts/smoke/mock-server.mjs new file mode 100644 index 000000000..76104fcb8 --- /dev/null +++ b/scripts/smoke/mock-server.mjs @@ -0,0 +1,120 @@ +import { createServer } from 'node:http' +import { Buffer } from 'node:buffer' +import { bounded } from './lifecycle.mjs' + +export async function startMockServer({ lifecycle, signal }) { + signal?.throwIfAborted() + const requests = [] + const sockets = new Set() + const pending = new Set() + let released = false + let closing + const event = (content, finishReason = null) => + `data: ${JSON.stringify({ + choices: [{ delta: { content }, finish_reason: finishReason }], + })}\r\n\r\n` + const finish = (response) => { + pending.delete(response) + if (response.destroyed) return + const bytes = Buffer.from(event('δΈ–η•ŒπŸ™‚', 'stop') + 'data: [DONE]\r\n\r\n') + // Exercise split UTF-8 and CRLF writes without assuming network read boundaries. + for (const byte of bytes) response.write(Buffer.from([byte])) + response.end() + } + const server = createServer((request, response) => { + response.on('error', () => response.destroy()) + response.setHeader('Access-Control-Allow-Origin', '*') + response.setHeader('Access-Control-Allow-Headers', 'content-type, authorization') + response.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS') + if (request.method === 'OPTIONS') { + response.writeHead(204).end() + return + } + const entry = { method: request.method, path: request.url, body: null, status: null } + requests.push(entry) + let body = '' + request.setEncoding('utf8') + request.on('error', () => response.destroy()) + request.on('data', (chunk) => { + body += chunk + if (Buffer.byteLength(body) > 65536) request.destroy() + }) + request.on('end', () => { + const json = (status, value) => { + entry.status = status + response.writeHead(status, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(value)) + } + if ( + request.method !== 'POST' || + !['/success/v1/chat/completions', '/error/v1/chat/completions'].includes(request.url) + ) { + json(404, { error: { message: 'Unknown smoke endpoint' } }) + return + } + try { + entry.body = JSON.parse(body) + } catch { + json(400, { error: { message: 'Invalid JSON' } }) + return + } + if ( + request.headers.authorization !== 'Bearer smoke-local-only' || + entry.body?.model !== 'smoke-model' || + entry.body?.stream !== true || + !Array.isArray(entry.body?.messages) + ) { + json(400, { error: { message: 'Invalid Chat Completions request' } }) + return + } + if (request.url === '/error/v1/chat/completions') { + json(503, { error: { message: 'Smoke upstream unavailable', code: 'smoke_503' } }) + return + } + entry.status = 200 + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + }) + pending.add(response) + response.once('close', () => pending.delete(response)) + response.write(event('Hello ')) + if (released) finish(response) + }) + }) + server.on('connection', (socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + }) + let listening + const close = () => { + closing ??= (async () => { + // A cancelled startup may still finish listening; close it before returning. + await listening?.catch(() => {}) + const closed = new Promise((resolve, reject) => { + server.close((error) => { + if (error && error.code !== 'ERR_SERVER_NOT_RUNNING') reject(error) + else resolve() + }) + }) + for (const socket of sockets) socket.destroy() + await closed + })() + return closing + } + lifecycle.defer('mock HTTP server', close) + listening = new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + await bounded(listening, 10000, 'mock HTTP listen', signal) + return { + baseUrl: `http://127.0.0.1:${server.address().port}`, + requests, + release() { + released = true + for (const response of pending) finish(response) + }, + close, + } +} diff --git a/scripts/smoke/runner.mjs b/scripts/smoke/runner.mjs new file mode 100644 index 000000000..762902cb4 --- /dev/null +++ b/scripts/smoke/runner.mjs @@ -0,0 +1,357 @@ +import process from 'node:process' +import { constants, mkdtempSync, writeFileSync } from 'node:fs' +import { access, lstat, mkdir, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { fileURLToPath } from 'node:url' +import { bounded, createLifecycle } from './lifecycle.mjs' +import { startChromium } from './chromium.mjs' +import { startFirefox } from './firefox.mjs' +import { runScenarios } from './scenarios.mjs' +import { hashArtifact } from './artifacts.mjs' + +export { hashArtifact } + +export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +export const HELP = `Usage: npm run smoke -- [options] + --browser all|chromium|firefox Selected browsers (default: all, run serially) + --chromium-path PATH Full Chromium or Chrome for Testing executable + --firefox-path PATH Firefox executable + --geckodriver-path PATH Native geckodriver executable (not an npm wrapper) + --artifacts-dir PATH Parent directory for a unique retained run directory + --help Show this help + +Requires Linux, Node 22+, and production builds in this worktree. +No downloads, dependency installation, or builds are performed automatically. +Exit: 0 pass; 1 test/runtime/cleanup failure; 2 preflight; 130 SIGINT; 143 SIGTERM. +` + +export function parseArgs(args) { + const options = { browser: 'all' } + const allowed = new Set([ + 'browser', + 'chromium-path', + 'firefox-path', + 'geckodriver-path', + 'artifacts-dir', + ]) + const seen = new Set() + for (let index = 0; index < args.length; index++) { + const key = args[index].slice(2) + if (args[index] === '--help') { + options.help = true + continue + } + if (!args[index].startsWith('--') || !allowed.has(key) || seen.has(key)) { + throw new Error(`Unknown or repeated option: ${args[index]}`) + } + const value = args[++index] + if (!value || value.startsWith('--')) throw new Error(`Missing value for --${key}`) + seen.add(key) + options[key] = value + } + if (!['all', 'chromium', 'firefox'].includes(options.browser)) + throw new Error(`Invalid browser: ${options.browser}`) + return options +} + +export async function findExecutable(explicit, candidates, pathValue = process.env.PATH || '') { + const paths = explicit + ? [resolve(explicit)] + : pathValue + .split(delimiter) + .filter(Boolean) + .flatMap((dir) => candidates.map((name) => join(dir, name))) + for (const path of paths) { + try { + await access(path, constants.X_OK) + if ((await stat(path)).isFile()) return resolve(path) + } catch (error) { + if (!['ENOENT', 'EACCES', 'ENOTDIR'].includes(error.code)) throw error + } + } + throw new Error( + `Executable not found: ${explicit || candidates.join(' / ')}; specify its --*-path`, + ) +} + +export async function preflight(options, root = ROOT, signal) { + signal?.throwIfAborted() + if (process.platform !== 'linux') throw new Error('Smoke currently supports Linux only') + if (Number(process.versions.node.split('.')[0]) < 22) throw new Error('Node 22+ is required') + const browsers = options.browser === 'all' ? ['chromium', 'firefox'] : [options.browser] + const jobs = [] + for (const browser of browsers) { + signal?.throwIfAborted() + const executable = await findExecutable( + options[`${browser}-path`], + browser === 'chromium' ? ['chromium', 'chromium-browser', 'chrome-for-testing'] : ['firefox'], + ) + const extensionDir = join(root, 'build', browser) + if (browser === 'firefox' && !(await lstat(extensionDir)).isDirectory()) + throw new Error('Firefox build source must be a directory') + for (const name of ['manifest.json', 'background.js', 'popup.html', 'popup.js']) { + if (!(await stat(join(extensionDir, name))).isFile()) + throw new Error(`Missing build file: ${name}`) + } + const manifest = JSON.parse( + await readFile(join(extensionDir, 'manifest.json'), { encoding: 'utf8', signal }), + ) + if (!manifest.version || !manifest.name) throw new Error(`Invalid ${browser} manifest`) + const job = { + browser, + executable, + extensionDir, + manifest, + artifactSha256: await hashArtifact(extensionDir, signal), + } + if (browser === 'firefox') { + job.geckodriver = await findExecutable(options['geckodriver-path'], ['geckodriver']) + // npm launchers may download binaries; require an already installed native driver. + const header = (await readFile(job.geckodriver, { signal })).subarray(0, 4) + if (header.toString('hex') !== '7f454c46') + throw new Error('Specify a native Linux geckodriver binary, not a downloading wrapper') + } + signal?.throwIfAborted() + jobs.push(job) + } + return jobs +} + +// Resolve existing symlinks even when the requested directory has not been created yet. +async function resolveFuturePath(path) { + try { + return await realpath(path) + } catch (error) { + if (error.code !== 'ENOENT') throw error + // A dangling symlink is not a missing directory that mkdir can safely create. + const entry = await lstat(path).catch((failure) => { + if (failure.code !== 'ENOENT') throw failure + }) + if (entry || dirname(path) === path) throw error + return join(await resolveFuturePath(dirname(path)), basename(path)) + } +} + +export async function validateArtifactParent(parent, browsers, root = ROOT) { + const resolvedParent = await resolveFuturePath(resolve(parent)) + for (const browser of browsers) { + const build = await resolveFuturePath(join(root, 'build', browser)) + const nesting = relative(build, resolvedParent) + if (!nesting || (!isAbsolute(nesting) && nesting !== '..' && !nesting.startsWith(`..${sep}`))) + throw new Error(`Artifact parent must be outside the selected ${browser} build`) + } + // Ancestors are safe: mkdtemp creates a new, exclusive sibling of any existing build. + return resolvedParent +} + +export async function run(args) { + let options + try { + options = parseArgs(args) + } catch (error) { + console.error(error.message) + return 2 + } + if (options.help) { + console.log(HELP) + return 0 + } + const controller = new AbortController() + let lifecycle = createLifecycle({ signal: controller.signal }) + let signalCode + let activeAdapter + let artifactsDir + const report = { + result: 'FAIL', + selectedBrowsers: options.browser === 'all' ? ['chromium', 'firefox'] : [options.browser], + nodeVersion: process.version, + root: ROOT, + browsers: [], + failedStage: 'preflight', + cleanup: [], + } + function abort(error, code) { + signalCode ||= code + if (controller.signal.aborted) { + lifecycle.force() + return + } + controller.abort(error) + } + const onInterrupt = () => abort(new Error('Interrupted by SIGINT'), 130) + const onTerminate = () => abort(new Error('Interrupted by SIGTERM'), 143) + const onFatal = (error) => abort(error instanceof Error ? error : new Error(String(error)), 1) + process.on('SIGINT', onInterrupt) + process.on('SIGTERM', onTerminate) + process.on('uncaughtException', onFatal) + process.on('unhandledRejection', onFatal) + process.stdout.on('error', onFatal) + process.stderr.on('error', onFatal) + let exitCode = 1 + async function cleanupBrowser() { + const errors = await lifecycle.cleanup() + report.cleanup = [ + ...new Set([...report.cleanup, ...errors.map((error) => String(error.stack || error))]), + ] + return errors + } + function applyLateSignal() { + if (!signalCode || exitCode === signalCode) return + exitCode = signalCode + report.exitCode = exitCode + report.result = 'FAIL' + report.failedStage ||= 'reporting' + // No await between this final signal snapshot, report update, and returning the status. + if (artifactsDir) + writeFileSync(join(artifactsDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`) + } + try { + const parent = await validateArtifactParent( + options['artifacts-dir'] || tmpdir(), + report.selectedBrowsers, + ) + controller.signal.throwIfAborted() + await mkdir(parent, { recursive: true }) + controller.signal.throwIfAborted() + artifactsDir = mkdtempSync(join(parent, 'chatgptbox-smoke-')) + report.artifactsDir = artifactsDir + const jobs = await bounded( + preflight(options, ROOT, controller.signal), + 60000, + 'Preflight', + controller.signal, + ) + report.failedStage = 'startup' + await bounded( + (async () => { + for (const job of jobs) { + controller.signal.throwIfAborted() + if (report.browsers.length) lifecycle = createLifecycle({ signal: controller.signal }) + const browserDir = join(artifactsDir, job.browser) + await mkdir(browserDir) + controller.signal.throwIfAborted() + // Keep acquisition and registration synchronous so cancellation cannot leak a profile. + const profileDir = mkdtempSync(join(tmpdir(), `chatgptbox-${job.browser}-`)) + const profileLifecycle = lifecycle + lifecycle.defer(`${job.browser} profile`, async () => { + await profileLifecycle.assertProcessesStopped() + await rm(profileDir, { recursive: true, force: true }) + }) + const browserReport = { + browser: job.browser, + executable: job.executable, + artifactSha256: job.artifactSha256, + ...(job.browser === 'firefox' + ? { + preflightManifestVersion: job.manifest.version, + snapshotDir: join(browserDir, 'extension'), + } + : { manifestVersion: job.manifest.version }), + profileDir, + checks: [], + } + report.browsers.push(browserReport) + report.failedStage = `${job.browser}:startup` + activeAdapter = await bounded( + (job.browser === 'chromium' ? startChromium : startFirefox)({ + ...job, + profileDir, + artifactsDir: browserDir, + lifecycle, + signal: controller.signal, + }), + 60000, + `${job.browser} startup`, + controller.signal, + ) + Object.assign(browserReport, activeAdapter.metadata) + report.failedStage = `${job.browser}:scenarios` + Object.assign( + browserReport, + await runScenarios(activeAdapter, { + lifecycle, + signal: controller.signal, + checks: browserReport.checks, + }), + ) + report.failedStage = `${job.browser}:shutdown` + await activeAdapter.close() + activeAdapter = undefined + report.failedStage = `${job.browser}:cleanup` + const cleanupErrors = await cleanupBrowser() + if (cleanupErrors.length) + throw new AggregateError(cleanupErrors, `${job.browser} cleanup failed`) + } + })(), + 300000, + 'Smoke run', + controller.signal, + ) + exitCode = 0 + report.failedStage = null + } catch (error) { + exitCode = signalCode || (report.failedStage === 'preflight' ? 2 : 1) + report.error = String(error.stack || error) + if (!artifactsDir && !process.stderr.destroyed) console.error(String(error.message || error)) + report.cleanup.push( + ...(error.cleanupErrors || []).map((failure) => String(failure.stack || failure)), + ) + if (activeAdapter && !controller.signal.aborted) { + try { + await bounded(activeAdapter.capture('failure'), 3000, 'Failure capture') + } catch (captureError) { + report.captureError = String(captureError) + } + } + } finally { + controller.abort(new Error('Smoke run finished')) + await cleanupBrowser() + if (report.cleanup.length && exitCode === 0) { + exitCode = 1 + report.failedStage = 'cleanup' + } + exitCode = signalCode || exitCode + report.exitCode = exitCode + report.result = exitCode === 0 ? 'PASS' : 'FAIL' + try { + if (artifactsDir) + await writeFile(join(artifactsDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`) + applyLateSignal() + await new Promise((resolve, reject) => { + process.stdout.write( + `Smoke report; artifacts: ${artifactsDir || 'unavailable'}\n`, + (error) => (error ? reject(error) : resolve()), + ) + }) + } catch (error) { + exitCode = signalCode || 1 + report.exitCode = exitCode + report.result = 'FAIL' + report.failedStage ||= 'reporting' + report.reportingError = String(error.stack || error) + if (artifactsDir) { + try { + await writeFile(join(artifactsDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`) + } catch (writeError) { + if (!process.stderr.destroyed) + console.error(`Cannot write smoke report: ${writeError.message}`) + } + } + if (!process.stderr.destroyed) console.error(`Smoke reporting failed: ${error.message}`) + } + try { + applyLateSignal() + } catch (error) { + exitCode = signalCode || 1 + if (!process.stderr.destroyed) console.error(`Cannot finalize smoke report: ${error.message}`) + } + process.removeListener('SIGINT', onInterrupt) + process.removeListener('SIGTERM', onTerminate) + process.removeListener('uncaughtException', onFatal) + process.removeListener('unhandledRejection', onFatal) + process.stdout.removeListener('error', onFatal) + process.stderr.removeListener('error', onFatal) + } + return exitCode +} diff --git a/scripts/smoke/scenarios.mjs b/scripts/smoke/scenarios.mjs new file mode 100644 index 000000000..bd5a0cf88 --- /dev/null +++ b/scripts/smoke/scenarios.mjs @@ -0,0 +1,309 @@ +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import { bounded, waitFor } from './lifecycle.mjs' +import { startMockServer } from './mock-server.mjs' + +export function assertPopupIdentity(identity, metadata) { + assert.ok(metadata.expectedName, 'Adapter metadata must include expectedName') + assert.ok(metadata.expectedVersion, 'Adapter metadata must include expectedVersion') + assert.ok(metadata.extensionId, 'Adapter metadata must include the installed extensionId') + assert.ok(metadata.popupUrl, 'Adapter metadata must include the installed popupUrl') + assert.equal(identity.visible, true, 'Installed popup tabs must be visible') + assert.equal( + identity.extensionId, + metadata.extensionId, + 'Runtime identity must match installation', + ) + assert.equal( + identity.name, + metadata.expectedName, + 'Runtime name must match the installed manifest', + ) + assert.equal( + identity.version, + metadata.expectedVersion, + 'Runtime version must match the installed manifest', + ) + assert.equal( + identity.location, + identity.popupUrl, + 'The evaluated page must be the manifest popup', + ) + assert.equal(identity.popupUrl, metadata.popupUrl) +} + +export function assertScenarioMessages(messages, { phase, question, history = [] }) { + const errors = messages.filter((message) => message.error) + const done = messages.filter((message) => message.done === true) + const answers = messages.filter((message) => message.answer != null) + const acknowledgements = messages.filter((message) => message.session && message.done !== true) + assert.equal( + acknowledgements.length, + 1, + 'The background must acknowledge the initial session once', + ) + const acknowledgement = acknowledgements[0] + assert.equal(acknowledgement.answer == null && !acknowledgement.error, true) + assert.deepEqual( + acknowledgement.session.conversationRecords, + history, + 'The initial acknowledgement must preserve history', + ) + assert.ok( + messages.indexOf(acknowledgement) < + messages.findIndex((message) => message.answer != null || message.error), + 'The initial acknowledgement must precede the first answer or error', + ) + if (phase === 'error') { + assert.equal(errors.length, 1, 'HTTP failure must emit one error') + assert.match(String(errors[0].error), /smoke_503|Smoke upstream unavailable/) + assert.equal(done.length, 0, 'HTTP failure must not complete successfully') + assert.equal(answers.length, 0, 'HTTP failure must not emit an answer') + return + } + assert.equal(errors.length, 0, 'Successful stream must not emit errors') + if (phase === 'partial') { + assert.equal(done.length, 0, 'The gated stream must still be live') + assert.deepEqual( + answers.map(({ answer }) => answer), + ['Hello '], + 'The browser must observe exactly the first delta', + ) + return + } + assert.equal(phase, 'final') + assert.equal(done.length, 1, 'The stream must complete exactly once') + // Mock content events, not network chunks, determine the cumulative answers. + assert.deepEqual( + answers.map(({ answer }) => answer), + ['Hello ', 'Hello δΈ–η•ŒπŸ™‚'], + 'Every cumulative answer must match the mock stream exactly once and in order', + ) + assert.equal(done[0].answer, null, 'Completion must not carry answer content') + assert.ok( + messages.indexOf(done[0]) > messages.indexOf(answers.at(-1)), + 'Completion must follow every answer', + ) + assert.deepEqual( + done[0].session?.conversationRecords, + [...history, { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }], + 'The stream must add exactly one complete history record', + ) +} + +// These functions are serialized into the installed popup and must remain self-contained. +function popupIdentity() { + const api = globalThis.browser || globalThis.chrome + if (!api?.runtime?.id) throw new Error('Missing extension runtime in popup') + const manifest = api.runtime.getManifest() + const visible = [...document.querySelectorAll('#app [role="tab"]')].some((tab) => { + const style = globalThis.getComputedStyle(tab) + return ( + [...tab.getClientRects()].some((rect) => rect.width > 0 && rect.height > 0) && + style.visibility !== 'hidden' && + style.display !== 'none' && + style.opacity !== '0' + ) + }) + return { + visible, + extensionId: api.runtime.id, + name: manifest.name, + version: manifest.version, + popupUrl: api.runtime.getURL( + manifest.action?.default_popup || manifest.browser_action?.default_popup, + ), + location: globalThis.location.href, + } +} + +async function configure(baseUrl) { + const api = globalThis.browser || globalThis.chrome + await api.storage.local.set({ + preferredLanguage: 'en', + customOpenAIProviders: ['success', 'error'].map((name) => ({ + id: `smoke-${name}`, + name: `Smoke ${name}`, + baseUrl: `${baseUrl}/${name}/v1`, + chatCompletionsPath: '/chat/completions', + enabled: true, + })), + providerSecrets: { + 'smoke-success': 'smoke-local-only', + 'smoke-error': 'smoke-local-only', + }, + maxConversationContextLength: 4, + maxResponseTokenLength: 64, + temperatureOverrideEnabled: false, + }) +} + +export function beginRequest(key, question, history) { + const api = globalThis.browser || globalThis.chrome + const state = (window.__chatGPTBoxSmoke ??= {}) + if (state[key]) throw new Error(`Smoke request already started: ${key}`) + const port = api.runtime.connect({ name: `smoke-${key}` }) + const request = (state[key] = { port, messages: [], disconnected: false }) + port.onMessage.addListener((message) => request.messages.push(message)) + port.onDisconnect.addListener(() => { + request.disconnected = true + request.disconnectError = api.runtime.lastError?.message || 'Unexpected port disconnect' + }) + port.postMessage({ + session: { + question, + modelName: 'customModel', + conversationRecords: history, + isRetry: false, + apiMode: { + groupName: 'customApiModelKeys', + itemName: `Smoke ${key}`, + isCustom: true, + providerId: `smoke-${key}`, + customName: 'smoke-model', + customUrl: '', + apiKey: '', + active: true, + }, + }, + }) +} + +export function snapshot(key) { + const request = window.__chatGPTBoxSmoke?.[key] + if (!request) throw new Error(`Missing smoke request: ${key}`) + if (request.disconnected) throw new Error(request.disconnectError) + return request.messages +} + +export function disconnectPorts() { + const state = window.__chatGPTBoxSmoke + if (!state) return + const errors = [] + try { + for (const request of Object.values(state)) { + try { + request.port.disconnect() + } catch (error) { + errors.push(error) + } + } + } finally { + delete window.__chatGPTBoxSmoke + } + if (errors.length) throw new AggregateError(errors, 'Failed to disconnect smoke ports') +} + +export async function runScenarios(adapter, { lifecycle, signal, checks = [] }) { + const evaluate = (fn, ...args) => { + signal?.throwIfAborted() + return bounded(adapter.evaluate(fn, ...args), 10000, `popup ${fn.name}`, signal) + } + const identity = await waitFor( + async () => { + const value = await evaluate(popupIdentity) + return value.visible && value + }, + { signal, label: 'visible installed popup tabs' }, + ) + assertPopupIdentity(identity, adapter.metadata) + checks.push('Installed popup visible with matching runtime identity and version') + + const mock = await startMockServer({ lifecycle, signal }) + let disconnected = false + let disconnecting + const disconnect = async () => { + if (disconnected) return + if (disconnecting) return disconnecting + disconnecting = bounded( + Promise.resolve().then(() => adapter.evaluateCleanup(disconnectPorts)), + 5000, + 'disconnect smoke ports', + ) + try { + await disconnecting + disconnected = true + } finally { + disconnecting = undefined + } + } + lifecycle.defer('smoke runtime ports', disconnect) + let primaryError + try { + await evaluate(configure, mock.baseUrl) + const question = 'Smoke success question' + await evaluate(beginRequest, 'success', question, []) + const partial = await waitFor( + async () => { + const messages = await evaluate(snapshot, 'success') + if (messages.some((message) => message.error || message.done)) { + assertScenarioMessages(messages, { phase: 'partial', question }) + } + return messages.some((message) => typeof message.answer === 'string') && messages + }, + { signal, label: 'first streamed answer before release' }, + ) + assertScenarioMessages(partial, { phase: 'partial', question }) + checks.push('Partial Hello answer observed while the SSE response remains gated') + mock.release() + await waitFor( + async () => { + const messages = await evaluate(snapshot, 'success') + assert.equal(messages.filter((message) => message.error).length, 0) + return messages.some((message) => message.done) + }, + { signal, label: 'completed streamed answer' }, + ) + const history = [{ question, answer: 'Hello δΈ–η•ŒπŸ™‚' }] + await evaluate(beginRequest, 'error', 'Smoke error question', history) + await waitFor( + async () => { + const messages = await evaluate(snapshot, 'error') + assert.equal(messages.filter((message) => message.done).length, 0) + return messages.some((message) => message.error) + }, + { signal, label: 'HTTP 503 error from background' }, + ) + // Keep both ports open through the error round and a short drain window to catch late done messages. + await delay(150, undefined, { signal }) + assertScenarioMessages(await evaluate(snapshot, 'success'), { phase: 'final', question }) + checks.push('Exact Unicode answer, one completion, and one conversation record') + assertScenarioMessages(await evaluate(snapshot, 'error'), { phase: 'error', history }) + checks.push('HTTP 503 reported without completion or a new conversation record') + assert.equal(mock.requests.length, 2, 'Each scenario must issue exactly one HTTP request') + assert.deepEqual( + mock.requests.map(({ status }) => status), + [200, 503], + ) + assert.deepEqual(mock.requests[0].body.messages, [{ role: 'user', content: question }]) + assert.deepEqual(mock.requests[1].body.messages, [ + { role: 'user', content: question }, + { role: 'assistant', content: 'Hello δΈ–η•ŒπŸ™‚' }, + { role: 'user', content: 'Smoke error question' }, + ]) + } catch (error) { + primaryError = error instanceof Error ? error : new Error(String(error), { cause: error }) + } + const cleanupErrors = [] + for (const cleanup of [disconnect, () => mock.close()]) { + try { + await cleanup() + } catch (error) { + cleanupErrors.push( + error instanceof Error ? error : new Error(String(error), { cause: error }), + ) + } + } + if (primaryError) { + if (cleanupErrors.length) { + primaryError.cleanupErrors = [...(primaryError.cleanupErrors ?? []), ...cleanupErrors] + } + throw primaryError + } + if (cleanupErrors.length) { + const error = new AggregateError(cleanupErrors, 'Smoke scenario cleanup failed') + error.cleanupErrors = cleanupErrors + throw error + } + return { checks, requests: mock.requests } +} diff --git a/scripts/smoke/supervisor.mjs b/scripts/smoke/supervisor.mjs new file mode 100644 index 000000000..74d45a27d --- /dev/null +++ b/scripts/smoke/supervisor.mjs @@ -0,0 +1,102 @@ +// Internal entry point: lifecycle starts this process in a new POSIX session. +// Never use a remembered PID to signal a group, including on error paths. +import { spawn } from 'node:child_process' +import process from 'node:process' +import { clearTimeout, setTimeout } from 'node:timers' + +if (process.platform === 'win32' || typeof process.send !== 'function') { + throw new Error('The smoke supervisor requires an isolated POSIX IPC child') +} + +let started = false +let stopping = false +let forcing = false +let timer + +function force() { + if (forcing) return + forcing = true + stopping = true + clearTimeout(timer) + // The sender is itself a member, so this cannot select a recycled group. + // SIGKILL includes us: do not promise a subsequent target-exit message. + const kill = () => process.kill(0, 'SIGKILL') + timer = setTimeout(kill, 100) + if (process.connected) { + try { + process.send({ version: 1, type: 'terminating' }, kill) + } catch { + kill() + } + } else kill() +} + +function stop(graceMs = 100) { + if (stopping) return + stopping = true + timer = setTimeout(force, graceMs) + process.kill(0, 'SIGTERM') +} + +function send(message) { + if (!process.connected) { + stop() + return + } + process.send({ version: 1, ...message }, (error) => { + if (error) stop() + }) +} + +// Remain alive through the graceful group signal, including after target exit. +process.on('SIGTERM', () => {}) +process.on('SIGINT', () => stop()) +process.on('disconnect', () => stop()) +process.on('error', () => stop()) +process.on('message', (message) => { + if (!message || message.version !== 1) { + stop() + return + } + if (message.type === 'force') { + force() + return + } + if (message.type === 'stop') { + if (!Number.isFinite(message.graceMs) || message.graceMs < 0 || message.graceMs > 1000) { + stop() + return + } + stop(message.graceMs) + return + } + if (stopping) return + if ( + message.type !== 'start' || + started || + typeof message.command !== 'string' || + !Array.isArray(message.args) || + !message.args.every((arg) => typeof arg === 'string') + ) { + stop() + return + } + started = true + try { + const target = spawn(message.command, message.args, { + cwd: message.cwd, + env: message.env, + detached: false, + // The target must not inherit the private control channel. + stdio: ['ignore', 'inherit', 'inherit'], + }) + target.once('spawn', () => send({ type: 'started', pid: target.pid })) + target.once('error', (error) => + send({ type: 'start-error', message: error.message, code: error.code }), + ) + target.once('exit', (code, signal) => send({ type: 'target-exit', code, signal })) + } catch (error) { + send({ type: 'start-error', message: error.message, code: error.code }) + } +}) +send({ type: 'ready' }) diff --git a/scripts/xvfb-smoke.mjs b/scripts/xvfb-smoke.mjs new file mode 100644 index 000000000..d41abc153 --- /dev/null +++ b/scripts/xvfb-smoke.mjs @@ -0,0 +1,5 @@ +// Compatibility name: native headless browsers no longer require Xvfb. +import process from 'node:process' +import { run } from './smoke/runner.mjs' + +process.exitCode = await run(process.argv.slice(2)) diff --git a/tests/fixtures/smoke/fake-browser.fixture b/tests/fixtures/smoke/fake-browser.fixture new file mode 100644 index 000000000..cd320e60a --- /dev/null +++ b/tests/fixtures/smoke/fake-browser.fixture @@ -0,0 +1,11 @@ +#!/usr/bin/env node +// A deliberately unready browser used to exercise the production runner's cleanup. +const process = require('node:process') +const profile = process.argv + .find((arg) => arg.startsWith('--user-data-dir=')) + ?.slice('--user-data-dir='.length) +process.on('SIGTERM', () => {}) +// Failure-only safety net; successful tests must observe runner cleanup first. +setTimeout(() => process.exit(99), 30000).unref() +process.stdout.write(`${JSON.stringify({ pid: process.pid, profile })}\n`) +setInterval(() => {}, 1000) diff --git a/tests/fixtures/smoke/process.fixture b/tests/fixtures/smoke/process.fixture new file mode 100644 index 000000000..567b69b2c --- /dev/null +++ b/tests/fixtures/smoke/process.fixture @@ -0,0 +1,41 @@ +const { spawn } = require('node:child_process') +const process = require('node:process') +const { clearInterval, setInterval, setTimeout } = require('node:timers') + +const mode = process.argv[2] +if (mode === 'early-exit') { + process.stdout.write('stdout before exit\n') + process.stderr.write('stderr before exit\n') + process.exitCode = 7 +} else if (mode === 'descendant') { + const child = spawn(process.execPath, [__filename, 'ignore-term'], { + stdio: ['ignore', 'inherit', 'inherit', 'ipc'], + }) + child.on('message', (message) => { + if (message === 'ready') { + process.stdout.write(`DESCENDANT ${child.pid}\n`, () => process.exit(0)) + } + }) + child.on('error', () => process.exit(1)) +} else { + if (mode === 'ignore-term') { + process.on('SIGTERM', () => process.stdout.write('IGNORED TERM\n')) + } + function announce() { + process.stdout.write(`READY ${process.pid}\n`) + process.stderr.write('stderr ready\n') + if (process.send) process.send('ready') + } + if (mode === 'flood') { + Promise.all([ + new Promise((resolve) => process.stdout.write('O'.repeat(2 * 1024 * 1024), resolve)), + new Promise((resolve) => process.stderr.write('E'.repeat(2 * 1024 * 1024), resolve)), + ]).then(announce) + } else announce() + const timer = setInterval(() => {}, 1000) + // Fixture safety net: a failed test cannot leave a live process indefinitely. + setTimeout(() => { + clearInterval(timer) + process.exit(99) + }, 15000).unref() +} diff --git a/tests/unit/scripts/smoke-artifacts.test.mjs b/tests/unit/scripts/smoke-artifacts.test.mjs new file mode 100644 index 000000000..b5f1893cc --- /dev/null +++ b/tests/unit/scripts/smoke-artifacts.test.mjs @@ -0,0 +1,141 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import streams from 'node:fs' +import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { PassThrough } from 'node:stream' +import { setImmediate } from 'node:timers/promises' +import test from 'node:test' +import { hashArtifact, snapshotArtifact } from '../../../scripts/smoke/artifacts.mjs' +import { bounded } from '../../../scripts/smoke/lifecycle.mjs' + +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), 'smoke-artifacts-')) + t.after(() => rm(root, { recursive: true, force: true })) + const source = join(root, 'source') + const destination = join(root, 'snapshot') + await mkdir(source) + await mkdir(join(source, 'nested')) + await writeFile(join(source, 'nested/file'), 'original') + const hash = await hashArtifact(source) + return { root, source, destination, hash } +} + +test('snapshot includes nested files and is independent from later build writes', async (t) => { + const { source, destination, hash } = await fixture(t) + assert.equal(await snapshotArtifact(source, destination, hash), hash) + await writeFile(join(source, 'nested/file'), 'new build') + assert.equal(await readFile(join(destination, 'nested/file'), 'utf8'), 'original') + assert.equal(await hashArtifact(destination), hash) +}) + +for (const operation of ['add', 'remove']) { + test(`snapshot rejects an empty directory ${operation} after preflight`, async (t) => { + const { source, destination } = await fixture(t) + const empty = join(source, 'empty') + if (operation === 'remove') await mkdir(empty) + const hash = await hashArtifact(source) + if (operation === 'add') await mkdir(empty) + else await rm(empty, { recursive: true }) + assert.notEqual(await hashArtifact(source), hash) + await assert.rejects( + snapshotArtifact(source, destination, hash), + /Build changed since preflight/, + ) + }) +} + +test('binary contents cannot impersonate a second artifact entry', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'smoke-hash-framing-')) + t.after(() => rm(root, { recursive: true, force: true })) + const first = join(root, 'first') + const second = join(root, 'second') + await mkdir(first) + await mkdir(second) + await writeFile(join(first, 'a'), Buffer.from('first\0b\0second')) + await writeFile(join(second, 'a'), 'first') + await writeFile(join(second, 'b'), 'second') + const hash = await hashArtifact(first) + assert.notEqual(await hashArtifact(second), hash) + await assert.rejects( + snapshotArtifact(second, join(root, 'snapshot'), hash), + /Build changed since preflight/, + ) +}) + +test( + 'snapshot rejects symlinks instead of reading outside the build', + { skip: process.platform !== 'linux' }, + async (t) => { + const { root, source, destination, hash } = await fixture(t) + await writeFile(join(root, 'outside'), 'not a build artifact') + await symlink(join(root, 'outside'), join(source, 'link')) + await assert.rejects(snapshotArtifact(source, destination, hash), /non-regular build artifact/) + }, +) + +test('snapshot never overwrites an existing destination', async (t) => { + const { source, destination, hash } = await fixture(t) + await mkdir(destination) + await writeFile(join(destination, 'existing'), 'retain') + await assert.rejects(snapshotArtifact(source, destination, hash), { code: 'EEXIST' }) + assert.equal(await readFile(join(destination, 'existing'), 'utf8'), 'retain') +}) + +test('snapshot checks preflight hash and cancellation before acquiring a destination', async (t) => { + const { source, destination, hash } = await fixture(t) + await assert.rejects(snapshotArtifact(source, destination), /Missing preflight build hash/) + const controller = new AbortController() + const reason = new Error('Cancelled before snapshot') + controller.abort(reason) + await assert.rejects( + snapshotArtifact(source, destination, hash, controller.signal), + (error) => error === reason, + ) + await assert.rejects(stat(destination), { code: 'ENOENT' }) +}) + +test('snapshot cancellation waits for the transfer to close', { timeout: 5000 }, async (t) => { + const { source, destination, hash } = await fixture(t) + const started = Promise.withResolvers() + const destroying = Promise.withResolvers() + const release = Promise.withResolvers() + const input = new PassThrough({ + destroy(error, callback) { + destroying.resolve() + release.promise.then(() => callback(error)) + }, + }) + const mockRead = t.mock.method(streams, 'createReadStream', () => { + input.write('partial') + started.resolve() + return input + }) + syncBuiltinESMExports() + t.after(() => { + release.resolve() + input.destroy() + mockRead.mock.restore() + syncBuiltinESMExports() + }) + const controller = new AbortController() + const reason = new Error('Cancel the active transfer') + let settled = false + const rejected = assert.rejects( + snapshotArtifact(source, destination, hash, controller.signal).finally(() => { + settled = true + }), + (error) => error === reason, + ) + await bounded(started.promise, 1000, 'Snapshot transfer') + controller.abort(reason) + await bounded(destroying.promise, 1000, 'Transfer destruction') + await setImmediate() + assert.equal(settled, false, 'A pending close must not be abandoned') + release.resolve() + await bounded(rejected, 1000, 'Snapshot cancellation') + assert.equal(input.closed, true) +}) diff --git a/tests/unit/scripts/smoke-chromium.test.mjs b/tests/unit/scripts/smoke-chromium.test.mjs new file mode 100644 index 000000000..5aaf57655 --- /dev/null +++ b/tests/unit/scripts/smoke-chromium.test.mjs @@ -0,0 +1,746 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import { createHash } from 'node:crypto' +import { once } from 'node:events' +import fs, { mkdtemp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { writeFileSync } from 'node:fs' +import { createServer } from 'node:http' +import { syncBuiltinESMExports } from 'node:module' +import os from 'node:os' +import path from 'node:path' +import { test } from 'node:test' +import { clearTimeout, setTimeout } from 'node:timers' +import { runInNewContext } from 'node:vm' +import { connectCDP } from '../../../scripts/smoke/cdp.mjs' +import { startChromium } from '../../../scripts/smoke/chromium.mjs' +import { createLifecycle } from '../../../scripts/smoke/lifecycle.mjs' +import { hashArtifact } from '../../../scripts/smoke/artifacts.mjs' + +function fakeSocket(onSend, { open = true } = {}) { + const sockets = [] + class FakeWebSocket { + constructor(url) { + this.url = url + this.listeners = new Map() + this.closeCount = 0 + this.sent = [] + sockets.push(this) + if (open) globalThis.queueMicrotask(() => this.emit('open')) + } + addEventListener(type, fn) { + const listeners = this.listeners.get(type) ?? [] + listeners.push(fn) + this.listeners.set(type, listeners) + } + emit(type, event = {}) { + for (const listener of this.listeners.get(type) ?? []) listener(event) + } + reply(message) { + this.emit('message', { data: JSON.stringify(message) }) + } + send(data) { + const message = JSON.parse(data) + this.sent.push(message) + onSend?.(message, this) + } + close() { + this.closeCount++ + this.emit('close') + } + } + return { FakeWebSocket, sockets } +} + +function cleanupRegistry(t) { + const callbacks = [] + t.after(async () => { + for (const callback of callbacks.reverse()) await callback() + }) + return { + callbacks, + defer(label, cleanup) { + callbacks.push(cleanup) + }, + } +} + +test('CDP registers cleanup before handshake and closes a stalled connection', async (t) => { + const lifecycle = cleanupRegistry(t) + const { FakeWebSocket, sockets } = fakeSocket(undefined, { open: false }) + const connecting = connectCDP('ws://127.0.0.1:1234', { + lifecycle, + timeoutMs: 10, + WebSocketImpl: FakeWebSocket, + }) + assert.equal(lifecycle.callbacks.length, 1) + await assert.rejects(connecting, /Connect Chromium CDP timed out/) + await lifecycle.callbacks[0]() + await lifecycle.callbacks[0]() + assert.equal(sockets[0].closeCount, 1) +}) + +test('CDP does not open a transport when cleanup registration is rejected', async () => { + const { FakeWebSocket, sockets } = fakeSocket() + await assert.rejects( + connectCDP('ws://127.0.0.1:1234', { + lifecycle: { + defer() { + throw new Error('Lifecycle is already closing') + }, + }, + WebSocketImpl: FakeWebSocket, + }), + /Lifecycle is already closing/, + ) + assert.equal(sockets.length, 0) +}) + +test('CDP correlates concurrent responses and surfaces protocol errors without retry', async (t) => { + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + WebSocketImpl: FakeWebSocket, + }) + const socket = sockets[0] + const first = client.request('Target.createTarget', { url: 'about:blank' }) + const second = client.request('Runtime.evaluate', {}, 'popup-session') + socket.reply({ method: 'Runtime.consoleAPICalled', params: {} }) + socket.reply({ id: socket.sent[1].id, result: { value: 42 } }) + socket.reply({ id: socket.sent[0].id, error: { code: -1, message: 'Target denied' } }) + assert.deepEqual(await second, { value: 42 }) + await assert.rejects(first, /Target.createTarget: Target denied/) + assert.equal(socket.sent[1].sessionId, 'popup-session') + assert.equal(socket.sent.length, 2) +}) + +test('CDP request timeout ignores late responses and rejects pending requests on disconnect', async (t) => { + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + timeoutMs: 10, + WebSocketImpl: FakeWebSocket, + }) + await assert.rejects(client.request('Target.createTarget'), /Target.createTarget timed out/) + const next = client.request('Browser.getVersion') + sockets[0].reply({ id: sockets[0].sent[0].id, result: { wrong: true } }) + sockets[0].emit('close') + await assert.rejects(next, /disconnected/) + await assert.rejects(client.request('Browser.getVersion'), /disconnected/) + assert.equal(sockets[0].sent.length, 2) +}) + +test('CDP cancellation prevents new RPCs and permits bounded diagnostic requests', async (t) => { + const controller = new AbortController() + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + signal: controller.signal, + WebSocketImpl: FakeWebSocket, + }) + const pending = client.request('Runtime.evaluate') + controller.abort(new Error('Cancelled by test')) + await assert.rejects(pending, /Cancelled by test/) + await assert.rejects(client.request('Target.createTarget'), /Cancelled by test/) + const capture = client.request('Page.captureScreenshot', {}, 'popup', { signal: undefined }) + sockets[0].reply({ id: sockets[0].sent.at(-1).id, result: { data: 'png' } }) + assert.deepEqual(await capture, { data: 'png' }) + assert.equal(sockets[0].sent.length, 2) +}) + +test('CDP rejects malformed messages and missing results', async (t) => { + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + WebSocketImpl: FakeWebSocket, + }) + const missing = client.request('Browser.getVersion') + sockets[0].reply({ id: sockets[0].sent.at(-1).id }) + await assert.rejects(missing, /missing CDP result/) + const malformed = client.request('Runtime.evaluate') + sockets[0].emit('message', { data: 'not JSON' }) + await assert.rejects(malformed, /Invalid CDP WebSocket message/) +}) + +test('CDP run cancellation preserves an already pending independent cleanup RPC', async (t) => { + const controller = new AbortController() + const cleanup = new AbortController() + const { FakeWebSocket, sockets } = fakeSocket() + const client = await connectCDP('ws://127.0.0.1:1234', { + lifecycle: cleanupRegistry(t), + signal: controller.signal, + WebSocketImpl: FakeWebSocket, + }) + const normal = client.request('Runtime.evaluate') + const independent = client.request('Runtime.evaluate', {}, 'popup', { + signal: cleanup.signal, + timeoutMs: 1000, + }) + controller.abort(new Error('Run cancelled')) + await assert.rejects(normal, /Run cancelled/) + sockets[0].reply({ id: sockets[0].sent[0].id, result: { late: true } }) + sockets[0].reply({ id: sockets[0].sent[1].id, result: { cleaned: true } }) + assert.deepEqual(await independent, { cleaned: true }) + await assert.rejects( + client.request('Runtime.evaluate', {}, 'popup', { + signal: cleanup.signal, + timeoutMs: 10, + }), + /timed out after 10 ms/, + ) + await assert.rejects(client.request('Runtime.evaluate'), /Run cancelled/) + assert.equal(sockets[0].sent.length, 3) +}) + +test('Chromium cleanup evaluation survives cancellation and retains a five-second deadline', async (t) => { + const controller = new AbortController() + const fixture = await chromiumFixture(t, { signal: controller.signal }) + const adapter = await startChromium(fixture.options) + const normal = adapter.evaluate(() => new Promise(() => {})) + controller.abort(new Error('Run cancelled')) + await assert.rejects(normal, /Run cancelled/) + await assert.rejects( + Promise.resolve().then(() => adapter.evaluate(() => 1)), + /Run cancelled/, + ) + assert.equal(await adapter.evaluateCleanup((a, b) => a + b, 2, 3), 5) + await assert.rejects( + adapter.evaluateCleanup(async () => { + throw new Error('Cleanup failed') + }), + /Cleanup failed/, + ) + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + const stalled = adapter.evaluateCleanup(() => new Promise(() => {})) + const rejected = assert.rejects(stalled, /timed out after 5000 ms/) + t.mock.timers.tick(5001) + await rejected +}) + +test('CDP uses the Node builtin WebSocket against a local protocol peer', async (t) => { + const server = createServer() + const peers = new Set() + t.after(async () => { + for (const peer of peers) peer.destroy() + await new Promise((resolve) => server.close(resolve)) + }) + server.on('upgrade', (request, peer) => { + peers.add(peer) + peer.on('close', () => peers.delete(peer)) + const accept = createHash('sha1') + .update(`${request.headers['sec-websocket-key']}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`) + .digest('base64') + peer.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`, + ) + let input = Buffer.alloc(0) + peer.on('data', (chunk) => { + input = Buffer.concat([input, chunk]) + while (input.length >= 2) { + const opcode = input[0] & 15 + const length = input[1] & 127 + // The test sends only small, single-frame CDP commands and a close frame. + assert.ok(length < 126) + assert.ok(input[1] & 128) + if (input.length < 6 + length) return + const mask = input.subarray(2, 6) + const payload = Buffer.from(input.subarray(6, 6 + length)) + for (let i = 0; i < payload.length; i++) payload[i] ^= mask[i % 4] + input = input.subarray(6 + length) + if (opcode === 8) { + peer.end(Buffer.from([0x88, 0])) + return + } + assert.equal(opcode, 1) + const message = JSON.parse(payload.toString()) + assert.equal(message.method, 'Browser.getVersion') + const response = Buffer.from( + JSON.stringify({ id: message.id, result: { product: 'Chromium/native-transport' } }), + ) + peer.write(Buffer.concat([Buffer.from([0x81, response.length]), response])) + } + }) + }) + const listening = once(server, 'listening') + server.listen(0, '127.0.0.1') + await listening + const client = await connectCDP(`ws://127.0.0.1:${server.address().port}/devtools/browser/test`, { + lifecycle: cleanupRegistry(t), + }) + assert.deepEqual(await client.request('Browser.getVersion'), { + product: 'Chromium/native-transport', + }) + await client.close() +}) + +async function chromiumFixture( + t, + { + protocolError, + evaluationError, + screenshotError, + contextTransition, + partialPort, + signal, + key, + } = {}, +) { + const directory = await fs.realpath(await mkdtemp(path.join(os.tmpdir(), 'smoke-chromium-test-'))) + t.after(() => rm(directory, { recursive: true, force: true })) + const extensionDir = path.join(directory, 'extension') + const profileDir = path.join(directory, 'profile') + const artifactsDir = path.join(directory, 'artifacts') + await mkdir(extensionDir) + await mkdir(profileDir) + await mkdir(artifactsDir) + await writeFile( + path.join(extensionDir, 'manifest.json'), + JSON.stringify({ + version: '1.2.3', + name: 'ChatGPTBox', + ...(key ? { key } : {}), + background: { service_worker: 'background.js' }, + action: { default_popup: 'popup.html?popup=true' }, + }), + ) + const extensionId = createHash('sha256') + .update(key ? Buffer.from(key, 'base64') : extensionDir) + .digest('hex') + .slice(0, 32) + .replace(/[0-9a-f]/g, (digit) => String.fromCharCode(97 + Number.parseInt(digit, 16))) + const popupUrl = `chrome-extension://${extensionId}/popup.html?popup=true` + const lifecycle = cleanupRegistry(t) + const launches = [] + const portRecord = '43210\n/devtools/browser/12345678-1234-1234-1234-123456789abc\n' + lifecycle.spawn = (executable, args, options) => { + launches.push({ executable, args, options }) + const portFile = path.join(profileDir, 'DevToolsActivePort') + writeFileSync(portFile, partialPort ? '43210\n/devtools/browser/1234' : portRecord) + if (partialPort) { + const timer = setTimeout(() => writeFileSync(portFile, portRecord), 10) + t.after(() => clearTimeout(timer)) + } + return { + assertRunning() { + signal?.throwIfAborted() + }, + output: () => 'Fake browser output', + } + } + const { FakeWebSocket, sockets } = fakeSocket((message, socket) => { + if (message.method === protocolError) { + socket.reply({ id: message.id, error: { message: 'Deliberate protocol failure' } }) + return + } + let result = {} + switch (message.method) { + case 'Browser.getVersion': + result = { product: 'Chromium/150.0.0.0' } + break + case 'Target.getTargets': + result = { + targetInfos: [ + { type: 'page', targetId: 'unrelated', url: 'https://example.test/' }, + { + type: 'service_worker', + targetId: 'built-in-component', + url: `chrome-extension://${'b'.repeat(32)}/background.js`, + }, + { + type: 'service_worker', + targetId: 'worker', + url: `chrome-extension://${extensionId}/background.js`, + }, + ], + } + break + case 'Target.createTarget': + assert.equal(message.params.url, popupUrl) + result = { targetId: 'popup' } + break + case 'Target.attachToTarget': + assert.equal(message.params.targetId, 'popup') + result = { sessionId: 'popup-session' } + break + case 'Runtime.evaluate': { + assert.equal(message.sessionId, 'popup-session') + assert.equal(message.params.awaitPromise, true) + assert.equal(message.params.returnByValue, true) + if (contextTransition && message.params.expression.includes('readyState')) { + contextTransition = false + socket.reply({ + id: message.id, + error: { code: -32000, message: 'Execution context was destroyed.' }, + }) + return + } + if (evaluationError && message.params.expression.includes('readyState')) { + result = { + exceptionDetails: { exception: { description: 'Popup initialization failed' } }, + } + break + } + const value = runInNewContext(message.params.expression, { + location: { href: popupUrl }, + document: { + readyState: 'complete', + documentElement: { outerHTML: 'Extension popup' }, + }, + chrome: { runtime: { id: extensionId } }, + }) + Promise.resolve(value).then( + (value) => socket.reply({ id: message.id, result: { result: { value } } }), + (error) => + socket.reply({ + id: message.id, + result: { exceptionDetails: { exception: { description: error.message } } }, + }), + ) + return + } + case 'Page.captureScreenshot': + if (screenshotError) { + socket.reply({ id: message.id, error: { message: 'Screenshot failed' } }) + return + } + result = { data: Buffer.from('fake-png').toString('base64') } + break + } + socket.reply({ id: message.id, result }) + }) + t.mock.method(globalThis, 'WebSocket', function (url) { + return new FakeWebSocket(url) + }) + return { + options: { + executable: '/fake/chromium', + extensionDir, + profileDir, + artifactsDir, + lifecycle, + signal, + artifactSha256: await hashArtifact(extensionDir), + }, + launches, + sockets, + extensionId, + popupUrl, + } +} + +test('Chromium launches isolated native headless and evaluates async code in its extension popup', async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium(fixture.options) + const launch = fixture.launches[0] + assert.equal(launch.executable, '/fake/chromium') + assert.ok(launch.args.includes('--headless=new')) + assert.ok(launch.args.includes('--remote-debugging-port=0')) + assert.ok(launch.args.includes(`--load-extension=${fixture.options.extensionDir}`)) + assert.ok(launch.args.includes(`--disable-extensions-except=${fixture.options.extensionDir}`)) + assert.ok(launch.args.includes(`--user-data-dir=${fixture.options.profileDir}`)) + assert.ok(!launch.args.includes('--no-sandbox')) + assert.equal( + fixture.sockets[0].url, + 'ws://127.0.0.1:43210/devtools/browser/12345678-1234-1234-1234-123456789abc', + ) + assert.deepEqual(adapter.metadata, { + browserVersion: 'Chromium/150.0.0.0', + extensionId: fixture.extensionId, + popupUrl: fixture.popupUrl, + expectedVersion: '1.2.3', + expectedName: 'ChatGPTBox', + manifestVersion: '1.2.3', + }) + const identity = await adapter.evaluate(async (arg) => { + await Promise.resolve() + return { id: globalThis.chrome.runtime.id, arg } + }, 'quotes " and newlines\n') + assert.equal(identity.id, fixture.extensionId) + assert.equal(identity.arg, 'quotes " and newlines\n') + await assert.rejects( + adapter.evaluate(async () => { + throw new Error('Scenario failed') + }), + /Scenario failed/, + ) + await adapter.capture('failure') + assert.equal( + await readFile(path.join(fixture.options.artifactsDir, 'failure.png'), 'utf8'), + 'fake-png', + ) + assert.match( + await readFile(path.join(fixture.options.artifactsDir, 'failure.html'), 'utf8'), + /Extension popup/, + ) + await assert.rejects(adapter.capture('../outside'), /inside artifactsDir/) + await adapter.close() + await adapter.close() + assert.equal(fixture.sockets[0].closeCount, 1) +}) + +for (const event of ['close', 'error']) { + test(`Chromium close rejects an idle transport ${event} while the browser is alive`, async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium(fixture.options) + fixture.sockets[0].emit(event) + const expected = event === 'close' ? /disconnected/ : /transport error/ + await assert.rejects(adapter.close(), expected) + await assert.rejects(adapter.close(), expected) + await fixture.options.lifecycle.callbacks[0]() + assert.equal(fixture.sockets[0].closeCount, event === 'close' ? 0 : 1) + }) +} + +test('Chromium retries only the popup readiness context transition', async (t) => { + const fixture = await chromiumFixture(t, { contextTransition: true }) + await startChromium(fixture.options) + assert.equal( + fixture.sockets[0].sent.filter( + (message) => + message.method === 'Runtime.evaluate' && message.params.expression.includes('readyState'), + ).length, + 2, + ) + assert.equal( + fixture.sockets[0].sent.filter((message) => message.method === 'Target.createTarget').length, + 1, + ) +}) + +test('Chromium waits for a complete DevToolsActivePort record before connecting', async (t) => { + const fixture = await chromiumFixture(t, { partialPort: true }) + await startChromium(fixture.options) + assert.equal(fixture.sockets.length, 1) + assert.equal( + fixture.sockets[0].url, + 'ws://127.0.0.1:43210/devtools/browser/12345678-1234-1234-1234-123456789abc', + ) +}) + +test('Chromium diagnostics bypass cancelled liveness checks while the transport remains open', async (t) => { + const controller = new AbortController() + const fixture = await chromiumFixture(t, { signal: controller.signal }) + const adapter = await startChromium(fixture.options) + controller.abort(new Error('Scenario cancelled')) + await assert.rejects( + adapter.evaluate(() => true), + /Scenario cancelled/, + ) + await adapter.capture('cancelled') + assert.equal( + await readFile(path.join(fixture.options.artifactsDir, 'cancelled.png'), 'utf8'), + 'fake-png', + ) + assert.match( + await readFile(path.join(fixture.options.artifactsDir, 'cancelled.html'), 'utf8'), + /Extension popup/, + ) +}) + +test('Chromium ignores component workers with the same script path', async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium(fixture.options) + assert.equal(adapter.metadata.extensionId, fixture.extensionId) + assert.notEqual(adapter.metadata.extensionId, 'b'.repeat(32)) +}) + +test('Chromium uses the manifest public key instead of the load path for identity', async (t) => { + const fixture = await chromiumFixture(t, { + key: Buffer.from('Fixture public key').toString('base64'), + }) + const adapter = await startChromium(fixture.options) + assert.equal(adapter.metadata.extensionId, fixture.extensionId) +}) + +test( + 'Chromium resolves a symlink before selecting its worker and loading the extension', + { skip: os.platform() === 'win32' }, + async (t) => { + const fixture = await chromiumFixture(t) + const alias = path.join(path.dirname(fixture.options.extensionDir), 'extension alias') + await fs.symlink(fixture.options.extensionDir, alias, 'dir') + const adapter = await startChromium({ ...fixture.options, extensionDir: alias }) + assert.equal(adapter.metadata.extensionId, fixture.extensionId) + assert.ok(fixture.launches[0].args.includes(`--load-extension=${fixture.options.extensionDir}`)) + }, +) + +test('Chromium uses the verified build manifest instead of stale caller metadata', async (t) => { + const fixture = await chromiumFixture(t) + const adapter = await startChromium({ + ...fixture.options, + manifest: { + name: 'Runner build', + version: '2.3.4', + background: { service_worker: 'background.js' }, + action: { default_popup: 'popup.html?popup=true' }, + }, + }) + assert.equal(adapter.metadata.expectedName, 'ChatGPTBox') + assert.equal(adapter.metadata.expectedVersion, '1.2.3') + assert.equal(adapter.metadata.manifestVersion, '1.2.3') +}) + +for (const change of [ + 'manifest', + 'invalid manifest', + 'missing manifest', + 'file', + 'empty directory', +]) { + test(`Chromium rejects a changed ${change} after preflight before spawning`, async (t) => { + const fixture = await chromiumFixture(t) + const { extensionDir } = fixture.options + const manifestPath = path.join(extensionDir, 'manifest.json') + if (change === 'manifest') { + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) + manifest.version = '9.9.9' + await writeFile(manifestPath, JSON.stringify(manifest)) + } else if (change === 'invalid manifest') { + await writeFile(manifestPath, '{') + } else if (change === 'missing manifest') { + await rm(manifestPath) + } else if (change === 'file') { + await writeFile(path.join(extensionDir, 'new.js'), 'changed') + } else { + await mkdir(path.join(extensionDir, 'new-directory')) + } + await assert.rejects(startChromium(fixture.options), /Build changed since preflight/) + assert.equal(fixture.launches.length, 0) + }) +} + +test('Chromium requires a valid preflight hash before spawning', async (t) => { + const fixture = await chromiumFixture(t) + for (const artifactSha256 of [undefined, '', 'invalid']) { + await assert.rejects( + startChromium({ ...fixture.options, artifactSha256 }), + /Missing preflight build hash/, + ) + } + assert.equal(fixture.launches.length, 0) +}) + +test('Chromium captures PNG and DOM when popup startup fails', async (t) => { + const fixture = await chromiumFixture(t, { evaluationError: true }) + await assert.rejects( + startChromium(fixture.options), + /Popup initialization failed.*\nFake browser output/, + ) + assert.equal( + await readFile(path.join(fixture.options.artifactsDir, 'chromium-startup-failure.png'), 'utf8'), + 'fake-png', + ) + assert.match( + await readFile( + path.join(fixture.options.artifactsDir, 'chromium-startup-failure.html'), + 'utf8', + ), + /Extension popup/, + ) +}) + +test('Chromium captures DOM even if screenshot fails', async (t) => { + const fixture = await chromiumFixture(t, { screenshotError: true }) + const adapter = await startChromium(fixture.options) + await assert.rejects(adapter.capture('failure'), /Chromium capture failed/) + assert.match( + await readFile(path.join(fixture.options.artifactsDir, 'failure.html'), 'utf8'), + /Extension popup/, + ) +}) + +test('Chromium does not retry target creation failures and retains cleanup', async (t) => { + const fixture = await chromiumFixture(t, { protocolError: 'Target.createTarget' }) + await assert.rejects( + startChromium(fixture.options), + /Target.createTarget: Deliberate protocol failure/, + ) + assert.equal( + fixture.sockets[0].sent.filter((message) => message.method === 'Target.createTarget').length, + 1, + ) + assert.equal(fixture.options.lifecycle.callbacks.length, 1) +}) + +test('Chromium rejects stale profile port files without launching', async (t) => { + const fixture = await chromiumFixture(t) + await writeFile( + path.join(fixture.options.profileDir, 'DevToolsActivePort'), + '1234\n/devtools/browser/stale', + ) + await assert.rejects(startChromium(fixture.options), /use a fresh profile/) + assert.equal(fixture.launches.length, 0) +}) + +test('Chromium cancellation before startup does not spawn a process', async (t) => { + const fixture = await chromiumFixture(t) + const controller = new AbortController() + controller.abort(new Error('Stop now')) + await assert.rejects(startChromium({ ...fixture.options, signal: controller.signal }), /Stop now/) + assert.equal(fixture.launches.length, 0) +}) + +test( + 'Chromium does not recreate an owned profile after cancellation during manifest reading', + { timeout: 5000 }, + async (t) => { + const controller = new AbortController() + const fixture = await chromiumFixture(t, { signal: controller.signal }) + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.force()) + lifecycle.defer('Chromium profile', () => + rm(fixture.options.profileDir, { recursive: true, force: true }), + ) + const started = Promise.withResolvers() + const release = Promise.withResolvers() + const originalReadFile = readFile + const manifestPath = path.join(fixture.options.extensionDir, 'manifest.json') + const mockRead = t.mock.method(fs, 'readFile', async (...args) => { + const bytes = await originalReadFile(...args) + if (args[0] === manifestPath) { + started.resolve() + await release.promise + } + return bytes + }) + syncBuiltinESMExports() + t.after(() => { + release.resolve() + mockRead.mock.restore() + syncBuiltinESMExports() + }) + const reason = new Error('Cancelled while reading manifest') + const rejected = assert.rejects( + startChromium({ ...fixture.options, lifecycle }), + (error) => error === reason, + ) + await started.promise + controller.abort(reason) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(stat(fixture.options.profileDir), { code: 'ENOENT' }) + release.resolve() + await rejected + await assert.rejects(stat(fixture.options.profileDir), { code: 'ENOENT' }) + assert.equal(fixture.sockets.length, 0) + }, +) + +test('Chromium normal close rejects a late browser exit while lifecycle cleanup remains available', async (t) => { + const fixture = await chromiumFixture(t) + const originalSpawn = fixture.options.lifecycle.spawn + let browser + fixture.options.lifecycle.spawn = (...args) => (browser = originalSpawn(...args)) + const adapter = await startChromium(fixture.options) + const failure = new Error('Process Chromium exited unexpectedly (7)') + t.mock.method(browser, 'assertRunning', () => { + throw failure + }) + fixture.sockets[0].emit('close') + await assert.rejects(adapter.close(), (error) => error === failure) + await fixture.options.lifecycle.callbacks[0]() +}) diff --git a/tests/unit/scripts/smoke-firefox.test.mjs b/tests/unit/scripts/smoke-firefox.test.mjs new file mode 100644 index 000000000..9640cace0 --- /dev/null +++ b/tests/unit/scripts/smoke-firefox.test.mjs @@ -0,0 +1,675 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import streams from 'node:fs' +import fs, { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import { setTimeout } from 'node:timers/promises' +import { URL } from 'node:url' +import { runInNewContext } from 'node:vm' +import test from 'node:test' +import { startFirefox } from '../../../scripts/smoke/firefox.mjs' +import { bounded, createLifecycle, waitFor } from '../../../scripts/smoke/lifecycle.mjs' +import { hashArtifact } from '../../../scripts/smoke/runner.mjs' + +const popupUrl = 'moz-extension://owned-extension/popup.html?popup=true' + +async function fixture(t, override = () => {}, popup = 'popup.html?popup=true') { + const root = await mkdtemp(join(tmpdir(), 'smoke-firefox-test-')) + t.after(() => rm(root, { recursive: true, force: true })) + const calls = [] + const cleanups = [] + const spawns = [] + let handlesReads = 0 + const controller = new AbortController() + const options = { + executable: '/usr/lib/firefox/firefox', + geckodriver: '/owned/geckodriver', + archive: join(root, 'extension.zip'), + extensionDir: join(root, 'build'), + manifest: { + version: '2.6.1', + name: 'ChatGPTBox', + browser_action: { default_popup: popup }, + }, + profileDir: join(root, 'profile'), + artifactsDir: join(root, 'artifacts'), + signal: controller.signal, + lifecycle: { + spawn(...args) { + spawns.push(args) + return { + child: {}, + exited: new Promise(() => {}), + output: () => '1720000000000\tgeckodriver\tINFO\tListening on 127.0.0.1:41239\n', + assertRunning() {}, + } + }, + defer(label, cleanup) { + cleanups.push({ label, cleanup }) + }, + }, + } + await mkdir(options.profileDir) + await mkdir(options.artifactsDir) + await writeFile(options.archive, 'owned archive bytes') + await mkdir(options.extensionDir) + await writeFile(join(options.extensionDir, 'manifest.json'), JSON.stringify(options.manifest)) + await writeFile(join(options.extensionDir, 'background.js'), 'current build') + options.artifactSha256 = await hashArtifact(options.extensionDir) + t.mock.method(globalThis, 'fetch', async (url, init) => { + assert.equal(new URL(url).origin, 'http://127.0.0.1:41239') + const path = new URL(url).pathname + const call = { path, method: init.method, body: init.body && JSON.parse(init.body), init } + calls.push(call) + const overridden = await override(call, { controller, cleanups }) + if (overridden) return overridden + let value = null + if (path === '/status') value = { ready: true } + else if (path === '/session') { + assert.equal(cleanups.length, 1, 'cleanup must be registered before creating a session') + value = { sessionId: 'owned-session', capabilities: { browserVersion: '154.0' } } + } else if (path.endsWith('/moz/addon/install')) value = 'addon@example.test' + else if (path.endsWith('/window/handles')) { + value = handlesReads++ ? ['original', 'popup'] : ['original'] + } else if (path.endsWith('/execute/sync') && call.body.script.includes('getByID')) { + value = 'moz-extension://owned-extension/' + } else if (path.endsWith('/execute/async')) { + value = await new Promise((resolve, reject) => { + try { + runInNewContext(`(function () { ${call.body.script} }).apply(null, args)`, { + args: [...call.body.args, resolve], + location: { + href: calls.find( + (entry) => + typeof entry.body?.script === 'string' && + entry.body.script.includes('gBrowser.addTab'), + )?.body.args[0], + }, + document: { readyState: 'complete' }, + browser: { storage: { local: { get: async () => ({ stored: true }) } } }, + }) + } catch (error) { + reject(error) + } + }) + } else if (path.endsWith('/screenshot')) value = Buffer.from('image bytes').toString('base64') + else if (path.endsWith('/source')) value = 'popup' + return Response.json({ value }) + }) + return { options, calls, cleanups, spawns, controller } +} + +test('Firefox opens the popup declared by the installed manifest including its query', async (t) => { + const declared = 'custom/declared-popup.html?from=manifest' + const { options, calls } = await fixture(t, undefined, declared) + const adapter = await startFirefox(options) + assert.equal(adapter.metadata.popupUrl, `moz-extension://owned-extension/${declared}`) + assert.deepEqual( + calls.find( + (call) => + typeof call.body?.script === 'string' && call.body.script.includes('gBrowser.addTab'), + ).body.args, + [`moz-extension://owned-extension/${declared}`], + ) + await adapter.close() +}) + +test('Firefox installs the verified directory snapshot instead of a stale archive', async (t) => { + const { options, calls } = await fixture(t) + const adapter = await startFirefox(options) + const installed = calls.find((call) => call.path.endsWith('/moz/addon/install')).body.path + assert.equal(installed, join(options.artifactsDir, 'extension')) + assert.equal(await hashArtifact(installed), options.artifactSha256) + await writeFile(join(options.extensionDir, 'background.js'), 'later build') + assert.equal(await readFile(join(installed, 'background.js'), 'utf8'), 'current build') + assert.equal(adapter.metadata.artifactSha256, options.artifactSha256) + assert.equal(Object.hasOwn(adapter.metadata, 'archiveSha256'), false) + await adapter.close() +}) + +test('Firefox rejects a build changed since preflight before launching processes', async (t) => { + const { options, calls, spawns } = await fixture(t) + await writeFile(join(options.extensionDir, 'background.js'), 'changed build') + await assert.rejects(startFirefox(options), /Build changed since preflight/) + assert.equal(spawns.length, 0) + assert.equal(calls.length, 0) +}) + +for (const popup of [ + null, + '', + ' ', + ' popup.html', + 'https://example.test/popup.html', + 'javascript:alert(1)', + '//other-extension/popup.html', + 'moz-extension://other-extension/popup.html', + 'moz-extension://owned-extension/popup.html', +]) { + test(`Firefox rejects invalid manifest popup ${JSON.stringify( + popup, + )} before startup`, async (t) => { + const { options, calls, spawns } = await fixture(t, undefined, popup) + await assert.rejects(startFirefox(options), /Invalid Firefox manifest popup path/) + assert.equal(calls.length, 0) + assert.equal(spawns.length, 0) + }) +} + +test('Firefox uses snapshot manifest identity and action popup precedence', async (t) => { + const { options } = await fixture(t) + const manifest = { + ...options.manifest, + name: 'Current snapshot', + version: '3.0', + action: { default_popup: 'action-popup.html?source=action#tab' }, + } + await writeFile(join(options.extensionDir, 'manifest.json'), JSON.stringify(manifest)) + options.artifactSha256 = await hashArtifact(options.extensionDir) + // The earlier metadata may predate the directory hash; use the snapshot instead. + const adapter = await startFirefox(options) + assert.equal(adapter.metadata.expectedName, manifest.name) + assert.equal(adapter.metadata.expectedVersion, manifest.version) + assert.equal(adapter.metadata.manifestVersion, manifest.version) + assert.equal( + adapter.metadata.popupUrl, + 'moz-extension://owned-extension/action-popup.html?source=action#tab', + ) + await adapter.close() +}) + +test( + 'Firefox does not recreate an owned profile after cancellation during snapshot validation', + { timeout: 5000 }, + async (t) => { + const { options, controller, calls } = await fixture(t) + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.force()) + lifecycle.defer('Firefox profile', () => + rm(options.profileDir, { recursive: true, force: true }), + ) + const started = Promise.withResolvers() + const release = Promise.withResolvers() + const originalReadFile = readFile + const mockRead = t.mock.method(fs, 'readFile', async (...args) => { + const bytes = await originalReadFile(...args) + if (args[0] === join(options.artifactsDir, 'extension/manifest.json')) { + started.resolve() + await release.promise + } + return bytes + }) + syncBuiltinESMExports() + t.after(() => { + release.resolve() + mockRead.mock.restore() + syncBuiltinESMExports() + }) + const reason = new Error('Cancelled while validating snapshot') + const rejected = assert.rejects( + startFirefox({ ...options, lifecycle }), + (error) => error === reason, + ) + await started.promise + controller.abort(reason) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + release.resolve() + await rejected + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + assert.equal(calls.length, 0) + }, +) + +test( + 'Firefox cancellation closes an in-flight snapshot transfer before rejecting startup', + { timeout: 5000 }, + async (t) => { + const { options, controller, calls } = await fixture(t) + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.force()) + lifecycle.defer('Firefox profile', () => + rm(options.profileDir, { recursive: true, force: true }), + ) + const spawn = t.mock.method(lifecycle, 'spawn', () => { + throw new Error('Unexpected browser startup') + }) + const started = Promise.withResolvers() + const originalRead = streams.createReadStream + const originalWrite = streams.createWriteStream + const input = new PassThrough() + let output + const mockRead = t.mock.method(streams, 'createReadStream', (path, ...args) => { + if (path === join(options.extensionDir, 'background.js')) { + input.write('partial') + started.resolve() + return input + } + return originalRead(path, ...args) + }) + const mockWrite = t.mock.method(streams, 'createWriteStream', (path, ...args) => { + const result = originalWrite(path, ...args) + if (path === join(options.artifactsDir, 'extension/background.js')) output = result + return result + }) + syncBuiltinESMExports() + t.after(() => { + input.destroy() + output?.destroy() + mockRead.mock.restore() + mockWrite.mock.restore() + syncBuiltinESMExports() + }) + const reason = new Error('Cancelled during snapshot copy') + const rejected = assert.rejects( + startFirefox({ ...options, lifecycle }), + (error) => error === reason, + ) + await bounded(started.promise, 1000, 'Snapshot transfer start') + await waitFor(() => output?.bytesWritten > 0, { timeoutMs: 1000 }) + controller.abort(reason) + await bounded(rejected, 1000, 'Snapshot cancellation') + assert.equal(input.closed, true) + assert.equal(output.closed, true) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + assert.equal( + await readFile(join(options.artifactsDir, 'extension/background.js'), 'utf8'), + 'partial', + ) + await assert.rejects(stat(options.profileDir), { code: 'ENOENT' }) + assert.equal(spawn.mock.callCount(), 0) + assert.equal(calls.length, 0) + }, +) + +test('Firefox uses an owned headless session and privileged extension popup context', async (t) => { + const { options, calls, cleanups, spawns } = await fixture(t) + const adapter = await startFirefox(options) + assert.deepEqual(spawns[0].slice(0, 2), [ + options.geckodriver, + [ + '--host', + '127.0.0.1', + '--port', + '0', + '--websocket-port', + '0', + '--profile-root', + options.profileDir, + '--allow-system-access', + ], + ]) + assert.equal(spawns[0][2].logPath, join(options.artifactsDir, 'geckodriver.log')) + const session = calls.find((call) => call.path === '/session') + assert.equal( + session.body.capabilities.alwaysMatch['moz:firefoxOptions'].binary, + options.executable, + ) + assert.deepEqual(session.body.capabilities.alwaysMatch['moz:firefoxOptions'].args, ['-headless']) + assert.deepEqual(calls.find((call) => call.path.endsWith('/moz/addon/install')).body, { + path: join(options.artifactsDir, 'extension'), + temporary: true, + }) + assert.deepEqual( + calls.filter((call) => call.path.endsWith('/moz/context')).map((call) => call.body.context), + ['chrome', 'content'], + ) + const tab = calls.find( + (call) => call.path.endsWith('/execute/sync') && call.body.script.includes('gBrowser.addTab'), + ) + assert.match(tab.body.script, /Services\.scriptSecurityManager\.getSystemPrincipal\(\)/) + assert.deepEqual(tab.body.args, [popupUrl]) + assert.deepEqual(calls.find((call) => call.path.endsWith('/window')).body, { handle: 'popup' }) + assert.deepEqual(adapter.metadata, { + browserVersion: '154.0', + extensionId: 'addon@example.test', + popupUrl, + expectedVersion: '2.6.1', + expectedName: 'ChatGPTBox', + artifactSha256: options.artifactSha256, + snapshotDir: join(options.artifactsDir, 'extension'), + manifestVersion: '2.6.1', + }) + assert.equal(await adapter.evaluate(async (a, b) => Promise.resolve(a + b), 3, 4), 7) + assert.equal( + await adapter.evaluate(async () => (await globalThis.browser.storage.local.get()).stored), + true, + ) + await assert.rejects( + adapter.evaluate(async () => { + throw new Error('page failure') + }), + /page failure/, + ) + await adapter.capture('popup-initial') + await assert.rejects(adapter.capture('../escape'), /inside artifactsDir/) + assert.equal( + await readFile(join(options.artifactsDir, 'popup-initial.png'), 'utf8'), + 'image bytes', + ) + assert.equal( + await readFile(join(options.artifactsDir, 'popup-initial.html'), 'utf8'), + 'popup', + ) + await adapter.close() + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) +}) + +test('Firefox capture retains DOM when screenshot fails', async (t) => { + const { options } = await fixture(t, ({ path }) => { + if (path.endsWith('/screenshot')) { + return Response.json({ value: { error: 'unknown error', message: 'Screenshot failed' } }) + } + }) + const adapter = await startFirefox(options) + await assert.rejects(adapter.capture('failure'), (error) => { + assert.ok(error instanceof AggregateError) + assert.equal(error.errors.length, 1) + assert.match(error.errors[0].message, /Screenshot failed/) + assert.match(String(error), /Screenshot failed/) + return true + }) + assert.equal( + await readFile(join(options.artifactsDir, 'failure.html'), 'utf8'), + 'popup', + ) + await adapter.close() +}) + +test('Firefox capture preserves both diagnostic errors', async (t) => { + const { options } = await fixture(t, ({ path }) => { + if (path.endsWith('/screenshot') || path.endsWith('/source')) { + return Response.json({ value: { error: 'unknown error', message: `Failed ${path}` } }) + } + }) + const adapter = await startFirefox(options) + await assert.rejects(adapter.capture('failure'), (error) => { + assert.ok(error instanceof AggregateError) + assert.equal(error.errors.length, 2) + assert.match(error.errors[0].message, /screenshot/) + assert.match(error.errors[1].message, /source/) + assert.match(String(error), /Failed \/session\/owned-session\/screenshot/) + assert.match(String(error), /Failed \/session\/owned-session\/source/) + return true + }) + await adapter.close() +}) + +test('Firefox capture cancels both pending diagnostics with the run signal', async (t) => { + const { options, controller, calls } = await fixture(t, ({ path, init }) => { + if (path.endsWith('/screenshot') || path.endsWith('/source')) { + return new Promise((resolve, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason), { once: true }) + }) + } + }) + const adapter = await startFirefox(options) + const reason = new Error('Run cancelled during capture') + const captured = adapter.capture('failure') + const rejected = assert.rejects(captured, (error) => { + assert.ok(error instanceof AggregateError) + assert.deepEqual(error.errors, [reason, reason]) + return true + }) + const diagnostics = calls.filter( + ({ path }) => path.endsWith('/screenshot') || path.endsWith('/source'), + ) + assert.equal(diagnostics.length, 2) + controller.abort(reason) + await rejected + assert.ok(diagnostics.every(({ init }) => init.signal.aborted)) + await adapter.close() +}) + +for (const path of ['/session', '/session/owned-session/moz/addon/install']) { + for (const status of [200, 500]) { + test(`Firefox rejects WebDriver error at ${path} with HTTP ${status} without retry`, async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path === path) { + return Response.json( + { value: { error: 'unknown error', message: 'broken operation' } }, + { status }, + ) + } + }) + await assert.rejects(startFirefox(options), /broken operation/) + assert.equal(calls.filter((call) => call.path === path).length, 1) + await cleanups[0].cleanup() + assert.equal( + calls.filter((call) => call.method === 'DELETE').length, + path === '/session' ? 0 : 1, + ) + }) + } +} + +test('Firefox rejects failed HTTP status even with a successful-looking value', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/session') + return Response.json({ value: { sessionId: 'bad' } }, { status: 502 }) + }) + await assert.rejects(startFirefox(options), /HTTP 502/) + assert.equal(calls.filter((call) => call.path === '/session').length, 1) +}) + +test('Firefox rejects malformed HTTP JSON without retrying session creation', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/session') return new Response('error', { status: 503 }) + }) + await assert.rejects(startFirefox(options), /HTTP 503, invalid JSON/) + assert.equal(calls.filter((call) => call.path === '/session').length, 1) +}) + +for (const value of ['', ' ', null]) { + test(`Firefox rejects empty addon ID ${JSON.stringify(value)} and retains cleanup`, async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path.endsWith('/moz/addon/install')) return Response.json({ value }) + }) + await assert.rejects(startFirefox(options), /empty addon ID/) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/addon/install')).length, 1) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/context')).length, 0) + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) + }) +} + +test('Firefox retries explicit read-only not-ready status', async (t) => { + let statusReads = 0 + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/status') return Response.json({ value: { ready: ++statusReads > 1 } }) + }) + const adapter = await startFirefox(options) + assert.equal(statusReads, 2) + assert.equal(calls.filter((call) => call.path === '/session').length, 1) + await adapter.close() +}) + +test('Firefox preserves evaluation messages when the stack contains only locations', async (t) => { + const { options } = await fixture(t) + const adapter = await startFirefox(options) + await assert.rejects( + adapter.evaluate(() => { + const error = new Error('Popup snapshot is missing') + error.stack = 'pageFunction@moz-extension://owned-extension/popup.html:4:2' + throw error + }), + (error) => { + assert.match(error.message, /^Firefox evaluation failed: Error: Popup snapshot is missing/) + assert.match(error.message, /pageFunction@moz-extension:/) + assert.doesNotMatch(error.message, /WebDriver POST/) + return true + }, + ) + await adapter.close() +}) + +test('Firefox waits for a complete listening-port log record', async (t) => { + const { options, calls } = await fixture(t) + const originalSpawn = options.lifecycle.spawn + let reads = 0 + options.lifecycle.spawn = (...args) => ({ + ...originalSpawn(...args), + output: () => + ++reads === 1 + ? 'geckodriver INFO Listening on 127.0.0.1:4' + : 'geckodriver INFO Listening on 127.0.0.1:41239\n', + }) + const adapter = await startFirefox(options) + assert.equal(reads, 2) + assert.equal(calls.filter((call) => call.path === '/status').length, 1) + await adapter.close() +}) + +test('Firefox rejects malformed readiness instead of retrying', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.path === '/status') return Response.json({ value: {} }) + }) + await assert.rejects(startFirefox(options), /Invalid geckodriver readiness status/) + assert.equal(calls.length, 1) +}) + +test('Firefox refuses an empty session ID before installing the addon', async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path === '/session') return Response.json({ value: { sessionId: '' } }) + }) + await assert.rejects(startFirefox(options), /empty session ID/) + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'POST').length, 1) + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 0) +}) + +test('Firefox surfaces failed session cleanup and does not retry it', async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.method === 'DELETE') { + return Response.json({ value: { error: 'unknown error', message: 'cannot close' } }) + } + }) + const adapter = await startFirefox(options) + await assert.rejects(adapter.close(), /cannot close/) + await assert.rejects(cleanups[0].cleanup(), /cannot close/) + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) +}) + +test('Firefox never retries a network failure during addon installation', async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path.endsWith('/moz/addon/install')) throw new TypeError('fetch failed') + }) + await assert.rejects(startFirefox(options), /fetch failed/) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/addon/install')).length, 1) + await cleanups[0].cleanup() +}) + +test('Firefox aborts a stalled installation and still deletes its session once', async (t) => { + const { options, calls, cleanups, controller } = await fixture(t, async (call) => { + if (call.path.endsWith('/moz/addon/install')) { + await setTimeout(10) + controller.abort(new Error('test operation timeout')) + return new Promise(() => {}) + } + }) + await assert.rejects(startFirefox(options), /test operation timeout/) + assert.equal(calls.filter((call) => call.path.endsWith('/moz/addon/install')).length, 1) + assert.equal( + calls.find((call) => call.path.endsWith('/moz/addon/install')).init.signal.aborted, + true, + ) + await cleanups[0].cleanup() + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) + assert.equal( + calls.find((call) => call.method === 'DELETE').init.signal.reason?.message === + 'test operation timeout', + false, + ) +}) + +for (const path of ['/session', '/session/owned-session/moz/addon/install']) { + test(`Firefox bounds a stalled ${path} request without retrying`, async (t) => { + const { options, calls, cleanups } = await fixture(t, (call) => { + if (call.path === path) { + queueMicrotask(() => t.mock.timers.tick(path === '/session' ? 60001 : 30001)) + return new Promise(() => {}) + } + }) + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + await assert.rejects(startFirefox(options), path === '/session' ? /60000 ms/ : /30000 ms/) + assert.equal(calls.filter((call) => call.path === path).length, 1) + assert.equal(calls.find((call) => call.path === path).init.signal.aborted, true) + await cleanups[0].cleanup() + }) +} + +test('Firefox bounds session shutdown to five seconds without retry', async (t) => { + const { options, calls } = await fixture(t, (call) => { + if (call.method === 'DELETE') { + queueMicrotask(() => t.mock.timers.tick(5001)) + return new Promise(() => {}) + } + }) + const adapter = await startFirefox(options) + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + await assert.rejects(adapter.close(), /5000 ms/) + assert.equal(calls.filter((call) => call.method === 'DELETE').length, 1) + assert.equal(calls.find((call) => call.method === 'DELETE').init.signal.aborted, true) +}) + +test('Firefox cleanup evaluation survives cancellation and retains a five-second deadline', async (t) => { + let stall = false + const started = Promise.withResolvers() + const { options, calls, controller } = await fixture(t, (call) => { + if (stall && call.path.endsWith('/execute/async')) { + started.resolve() + return new Promise(() => {}) + } + }) + const adapter = await startFirefox(options) + stall = true + const normal = adapter.evaluate(() => 1) + await started.promise + controller.abort(new Error('Run cancelled')) + await assert.rejects(normal, /Run cancelled/) + const count = calls.length + await assert.rejects( + adapter.evaluate(() => 1), + /Run cancelled/, + ) + assert.equal(calls.length, count) + stall = false + assert.equal(await adapter.evaluateCleanup((a, b) => a + b, 2, 3), 5) + await assert.rejects( + adapter.evaluateCleanup(async () => { + throw new Error('Cleanup failed') + }), + /Cleanup failed/, + ) + stall = true + t.mock.timers.enable({ apis: ['setTimeout'] }) + syncBuiltinESMExports() + t.after(() => { + t.mock.timers.reset() + syncBuiltinESMExports() + }) + const stalled = adapter.evaluateCleanup(() => 1) + const cleanupSignal = calls.at(-1).init.signal + assert.equal(cleanupSignal.aborted, false) + assert.notEqual(cleanupSignal, controller.signal) + const rejected = assert.rejects(stalled, /timed out after 5000 ms/) + t.mock.timers.tick(5001) + await rejected + assert.equal(cleanupSignal.aborted, true) + await adapter.close() +}) diff --git a/tests/unit/scripts/smoke-interruption.test.mjs b/tests/unit/scripts/smoke-interruption.test.mjs new file mode 100644 index 000000000..de7b51a76 --- /dev/null +++ b/tests/unit/scripts/smoke-interruption.test.mjs @@ -0,0 +1,554 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import process from 'node:process' +import { spawn } from 'node:child_process' +import { + chmod, + cp, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + stat, + writeFile, +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { ROOT } from '../../../scripts/smoke/runner.mjs' +import { bounded, waitFor } from '../../../scripts/smoke/lifecycle.mjs' + +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), 'smoke checkout = with spaces ')) + const cleanups = [] + t.after(async () => { + const results = await Promise.allSettled(cleanups.map(async (cleanup) => cleanup())) + const errors = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason) + if (errors.length) { + t.diagnostic(`Preserved fixture after cleanup failure: ${root}`) + throw new AggregateError(errors, 'Fixture cleanup failed') + } + await rm(root, { recursive: true, force: true }) + }) + await cp(join(ROOT, 'scripts/smoke'), join(root, 'scripts/smoke'), { recursive: true }) + for (const name of ['xvfb-smoke.mjs', 'run-smoke.sh']) { + await cp(join(ROOT, 'scripts', name), join(root, 'scripts', name)) + } + const build = join(root, 'build/chromium') + await mkdir(build, { recursive: true }) + await writeFile( + join(build, 'manifest.json'), + JSON.stringify({ + name: 'Smoke fixture', + version: '1.0', + background: { service_worker: 'background.js' }, + action: { default_popup: 'popup.html' }, + }), + ) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), '') + const executable = join(root, 'fake browser') + await cp(join(ROOT, 'tests/fixtures/smoke/fake-browser.fixture'), executable) + await chmod(executable, 0o700) + return { root, executable, cleanups } +} + +async function isLive(pid) { + try { + const contents = await readFile(`/proc/${pid}/stat`, 'utf8') + const state = contents.slice(contents.lastIndexOf(')') + 2).split(' ')[0] + if (!/^[A-Z]$/.test(state)) throw new Error('Invalid process state') + return !['Z', 'X'].includes(state) + } catch (error) { + if (error.code === 'ENOENT' || error.code === 'ESRCH') return false + throw error + } +} + +async function stopRunner(child, exited) { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM') + try { + await bounded(exited, 15000, 'Fixture runner exit') + } catch { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + await bounded(exited, 2000, 'Reap fixture runner') + } +} + +async function launch(t, fixture, tag) { + const parent = join(fixture.root, tag) + const child = spawn( + 'sh', + [ + join(fixture.root, 'scripts/run-smoke.sh'), + '--browser', + 'chromium', + '--chromium-path', + fixture.executable, + '--artifacts-dir', + parent, + ], + { + cwd: tmpdir(), + env: { ...process.env, TMPDIR: fixture.root }, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let output = '' + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.on('data', (chunk) => { + output += chunk + }) + const exited = new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (code, signal) => resolve({ code, signal })) + }) + exited.catch(() => {}) + let browser + fixture.cleanups.push(async () => { + await stopRunner(child, exited) + if (!browser) throw new Error('Cannot verify fixture browser cleanup') + // Never signal a remembered target PID as a process group. The fixture's + // watchdog is only a final safety net after a failed test, not a pass condition. + await waitFor(async () => !(await isLive(browser.pid)), { timeoutMs: 35000 }) + }) + const artifacts = await waitFor( + async () => { + if (child.exitCode !== null) throw new Error(`Runner exited early: ${output}`) + try { + const entries = await readdir(parent) + if (!entries.length) return false + const directory = join(parent, entries[0]) + const log = await readFile(join(directory, 'chromium/chromium.log'), 'utf8') + if (!log.endsWith('\n')) return false + browser = JSON.parse(log.trim()) + return directory + } catch (error) { + if (error.code === 'ENOENT') return false + throw error + } + }, + { timeoutMs: 10000, label: 'Fixture browser startup' }, + ) + return { child, browser, artifacts, exited, output: () => output } +} + +for (const [signal, expectedCode] of [ + ['SIGINT', 130], + ['SIGTERM', 143], +]) { + test( + `runner forwards ${signal} to the active preflight artifact walk`, + { skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + const ready = join(setup.root, 'hash-ready') + const stopped = join(setup.root, 'hash-stopped') + await writeFile( + join(setup.root, 'scripts/smoke/artifacts.mjs'), + ` + import { writeFileSync } from 'node:fs' + export async function hashArtifact(directory, signal) { + return new Promise((resolve, reject) => { + signal?.addEventListener('abort', () => { + writeFileSync(${JSON.stringify(stopped)}, 'cancelled') + reject(signal.reason) + }, { once: true }) + writeFileSync(${JSON.stringify(ready)}, 'ready') + }) + } + export async function snapshotArtifact() { throw new Error('Must not start a browser') } + `, + ) + const child = spawn( + process.execPath, + [ + join(setup.root, 'scripts/xvfb-smoke.mjs'), + '--browser', + 'chromium', + '--chromium-path', + setup.executable, + '--artifacts-dir', + join(setup.root, 'artifacts'), + ], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ) + let output = '' + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.on('data', (chunk) => { + output += chunk + }) + const exited = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', (code) => resolve(code)) + }) + exited.catch(() => {}) + setup.cleanups.push(() => stopRunner(child, exited)) + await waitFor( + async () => { + try { + return await readFile(ready, 'utf8') + } catch (error) { + if (error.code === 'ENOENT') return false + throw error + } + }, + { timeoutMs: 10000, label: 'Preflight hash entry' }, + ) + child.kill(signal) + assert.equal(await bounded(exited, 10000, 'Cancelled preflight exit'), expectedCode, output) + assert.equal(await readFile(stopped, 'utf8'), 'cancelled') + const [runDirectory] = await readdir(join(setup.root, 'artifacts')) + const report = JSON.parse( + await readFile(join(setup.root, 'artifacts', runDirectory, 'report.json'), 'utf8'), + ) + assert.equal(report.failedStage, 'preflight') + assert.equal(report.exitCode, expectedCode) + assert.deepEqual(report.browsers, []) + assert.deepEqual(report.cleanup, []) + }, + ) +} + +test( + 'shell runner preserves SIGINT/SIGTERM, cleans owned profiles and isolates simultaneous runs', + { timeout: 60000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + const first = await launch(t, setup, 'first') + const second = await launch(t, setup, 'second') + assert.notEqual(first.browser.profile, second.browser.profile) + // Foreground Ctrl+C reaches the runner group; detached browser groups are owned separately. + process.kill(-first.child.pid, 'SIGINT') + await delay(50) + first.child.kill('SIGINT') + assert.deepEqual(await bounded(first.exited, 15000, 'SIGINT exit'), { code: 130, signal: null }) + assert.equal(await isLive(second.browser.pid), true) + assert.doesNotThrow(() => process.kill(second.child.pid, 0)) + assert.equal((await stat(second.browser.profile)).isDirectory(), true) + second.child.kill('SIGTERM') + assert.deepEqual(await bounded(second.exited, 15000, 'SIGTERM exit'), { + code: 143, + signal: null, + }) + for (const [run, code] of [ + [first, 130], + [second, 143], + ]) { + await assert.rejects(stat(run.browser.profile), { code: 'ENOENT' }) + assert.equal(await isLive(run.browser.pid), false) + const report = JSON.parse(await readFile(join(run.artifacts, 'report.json'), 'utf8')) + assert.equal(report.exitCode, code, run.output()) + assert.equal(report.result, 'FAIL') + assert.deepEqual(report.cleanup, []) + assert.equal(report.root, await realpath(setup.root)) + } + }, +) + +test( + 'all-browser runner reaps the previous browser and removes its profile before starting the next', + { timeout: 75000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + await cp(join(setup.root, 'build/chromium'), join(setup.root, 'build/firefox'), { + recursive: true, + }) + await writeFile(join(setup.root, 'build/firefox.zip'), 'Fixture archive') + const marker = join(setup.root, 'previous-browser.json') + setup.cleanups.push(async () => { + const previous = JSON.parse(await readFile(marker, 'utf8')) + await waitFor(async () => !(await isLive(previous.targetPid)), { timeoutMs: 35000 }) + }) + await writeFile( + join(setup.root, 'scripts/smoke/chromium.mjs'), + ` + import process from 'node:process' + import { writeFile } from 'node:fs/promises' + import { waitFor } from './lifecycle.mjs' + export async function startChromium({ lifecycle, profileDir }) { + const managed = lifecycle.spawn(process.execPath, ['-e', + 'setTimeout(() => process.exit(99), 30000); console.log(process.pid)']) + const targetPid = await waitFor(() => { + managed.assertRunning() + const output = managed.output() + return /^\\d+\\n$/.test(output) && Number(output.trim()) + }, { timeoutMs: 3000 }) + await writeFile(${JSON.stringify( + marker, + )}, JSON.stringify({ targetPid, supervisorPid: managed.child.pid, profileDir })) + return { metadata: {}, close: async () => {}, capture: async () => {} } + } + `, + ) + await writeFile( + join(setup.root, 'scripts/smoke/firefox.mjs'), + ` + import process from 'node:process' + import assert from 'node:assert/strict' + import { readFile, stat } from 'node:fs/promises' + const isLive = ${isLive.toString()} + export async function startFirefox() { + const previous = JSON.parse(await readFile(${JSON.stringify(marker)}, 'utf8')) + assert.throws(() => process.kill(previous.supervisorPid, 0), { code: 'ESRCH' }) + assert.equal(await isLive(previous.targetPid), false) + await assert.rejects(stat(previous.profileDir), { code: 'ENOENT' }) + return { metadata: {}, close: async () => {}, capture: async () => {} } + } + `, + ) + await writeFile( + join(setup.root, 'scripts/smoke/scenarios.mjs'), + ` + export async function runScenarios() { return { checks: ['Fixture check'], requests: [] } } + `, + ) + const child = spawn( + process.execPath, + [ + join(setup.root, 'scripts/xvfb-smoke.mjs'), + '--browser', + 'all', + '--chromium-path', + setup.executable, + '--firefox-path', + process.execPath, + '--geckodriver-path', + process.execPath, + '--artifacts-dir', + join(setup.root, 'artifacts'), + ], + { + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let output = '' + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.on('data', (chunk) => { + output += chunk + }) + const exit = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', resolve) + }) + setup.cleanups.push(() => stopRunner(child, exit)) + assert.equal(await bounded(exit, 10000, 'Sequential browsers'), 0, output) + }, +) + +test( + 'runner reports success, failures, EPIPE and late signals truthfully', + { timeout: 60000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + // Substitute protocol boundaries only; exercise the real CLI, report and lifecycle. + await writeFile( + join(setup.root, 'scripts/smoke/chromium.mjs'), + ` + import process from 'node:process' + export async function startChromium({ lifecycle }) { + lifecycle.defer('fixture cleanup', async () => { + if (process.env.SMOKE_FIXTURE_MODE === 'cleanup') throw new Error('Fixture cleanup failure') + }) + return { metadata: {}, close: async () => {}, capture: async () => {} } + } + `, + ) + await writeFile( + join(setup.root, 'scripts/smoke/scenarios.mjs'), + ` + import process from 'node:process' + import { setTimeout } from 'node:timers/promises' + export async function runScenarios() { + await setTimeout(50) + if (process.env.SMOKE_FIXTURE_MODE === 'scenario-cleanup') { + throw Object.assign(new Error('Fixture primary failure'), { + cleanupErrors: [new Error('Fixture secondary cleanup failure')] + }) + } + if (process.env.SMOKE_FIXTURE_MODE === 'scenario') throw new Error('Fixture scenario failure') + return { checks: ['Fixture check'], requests: [] } + } + `, + ) + const interruptImport = join(setup.root, 'interrupt-report.mjs') + await writeFile( + interruptImport, + ` + import fs from 'node:fs/promises' + import process from 'node:process' + import { syncBuiltinESMExports } from 'node:module' + import { setTimeout } from 'node:timers/promises' + const original = fs.writeFile + let sent = false + fs.writeFile = async (...args) => { + const result = await original(...args) + if (!sent && String(args[0]).endsWith('report.json')) { + sent = true + process.kill(process.pid, 'SIGTERM') + await setTimeout(20) + } + return result + } + syncBuiltinESMExports() + `, + ) + const stdoutInterruptImport = join(setup.root, 'interrupt-stdout.mjs') + await writeFile( + stdoutInterruptImport, + ` + import process from 'node:process' + import { setInterval, clearInterval } from 'node:timers' + const original = process.stdout.write + let sent = false + process.stdout.write = function (chunk, callback) { + return original.call(this, chunk, (error) => { + if (error || sent) return callback(error) + sent = true + const signal = process.env.SMOKE_FIXTURE_MODE === 'stdout-int' ? 'SIGINT' : 'SIGTERM' + // Release the write only after the runner has observed the signal. + const keepAlive = setInterval(() => {}, 1000) + process.once(signal, () => { + clearInterval(keepAlive) + callback() + }) + process.kill(process.pid, signal) + }) + } + `, + ) + for (const [mode, expected] of [ + ['success', 0], + ['scenario', 1], + ['scenario-cleanup', 1], + ['cleanup', 1], + ['epipe', 1], + ['late-signal', 143], + ['stdout-int', 130], + ['stdout-term', 143], + ]) { + const parent = join(setup.root, mode) + const preload = + mode === 'late-signal' + ? interruptImport + : mode.startsWith('stdout-') + ? stdoutInterruptImport + : undefined + const child = spawn( + preload ? process.execPath : 'sh', + [ + ...(preload + ? ['--import', preload, join(setup.root, 'scripts/xvfb-smoke.mjs')] + : [join(setup.root, 'scripts/run-smoke.sh')]), + '--browser', + 'chromium', + '--chromium-path', + setup.executable, + '--artifacts-dir', + parent, + ], + { + env: { ...process.env, SMOKE_FIXTURE_MODE: mode }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) + let output = '' + if (mode === 'epipe') child.stdout.destroy() + else + child.stdout.on('data', (chunk) => { + output += chunk + }) + child.stderr.resume() + const exit = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', (code) => resolve(code)) + }) + setup.cleanups.push(() => stopRunner(child, exit)) + assert.equal(await bounded(exit, 5000, `${mode} exit`), expected, mode) + const directories = await readdir(parent) + const report = JSON.parse(await readFile(join(parent, directories[0], 'report.json'), 'utf8')) + assert.equal(report.exitCode, expected, mode) + assert.equal(report.result, expected === 0 ? 'PASS' : 'FAIL', mode) + if (mode.startsWith('stdout-')) { + assert.doesNotMatch(output, /PASS|exit 0/, mode) + assert.match(output, /artifacts:/, mode) + } + await assert.rejects(stat(report.browsers[0].profileDir), { code: 'ENOENT' }) + if (mode === 'epipe') assert.equal(report.failedStage, 'reporting') + if (mode === 'cleanup') assert.match(report.cleanup[0], /Fixture cleanup failure/) + if (mode === 'scenario-cleanup') { + assert.match(report.error, /Fixture primary failure/) + assert.match(report.cleanup[0], /Fixture secondary cleanup failure/) + } + } + }, +) + +test( + 'Firefox startup failures report preflight identity separately from the snapshot', + { timeout: 10000, skip: process.platform !== 'linux' }, + async (t) => { + const setup = await fixture(t) + await cp(join(setup.root, 'build/chromium'), join(setup.root, 'build/firefox'), { + recursive: true, + }) + await writeFile( + join(setup.root, 'scripts/smoke/firefox.mjs'), + ` + import { join } from 'node:path' + import { snapshotArtifact } from './artifacts.mjs' + export async function startFirefox(options) { + await snapshotArtifact(options.extensionDir, join(options.artifactsDir, 'extension'), + options.artifactSha256, options.signal) + throw new Error('Fixture Firefox startup failure') + } + `, + ) + const parent = join(setup.root, 'startup-failure') + const child = spawn( + process.execPath, + [ + join(setup.root, 'scripts/xvfb-smoke.mjs'), + '--browser', + 'firefox', + '--firefox-path', + setup.executable, + '--geckodriver-path', + process.execPath, + '--artifacts-dir', + parent, + ], + { stdio: 'ignore' }, + ) + const exit = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', resolve) + }) + setup.cleanups.push(() => stopRunner(child, exit)) + assert.equal(await bounded(exit, 5000, 'Firefox startup failure exit'), 1) + const directories = await readdir(parent) + const report = JSON.parse(await readFile(join(parent, directories[0], 'report.json'), 'utf8')) + assert.equal(report.failedStage, 'firefox:startup') + assert.match(report.error, /Fixture Firefox startup failure/) + assert.equal(report.browsers[0].preflightManifestVersion, '1.0') + assert.equal(Object.hasOwn(report.browsers[0], 'manifestVersion'), false) + assert.equal( + report.browsers[0].snapshotDir, + join(await realpath(parent), directories[0], 'firefox/extension'), + ) + assert.equal((await stat(join(report.browsers[0].snapshotDir, 'manifest.json'))).isFile(), true) + await assert.rejects(stat(report.browsers[0].profileDir), { code: 'ENOENT' }) + assert.deepEqual(report.cleanup, []) + }, +) diff --git a/tests/unit/scripts/smoke-lifecycle.test.mjs b/tests/unit/scripts/smoke-lifecycle.test.mjs new file mode 100644 index 000000000..f9ab3442f --- /dev/null +++ b/tests/unit/scripts/smoke-lifecycle.test.mjs @@ -0,0 +1,645 @@ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { performance } from 'node:perf_hooks' +import process from 'node:process' +import test from 'node:test' +import timers from 'node:timers' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' +import { bounded, createLifecycle, waitFor } from '../../../scripts/smoke/lifecycle.mjs' + +const fixture = fileURLToPath(new URL('../../fixtures/smoke/process.fixture', import.meta.url)) +const options = { timeout: 10000, skip: process.platform === 'win32' } + +function owned(t, options = {}) { + const lifecycle = createLifecycle({ cleanupTimeoutMs: 1500, ...options }) + t.after(() => lifecycle.force()) + return lifecycle +} + +function stalledCleanupClock(t) { + const clock = { elapsed: 0, delays: [] } + const setTimeout = timers.setTimeout + const time = t.mock.method(performance, 'now', () => clock.elapsed) + const timeout = t.mock.method(timers, 'setTimeout', (callback, ms, ...args) => { + clock.delays.push(ms) + // Only the two stalled-callback tests use this clock. Advance by the + // requested budget when its timer fires, independently of worker pauses. + return setTimeout(() => { + clock.elapsed += ms + callback(...args) + }, 0) + }) + syncBuiltinESMExports() + t.after(() => { + time.mock.restore() + timeout.mock.restore() + syncBuiltinESMExports() + }) + return clock +} + +async function ready(managed) { + await waitFor( + () => { + managed.assertRunning() + return managed.output().includes('READY') && managed.output().includes('stderr ready') + }, + { timeoutMs: 3000, label: 'Fixture startup' }, + ) +} + +async function isLive(pid) { + try { + process.kill(pid, 0) + if (process.platform === 'linux') { + const stat = await readFile(`/proc/${pid}/stat`, 'utf8') + return !['Z', 'X'].includes(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0]) + } + return true + } catch (error) { + if (error.code === 'ESRCH' || error.code === 'ENOENT') return false + throw error + } +} + +test( + 'bounded settles success, failure, timeout, and cancellation without late rejections', + options, + async () => { + assert.equal(await bounded(Promise.resolve(42), 100, 'Success'), 42) + const failure = new Error('Original failure') + await assert.rejects(bounded(Promise.reject(failure), 100, 'Failure'), (e) => e === failure) + await assert.rejects(bounded(new Promise(() => {}), 20, 'Stalled'), /Stalled timed out/) + const controller = new AbortController() + const pending = bounded( + delay(50).then(() => Promise.reject(failure)), + 500, + 'Abort', + controller.signal, + ) + controller.abort(failure) + await assert.rejects(pending, (e) => e === failure) + await assert.rejects( + bounded( + Promise.reject(new Error('Late rejection')), + 500, + 'Already aborted', + controller.signal, + ), + (e) => e === failure, + ) + await delay(75) + for (const ms of [0, -1, Infinity, NaN, 2147483648]) { + await assert.rejects(bounded(Promise.resolve(), ms, 'Invalid'), RangeError) + } + }, +) + +test('waitFor retries only falsy readiness and bounds a hung probe', options, async () => { + let attempts = 0 + assert.equal( + await waitFor(() => (++attempts === 3 ? 'ready' : false), { timeoutMs: 1000 }), + 'ready', + ) + assert.equal(attempts, 3) + attempts = 0 + await assert.rejects( + waitFor(() => { + attempts++ + throw new Error('Fatal probe') + }), + /Fatal probe/, + ) + assert.equal(attempts, 1) + await assert.rejects( + waitFor(() => new Promise(() => {}), { timeoutMs: 20, label: 'Hung probe' }), + /Hung probe timed out/, + ) + await assert.rejects( + waitFor(() => false, { timeoutMs: 20 }), + /timed out/, + ) + const controller = new AbortController() + controller.abort() + await assert.rejects( + waitFor(() => assert.fail('Aborted probe ran'), { signal: controller.signal }), + { name: 'AbortError' }, + ) +}) + +test( + 'cleanup is LIFO and idempotent, and continues after failed or hung callbacks', + options, + async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 180 }) + const seen = [] + lifecycle.defer('First', (...args) => { + assert.deepEqual(args, []) + seen.push('first') + }) + lifecycle.defer('Failed', () => { + seen.push('failed') + throw new Error('Cleanup failed') + }) + lifecycle.defer('Hung', () => { + seen.push('hung') + return new Promise(() => {}) + }) + const started = performance.now() + const cleanup = lifecycle.cleanup() + assert.equal(lifecycle.cleanup(), cleanup) + assert.throws(() => lifecycle.defer('Late', () => {}), /closing/) + assert.throws(() => lifecycle.spawn(process.execPath, [fixture, 'hang']), /closing/) + const errors = await cleanup + assert.equal(errors.length, 2) + assert.match(errors[0].message, /Hung.*timed out/) + assert.match(errors[1].message, /Failed.*Cleanup failed/) + assert.deepEqual(seen, ['hung', 'failed', 'first']) + assert.ok(performance.now() - started < 1000) + assert.equal(await lifecycle.cleanup(), errors) + }, +) + +test('spawn failure is observable and does not prevent independent cleanup', options, async (t) => { + const lifecycle = owned(t) + let cleaned = false + lifecycle.defer('Independent', () => { + cleaned = true + }) + const managed = lifecycle.spawn('/nonexistent/chatgptbox-smoke-executable', []) + await assert.rejects(managed.exited, /ENOENT/) + assert.throws(() => managed.assertRunning(), /ENOENT/) + const errors = await lifecycle.cleanup() + assert.equal(cleaned, true) + assert.ok(errors.some((error) => /ENOENT/.test(error.message))) +}) + +test('multiple stalled resources share one aggregate cleanup budget', options, async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 200 }) + const clock = stalledCleanupClock(t) + const seen = [] + for (let index = 0; index < 4; index++) { + lifecycle.defer(`Stalled ${index}`, () => { + seen.push(index) + return new Promise(() => {}) + }) + } + assert.equal((await lifecycle.cleanup()).length, 4) + assert.deepEqual(seen, [3, 2, 1, 0]) + assert.equal(clock.elapsed, 200, 'Cleanup multiplied the timeout by resource count') +}) + +test( + 'an exhausted aggregate deadline does not add a timer for every remaining resource', + options, + async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 20 }) + const clock = stalledCleanupClock(t) + let invoked = 0 + for (let index = 0; index < 250; index++) { + lifecycle.defer(`Stalled ${index}`, () => { + invoked++ + return new Promise(() => {}) + }) + } + const errors = await lifecycle.cleanup() + assert.equal(errors.length, 250) + assert.equal(invoked, 250) + assert.ok(errors.some((error) => /Overall cleanup deadline exceeded/.test(error.message))) + assert.equal(clock.elapsed, 20, 'Cleanup added per-resource delays after its deadline') + assert.ok(clock.delays.length <= 20, 'Expired resources must not allocate more timers') + }, +) + +test( + 'a failed callback cannot strand processes or remove their profile before reaping', + options, + async (t) => { + const lifecycle = owned(t) + let managed + let profileCleaned = false + lifecycle.defer('Profile', async () => { + assert.equal(await isLive(managed.child.pid), false) + assert.notEqual(managed.child.signalCode, null) + profileCleaned = true + }) + managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + lifecycle.defer('Broken protocol close', () => { + throw new Error('Protocol unavailable') + }) + const errors = await lifecycle.cleanup() + assert.equal(errors.length, 1) + assert.match(errors[0].message, /Protocol unavailable/) + assert.equal(profileCleaned, true) + }, +) + +test( + 'early exit preserves status and both output streams in diagnostics and log', + options, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-lifecycle-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const lifecycle = owned(t) + const logPath = join(directory, 'process.log') + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit'], { logPath }) + assert.deepEqual(await bounded(managed.exited, 3000, 'Early exit'), { code: 7, signal: null }) + assert.throws(() => managed.assertRunning(), /exited unexpectedly/) + assert.deepEqual(await lifecycle.cleanup(), []) + for (const output of [managed.output(), await readFile(logPath, 'utf8')]) { + assert.match(output, /stdout before exit/) + assert.match(output, /stderr before exit/) + } + }, +) + +test('cleanup terminates and reaps a hanging owned child', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.deepEqual(await managed.exited, { code: null, signal: 'SIGTERM' }) + assert.equal(await isLive(managed.child.pid), false) + await lifecycle.assertProcessesStopped() +}) + +test('parent cleanup never signals a remembered numeric process group', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + const originalKill = process.kill + const destructiveSignals = [] + try { + process.kill = (pid, signal) => { + if (signal !== 0) destructiveSignals.push({ pid, signal }) + return originalKill.call(process, pid, signal) + } + assert.deepEqual(await lifecycle.cleanup(), []) + assert.deepEqual(destructiveSignals, []) + } finally { + process.kill = originalKill + } +}) + +test('force before supervisor readiness prevents target startup', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit']) + assert.deepEqual(await lifecycle.force(), []) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.equal(managed.output(), '') + assert.equal(await isLive(managed.child.pid), false) +}) + +test('cleanup waiting on another resource cannot start a late-ready target', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit']) + lifecycle.defer('Protocol close', () => delay(300)) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.equal(managed.output(), '') +}) + +test('the target does not inherit the supervisor control channel', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, ['-e', 'console.log(typeof process.send)']) + assert.deepEqual(await bounded(managed.exited, 3000), { code: 0, signal: null }) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.equal(managed.output().trim(), 'undefined') +}) + +test( + 'control EOF invokes supervisor cleanup without waiting for ChildProcess close', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + managed.child.disconnect() + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /disconnected|ownership was lost/.test(error.message))) + await waitFor(async () => !(await isLive(managed.child.pid)), { timeoutMs: 2000 }) + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) + }, +) + +test('invalid supervisor results cannot resolve the target successfully', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + managed.child.emit('message', { version: 1, type: 'target-exit' }) + await assert.rejects(managed.exited, /Invalid supervisor message/) + assert.ok((await lifecycle.cleanup()).length > 0) + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) +}) + +test( + 'supervisor exit before final IPC delivery preserves the target result', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + const originalEmit = managed.child.emit + const pending = [] + let exited = false + let disconnected = false + let sawTerminating = false + managed.child.emit = function (event, ...args) { + if (event === 'message' && args[0]?.type === 'terminating') sawTerminating = true + if ( + !exited && + event === 'message' && + ['target-exit', 'terminating'].includes(args[0]?.type) + ) { + pending.push(args) + return true + } + if (event === 'disconnect' && !exited) { + disconnected = true + return true + } + const result = originalEmit.call(this, event, ...args) + if (event === 'exit') { + exited = true + for (const message of pending) originalEmit.call(this, 'message', ...message) + if (disconnected) originalEmit.call(this, 'disconnect') + } + return result + } + try { + assert.deepEqual(await lifecycle.cleanup(), []) + assert.equal(sawTerminating, true) + assert.deepEqual(await managed.exited, { code: null, signal: 'SIGTERM' }) + } finally { + managed.child.emit = originalEmit + } + }, +) + +test( + 'unexpected supervisor exit remains uncertain even after its group disappears', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'early-exit']) + assert.deepEqual(await bounded(managed.exited, 3000), { code: 7, signal: null }) + // The fixture has already ended; only our directly owned supervisor remains. + const exited = new Promise((resolve) => managed.child.once('exit', resolve)) + managed.child.kill('SIGKILL') + await bounded(exited, 2000) + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /Supervisor/.test(error.message))) + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) + }, +) + +test( + 'a timed-out process cleanup preserves its guarded profile until the process stops', + options, + async (t) => { + const profileDir = await mkdtemp(join(tmpdir(), 'smoke-lifecycle-preserved-profile-')) + const lifecycle = owned(t, { cleanupTimeoutMs: 1000 }) + lifecycle.defer('Profile', async () => { + await lifecycle.assertProcessesStopped() + await rm(profileDir, { recursive: true }) + }) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + const originalSend = managed.child.send + const originalDisconnect = managed.child.disconnect + let suppressedCommands = 0 + try { + await ready(managed) + managed.child.send = () => { + suppressedCommands++ + return true + } + managed.child.disconnect = () => {} + const errors = await lifecycle.cleanup() + assert.ok(suppressedCommands > 0) + assert.ok(errors.length > 0) + await assert.rejects( + bounded(lifecycle.assertProcessesStopped(), 2000, 'Check surviving process'), + /ownership was lost/, + ) + assert.equal(await isLive(managed.child.pid), true) + assert.equal((await stat(profileDir)).isDirectory(), true) + } finally { + managed.child.send = originalSend + managed.child.disconnect = originalDisconnect + if (managed.child.connected) managed.child.disconnect() + await lifecycle.force() + await waitFor(async () => !(await isLive(managed.child.pid)), { timeoutMs: 2000 }) + await rm(profileDir, { recursive: true, force: true }) + } + await assert.rejects(lifecycle.assertProcessesStopped(), /ownership was lost/) + }, +) + +test( + 'the stopped-process guard preserves ownership on missing or unknown process state', + options, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + const originalKill = process.kill + let inspectionError + const checks = [] + try { + await ready(managed) + process.kill = (pid, signal) => { + if (pid === -managed.child.pid) { + checks.push(signal) + if (inspectionError) throw inspectionError + } + return originalKill.call(process, pid, signal) + } + inspectionError = Object.assign(new Error('Process not visible'), { code: 'ESRCH' }) + await lifecycle.assertProcessesStopped() + inspectionError = Object.assign(new Error('Process inspection denied'), { code: 'EPERM' }) + await assert.rejects(lifecycle.assertProcessesStopped(), (error) => error === inspectionError) + inspectionError = undefined + await assert.rejects(lifecycle.assertProcessesStopped(), /still has live members/) + assert.deepEqual(checks, [0, 0, 0]) + } finally { + process.kill = originalKill + await lifecycle.force() + await assert.rejects( + bounded(managed.exited, 2000, 'Reap inspection fixture'), + /status unavailable/, + ) + } + await lifecycle.assertProcessesStopped() + }, +) + +test('cleanup escalates an owned group ignoring TERM to KILL', options, async (t) => { + const lifecycle = owned(t, { cleanupTimeoutMs: 800 }) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + assert.deepEqual(await lifecycle.cleanup(), []) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.match(managed.output(), /IGNORED TERM/) + assert.equal(await isLive(managed.child.pid), false) +}) + +test('descendants remain owned after their group leader exits', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'descendant']) + assert.deepEqual(await bounded(managed.exited, 3000, 'Leader exit'), { code: 0, signal: null }) + const descendant = Number(managed.output().match(/DESCENDANT (\d+)/)?.[1]) + assert.ok(Number.isInteger(descendant)) + assert.equal(await isLive(descendant), true) + assert.deepEqual(await lifecycle.cleanup(), []) + await waitFor(async () => !(await isLive(descendant)), { timeoutMs: 1000 }) +}) + +test( + 'independent lifecycles and an unrelated sentinel survive another lifecycle cleanup', + options, + async (t) => { + const sentinel = spawn(process.execPath, [fixture, 'hang'], { stdio: 'ignore' }) + const sentinelExit = new Promise((resolve) => sentinel.once('exit', resolve)) + t.after(async () => { + sentinel.kill('SIGKILL') + await bounded(sentinelExit, 2000, 'Sentinel exit') + }) + const first = owned(t) + const second = owned(t) + const one = first.spawn(process.execPath, [fixture, 'hang']) + const two = second.spawn(process.execPath, [fixture, 'hang']) + await Promise.all([ready(one), ready(two)]) + assert.deepEqual(await first.cleanup(), []) + two.assertRunning() + assert.equal(await isLive(two.child.pid), true) + assert.equal(await isLive(sentinel.pid), true) + assert.deepEqual(await second.cleanup(), []) + assert.equal(await isLive(sentinel.pid), true) + }, +) + +test( + 'abort starts cleanup and force acts as the second signal without global handlers', + options, + async (t) => { + const termListeners = process.listenerCount('SIGTERM') + const intListeners = process.listenerCount('SIGINT') + const controller = new AbortController() + const lifecycle = owned(t, { signal: controller.signal, cleanupTimeoutMs: 5000 }) + const managed = lifecycle.spawn(process.execPath, [fixture, 'ignore-term']) + await ready(managed) + controller.abort() + assert.throws(() => lifecycle.spawn(process.execPath, [fixture, 'hang']), { + name: 'AbortError', + }) + await waitFor(() => managed.output().includes('IGNORED TERM'), { timeoutMs: 1000 }) + const started = performance.now() + assert.equal(lifecycle.force(), lifecycle.cleanup()) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.ok(performance.now() - started < 750) + await assert.rejects(managed.exited, /Target exit status unavailable/) + assert.equal(process.listenerCount('SIGTERM'), termListeners) + assert.equal(process.listenerCount('SIGINT'), intListeners) + }, +) + +test( + 'pre-aborted lifecycle and force before spawn forbid acquiring resources', + options, + async (t) => { + const controller = new AbortController() + controller.abort() + const aborted = owned(t, { signal: controller.signal }) + assert.throws(() => aborted.spawn(process.execPath, [fixture, 'hang']), { name: 'AbortError' }) + assert.deepEqual(await aborted.cleanup(), []) + const forced = owned(t) + assert.deepEqual(await forced.force(), []) + assert.throws(() => forced.spawn(process.execPath, [fixture, 'hang']), /closing/) + }, +) + +test('log errors are handled, visible, and do not strand a running process', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang'], { + logPath: '/nonexistent/chatgptbox-smoke-directory/output.log', + }) + await waitFor(() => managed.output().includes('READY'), { timeoutMs: 3000 }) + await waitFor( + () => { + try { + managed.assertRunning() + return false + } catch (error) { + assert.match(error.message, /Log.*ENOENT/) + return true + } + }, + { timeoutMs: 3000, label: 'Log stream failure' }, + ) + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /Log.*ENOENT/.test(error.message))) + assert.equal(await isLive(managed.child.pid), false) +}) + +test('output stream errors are handled and retained during cleanup', options, async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'hang']) + await ready(managed) + managed.child.stdout.destroy(new Error('Read stream failure')) + await waitFor( + () => { + try { + managed.assertRunning() + return false + } catch (error) { + assert.match(error.message, /Read stream failure/) + return true + } + }, + { timeoutMs: 3000, label: 'Output stream failure' }, + ) + const errors = await lifecycle.cleanup() + assert.equal(errors.length, 1) + assert.match(errors[0].message, /Read stream failure/) + assert.equal(await isLive(managed.child.pid), false) +}) + +test( + 'a real log write error releases pipe backpressure and still allows cleanup', + { + ...options, + skip: process.platform !== 'linux', + }, + async (t) => { + const lifecycle = owned(t) + const managed = lifecycle.spawn(process.execPath, [fixture, 'flood'], { logPath: '/dev/full' }) + await waitFor(() => managed.output().includes('stderr ready'), { timeoutMs: 3000 }) + assert.throws(() => managed.assertRunning(), /ENOSPC/) + const errors = await lifecycle.cleanup() + assert.ok(errors.some((error) => /ENOSPC/.test(error.message))) + assert.equal(await isLive(managed.child.pid), false) + }, +) + +test( + 'large output is drained to disk while the diagnostic tail stays bounded', + options, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-lifecycle-flood-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const lifecycle = owned(t) + const logPath = join(directory, 'flood.log') + const managed = lifecycle.spawn(process.execPath, [fixture, 'flood'], { logPath }) + await ready(managed) + assert.ok(managed.output().length <= 1024 * 1024) + assert.deepEqual(await lifecycle.cleanup(), []) + const log = await readFile(logPath, 'utf8') + assert.equal(log.match(/O/g).length, 2 * 1024 * 1024) + assert.equal(log.match(/E/g).length, 2 * 1024 * 1024 + 1) + }, +) diff --git a/tests/unit/scripts/smoke-mock-server.test.mjs b/tests/unit/scripts/smoke-mock-server.test.mjs new file mode 100644 index 000000000..1201dbe10 --- /dev/null +++ b/tests/unit/scripts/smoke-mock-server.test.mjs @@ -0,0 +1,123 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import { TextDecoder } from 'node:util' +import { startMockServer } from '../../../scripts/smoke/mock-server.mjs' + +async function setup(t) { + const cleanups = [] + const mock = await startMockServer({ + lifecycle: { defer: (label, cleanup) => cleanups.push({ label, cleanup }) }, + }) + assert.equal(cleanups.length, 1) + t.after(() => cleanups[0].cleanup()) + return mock +} + +function post(mock, path, body = { model: 'smoke-model', stream: true, messages: [] }) { + return globalThis.fetch(mock.baseUrl + path, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: 'Bearer smoke-local-only' }, + body: JSON.stringify(body), + signal: globalThis.AbortSignal.timeout(5000), + }) +} + +test( + 'mock gates the remainder until a client has observed the complete first event', + { timeout: 10000 }, + async (t) => { + const mock = await setup(t) + assert.match(mock.baseUrl, /^http:\/\/127\.0\.0\.1:\d+$/) + const response = await post(mock, '/success/v1/chat/completions') + assert.equal(response.status, 200) + const reader = response.body.getReader() + const decoder = new TextDecoder() + let text = '' + while (!text.includes('\r\n\r\n')) { + const part = await reader.read() + assert.equal(part.done, false) + text += decoder.decode(part.value, { stream: true }) + } + assert.match(text, /Hello /) + assert.doesNotMatch(text, /δΈ–η•Œ|DONE/) + const waiting = reader.read() + assert.equal(await Promise.race([waiting, delay(30, 'still gated')]), 'still gated') + mock.release() + let part = await waiting + while (!part.done) { + text += decoder.decode(part.value, { stream: true }) + part = await reader.read() + } + text += decoder.decode() + const events = text + .split('\r\n\r\n') + .filter(Boolean) + .map((line) => line.slice(6)) + assert.equal(events.length, 3) + assert.equal(JSON.parse(events[0]).choices[0].delta.content, 'Hello ') + assert.equal(JSON.parse(events[1]).choices[0].delta.content, 'δΈ–η•ŒπŸ™‚') + assert.equal(events[2], '[DONE]') + assert.equal(mock.requests.length, 1) + assert.equal(mock.requests[0].status, 200) + await mock.close() + await mock.close() + }, +) + +test('mock reports HTTP 503 and records the Chat Completions body', async (t) => { + const mock = await setup(t) + const response = await post(mock, '/error/v1/chat/completions') + assert.equal(response.status, 503) + assert.deepEqual(await response.json(), { + error: { message: 'Smoke upstream unavailable', code: 'smoke_503' }, + }) + assert.deepEqual(mock.requests, [ + { + method: 'POST', + path: '/error/v1/chat/completions', + body: { model: 'smoke-model', stream: true, messages: [] }, + status: 503, + }, + ]) +}) + +test('mock rejects a Responses endpoint and invalid Chat Completions payload', async (t) => { + const mock = await setup(t) + for (const [path, body, status] of [ + ['/v1/responses', {}, 404], + ['/success/v1/chat/completions', { input: 'wrong API' }, 400], + ]) { + const response = await post(mock, path, body) + assert.equal(response.status, status) + await response.text() + } +}) + +test( + 'cleanup closes an unreleased response and its listening socket', + { timeout: 10000 }, + async (t) => { + const mock = await setup(t) + const response = await post(mock, '/success/v1/chat/completions') + const consumed = response.text().then( + () => 'ended', + () => 'closed', + ) + await mock.close() + assert.equal(await consumed, 'closed') + await assert.rejects(post(mock, '/error/v1/chat/completions')) + }, +) + +test('an already aborted startup allocates no server', async () => { + let registrations = 0 + await assert.rejects( + startMockServer({ + lifecycle: { defer: () => registrations++ }, + signal: globalThis.AbortSignal.abort(new Error('cancelled')), + }), + /cancelled/, + ) + assert.equal(registrations, 0) +}) diff --git a/tests/unit/scripts/smoke-runner.test.mjs b/tests/unit/scripts/smoke-runner.test.mjs new file mode 100644 index 000000000..b4dcac48f --- /dev/null +++ b/tests/unit/scripts/smoke-runner.test.mjs @@ -0,0 +1,404 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import process from 'node:process' +import { mkdtemp, mkdir, writeFile, rm, chmod, symlink, rename } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import fs from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { bounded, createLifecycle, waitFor } from '../../../scripts/smoke/lifecycle.mjs' +import { + parseArgs, + findExecutable, + hashArtifact, + preflight, + ROOT, + validateArtifactParent, +} from '../../../scripts/smoke/runner.mjs' + +test('artifact parent validation resolves aliases and missing descendants without writing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'smoke-parent-')) + t.after(() => rm(root, { recursive: true, force: true })) + const build = join(root, 'build/chromium') + await mkdir(build, { recursive: true }) + await writeFile(join(build, 'keep'), 'unchanged') + const before = await hashArtifact(build) + await symlink(build, join(root, 'alias')) + await symlink(join(root, 'build'), join(root, 'build-alias')) + for (const parent of [ + build, + join(build, 'new/deep'), + join(root, 'alias'), + join(root, 'alias/new/deep'), + join(root, 'build-alias/chromium/new'), + ]) { + await assert.rejects(validateArtifactParent(parent, ['chromium'], root), /outside.*build/) + } + assert.equal(await hashArtifact(build), before) + const realRoot = await fs.realpath(root) + assert.equal(await validateArtifactParent(root, ['chromium'], root), realRoot) + const sibling = join(root, 'build/chromium-other/new') + assert.equal( + await validateArtifactParent(sibling, ['chromium'], root), + join(realRoot, 'build/chromium-other/new'), + ) + await assert.rejects(fs.stat(sibling), { code: 'ENOENT' }) + await symlink(build, join(root, 'build/firefox')) + await assert.rejects( + validateArtifactParent(join(build, 'new'), ['firefox'], root), + /outside.*build/, + ) +}) + +test( + 'real runner rejects unsafe artifact parents before writes and accepts build ancestors', + { skip: process.platform !== 'linux' }, + async (t) => { + const root = await mkdtemp(join(tmpdir(), 'smoke-parent-runner-')) + let retainFixture = false + t.after(async () => { + if (retainFixture) { + t.diagnostic(`Retaining CLI fixture after incomplete cleanup: ${root}`) + return + } + await rm(root, { recursive: true, force: true }) + }) + await fs.cp(join(ROOT, 'scripts/smoke'), join(root, 'scripts/smoke'), { recursive: true }) + await fs.cp(join(ROOT, 'scripts/xvfb-smoke.mjs'), join(root, 'scripts/xvfb-smoke.mjs')) + await writeFile( + join(root, 'scripts/smoke/scenarios.mjs'), + 'export async function runScenarios() { return { checks: [], requests: [] } }\n', + ) + for (const browser of ['chromium', 'firefox']) { + const build = join(root, 'build', browser) + await mkdir(build, { recursive: true }) + await writeFile( + join(build, 'manifest.json'), + JSON.stringify({ name: 'Fixture', version: '1' }), + ) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'fixture') + await writeFile( + join(root, 'scripts/smoke', `${browser}.mjs`), + `export async function start${browser === 'chromium' ? 'Chromium' : 'Firefox'}() { + return { metadata: {}, close: async () => {}, capture: async () => {} } + }`, + ) + const before = await hashArtifact(build) + const alias = join(root, `${browser}-alias`) + await symlink(build, alias) + for (const parent of [build, join(build, 'new/deep'), alias, join(alias, 'new/deep'), root]) { + const result = await invoke( + process.execPath, + [ + join(root, 'scripts/xvfb-smoke.mjs'), + '--browser', + browser, + `--${browser}-path`, + process.execPath, + '--geckodriver-path', + process.execPath, + '--artifacts-dir', + parent, + ], + root, + ).catch((error) => { + if (error.cleanupErrors?.length) retainFixture = true + throw error + }) + assert.equal(result.code, parent === root ? 0 : 2, result.output) + if (parent !== root) { + assert.match(result.output, /artifacts: unavailable/) + assert.match(result.output, /Artifact parent must be outside the selected/) + } + assert.equal(await hashArtifact(build), before) + } + } + }, +) + +test('smoke CLI accepts explicit selections and rejects typos, duplicates and missing values', () => { + assert.deepEqual(parseArgs([]), { browser: 'all' }) + assert.deepEqual(parseArgs(['--browser', 'firefox', '--firefox-path', '/a path/firefox']), { + browser: 'firefox', + 'firefox-path': '/a path/firefox', + }) + for (const args of [ + ['--browser'], + ['--browser', 'chrome'], + ['--no-sandbox'], + ['x'], + ['--browser', 'all', '--browser', 'all'], + ]) { + assert.throws(() => parseArgs(args)) + } +}) + +test('explicit executable wins PATH; invalid explicit paths never fall back', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-path-test-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const executable = join(directory, 'a browser') + await writeFile(executable, '#!/bin/sh\nexit 0\n') + await chmod(executable, 0o700) + assert.equal(await findExecutable(executable, ['node'], process.env.PATH), executable) + assert.equal(await findExecutable(undefined, ['a browser'], directory), executable) + await assert.rejects( + findExecutable(join(directory, 'missing'), ['node'], process.env.PATH), + /not found/, + ) + await assert.rejects(findExecutable(directory, [], ''), /not found/) +}) + +test('build hash is stable and covers nested content and file names', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-hash-test-')) + t.after(() => rm(directory, { recursive: true, force: true })) + await mkdir(join(directory, 'nested')) + await writeFile(join(directory, 'nested/a'), 'one') + const first = await hashArtifact(directory) + assert.equal(await hashArtifact(directory), first) + await writeFile(join(directory, 'nested/a'), 'two') + const changedContent = await hashArtifact(directory) + assert.notEqual(changedContent, first) + await rename(join(directory, 'nested/a'), join(directory, 'nested/renamed')) + const changedName = await hashArtifact(directory) + assert.notEqual(changedName, changedContent) + await writeFile(join(directory, 'nested/b'), 'two') + assert.notEqual(await hashArtifact(directory), changedName) +}) + +test('preflight rejects missing artifacts', { skip: process.platform !== 'linux' }, async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-preflight-test-')) + t.after(() => rm(directory, { recursive: true, force: true })) + await assert.rejects( + preflight({ browser: 'chromium', 'chromium-path': process.execPath }, directory), + /ENOENT/, + ) +}) + +test('preflight preserves cancellation before checking executables or artifacts', async () => { + const controller = new AbortController() + const reason = new Error('Cancelled before preflight') + controller.abort(reason) + await assert.rejects( + preflight({ browser: 'chromium', 'chromium-path': '/missing' }, '/missing', controller.signal), + (error) => error === reason, + ) +}) + +test( + 'preflight passes cancellation into artifact reads and stops the walk', + { skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-preflight-abort-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const build = join(directory, 'build/chromium') + await mkdir(build, { recursive: true }) + await writeFile(join(build, 'manifest.json'), JSON.stringify({ name: 'Fixture', version: '1' })) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'current build') + const controller = new AbortController() + const reason = new Error('Cancelled during artifact read') + const originalReadFile = fs.readFile + let cancelledRead = false + let readsAfterAbort = 0 + t.mock.method(fs, 'readFile', async (path, options) => { + if (controller.signal.aborted) readsAfterAbort++ + if (path === join(build, 'background.js')) { + cancelledRead = true + assert.equal(options?.signal, controller.signal) + controller.abort(reason) + } + return originalReadFile(path, options) + }) + syncBuiltinESMExports() + t.after(() => { + t.mock.restoreAll() + syncBuiltinESMExports() + }) + await assert.rejects( + preflight( + { browser: 'chromium', 'chromium-path': process.execPath }, + directory, + controller.signal, + ), + (error) => error.name === 'AbortError' && error.cause === reason, + ) + assert.equal(cancelledRead, true) + assert.equal(readsAfterAbort, 0) + }, +) + +test( + 'Firefox preflight selects the directory independently of distribution ZIPs', + { skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-firefox-preflight-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const build = join(directory, 'build/firefox') + await mkdir(build, { recursive: true }) + await writeFile(join(build, 'manifest.json'), JSON.stringify({ name: 'Fixture', version: '1' })) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'current build') + const options = { + browser: 'firefox', + 'firefox-path': process.execPath, + 'geckodriver-path': process.execPath, + } + const [withoutZip] = await preflight(options, directory) + await writeFile(join(directory, 'build/firefox.zip'), 'stale archive') + const [withStaleZip] = await preflight(options, directory) + assert.equal(withoutZip.artifactSha256, await hashArtifact(build)) + assert.deepEqual(withStaleZip, withoutZip) + assert.equal(Object.hasOwn(withStaleZip, 'archive'), false) + assert.equal(Object.hasOwn(withStaleZip, 'archiveSha256'), false) + }, +) + +test( + 'Firefox preflight rejects a symlink build root before snapshot startup', + { skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-firefox-symlink-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const build = join(directory, 'actual-build') + await mkdir(build) + await mkdir(join(directory, 'build')) + await writeFile(join(build, 'manifest.json'), JSON.stringify({ name: 'Fixture', version: '1' })) + for (const name of ['background.js', 'popup.js', 'popup.html']) + await writeFile(join(build, name), 'current build') + await symlink(build, join(directory, 'build/firefox')) + await assert.rejects( + preflight( + { + browser: 'firefox', + 'firefox-path': process.execPath, + 'geckodriver-path': process.execPath, + }, + directory, + ), + /Firefox build source must be a directory/, + ) + }, +) + +async function invoke(command, args, cwd) { + const lifecycle = createLifecycle() + let managed + let result + let failure + try { + managed = lifecycle.spawn(command, args, { cwd }) + result = await bounded(managed.exited, 10000, 'CLI timeout') + } catch (error) { + failure = error + } + // Reuse owned-group termination, child reaping and pipe draining before settling. + const cleanupErrors = await lifecycle.cleanup() + if (cleanupErrors.length) + throw Object.assign( + new AggregateError( + failure ? [failure, ...cleanupErrors] : cleanupErrors, + failure?.message || 'CLI cleanup failed', + failure ? { cause: failure } : undefined, + ), + { cleanupErrors }, + ) + if (failure) throw failure + return { code: result.code, output: managed.output() } +} + +test( + 'CLI invocation preserves exit status and output and reports spawn failure', + { skip: process.platform !== 'linux' }, + async () => { + const result = await invoke( + process.execPath, + [ + '-e', + 'process.stdout.write("stdout"); process.stderr.write("stderr"); process.exitCode = 7', + ], + tmpdir(), + ) + assert.equal(result.code, 7) + assert.match(result.output, /stdout/) + assert.match(result.output, /stderr/) + await assert.rejects( + invoke(join(import.meta.dirname, 'missing-cli-executable'), [], tmpdir()), + /ENOENT/, + ) + }, +) + +test( + 'CLI timeouts stop owned descendants before rejecting', + { timeout: 30000, skip: process.platform !== 'linux' }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'smoke-cli-timeout-')) + const marker = join(directory, 'pids.json') + async function running(pid) { + try { + const stat = await fs.readFile(`/proc/${pid}/stat`, 'utf8') + return !['Z', 'X'].includes(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0]) + } catch (error) { + if (error.code === 'ENOENT' || error.code === 'ESRCH') return false + throw error + } + } + t.after(async () => { + let pids = [] + try { + pids = JSON.parse(await fs.readFile(marker, 'utf8')) + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + for (const pid of pids) { + assert.ok(Number.isInteger(pid) && pid > 0) + if (await running(pid)) { + try { + process.kill(pid, 'SIGKILL') + } catch (error) { + if (error.code !== 'ESRCH') throw error + } + } + } + await waitFor(async () => (await Promise.all(pids.map(running))).every((value) => !value), { + timeoutMs: 5000, + label: 'owned CLI fixture processes stopped', + }) + await rm(directory, { recursive: true, force: true }) + }) + const source = ` + const { spawn } = require('node:child_process') + const { writeFileSync } = require('node:fs') + const child = spawn(process.execPath, ['-e', + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)" + ], { stdio: ['ignore', 'inherit', 'inherit'] }) + writeFileSync(process.argv[1], JSON.stringify([process.pid, child.pid])) + process.on('SIGTERM', () => {}) + setInterval(() => {}, 1000) + ` + await assert.rejects(invoke(process.execPath, ['-e', source, marker], directory), /CLI timeout/) + const pids = JSON.parse(await fs.readFile(marker, 'utf8')) + assert.equal(pids.length, 2) + for (const pid of pids) assert.equal(await running(pid), false, `Owned process ${pid} survived`) + }, +) + +test( + 'Node, shell and npm entries work outside the repository and propagate argument failures', + { skip: process.platform !== 'linux' }, + async () => { + const entries = [ + [process.execPath, [join(ROOT, 'scripts/xvfb-smoke.mjs')]], + ['sh', [join(ROOT, 'scripts/run-smoke.sh')]], + ['npm', ['--prefix', ROOT, 'run', 'smoke', '--']], + ] + for (const [command, args] of entries) { + const help = await invoke(command, [...args, '--help'], tmpdir()) + assert.equal(help.code, 0, help.output) + assert.match(help.output, /Usage: npm run smoke/) + const invalid = await invoke(command, [...args, '--invalid'], tmpdir()) + assert.equal(invalid.code, 2, invalid.output) + } + }, +) diff --git a/tests/unit/scripts/smoke-scenarios.test.mjs b/tests/unit/scripts/smoke-scenarios.test.mjs new file mode 100644 index 000000000..dfcd536d9 --- /dev/null +++ b/tests/unit/scripts/smoke-scenarios.test.mjs @@ -0,0 +1,438 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { runInNewContext } from 'node:vm' +import { Server } from 'node:http' +import { + assertPopupIdentity, + assertScenarioMessages, + beginRequest, + disconnectPorts, + runScenarios, + snapshot, +} from '../../../scripts/smoke/scenarios.mjs' +import { createLifecycle } from '../../../scripts/smoke/lifecycle.mjs' + +for (const { name, primaryFailure = false, corruptContext } of [ + { name: 'scenario cleanup retains all failures with a primary error', primaryFailure: true }, + { name: 'scenario cleanup retains all failures without a primary error' }, + ...['missing', 'reordered', 'corrupted', 'extra success'].map((corruptContext) => ({ + name: `runScenarios rejects ${corruptContext} outbound context and cleans up`, + corruptContext, + })), +]) { + test(name, async (t) => { + const lifecycle = createLifecycle() + t.after(() => lifecycle.cleanup()) + const primary = new Error('PRIMARY_CONFIGURATION_FAILURE') + const disconnectFailure = new Error('SECONDARY_DISCONNECT_FAILURE') + const closeFailure = new Error('SECONDARY_SERVER_CLOSE_FAILURE') + const cleanupCalls = [] + const close = Server.prototype.close + t.mock.method(Server.prototype, 'close', function (callback) { + cleanupCalls.push('close') + return close.call(this, (error) => callback(error || closeFailure)) + }) + const metadata = { + expectedName: 'ChatGPTBox', + expectedVersion: '2.7.0', + extensionId: 'smoke-extension', + popupUrl: 'moz-extension://smoke-uuid/popup.html', + } + let baseUrl + let completed = false + let streamed + let disconnectAttempts = 0 + const adapter = { + metadata, + evaluateCleanup(fn, ...args) { + return this.evaluate(fn, ...args) + }, + async evaluate(fn, ...args) { + if (fn.name === 'popupIdentity') + return { + visible: true, + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + } + if (fn.name === 'configure') { + if (primaryFailure) throw primary + baseUrl = args[0] + return + } + if (fn.name === 'disconnectPorts') { + cleanupCalls.push('disconnect') + if (++disconnectAttempts === 1) throw disconnectFailure + return + } + if (fn.name === 'beginRequest') { + const [key, question, history] = args + let messages = [ + ...history.flatMap(({ question, answer }) => [ + { role: 'user', content: question }, + { role: 'assistant', content: answer }, + ]), + { role: 'user', content: question }, + ] + if (key === 'error') { + if (corruptContext === 'missing') messages = messages.slice(-1) + if (corruptContext === 'reordered') { + messages = [messages[1], messages[0], messages[2]] + } + if (corruptContext === 'corrupted') messages[1].content = 'Hello δΈ–η•ŒοΏ½' + } else if (corruptContext === 'extra success') { + messages.unshift({ role: 'user', content: 'Unexpected previous question' }) + } + const response = await globalThis.fetch(`${baseUrl}/${key}/v1/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer smoke-local-only', + }, + body: JSON.stringify({ + model: 'smoke-model', + stream: true, + messages, + }), + signal: globalThis.AbortSignal.timeout(5000), + }) + if (key === 'success') { + streamed = response.text().then(() => { + completed = true + }) + streamed.catch(() => {}) + } else await response.text() + return + } + if (fn.name === 'snapshot') { + const history = [{ question: 'Smoke success question', answer: 'Hello δΈ–η•ŒπŸ™‚' }] + if (args[0] === 'error') return [session(history), { error: 'smoke_503' }] + return completed + ? [ + session([]), + { answer: 'Hello ', done: false }, + { answer: 'Hello δΈ–η•ŒπŸ™‚', done: false }, + { ...session(history), answer: null, done: true }, + ] + : [session([]), { answer: 'Hello ', done: false }] + } + throw new Error(`Unexpected evaluation: ${fn.name}`) + }, + } + const checks = [] + let reported + await assert.rejects(runScenarios(adapter, { lifecycle, checks }), (error) => { + reported = error + if (primaryFailure) assert.equal(error, primary) + else if (corruptContext) { + assert.ok(error instanceof assert.AssertionError) + assert.equal(error.code, 'ERR_ASSERTION') + assert.equal(error.operator, 'deepStrictEqual') + const expected = [{ role: 'user', content: 'Smoke success question' }] + if (corruptContext !== 'extra success') { + expected.push( + { role: 'assistant', content: 'Hello δΈ–η•ŒπŸ™‚' }, + { role: 'user', content: 'Smoke error question' }, + ) + } + assert.deepEqual(error.expected, expected) + assert.notDeepEqual(error.actual, expected) + assert.equal(checks.length, 4) + } else { + assert.ok(error instanceof AggregateError) + assert.deepEqual(error.errors, [disconnectFailure, closeFailure]) + assert.equal(checks.length, 4) + } + assert.deepEqual(error.cleanupErrors, [disconnectFailure, closeFailure]) + assert.ok(error.cleanupErrors.every((failure) => failure instanceof Error)) + return true + }) + assert.deepEqual(cleanupCalls, ['disconnect', 'close']) + await streamed + await lifecycle.cleanup() + assert.equal( + disconnectAttempts, + 2, + 'A later successful disconnect retry must not erase its first failure', + ) + assert.deepEqual(reported.cleanupErrors, [disconnectFailure, closeFailure]) + }) +} + +test('request state survives fresh evaluation sandboxes sharing the popup window', () => { + const window = {} + let onMessage + let disconnected = 0 + const sent = [] + const port = { + onMessage: { + addListener: (listener) => { + onMessage = listener + }, + }, + onDisconnect: { addListener() {} }, + postMessage: (message) => sent.push(message), + disconnect: () => disconnected++, + } + const browser = { runtime: { connect: () => port } } + const evaluate = (fn, ...args) => + runInNewContext(`(${fn.toString()})(...args)`, { + window, + browser, + args, + }) + evaluate(beginRequest, 'success', 'Question', []) + assert.equal(sent.length, 1) + onMessage({ answer: 'Hello ', done: false }) + assert.equal(evaluate(snapshot, 'success')[0].answer, 'Hello ') + onMessage({ answer: 'Hello δΈ–η•ŒπŸ™‚', done: false }) + assert.equal(evaluate(snapshot, 'success')[1].answer, 'Hello δΈ–η•ŒπŸ™‚') + assert.throws(() => evaluate(beginRequest, 'success', 'Duplicate', []), /already started/) + assert.equal(sent.length, 1) + evaluate(disconnectPorts) + assert.equal(disconnected, 1) + assert.equal(window.__chatGPTBoxSmoke, undefined) + evaluate(disconnectPorts) + assert.equal(disconnected, 1) +}) + +test('a failed scenario preserves completed checks in the caller array and disconnects ports', async (t) => { + const lifecycle = createLifecycle() + t.after(async () => assert.deepEqual(await lifecycle.cleanup(), [])) + const checks = ['Existing runner check'] + const failure = new Error('Configuration write failed') + const metadata = { + expectedName: 'ChatGPTBox', + expectedVersion: '2.7.0', + extensionId: 'smoke-extension', + popupUrl: 'moz-extension://smoke-uuid/popup.html', + } + const calls = [] + const adapter = { + metadata, + evaluateCleanup(fn, ...args) { + return this.evaluate(fn, ...args) + }, + async evaluate(fn) { + calls.push(fn.name) + if (fn.name === 'popupIdentity') { + return { + visible: true, + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + } + } + if (fn.name === 'disconnectPorts') return + throw failure + }, + } + await assert.rejects(runScenarios(adapter, { lifecycle, checks }), (error) => error === failure) + assert.deepEqual(checks, [ + 'Existing runner check', + 'Installed popup visible with matching runtime identity and version', + ]) + assert.deepEqual(calls, ['popupIdentity', 'configure', 'disconnectPorts']) +}) + +test('popup identity uses the adapter installation manifest and rejects mismatches or missing expectations', () => { + const metadata = { + expectedName: 'ChatGPTBox', + expectedVersion: '2.7.0', + extensionId: 'smoke-extension', + popupUrl: 'moz-extension://smoke-uuid/popup.html', + } + const identity = { + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + visible: true, + } + assertPopupIdentity(identity, metadata) + for (const override of [ + { name: 'Wrong extension' }, + { version: '0.0.0' }, + { extensionId: 'other-extension' }, + { location: 'https://example.com/' }, + { visible: false }, + ]) { + assert.throws(() => assertPopupIdentity({ ...identity, ...override }, metadata)) + } + for (const key of Object.keys(metadata)) { + assert.throws(() => assertPopupIdentity(identity, { ...metadata, [key]: undefined })) + } +}) + +const question = 'Smoke success question' +const history = [{ question: 'Previous question', answer: 'Previous answer' }] +const session = (records) => ({ session: { conversationRecords: records } }) +const partial = [session(history), { answer: 'Hello ', done: false }] +const completed = [ + ...partial, + { answer: 'Hello δΈ–η•ŒπŸ™‚', done: false }, + { ...session([...history, { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }]), answer: null, done: true }, +] +const failure = [session(history), { error: '{"error":{"code":"smoke_503"}}' }] + +test('scenario assertions accept partial, final, and HTTP failure messages', () => { + assertScenarioMessages(partial, { phase: 'partial', question, history }) + assertScenarioMessages(completed, { phase: 'final', question, history }) + assertScenarioMessages(failure, { phase: 'error', question, history }) +}) + +test('partial proof rejects completion before release and incorrect first answers', () => { + for (const messages of [ + completed, + [session(history), { answer: 'Hello δΈ–η•ŒπŸ™‚' }], + [session(history)], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'partial', question, history })) + } +}) + +test('partial proof requires one unchanged session acknowledgement before the answer', () => { + for (const messages of [ + [partial[1]], + [partial[1], partial[0]], + [partial[0], partial[0], partial[1]], + [{ ...partial[0], ...partial[1] }], + [session([]), partial[1]], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'partial', question, history })) + } +}) + +test('final proof rejects duplicate completion, corrupted Unicode, and duplicate history', () => { + const duplicateHistory = { + ...session([ + ...history, + { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }, + { question, answer: 'Hello δΈ–η•ŒπŸ™‚' }, + ]), + done: true, + } + for (const messages of [ + [...completed, completed.at(-1)], + [...partial, { answer: 'Hello δΈ–η•ŒοΏ½' }, completed.at(-1)], + [...completed.slice(0, -1), duplicateHistory], + partial, + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'final', question, history })) + } +}) + +test('final proof preserves the initial acknowledgement and completion ordering', () => { + for (const messages of [ + [...completed.slice(0, -1), session(history), completed.at(-1)], + [...completed.slice(0, -1), session([]), completed.at(-1)], + [session([]), ...completed.slice(1)], + [...partial, completed.at(-1), completed[2]], + [...partial, { ...completed.at(-1), answer: 'Hello δΈ–η•ŒπŸ™‚' }], + [...completed.slice(0, -1), { ...completed.at(-1), answer: undefined }], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'final', question, history })) + } +}) + +test('HTTP failure requires one standalone acknowledgement before the error', () => { + for (const messages of [ + [failure[1]], + [failure[1], failure[0]], + [failure[0], failure[0], failure[1]], + [{ ...failure[0], ...failure[1] }], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'error', question, history })) + } +}) + +test('HTTP failure proof rejects false success, unrelated errors, and modified history', () => { + for (const messages of [ + [...failure, { done: true }], + [...failure, { answer: 'unexpected' }], + [...failure, session([...history, { question, answer: '' }])], + [session(history), { error: 'Unexpected provider failure' }], + [session([]), failure[1]], + ]) { + assert.throws(() => assertScenarioMessages(messages, { phase: 'error', question, history })) + } +}) + +test('answer sequences reject corruption, duplication, omission and reordering before a valid final answer', () => { + for (const answers of [ + ['wrong', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 'corrupt', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 42, 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 'Hello ', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello ', 'Hello δΈ–η•ŒπŸ™‚', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello δΈ–η•ŒπŸ™‚', 'Hello ', 'Hello δΈ–η•ŒπŸ™‚'], + ['Hello δΈ–η•ŒπŸ™‚'], + ]) { + assert.throws(() => + assertScenarioMessages( + [session(history), ...answers.map((answer) => ({ answer, done: false })), completed.at(-1)], + { phase: 'final', question, history }, + ), + ) + } + for (const answers of [ + ['bad', 'Hello '], + ['Hello ', 'Hello '], + ]) { + assert.throws(() => + assertScenarioMessages( + answers.map((answer) => ({ answer })), + { phase: 'partial', question, history }, + ), + ) + } +}) + +test('scenario cancellation uses independent cleanup and successful disconnect is idempotent', async (t) => { + const controller = new AbortController() + const lifecycle = createLifecycle({ signal: controller.signal }) + t.after(() => lifecycle.cleanup()) + const reason = new Error('Cancelled during configuration') + let disconnects = 0 + const metadata = { + expectedName: 'Fixture', + expectedVersion: '1', + extensionId: 'fixture', + popupUrl: 'moz-extension://fixture/popup.html', + } + const adapter = { + metadata, + async evaluate(fn) { + controller.signal.throwIfAborted() + if (fn.name === 'popupIdentity') + return { + visible: true, + name: metadata.expectedName, + version: metadata.expectedVersion, + extensionId: metadata.extensionId, + popupUrl: metadata.popupUrl, + location: metadata.popupUrl, + } + assert.equal(fn.name, 'configure') + controller.abort(reason) + throw reason + }, + async evaluateCleanup(fn) { + assert.equal(controller.signal.aborted, true) + assert.equal(fn, disconnectPorts) + disconnects++ + }, + } + await assert.rejects(runScenarios(adapter, { lifecycle, signal: controller.signal }), (error) => { + assert.equal(error, reason) + assert.equal(error.cleanupErrors, undefined) + return true + }) + assert.deepEqual(await lifecycle.cleanup(), []) + assert.equal(disconnects, 1) +})