diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d136c41 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## 0.8.0 + +- Add declarative saved browser QA scenarios with isolated desktop/mobile + sessions, literal assertions, screenshots, and bounded error/request evidence. +- Record exact Git HEAD and dirty state plus observed changes during each run; + expose typed local JSON summaries for Console and PR workflows. +- Add scenario validation, bounded timeouts and cleanup, private output, and + real Chromium localhost regression coverage. Existing pane actions are unchanged. + +## 0.7.0 + +- Shared agent-browser sessions, local Chromium launch, CDP attach, streaming, + failed-request visibility, and readiness/build diagnostics. diff --git a/README.md b/README.md index 086e914..a33bbd9 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ guides to this plugin and its three siblings (Guard, Swarm, Conductor). ## Highlights +- **Repeatable QA scenarios** — run saved desktop/mobile checks in fresh browser + sessions and collect commit-bound screenshots, assertions, and error evidence. + See the [QA guide](docs/qa.md) (Browser 0.8.0). - **Shared agent sessions** — one isolated browser session per Herdr workspace. - **Attach to any CDP browser** — observe a Playwright, Puppeteer, or Browser Use run (or any Chrome started with `--remote-debugging-port`) without owning it. @@ -46,7 +49,7 @@ guides to this plugin and its three siblings (Guard, Swarm, Conductor). | --- | --- | --- | | Herdr | `>= 0.7.0` | Tested with Herdr 0.7.4 | | Node.js | `>= 20` | Node 22+ enables live WebSocket streaming, CDP attach mode, and launch mode | -| agent-browser | Optional | Required for shared agent sessions; tested with agent-browser 0.33.x; failed-request reporting needs the `network requests` command | +| agent-browser | Optional | Required for shared agent sessions, recording, and saved QA scenarios; QA requires 0.33.0+; tested with 0.33.x | | Chromium/Chrome | Optional | Any Chromium-based browser enables launch mode (`l`) and attach mode | | chafa | Optional | ANSI rendering and streamed JPEGs in Kitty mode | | carbonyl | Optional | Only required for the separate interactive Browse action | @@ -463,8 +466,9 @@ needed. ## Development -Use Node 22+ for full browser support (Node 20 supports polling only), Python -3.11+ for manifest validation, and ShellCheck for launcher validation. The +Use Node 22+ for the pane's full streaming, CDP attach, and launch support. +Node 20 supports pane polling and the standalone saved QA runner. Use Python +3.11+ for manifest validation and ShellCheck for launcher validation. The plugin runs its source directly; there is no bundled browser or compilation step. diff --git a/bin/qa.mjs b/bin/qa.mjs new file mode 100755 index 0000000..136c05b --- /dev/null +++ b/bin/qa.mjs @@ -0,0 +1,351 @@ +#!/usr/bin/env node +// Saved declarative QA against a fresh agent-browser session for each viewport. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { randomUUID, createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; + +const execute = promisify(execFile); +const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const MAX_CONFIG = 64 * 1024; +const MAX_OUTPUT = 2 * 1024 * 1024; +const MAX_RESULT = 1024 * 1024; +const MAX_ENTRIES = 100; +const TYPES = { + navigate: ["path"], click: ["selector"], fill: ["selector", "value"], + waitFor: ["selector"], assertText: ["selector", "contains"], + assertVisible: ["selector"], assertUrl: ["contains"], + assertTitle: ["contains"], screenshot: ["name"], +}; + +function object(value, keys, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`); + for (const key of Object.keys(value)) if (!keys.includes(key)) throw new Error(`${label}: unknown field ${key}`); +} +function string(value, label, max = 1000) { + if (typeof value !== "string" || !value.length || value.length > max || /[\u0000-\u001f\u007f]/.test(value)) { + throw new Error(`${label} must be nonempty text of at most ${max} characters without control characters`); + } +} +function slug(value, label) { + if (typeof value !== "string" || !/^[a-z][a-z0-9-]{0,39}$/.test(value)) throw new Error(`${label} must be a lowercase slug (max 40 characters)`); +} +function integer(value, min, max, label) { + if (!Number.isInteger(value) || value < min || value > max) throw new Error(`${label} must be an integer from ${min} to ${max}`); +} +function baseUrl(value) { + string(value, "baseUrl", 2000); + const url = new URL(value); + if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) { + throw new Error("baseUrl must be an HTTP(S) URL without credentials, query, or fragment"); + } + return url; +} + +export function validateScenario(value) { + object(value, ["schemaVersion", "name", "baseUrl", "viewports", "steps", "timeoutMs", "runTimeoutMs", "failOnConsoleError", "failOnPageError", "failOnFailedRequest"], "scenario"); + if (value.schemaVersion !== 1) throw new Error("schemaVersion must be 1"); + string(value.name, "name", 120); + const base = baseUrl(value.baseUrl); + if (!Array.isArray(value.viewports) || value.viewports.length < 1 || value.viewports.length > 4) throw new Error("viewports must contain 1–4 entries"); + const names = new Set(); + for (const viewport of value.viewports) { + object(viewport, ["name", "width", "height"], "viewport"); + slug(viewport.name, "viewport name"); + if (names.has(viewport.name)) throw new Error("viewport names must be unique"); + names.add(viewport.name); + integer(viewport.width, 320, 1920, "viewport width"); + integer(viewport.height, 240, 1600, "viewport height"); + } + if (!Array.isArray(value.steps) || value.steps.length < 1 || value.steps.length > 40) throw new Error("steps must contain 1–40 entries"); + if (value.steps[0]?.type !== "navigate") throw new Error("first step must navigate"); + for (const step of value.steps) { + const fields = TYPES[step?.type]; + if (!fields) throw new Error("unknown step type; scripts and eval are not supported"); + object(step, ["type", ...fields], "step"); + for (const field of fields) string(step[field], `step ${field}`); + for (const field of ["selector", "value"]) { + if (step[field]?.startsWith("-")) throw new Error(`step ${field} must not begin with '-'`); + } + if (step.type === "screenshot") slug(step.name, "screenshot name"); + if (step.type === "navigate") { + if (!step.path.startsWith("/") || step.path.startsWith("//") || step.path.includes("\\")) throw new Error("navigate path must be an origin-relative /path"); + if (new URL(step.path, base).origin !== base.origin) throw new Error("navigate must stay on baseUrl origin"); + } + } + for (const key of ["failOnConsoleError", "failOnPageError", "failOnFailedRequest"]) { + if (value[key] !== undefined && typeof value[key] !== "boolean") throw new Error(`${key} must be boolean`); + } + integer(value.timeoutMs ?? 10_000, 100, 30_000, "timeoutMs"); + integer(value.runTimeoutMs ?? 120_000, 1000, 300_000, "runTimeoutMs"); + return { + ...value, timeoutMs: value.timeoutMs ?? 10_000, runTimeoutMs: value.runTimeoutMs ?? 120_000, + failOnConsoleError: value.failOnConsoleError ?? true, + failOnPageError: value.failOnPageError ?? true, + failOnFailedRequest: value.failOnFailedRequest ?? true, + }; +} + +export function loadScenario(file, override) { + const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + let bytes; + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile()) throw new Error("scenario must be a regular file, not a pipe or device"); + if (stat.size > MAX_CONFIG) throw new Error("scenario exceeds 64 KiB"); + const buffer = Buffer.alloc(MAX_CONFIG + 1); + let length = 0; + while (length < buffer.length) { + const read = fs.readSync(fd, buffer, length, buffer.length - length, null); + if (!read) break; + length += read; + } + if (length > MAX_CONFIG) throw new Error("scenario exceeds 64 KiB"); + bytes = buffer.subarray(0, length); + } finally { fs.closeSync(fd); } + const parsed = JSON.parse(bytes); + if (override !== undefined) parsed.baseUrl = override; + const scenario = validateScenario(parsed); + return { scenario, sha256: hash(JSON.stringify(scenario)) }; +} + +export async function gitState(repo, env = process.env) { + const gitEnv = Object.fromEntries(Object.entries(env).filter(([key]) => !key.startsWith("GIT_"))); + Object.assign(gitEnv, { GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0", GIT_PAGER: "cat" }); + const git = async (...args) => (await execute("git", ["--no-pager", "-c", "core.fsmonitor=false", "-C", repo, ...args], { env: gitEnv, timeout: 5000, maxBuffer: MAX_OUTPUT })).stdout; + const commit = (await git("rev-parse", "HEAD")).trim(); + if (!/^[a-f0-9]{40}$/.test(commit)) throw new Error("QA requires a Git repository with a committed SHA-1 HEAD"); + const root = fs.realpathSync((await git("rev-parse", "--show-toplevel")).trim()); + const status = await git("status", "--porcelain=v1", "--untracked-files=all"); + const diff = await git("diff", "HEAD", "--binary", "--no-ext-diff", "--no-textconv"); + const branch = (await git("branch", "--show-current")).trim() || null; + return { root, commit, branch, dirty: status.length !== 0, fingerprint: hash(status + diff) }; +} + +// Engine defaults/extensions/profiles/remote providers from the user's environment +// must not turn a fresh QA session into an attached or authenticated session. +export function engineEnvironment(env, timeoutMs) { + const result = {}; + for (const key of ["PATH", "HOME", "TMPDIR", "LANG", "LC_ALL", "DISPLAY", "XAUTHORITY", "AGENT_BROWSER_EXECUTABLE_PATH"]) { + if (env[key] !== undefined) result[key] = env[key]; + } + result.AGENT_BROWSER_DEFAULT_TIMEOUT = String(timeoutMs); + result.AGENT_BROWSER_IDLE_TIMEOUT_MS = "60000"; + return result; +} + +function text(value) { return String(value ?? "").slice(0, 2000); } +function safeUrl(value) { + try { const url = new URL(value); url.username = ""; url.password = ""; url.search = ""; url.hash = ""; return text(url.href); } + catch { return "[unavailable URL]"; } +} +function entries(data, key) { + const items = Array.isArray(data) ? data : data?.[key]; + if (!Array.isArray(items)) throw new Error(`engine returned invalid ${key} telemetry`); + if (items.length > MAX_ENTRIES) throw new Error(`${key} telemetry exceeds ${MAX_ENTRIES} entries; evidence would be incomplete`); + return items; +} + +export function classifyTelemetry(consoleData, errorData, requestData) { + const messages = entries(consoleData, "messages"); + const errors = entries(errorData, "errors"); + const requests = entries(requestData, "requests"); + return { + consoleErrors: messages.filter((item) => item.type === "error" || item.level === "error").map((item) => ({ message: text(item.text ?? item.message) })), + pageErrors: errors.map((item) => ({ message: text(typeof item === "string" ? item : item.message ?? item.text) })), + failedRequests: requests.filter((item) => Number(item.status) >= 400 || item.failure || item.error).map((item) => ({ + method: text(item.method ?? "GET"), url: safeUrl(item.url), status: typeof item.status === "number" ? item.status : null, + })), + unresolvedRequests: requests.filter((item) => item.status == null && !item.failure && !item.error).length, + }; +} + +export function createEngine({ binary, session, configFile, cwd, env, timeoutMs, deadline, signal }) { + return async (args, { cleanup = false } = {}) => { + const remaining = cleanup ? 10_000 : Math.min(timeoutMs + 1000, deadline - Date.now()); + if (remaining <= 0) throw new Error("QA run deadline exceeded"); + try { + const { stdout } = await execute(binary, ["--config", configFile, "--session", session, "--json", ...args], { + cwd, env, timeout: remaining, maxBuffer: MAX_OUTPUT, signal: cleanup ? undefined : signal, + killSignal: "SIGKILL", + }); + const result = JSON.parse(stdout); + if (result.success !== true) throw new Error(text(result.error || "engine command failed")); + return result.data; + } catch (error) { + if (error.killed || error.name === "AbortError") throw new Error("engine command timed out or was interrupted"); + // execFile errors embed arguments/stdout; do not accidentally publish filled values. + if (error.code) throw new Error(`engine command failed (${error.code})`); + throw new Error(text(error.message)); + } + }; +} + +function saveJson(file, value) { fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); } + +export function boundResult(result) { + if (Buffer.byteLength(JSON.stringify(result, null, 2)) + 1 <= MAX_RESULT) return; + result.status = "failed"; + result.error = "Evidence exceeds 1 MiB; raw telemetry omitted. This run is incomplete."; + result.summary.passed = 0; + result.summary.failed = result.summary.viewports; + for (const run of result.runs) { + run.status = "failed"; + run.consoleErrors = []; + run.pageErrors = []; + run.failedRequests = []; + } +} + +export async function runQa({ config, repo, output, baseUrl: override, binary = "agent-browser", env = process.env, signal, engineFactory = createEngine }) { + const loaded = loadScenario(config, override); + const scenario = loaded.scenario; + const before = await gitState(repo, env); + const outputDir = output ? path.join(fs.realpathSync(path.dirname(path.resolve(output))), path.basename(path.resolve(output))) : fs.mkdtempSync(path.join(os.tmpdir(), "herdr-qa-")); + if (outputDir === before.root || outputDir.startsWith(`${before.root}${path.sep}`)) throw new Error("QA output must be outside the tested repository"); + if (output) fs.mkdirSync(outputDir, { mode: 0o700 }); + fs.chmodSync(outputDir, 0o700); + const configFile = path.join(outputDir, "engine-config.json"); + saveJson(configFile, {}); + const engineEnv = engineEnvironment(env, scenario.timeoutMs); + const runId = randomUUID(); + const deadline = Date.now() + scenario.runTimeoutMs; + const result = { + schemaVersion: 1, kind: "herdr-browser-qa", runId, status: "failed", + startedAt: new Date().toISOString(), finishedAt: null, + scenario: { name: scenario.name, sha256: loaded.sha256, policy: { + failOnConsoleError: scenario.failOnConsoleError, + failOnPageError: scenario.failOnPageError, + failOnFailedRequest: scenario.failOnFailedRequest, + } }, + git: { commit: before.commit, branch: before.branch, dirty: before.dirty, changedDuringRun: false }, + engine: { name: "agent-browser", version: null }, + runs: [], cleanup: { status: "passed" }, + }; + const resultPath = path.join(outputDir, "result.json"); + try { + const version = await execute(binary, ["--version"], { cwd: outputDir, env: engineEnv, timeout: 5000, maxBuffer: 4096 }); + result.engine.version = version.stdout.trim(); + const versionParts = /^agent-browser (\d+)\.(\d+)\.(\d+)$/.exec(result.engine.version); + if (!versionParts || !(Number(versionParts[1]) > 0 || Number(versionParts[2]) >= 33)) throw new Error("QA requires agent-browser >=0.33.0"); + for (const viewport of scenario.viewports) { + const run = { viewport, status: "failed", steps: [], artifacts: [], consoleErrors: [], pageErrors: [], failedRequests: [], unresolvedRequests: 0 }; + result.runs.push(run); + const session = `herdr-qa-${runId.replaceAll("-", "")}-${result.runs.length}`; + const engine = engineFactory({ binary, session, configFile, cwd: outputDir, env: engineEnv, timeoutMs: scenario.timeoutMs, deadline, signal }); + const screenshot = async (name) => { + const relative = `${viewport.name}-${name}.png`; + const target = path.join(outputDir, relative); + await engine(["screenshot", target]); + const stat = fs.lstatSync(target); + if (!stat.isFile() || stat.size > 10 * 1024 * 1024) throw new Error("screenshot missing or exceeds 10 MiB"); + fs.chmodSync(target, 0o600); + const bytes = fs.readFileSync(target); + if (!bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) throw new Error("engine screenshot is not PNG"); + run.artifacts.push({ kind: "screenshot", path: relative, sha256: hash(bytes), bytes: bytes.length }); + }; + let launched = false; + try { + launched = true; + await engine(["open", "about:blank"]); + await engine(["set", "viewport", String(viewport.width), String(viewport.height)]); + await engine(["console", "--clear"]); + await engine(["errors", "--clear"]); + await engine(["network", "requests", "--clear"]); + for (const [index, step] of scenario.steps.entries()) { + const outcome = { index, type: step.type, status: "failed" }; + run.steps.push(outcome); + try { + if (step.type === "navigate") await engine(["open", new URL(step.path, scenario.baseUrl).href]); + else if (step.type === "click") await engine(["click", step.selector]); + else if (step.type === "fill") await engine(["fill", step.selector, step.value]); + else if (step.type === "waitFor") await engine(["wait", step.selector]); + else if (step.type === "screenshot") await screenshot(`${index}-${step.name}`); + else { + const args = step.type === "assertVisible" ? ["is", "visible", step.selector] + : step.type === "assertText" ? ["get", "text", step.selector] + : ["get", step.type === "assertUrl" ? "url" : "title"]; + const data = await engine(args); + const key = { assertVisible: "visible", assertText: "text", assertUrl: "url", assertTitle: "title" }[step.type]; + const actual = data?.[key]; + if (step.type === "assertVisible" ? actual !== true : typeof actual !== "string" || !actual.includes(step.contains)) throw new Error(`${step.type} assertion failed`); + } + outcome.status = "passed"; + } catch (error) { outcome.error = text(error.message); throw error; } + } + run.status = "passed"; + } catch (error) { run.error = text(error.message); } + finally { + if (launched) { + try { await screenshot("final"); } catch (error) { run.status = "failed"; run.evidenceError = text(error.message); } + try { + Object.assign(run, classifyTelemetry(await engine(["console"]), await engine(["errors"]), await engine(["network", "requests"]))); + if ((scenario.failOnConsoleError && run.consoleErrors.length) || (scenario.failOnPageError && run.pageErrors.length) || (scenario.failOnFailedRequest && (run.failedRequests.length || run.unresolvedRequests))) run.status = "failed"; + } catch (error) { run.status = "failed"; run.telemetryError = text(error.message); } + try { await engine(["close"], { cleanup: true }); } + catch (error) { result.cleanup.status = "failed"; run.cleanupError = text(error.message); run.status = "failed"; } + } + } + if (signal?.aborted || Date.now() >= deadline) break; + } + } catch (error) { result.error = text(error.code ? `QA preflight failed (${error.code})` : error.message); } + try { + const after = await gitState(repo, env); + result.git.changedDuringRun = before.commit !== after.commit || before.branch !== after.branch || before.fingerprint !== after.fingerprint; + } catch { result.git.changedDuringRun = true; } + result.summary = { + viewports: scenario.viewports.length, + passed: result.runs.filter((run) => run.status === "passed").length, + failed: scenario.viewports.length - result.runs.filter((run) => run.status === "passed").length, + assertions: result.runs.reduce((sum, run) => sum + run.steps.filter((step) => step.type.startsWith("assert")).length, 0), + consoleErrors: result.runs.reduce((sum, run) => sum + run.consoleErrors.length, 0), + pageErrors: result.runs.reduce((sum, run) => sum + run.pageErrors.length, 0), + failedRequests: result.runs.reduce((sum, run) => sum + run.failedRequests.length, 0), + }; + result.status = result.summary.failed === 0 && !result.git.changedDuringRun && result.cleanup.status === "passed" && !result.error ? "passed" : "failed"; + if (signal?.aborted) { result.status = "failed"; result.error = "QA run interrupted"; } + result.finishedAt = new Date().toISOString(); + boundResult(result); + saveJson(resultPath, result); + return { ...result, resultPath }; +} + +export async function main(argv = process.argv.slice(2)) { + if (argv.includes("--help")) { + console.log("Usage: qa.mjs check --config scenario.json\n qa.mjs run --config scenario.json --repo /git/repo [--output /new/private/dir] [--base-url http://localhost:3000] [--json]\nQA uses fresh browser sessions; screenshots and messages may contain private data. Review evidence before sharing."); + return 0; + } + const action = argv.shift(); + if (!["run", "check"].includes(action)) throw new Error("expected run or check; use --help"); + const options = {}; + for (let i = 0; i < argv.length; i++) { + const key = argv[i]; + if (key === "--json") { options.json = true; continue; } + if (!["--config", "--repo", "--output", "--base-url"].includes(key) || !argv[i + 1] || argv[i + 1].startsWith("--")) throw new Error(`invalid option ${key}`); + if (options[key] !== undefined) throw new Error(`duplicate option ${key}`); + options[key] = argv[++i]; + } + if (!options["--config"]) throw new Error("--config is required"); + if (action === "check") { + const { scenario, sha256 } = loadScenario(options["--config"], options["--base-url"]); + console.log(JSON.stringify({ schemaVersion: 1, status: "valid", name: scenario.name, sha256 })); + return 0; + } + if (!options["--repo"]) throw new Error("--repo is required"); + const controller = new AbortController(); + const abort = () => controller.abort(); + process.once("SIGINT", abort); + process.once("SIGTERM", abort); + try { + const result = await runQa({ config: options["--config"], repo: options["--repo"], output: options["--output"], baseUrl: options["--base-url"], signal: controller.signal }); + console.log(options.json ? JSON.stringify(result) : `${result.status.toUpperCase()}: ${result.summary.passed}/${result.summary.viewports} viewports; evidence ${result.resultPath}`); + return result.status === "passed" ? 0 : 1; + } finally { process.removeListener("SIGINT", abort); process.removeListener("SIGTERM", abort); } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + main().then((code) => { process.exitCode = code; }, (error) => { console.error(`QA: ${text(error.message)}`); process.exitCode = 2; }); +} diff --git a/docs/qa.md b/docs/qa.md new file mode 100644 index 0000000..0e98b89 --- /dev/null +++ b/docs/qa.md @@ -0,0 +1,157 @@ +# Repeatable browser QA + +Browser 0.8 adds saved desktop/mobile scenarios backed by the installed +`agent-browser` CLI. Each viewport gets a new browser session; the runner +collects assertion outcomes, viewport screenshots, console errors, page errors, +and failed HTTP requests in a private evidence directory. No Herdr session is +needed for this standalone command. + +## First run + +Requirements: Node.js 20+, Git with a committed SHA-1 HEAD, agent-browser +0.33.0+, and its installed Chromium engine. Tested with agent-browser 0.33.2. + +```sh +npm install -g agent-browser +agent-browser install +``` + +Copy [the example scenario](../examples/qa.scenario.json) into the application +repository as `.herdr-browser-qa.json`. Set the expected heading and any other +checks to match the app. Commit the scenario and app changes to produce clean +commit-bound evidence. Start the app's development server yourself; QA does not +execute project scripts or start servers. + +From the Browser checkout: + +```sh +npm run qa -- check --config /path/to/app/.herdr-browser-qa.json +npm run qa -- run --config /path/to/app/.herdr-browser-qa.json \ + --repo /path/to/app --base-url http://localhost:3000 \ + --output /private/tmp/my-app-qa-run --json +``` + +The output directory must not exist yet, its parent must exist, and it must be +outside the tested repository. Omit `--output` to create a private temporary +directory. Each run prints its result location. `--json` prints the result as +one JSON object, with an additional absolute `resultPath` for local consumers. +The saved `result.json` contains the same evidence without that absolute path. +Exit 0 means all requested viewports passed and cleanup succeeded; exit 1 means +a failed run (including failed engine preflight); exit 2 means invalid command, +configuration, repository, or output setup. + +## Scenario contract + +The JSON file requires `schemaVersion: 1`, a `name`, HTTP(S) `baseUrl`, +`viewports`, and `steps`. Unknown fields and step types are refused. The first +step must be `navigate`. A viewport is `{ "name": "desktop", "width": 1440, +"height": 900 }`; names must be unique lowercase slugs. All steps run in order +for every viewport; the first failed step stops that viewport, then the runner +attempts a final screenshot, telemetry, and cleanup before the next viewport. + +| Step `type` | Other fields | Check or behavior | +| --- | --- | --- | +| `navigate` | `path` | Open `/path` on the configured base origin | +| `click` | `selector` | Click a CSS/engine selector | +| `fill` | `selector`, `value` | Replace an input's text with fixture data | +| `waitFor` | `selector` | Wait for the selected element to be visible | +| `assertVisible` | `selector` | Require a visible element | +| `assertText` | `selector`, `contains` | Require an element's text to contain the literal text | +| `assertTitle` | `contains` | Require the page title to contain literal text | +| `assertUrl` | `contains` | Require the current URL to contain literal text | +| `screenshot` | `name` | Save the current viewport as a named PNG | + +The runner always attempts a final viewport screenshot, including on assertion +failure. Screenshots use CSS viewport sizes, not full-page captures. Mobile +means a narrow viewport; it does not emulate touch input, device scale, mobile +user agent, or a physical device. Screenshot capture alone does not establish +visual correctness or accessibility compliance. + +Limits: 64 KiB config, 1–4 viewports, widths 320–1920, heights 240–1600, +1–40 steps, and 10 MiB per screenshot. `timeoutMs` defaults to 10000 (range +100–30000); `runTimeoutMs` defaults to 120000 (range 1000–300000), plus bounded +cleanup. Each command's captured output is limited to 2 MiB; each telemetry +category is limited to 100 entries. Exceeding a telemetry bound fails evidence +collection rather than silently dropping observations. Assertions are immediate; +use `waitFor` after transitions before asserting their result. +The final JSON is capped at 1 MiB. If raw telemetry would exceed that cap, the +runner removes it, marks the run incomplete and failed, and retains counts and +artifact references. Configuration must be a regular file; symlinks, devices, +and pipes are rejected. Git metadata ignores inherited repository-routing +environment variables and disables fsmonitor/external diff helpers. + +`failOnConsoleError`, `failOnPageError`, and `failOnFailedRequest` default to +`true`; an explicit `false` records that category without making it fail the +run. HTTP 4xx/5xx responses are failed requests. Some engine versions expose +no-response requests without distinguishing a transport failure from a still +pending request; these are separately counted as `unresolvedRequests`, and also +fail the run when `failOnFailedRequest` is true. Long-lived requests may need an +explicit project decision about that setting. Network capture is an observation +window, not proof of complete network coverage. + +No arbitrary JavaScript, `eval`, shell commands, uploaded files, persistent +profiles, saved authentication, or browser attach settings are accepted in the +scenario. CLI operands cannot begin with `-`. The engine uses an explicit empty +config and does not inherit agent-browser sessions, providers, extensions, +restore state, init scripts, or generic credential environment variables. +`AGENT_BROWSER_EXECUTABLE_PATH` remains available to select local Chromium. + +## Evidence and Console/PR use + +`result.json` has `schemaVersion: 1`, `kind: "herdr-browser-qa"`, unique `runId`, +`status` (`passed` or `failed`), start/end ISO timestamps, the resolved scenario +name and SHA-256, engine version, and: + +- `scenario.policy`: the three resolved `failOn*` booleans. A passing run with + a relaxed policy means only its configured criteria passed. Strict handoff + consumers should require all three booleans to be `true` and zero error counts. +- `git.commit`, `git.branch`, and `git.dirty`: the tested repository's observed + start state. Dirty runs are permitted but visibly labeled. +- `git.changedDuringRun`: a comparison of start/end HEAD, branch, status, and + tracked diff. A detected change fails the overall result. This comparison + does not detect temporary changes reverted before completion or changes to + the contents of already-untracked files. Dirty evidence is not clean-commit + verification. +- `summary`: requested `viewports`, `passed`/`failed` viewport counts, attempted + `assertions`, and `consoleErrors`, `pageErrors`, `failedRequests` counts. +- `runs[]`: viewport geometry, status, indexed step outcomes, error details, + bounded telemetry, unresolved request count, and artifact metadata. Each + artifact records a relative path, byte count, and SHA-256. +- `cleanup.status`: `passed` or `failed`. Failed cleanup fails the result. + +The Git record does **not** prove the served app was built from that commit. +Run the server from the intended checkout and review its build/deployment +provenance separately. Evidence is ordinary editable local JSON, not signed +attestation, an approval receipt, or permission to merge/deploy. + +Console consumers may read the typed summary and link local artifacts. PR +consumers should match `git.commit` to the proposed head and require +`dirty === false`, `changedDuringRun === false`, passed status, complete +viewport counts, and successful cleanup before describing a clean QA run. +Never interpolate raw browser output into executable commands. + +The output directory is `0700`; result and screenshots are `0600`. Console/page +messages and screenshots can contain sensitive app data. Network URLs omit +credentials, query strings, and fragments, but paths and messages can still +contain secrets. Review artifacts before sharing; do not commit them by default. +Use fixture accounts and non-sensitive form values. Browser interactions can +submit forms or navigate away through the app; this is not an OS sandbox or +network containment boundary. + +Only the newly generated session is closed, never a shared or attached browser. +SIGINT/SIGTERM request bounded cleanup and write failed evidence. A force kill +or host crash cannot guarantee evidence publication or cleanup; sessions have a +short idle timeout as a backstop. + +## Verification + +```sh +npm run validate +npm run test:qa +``` + +The opt-in QA integration test runs a localhost fixture in actual Chromium at +desktop and mobile sizes, checks PNG dimensions, exercises fill/click/assertions, +and proves error, HTTP failure, screenshot-on-failure, and cleanup behavior. +The ordinary suite includes schema, output bounds, isolation, Git drift, +failure/cleanup, and subprocess timeout tests without needing a browser engine. diff --git a/examples/qa.scenario.json b/examples/qa.scenario.json new file mode 100644 index 0000000..70ebc9b --- /dev/null +++ b/examples/qa.scenario.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "name": "Local app smoke", + "baseUrl": "http://localhost:3000", + "viewports": [ + { "name": "desktop", "width": 1440, "height": 900 }, + { "name": "mobile", "width": 390, "height": 844 } + ], + "steps": [ + { "type": "navigate", "path": "/" }, + { "type": "waitFor", "selector": "h1" }, + { "type": "assertVisible", "selector": "h1" }, + { "type": "assertText", "selector": "h1", "contains": "Your app heading" }, + { "type": "screenshot", "name": "home" } + ] +} diff --git a/herdr-plugin.toml b/herdr-plugin.toml index 9b38136..9affdd2 100644 --- a/herdr-plugin.toml +++ b/herdr-plugin.toml @@ -1,6 +1,6 @@ id = "structupath.browser" name = "Browser" -version = "0.7.0" +version = "0.8.0" min_herdr_version = "0.7.0" description = "Driveable browser pane: live screenshots, console output, localhost links; drives agent-browser sessions, attaches to any CDP browser, or launches its own Chromium" platforms = ["macos", "linux"] diff --git a/package.json b/package.json index 5041f4b..8d37e0f 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,14 @@ { "name": "herdr-browser", - "version": "0.7.0", + "version": "0.8.0", "private": true, "type": "module", "engines": { "node": ">=20" }, "scripts": { "build": "node scripts/build.mjs", "doctor": "node bin/doctor.mjs", + "qa": "node bin/qa.mjs", + "test:qa": "HERDR_BROWSER_REQUIRE_QA=1 node --test tests/qa.test.mjs tests/qa.integration.test.mjs", "test": "node --test tests/*.test.mjs", "test:integration": "HERDR_BROWSER_REQUIRE_INTEGRATION=1 node --test --test-name-pattern='end to end|e2e:' tests/launch.integration.test.mjs tests/renderer.test.mjs", "validate": "npm run build && shellcheck scripts/*.sh && npm test" diff --git a/tests/manifest.test.mjs b/tests/manifest.test.mjs index 70dd475..575bb48 100644 --- a/tests/manifest.test.mjs +++ b/tests/manifest.test.mjs @@ -57,8 +57,8 @@ test("release version and existing action IDs remain stable", () => { path.join(root, "herdr-plugin.toml"), "utf8", ); - assert.equal(packageJson.version, "0.7.0"); - assert.match(manifest, /^version = "0\.7\.0"$/m); + assert.equal(packageJson.version, "0.8.0"); + assert.match(manifest, /^version = "0\.8\.0"$/m); assert.deepEqual( [...manifest.matchAll(/^id = "([^"]+)"$/gm)] .slice(1, 6) diff --git a/tests/qa.integration.test.mjs b/tests/qa.integration.test.mjs new file mode 100644 index 0000000..89931d1 --- /dev/null +++ b/tests/qa.integration.test.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import test from "node:test"; +import { runQa } from "../bin/qa.mjs"; + +test("e2e: saved QA runs desktop/mobile assertions and records real browser failures", { + skip: process.env.HERDR_BROWSER_REQUIRE_QA !== "1" ? "opt in with npm run test:qa (agent-browser + Chromium required)" : false, + timeout: 180_000, +}, async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "hb-qa-e2e-")); + const repo = path.join(root, "repo"); + fs.mkdirSync(repo); + const git = (...args) => execFileSync("git", ["-C", repo, ...args], { stdio: "ignore" }); + git("init", "-q"); git("config", "user.email", "fixture@example.test"); git("config", "user.name", "Fixture"); + fs.writeFileSync(path.join(repo, "README.md"), "Synthetic browser QA fixture\n"); + git("add", "README.md"); git("-c", "commit.gpgsign=false", "commit", "-qm", "fixture"); + const server = http.createServer((req, res) => { + if (req.url === "/failure") { res.writeHead(503); res.end("unavailable"); return; } + if (req.url === "/favicon.ico") { res.writeHead(204); res.end(); return; } + res.setHeader("Content-Type", "text/html"); + res.end(`QA fixture + +

Ready

Waiting

+

`); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(async () => { await new Promise((resolve) => server.close(resolve)); fs.rmSync(root, { recursive: true, force: true }); }); + const config = path.join(root, "scenario.json"); + const scenario = { + schemaVersion: 1, name: "localhost fixture", baseUrl: `http://127.0.0.1:${server.address().port}`, + viewports: [{ name: "desktop", width: 1440, height: 900 }, { name: "mobile", width: 390, height: 844 }], + steps: [{ type: "navigate", path: "/" }, { type: "assertVisible", selector: "h1" }, + { type: "assertTitle", contains: "QA fixture" }, { type: "fill", selector: "#name", value: "Fixture" }, + { type: "click", selector: "#submit" }, { type: "assertText", selector: "#result", contains: "Hello Fixture" }], + }; + fs.writeFileSync(config, JSON.stringify(scenario)); + const passed = await runQa({ config, repo, output: path.join(root, "passed") }); + assert.equal(passed.status, "passed", JSON.stringify(passed)); + assert.equal(passed.git.dirty, false); + assert.equal(passed.summary.assertions, 6); + for (const run of passed.runs) { + const image = fs.readFileSync(path.join(root, "passed", run.artifacts[0].path)); + assert.equal(image.readUInt32BE(16), run.viewport.width); + assert.equal(image.readUInt32BE(20), run.viewport.height); + } + scenario.viewports = [scenario.viewports[0]]; + scenario.steps = [{ type: "navigate", path: "/broken" }, { type: "waitFor", selector: "#failure-observed" }, { type: "assertText", selector: "h1", contains: "Deliberately absent" }]; + fs.writeFileSync(config, JSON.stringify(scenario)); + const failed = await runQa({ config, repo, output: path.join(root, "failed") }); + assert.equal(failed.status, "failed"); + assert.equal(failed.cleanup.status, "passed"); + assert.equal(failed.runs[0].artifacts.length, 1); + assert.ok(failed.runs[0].steps.some((step) => step.status === "failed")); + assert.ok(failed.runs[0].consoleErrors.some((entry) => entry.message.includes("fixture console error")), JSON.stringify(failed)); + assert.ok(failed.runs[0].pageErrors.some((entry) => entry.message.includes("fixture page error")), JSON.stringify(failed)); + assert.ok(failed.runs[0].failedRequests.some((entry) => entry.status === 503), JSON.stringify(failed)); +}); diff --git a/tests/qa.test.mjs b/tests/qa.test.mjs new file mode 100644 index 0000000..50ef882 --- /dev/null +++ b/tests/qa.test.mjs @@ -0,0 +1,186 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import test from "node:test"; +import { validateScenario, loadScenario, gitState, engineEnvironment, classifyTelemetry, boundResult, createEngine, runQa } from "../bin/qa.mjs"; + +function scenario(overrides = {}) { + return { schemaVersion: 1, name: "Fixture", baseUrl: "http://127.0.0.1:3456", viewports: [{ name: "desktop", width: 1440, height: 900 }, { name: "mobile", width: 390, height: 844 }], steps: [{ type: "navigate", path: "/" }, { type: "assertText", selector: "h1", contains: "Ready" }], ...overrides }; +} +function fixture(t, settings = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "hb-qa-unit-")); + t.after(() => fs.rmSync(root, { force: true, recursive: true })); + const repo = path.join(root, "repo"); + fs.mkdirSync(repo); + const config = path.join(repo, "qa.json"); + fs.writeFileSync(config, JSON.stringify(scenario(settings))); + const git = (...args) => execFileSync("git", ["-C", repo, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + git("init", "-q"); git("config", "user.email", "fixture@example.test"); git("config", "user.name", "Fixture"); + git("add", "qa.json"); git("-c", "commit.gpgsign=false", "commit", "-qm", "fixture"); + const binary = path.join(root, "engine"); + fs.writeFileSync(binary, '#!/bin/sh\nprintf "agent-browser 0.33.2\\n"\n', { mode: 0o700 }); + return { root, repo, config, binary, git, output: path.join(root, "evidence") }; +} +function fakeFactory(log, mutate) { + return ({ session }) => async (args, options) => { + log.push({ session, args, options }); + await mutate?.(args); + if (args[0] === "screenshot") fs.writeFileSync(args[1], Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])); + if (args[0] === "get") return { text: "Ready", title: "Ready", url: "http://127.0.0.1:3456/" }; + if (args[0] === "is") return { visible: true }; + if (args[0] === "console") return { messages: [] }; + if (args[0] === "errors") return { errors: [] }; + if (args[0] === "network") return { requests: [] }; + return {}; + }; +} + +test("QA schema is declarative and bounds options, counts, URLs, filenames, and CLI operands", () => { + assert.equal(validateScenario(scenario()).timeoutMs, 10_000); + for (const invalid of [ + scenario({ eval: "anything" }), scenario({ steps: [{ type: "eval", code: "anything" }] }), + scenario({ steps: [{ type: "navigate", path: "//foreign.test" }] }), + scenario({ steps: [{ type: "navigate", path: "/\\foreign.test" }] }), + scenario({ steps: [{ type: "navigate", path: "/" }, { type: "fill", selector: "#a", value: "--cdp" }] }), + scenario({ steps: [{ type: "navigate", path: "/" }, { type: "screenshot", name: "../secret" }] }), + scenario({ baseUrl: "file:///tmp/anything" }), scenario({ baseUrl: "https://user:pass@example.test" }), + scenario({ timeoutMs: 100_000 }), scenario({ failOnConsoleError: "false" }), + scenario({ viewports: [{ name: "huge", width: 10_000, height: 800 }] }), + scenario({ viewports: [scenario().viewports[0], scenario().viewports[0]] }), + ]) assert.throws(() => validateScenario(invalid)); +}); + +test("QA config is byte-bounded and resolved overrides affect its evidence digest", (t) => { + const f = fixture(t); + const first = loadScenario(f.config); + assert.notEqual(loadScenario(f.config, "http://localhost:1234").sha256, first.sha256); + fs.writeFileSync(f.config, " ".repeat(65 * 1024)); + assert.throws(() => loadScenario(f.config), /64 KiB/); +}); + +test("QA config rejects symlinks and FIFOs without blocking", (t) => { + const f = fixture(t); + const link = path.join(f.root, "link.json"); + fs.symlinkSync(f.config, link); + assert.throws(() => loadScenario(link), /ELOOP/); + const fifo = path.join(f.root, "pipe"); + execFileSync("mkfifo", [fifo]); + assert.throws(() => loadScenario(fifo), /regular file/); + assert.throws(() => loadScenario(f.repo), /regular file/); +}); + +test("QA Git metadata ignores inherited repository/config overrides and never runs fsmonitor", async (t) => { + const f = fixture(t); const other = fixture(t); + const marker = path.join(f.root, "fsmonitor-ran"); + const monitor = path.join(f.root, "monitor"); + fs.writeFileSync(monitor, `#!/bin/sh\ntouch '${marker}'\n`, { mode: 0o700 }); + f.git("config", "core.fsmonitor", monitor); + fs.writeFileSync(path.join(other.repo, "different.txt"), "different"); + other.git("add", "different.txt"); other.git("-c", "commit.gpgsign=false", "commit", "-qm", "different"); + const state = await gitState(f.repo, { ...process.env, GIT_DIR: path.join(other.repo, ".git"), GIT_WORK_TREE: other.repo, + GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "core.worktree", GIT_CONFIG_VALUE_0: other.repo }); + assert.equal(state.commit, f.git("rev-parse", "HEAD")); + assert.equal(state.root, fs.realpathSync(f.repo)); + assert.equal(state.dirty, false); + assert.equal(fs.existsSync(marker), false); +}); + +test("QA engine does not inherit shared sessions, remote providers, profiles, init scripts, or credentials", () => { + const env = engineEnvironment({ PATH: "/bin", HOME: "/home/fixture", AGENT_BROWSER_EXECUTABLE_PATH: "/chrome", AGENT_BROWSER_SESSION: "shared", AGENT_BROWSER_PROVIDER: "paid", AGENT_BROWSER_RESTORE: "auth", AGENT_BROWSER_INIT_SCRIPTS: "/script", API_KEY: "secret", NODE_OPTIONS: "--require=bad" }, 1000); + assert.deepEqual(Object.keys(env).sort(), ["AGENT_BROWSER_DEFAULT_TIMEOUT", "AGENT_BROWSER_EXECUTABLE_PATH", "AGENT_BROWSER_IDLE_TIMEOUT_MS", "HOME", "PATH"].sort()); +}); + +test("QA telemetry distinguishes failed HTTP responses from incomplete requests and strips URL secrets", () => { + const observed = classifyTelemetry({ messages: [{ type: "error", text: "broken" }, { type: "log", text: "fine" }] }, { errors: ["page threw"] }, { requests: [ + { status: 500, url: "https://user:pass@example.test/api?token=private#secret", method: "POST" }, + { status: 200 }, { status: null }, + ] }); + assert.equal(observed.consoleErrors.length, 1); + assert.equal(observed.pageErrors.length, 1); + assert.equal(observed.failedRequests[0].url, "https://example.test/api"); + assert.equal(observed.unresolvedRequests, 1); + assert.throws(() => classifyTelemetry({}, [], []), /invalid messages/); + assert.throws(() => classifyTelemetry(Array(101).fill({}), [], []), /incomplete/); +}); + +test("QA bounds final JSON and marks oversized raw telemetry as incomplete evidence", () => { + const result = { status: "passed", summary: { viewports: 4, passed: 4, failed: 0, consoleErrors: 400 }, runs: Array.from({ length: 4 }, () => ({ status: "passed", consoleErrors: Array.from({ length: 100 }, () => ({ message: "\0".repeat(2000) })), pageErrors: [], failedRequests: [] })) }; + boundResult(result); + assert.equal(result.status, "failed"); + assert.equal(result.summary.failed, 4); + assert.equal(result.summary.consoleErrors, 400); + assert.match(result.error, /incomplete/); + assert.ok(Buffer.byteLength(JSON.stringify(result)) < 1024 * 1024); +}); + +test("QA passes both isolated viewports, writes private commit-bound evidence, and closes only owned sessions", async (t) => { + const f = fixture(t); const log = []; + const result = await runQa({ ...f, engineFactory: fakeFactory(log) }); + assert.equal(result.status, "passed"); + assert.equal(result.git.commit, f.git("rev-parse", "HEAD")); + assert.equal(result.git.dirty, false); + assert.equal(result.git.changedDuringRun, false); + assert.equal(result.summary.assertions, 2); + assert.deepEqual(result.scenario.policy, { failOnConsoleError: true, failOnPageError: true, failOnFailedRequest: true }); + assert.equal(new Set(log.map(({ session }) => session)).size, 2); + assert.ok(log.every(({ session }) => session.length < 50)); + assert.equal(log.filter(({ args }) => args[0] === "close").length, 2); + assert.ok(log.every(({ args }) => !args.includes("--all") && !args.includes("eval"))); + assert.equal(fs.statSync(result.resultPath).mode & 0o777, 0o600); + assert.equal(fs.statSync(f.output).mode & 0o777, 0o700); + assert.equal(JSON.parse(fs.readFileSync(result.resultPath)).runs[1].artifacts[0].path, "mobile-final.png"); +}); + +test("QA records explicitly relaxed telemetry policy alongside passing criteria", async (t) => { + const f = fixture(t, { failOnConsoleError: false }); + const factory = fakeFactory([]); + const result = await runQa({ ...f, engineFactory: (options) => { + const engine = factory(options); + return async (args, opts) => args[0] === "console" ? { messages: [{ type: "error", text: "intentional fixture error" }] } : engine(args, opts); + } }); + assert.equal(result.status, "passed"); + assert.equal(result.summary.consoleErrors, 2); + assert.equal(result.scenario.policy.failOnConsoleError, false); + assert.equal(result.scenario.policy.failOnPageError, true); +}); + +test("QA assertions fail visibly, still collect final evidence and clean up", async (t) => { + const f = fixture(t, { steps: [{ type: "navigate", path: "/" }, { type: "assertText", selector: "h1", contains: "missing" }] }); + const log = []; + const result = await runQa({ ...f, engineFactory: fakeFactory(log) }); + assert.equal(result.status, "failed"); + assert.match(result.runs[0].steps[1].error, /assertion failed/); + assert.equal(result.runs[0].artifacts.length, 1); + assert.equal(log.filter(({ args }) => args[0] === "close").length, 2); +}); + +test("QA detects Git changes and cleanup failures without claiming success", async (t) => { + const f = fixture(t); + const result = await runQa({ ...f, engineFactory: fakeFactory([], (args) => { + if (args[0] === "get") fs.writeFileSync(path.join(f.repo, "new.txt"), "changed"); + if (args[0] === "close") throw new Error("close failed"); + }) }); + assert.equal(result.git.changedDuringRun, true); + assert.equal(result.status, "failed"); + assert.equal(result.cleanup.status, "failed"); +}); + +test("QA refuses output reuse and repository-local output before browser commands", async (t) => { + const f = fixture(t); + const options = { ...f, engineFactory: () => { throw new Error("must not run"); } }; + await assert.rejects(runQa({ ...options, output: path.join(f.repo, "evidence") }), /outside/); + assert.equal(fs.existsSync(path.join(f.repo, "evidence")), false); + const link = path.join(f.root, "repo-link"); fs.symlinkSync(f.repo, link); + await assert.rejects(runQa({ ...options, output: path.join(link, "evidence") }), /outside/); + fs.mkdirSync(f.output); + await assert.rejects(runQa(options), /EEXIST/); +}); + +test("QA engine times out a hanging command and never exposes filled text in subprocess errors", async (t) => { + const f = fixture(t); + fs.writeFileSync(f.binary, '#!/usr/bin/env node\nsetTimeout(() => {}, 10000);\n'); + const engine = createEngine({ binary: f.binary, session: "owned", configFile: f.config, cwd: f.root, env: process.env, timeoutMs: 100, deadline: Date.now() + 100, signal: undefined }); + await assert.rejects(engine(["fill", "#input", "private-value"]), (error) => /timed out/.test(error.message) && !error.message.includes("private-value")); +});