diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f63286c..3bba435 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -105,6 +105,9 @@ jobs: - name: Run e2e tests run: bun test:e2e + - name: Run browser recovery test + run: bun test:recovery + - name: Print logs if: failure() run: docker compose logs \ No newline at end of file diff --git a/package.json b/package.json index e5636f7..93296ae 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "check": "eslint ./src ./tests", "test": "bun test", "test:e2e": "bun test tests/e2e", + "test:recovery": "bun test tests/recovery", "test:unit": "bun test tests/unit" }, "keywords": [], diff --git a/src/config/browser.ts b/src/config/browser.ts index 031b821..4a18840 100644 --- a/src/config/browser.ts +++ b/src/config/browser.ts @@ -1,5 +1,5 @@ import { chromium } from "playwright-core"; -import type { BrowserContextOptions } from "playwright-core"; +import type { Browser, BrowserContextOptions } from "playwright-core"; export const defaultContext: BrowserContextOptions = { viewport: { @@ -8,11 +8,35 @@ export const defaultContext: BrowserContextOptions = { }, }; +function launch(): Promise { + return chromium.launch({ + args: ["--remote-debugging-port=9222"], + executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, + }); +} + console.log("Chromium starting..."); -export const browser = await chromium.launch({ - args: ["--remote-debugging-port=9222"], - executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, -}); +export let instance = await launch(); console.log("Chromium started!"); + +let launching: Promise | null = null; + +// Chromium can die without the server dying too, so relaunch on demand. +export async function getBrowser(): Promise { + if (instance.isConnected()) { + return instance; + } + + if (!launching) { + console.log("Chromium disconnected, relaunching..."); + launching = launch().finally(() => { + launching = null; + }); + } + + instance = await launching; + + return instance; +} diff --git a/src/routes/health.ts b/src/routes/health.ts index 2ad664d..ceba911 100755 --- a/src/routes/health.ts +++ b/src/routes/health.ts @@ -1,11 +1,15 @@ -import { browser } from "../config"; +import { instance } from "../config"; export async function handleHealthRequest(_req: Request): Promise { + const connected = instance.isConnected(); + + // Non-2xx so an httpGet liveness probe restarts a pod whose Chromium died. return new Response( JSON.stringify({ - status: browser.isConnected() ? "pass" : "fail", + status: connected ? "pass" : "fail", }), { + status: connected ? 200 : 503, headers: { "Content-Type": "application/json" }, }, ); diff --git a/src/routes/reports.ts b/src/routes/reports.ts index 6d7f6a9..e3589b8 100755 --- a/src/routes/reports.ts +++ b/src/routes/reports.ts @@ -1,6 +1,6 @@ import type { BrowserContext, BrowserContextOptions } from "playwright-core"; import { playAudit } from "playwright-lighthouse"; -import { browser, defaultContext, lighthouseConfigs } from "../config"; +import { defaultContext, getBrowser, lighthouseConfigs } from "../config"; import { lighthouseSchema } from "../schemas"; export async function handleReportsRequest(req: Request): Promise { @@ -21,6 +21,7 @@ export async function handleReportsRequest(req: Request): Promise { if (body.locale) contextOptions.locale = body.locale; if (body.timezoneId) contextOptions.timezoneId = body.timezoneId; + const browser = await getBrowser(); context = await browser.newContext(contextOptions); // Grant permissions if specified diff --git a/src/routes/screenshots.ts b/src/routes/screenshots.ts index 0ebf6c4..d2608c5 100755 --- a/src/routes/screenshots.ts +++ b/src/routes/screenshots.ts @@ -3,7 +3,7 @@ import type { BrowserContextOptions, PageScreenshotOptions, } from "playwright-core"; -import { browser, defaultContext } from "../config"; +import { defaultContext, getBrowser } from "../config"; import { screenshotSchema } from "../schemas"; export async function handleScreenshotsRequest( @@ -31,6 +31,7 @@ export async function handleScreenshotsRequest( if (body.timezoneId) contextOptions.timezoneId = body.timezoneId; if (body.geolocation) contextOptions.geolocation = body.geolocation; + const browser = await getBrowser(); context = await browser.newContext(contextOptions); // Grant permissions if specified diff --git a/src/routes/test.ts b/src/routes/test.ts index c8c2a1d..77d248f 100755 --- a/src/routes/test.ts +++ b/src/routes/test.ts @@ -1,7 +1,8 @@ -import { browser, defaultContext } from "../config"; +import { defaultContext, getBrowser } from "../config"; import { generateTestHTML } from "../utils/test-page.js"; export async function handleTestRequest(_req: Request): Promise { + const browser = await getBrowser(); const context = await browser.newContext(defaultContext); try { const page = await context.newPage(); diff --git a/tests/recovery/browser.test.ts b/tests/recovery/browser.test.ts new file mode 100644 index 0000000..cd2f707 --- /dev/null +++ b/tests/recovery/browser.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, test } from "bun:test"; + +const BASE_URL = process.env.BASE_URL || "http://localhost:3000"; + +function resolveContainer(): string { + const result = Bun.spawnSync([ + "docker", + "compose", + "ps", + "-q", + "appwrite-browser", + ]); + + return result.success ? result.stdout.toString().trim() : ""; +} + +const container = resolveContainer(); + +describe.skipIf(container === "")("E2E Tests - browser recovery", () => { + test("should report 503 while Chromium is gone and recover on the next capture", async () => { + // SIGKILL is what the OOM killer sends in production, where Chromium + // dies while the server keeps running. + Bun.spawnSync([ + "docker", + "exec", + container, + "sh", + "-c", + "kill -9 $(pidof headless-shell)", + ]); + + // The server notices the dead browser asynchronously. + const deadline = Date.now() + 30_000; + let down = await fetch(`${BASE_URL}/v1/health`); + while (down.status !== 503 && Date.now() < deadline) { + await Bun.sleep(250); + down = await fetch(`${BASE_URL}/v1/health`); + } + expect(down.status).toBe(503); + + const capture = await fetch(`${BASE_URL}/v1/screenshots`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ url: "https://example.com" }), + }); + expect(capture.status).toBe(200); + + const up = await fetch(`${BASE_URL}/v1/health`); + expect(up.status).toBe(200); + }, 120_000); + + test("should share one relaunch between concurrent captures", async () => { + Bun.spawnSync([ + "docker", + "exec", + container, + "sh", + "-c", + "kill -9 $(pidof headless-shell)", + ]); + + const deadline = Date.now() + 30_000; + let health = await fetch(`${BASE_URL}/v1/health`); + while (health.status !== 503 && Date.now() < deadline) { + await Bun.sleep(250); + health = await fetch(`${BASE_URL}/v1/health`); + } + expect(health.status).toBe(503); + + const before = Bun.spawnSync(["docker", "logs", container]) + .stdout.toString() + .split("Chromium disconnected, relaunching...").length; + + const captures = await Promise.all( + Array.from({ length: 5 }, () => + fetch(`${BASE_URL}/v1/screenshots`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ url: "https://example.com" }), + }), + ), + ); + + for (const capture of captures) { + expect(capture.status).toBe(200); + } + + // A broken guard launches one Chromium per request instead of one total. + const after = Bun.spawnSync(["docker", "logs", container]) + .stdout.toString() + .split("Chromium disconnected, relaunching...").length; + expect(after).toBe(before + 1); + }, 120_000); +});