fix: relaunch Chromium on disconnect and fail health check when it is gone

.github/workflows/test.yml

@@ -105,6 +105,9 @@ jobs:
       - name: Run e2e tests
         run: bun test:e2e
 
+      - name: Run browser recovery test
+        run: bun test:recovery
+
       - name: Print logs
         if: failure()
         run: docker compose logs

package.json

@@ -14,6 +14,7 @@
 		"check": "eslint ./src ./tests",
 		"test": "bun test",
 		"test:e2e": "bun test tests/e2e",
+		"test:recovery": "bun test tests/recovery",
 		"test:unit": "bun test tests/unit"
 	},
 	"keywords": [],

src/config/browser.ts

@@ -1,5 +1,5 @@
 import { chromium } from "playwright-core";
-import type { BrowserContextOptions } from "playwright-core";
+import type { Browser, BrowserContextOptions } from "playwright-core";
 
 export const defaultContext: BrowserContextOptions = {
 	viewport: {
@@ -8,11 +8,35 @@ export const defaultContext: BrowserContextOptions = {
 	},
 };
 
+function launch(): Promise<Browser> {
+	return chromium.launch({
+		args: ["--remote-debugging-port=9222"],
+		executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH,
+	});
+}
+
 console.log("Chromium starting...");
 
-export const browser = await chromium.launch({
-	args: ["--remote-debugging-port=9222"],
-	executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH,
-});
+export let instance = await launch();
 
 console.log("Chromium started!");
+
+let launching: Promise<Browser> | null = null;
+
+// Chromium can die without the server dying too, so relaunch on demand.
+export async function getBrowser(): Promise<Browser> {
+	if (instance.isConnected()) {
+		return instance;
+	}
+
+	if (!launching) {
+		console.log("Chromium disconnected, relaunching...");
+		launching = launch().finally(() => {
+			launching = null;
+		});
+	}
+
+	instance = await launching;
+
+	return instance;
+}

src/routes/health.ts

@@ -1,11 +1,15 @@
-import { browser } from "../config";
+import { instance } from "../config";
 
 export async function handleHealthRequest(_req: Request): Promise<Response> {
+	const connected = instance.isConnected();
+
+	// Non-2xx so an httpGet liveness probe restarts a pod whose Chromium died.
 	return new Response(
 		JSON.stringify({
-			status: browser.isConnected() ? "pass" : "fail",
+			status: connected ? "pass" : "fail",
 		}),
 		{
+			status: connected ? 200 : 503,
 			headers: { "Content-Type": "application/json" },
 		},
 	);

src/routes/reports.ts

@@ -1,6 +1,6 @@
 import type { BrowserContext, BrowserContextOptions } from "playwright-core";
 import { playAudit } from "playwright-lighthouse";
-import { browser, defaultContext, lighthouseConfigs } from "../config";
+import { defaultContext, getBrowser, lighthouseConfigs } from "../config";
 import { lighthouseSchema } from "../schemas";
 
 export async function handleReportsRequest(req: Request): Promise<Response> {
@@ -21,6 +21,7 @@ export async function handleReportsRequest(req: Request): Promise<Response> {
 		if (body.locale) contextOptions.locale = body.locale;
 		if (body.timezoneId) contextOptions.timezoneId = body.timezoneId;
 
+		const browser = await getBrowser();
 		context = await browser.newContext(contextOptions);
 
 		// Grant permissions if specified

src/routes/screenshots.ts

@@ -3,7 +3,7 @@ import type {
 	BrowserContextOptions,
 	PageScreenshotOptions,
 } from "playwright-core";
-import { browser, defaultContext } from "../config";
+import { defaultContext, getBrowser } from "../config";
 import { screenshotSchema } from "../schemas";
 
 export async function handleScreenshotsRequest(
@@ -31,6 +31,7 @@ export async function handleScreenshotsRequest(
 		if (body.timezoneId) contextOptions.timezoneId = body.timezoneId;
 		if (body.geolocation) contextOptions.geolocation = body.geolocation;
 
+		const browser = await getBrowser();
 		context = await browser.newContext(contextOptions);
 
 		// Grant permissions if specified

src/routes/test.ts

@@ -1,7 +1,8 @@
-import { browser, defaultContext } from "../config";
+import { defaultContext, getBrowser } from "../config";
 import { generateTestHTML } from "../utils/test-page.js";
 
 export async function handleTestRequest(_req: Request): Promise<Response> {
+	const browser = await getBrowser();
 	const context = await browser.newContext(defaultContext);
 	try {
 		const page = await context.newPage();

tests/recovery/browser.test.ts

@@ -0,0 +1,94 @@
+import { describe, expect, test } from "bun:test";
+
+const BASE_URL = process.env.BASE_URL || "http://localhost:3000";
+
+function resolveContainer(): string {
+	const result = Bun.spawnSync([
+		"docker",
+		"compose",
+		"ps",
+		"-q",
+		"appwrite-browser",
+	]);
+
+	return result.success ? result.stdout.toString().trim() : "";
+}
+
+const container = resolveContainer();
+
+describe.skipIf(container === "")("E2E Tests - browser recovery", () => {
+	test("should report 503 while Chromium is gone and recover on the next capture", async () => {
+		// SIGKILL is what the OOM killer sends in production, where Chromium
+		// dies while the server keeps running.
+		Bun.spawnSync([
+			"docker",
+			"exec",
+			container,
+			"sh",
+			"-c",
+			"kill -9 $(pidof headless-shell)",
+		]);
+
+		// The server notices the dead browser asynchronously.
+		const deadline = Date.now() + 30_000;
+		let down = await fetch(`${BASE_URL}/v1/health`);
+		while (down.status !== 503 && Date.now() < deadline) {
+			await Bun.sleep(250);
+			down = await fetch(`${BASE_URL}/v1/health`);
+		}
+		expect(down.status).toBe(503);
+
+		const capture = await fetch(`${BASE_URL}/v1/screenshots`, {
+			method: "POST",
+			headers: { "Content-Type": "application/json" },
+			body: JSON.stringify({ url: "https://example.com" }),
+		});
+		expect(capture.status).toBe(200);
+
+		const up = await fetch(`${BASE_URL}/v1/health`);
+		expect(up.status).toBe(200);
+	}, 120_000);
+
+	test("should share one relaunch between concurrent captures", async () => {
+		Bun.spawnSync([
+			"docker",
+			"exec",
+			container,
+			"sh",
+			"-c",
+			"kill -9 $(pidof headless-shell)",
+		]);
+
+		const deadline = Date.now() + 30_000;
+		let health = await fetch(`${BASE_URL}/v1/health`);
+		while (health.status !== 503 && Date.now() < deadline) {
+			await Bun.sleep(250);
+			health = await fetch(`${BASE_URL}/v1/health`);
+		}
+		expect(health.status).toBe(503);
+
+		const before = Bun.spawnSync(["docker", "logs", container])
+			.stdout.toString()
+			.split("Chromium disconnected, relaunching...").length;
+
+		const captures = await Promise.all(
+			Array.from({ length: 5 }, () =>
+				fetch(`${BASE_URL}/v1/screenshots`, {
+					method: "POST",
+					headers: { "Content-Type": "application/json" },
+					body: JSON.stringify({ url: "https://example.com" }),
+				}),
+			),
+		);
+
+		for (const capture of captures) {
+			expect(capture.status).toBe(200);
+		}
+
+		// A broken guard launches one Chromium per request instead of one total.
+		const after = Bun.spawnSync(["docker", "logs", container])
+			.stdout.toString()
+			.split("Chromium disconnected, relaunching...").length;
+		expect(after).toBe(before + 1);
+	}, 120_000);
+});