Infra: send video.ready once HLS is ready

apps/web/__tests__/unit/developer-webhook-docs.test.ts

@@ -1,5 +1,6 @@
-import { execFileSync } from "node:child_process";
+import { execFileSync, spawn } from "node:child_process";
 import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { createServer } from "node:net";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { compileMDX } from "next-mdx-remote/rsc";
@@ -16,6 +17,7 @@ const { app } = await import("@/app/api/developer/v1/[...route]/webhooks");
 const { developerOpenApiSpec } = await import("@/lib/developer-openapi");
 const {
 	MAX_WEBHOOK_ENDPOINTS_PER_APP,
+	READY_EVENT_MAX_WAIT_MS,
 	WEBHOOK_DISABLE_AFTER_FAILURES,
 	WEBHOOK_DISABLE_AFTER_MS,
 	WEBHOOK_EVENT_TYPES,
@@ -143,6 +145,95 @@ describe("the webhooks guide", () => {
 		}
 	});
 
+	it("gives a receiver that answers Cap's request and refuses a forged one", async () => {
+		const jsBlocks = [...guide.matchAll(/```js\n([\s\S]*?)```/g)].map(
+			([, code]) => code ?? "",
+		);
+		const verify = jsBlocks.find((code) => code.includes("timingSafeEqual"));
+		const receiver = jsBlocks.find((code) => code.includes("createServer"));
+		expect(receiver).toContain('from "./verify-cap-webhook.mjs"');
+		const folder = mkdtempSync(join(tmpdir(), "cap-webhook-receiver-"));
+		const port = await new Promise<number>((resolve) => {
+			const probe = createServer().listen(0, () => {
+				const address = probe.address();
+				probe.close(() =>
+					resolve(typeof address === "object" && address ? address.port : 0),
+				);
+			});
+		});
+		const secret = "cws_0123456789abcdefghjkmnpqrstvwx";
+		writeFileSync(join(folder, "verify-cap-webhook.mjs"), verify ?? "");
+		writeFileSync(join(folder, "server.mjs"), receiver ?? "");
+		const child = spawn(process.execPath, [join(folder, "server.mjs")], {
+			env: { ...process.env, PORT: String(port), CAP_WEBHOOK_SECRET: secret },
+		});
+		let output = "";
+		child.stdout.on("data", (chunk) => {
+			output += chunk;
+		});
+		try {
+			const url = `http://127.0.0.1:${port}/webhooks/cap`;
+			const body = JSON.stringify({
+				id: "evt_1",
+				type: "video.ready",
+				data: { id: "video-1", hlsUrl: "https://play/master.m3u8" },
+			});
+			const post = async (signature: string) => {
+				for (let tries = 0; ; tries++) {
+					try {
+						return await fetch(url, {
+							method: "POST",
+							headers: { "Cap-Signature": signature },
+							body,
+						});
+					} catch (error) {
+						if (tries > 50) throw error;
+						await new Promise((resolve) => setTimeout(resolve, 50));
+					}
+				}
+			};
+			const oversized = await post("").then(() =>
+				fetch(url, { method: "POST", body: "x".repeat(2_000_000) }).then(
+					(response) => response.status,
+					() => "reset",
+				),
+			);
+			expect([413, "reset"]).toContain(oversized);
+			const now = Math.floor(Date.now() / 1000);
+			expect(
+				(await post(signWebhookBody(`${secret}x`, body, now))).status,
+			).toBe(400);
+			expect((await post(signWebhookBody(secret, body, now))).status).toBe(200);
+			expect((await post(signWebhookBody(secret, body, now))).status).toBe(200);
+			await new Promise((resolve) => setTimeout(resolve, 100));
+			expect(output.trim().split("\n")).toEqual([
+				"ready video-1 https://play/master.m3u8",
+			]);
+		} finally {
+			child.kill();
+			rmSync(folder, { recursive: true, force: true });
+		}
+	});
+
+	it("says when video.ready is sent, matching the code", () => {
+		expect(guide).not.toContain("No second event is sent");
+		expect(guide).toContain(
+			"`video.ready` arrives once every playback URL is ready, so `hlsUrl` is set unless HLS failed",
+		);
+		expect(READY_EVENT_MAX_WAIT_MS).toBe(30 * 60 * 1000);
+		expect(guide).toContain("still encoding 30 minutes after");
+		expect(
+			(spec.webhooks as Record<string, { post: { description: string } }>)[
+				"video.ready"
+			]?.post.description,
+		).toContain("30 minutes after");
+	});
+
+	it("indents code like the rest of the docs", () => {
+		const blocks = [...guide.matchAll(/```\w*\n([\s\S]*?)```/g)];
+		for (const [, code] of blocks) expect(code).not.toMatch(/^\t/m);
+	});
+
 	it("teaches the REST API only", () => {
 		expect(guide).not.toMatch(/\bcap (developers|caps|webhooks)\b/);
 		expect(guide).not.toMatch(/\bMCP\b/);

apps/web/__tests__/unit/developer-webhooks-api.test.ts

@@ -98,7 +98,9 @@ async function create(
 		body: (await response.json()) as {
 			data: Record<string, unknown> & { id: string; secret: string };
 			code?: string;
+			message?: string;
 			hint?: string;
+			docs?: string;
 		},
 	};
 }
@@ -175,6 +177,44 @@ describe("creating a webhook endpoint", () => {
 		expect(rows()).toHaveLength(0);
 	});
 
+	it("explains an unknown or missing event in plain words", async () => {
+		const unknown = await create(A, {
+			url: "https://a.example/x",
+			events: ["video.ready", "video.updated"],
+		});
+		expect(unknown.body.message).toBe(
+			'Invalid request: events.1: Unknown event "video.updated". Use video.ready or video.errored.',
+		);
+		const none = await create(A, { url: "https://a.example/x", events: [] });
+		expect(none.body.message).toBe(
+			"Invalid request: events: events must list at least one of: video.ready, video.errored",
+		);
+		expect(unknown.body.docs).toBe("https://cap.so/docs/api/rest-api");
+
+		const objectEvent = await create(A, {
+			url: "https://a.example/x",
+			events: [{ toString: null }],
+		});
+		expect(objectEvent.status).toBe(400);
+		expect(objectEvent.body.message).toBe(
+			"Invalid request: events.0: Unknown event. Use video.ready or video.errored.",
+		);
+		expect(rows()).toHaveLength(0);
+	});
+
+	it("links webhook errors to the webhooks guide", async () => {
+		const missing = await send("GET", "/wh_missing", A);
+		expect(await missing.json()).toMatchObject({
+			code: "webhook_not_found",
+			docs: "https://cap.so/docs/api/webhooks",
+		});
+		const refused = await create(A, { url: "http://hooks.example.com/cap" });
+		expect(refused.body).toMatchObject({
+			code: "invalid_webhook_url",
+			docs: "https://cap.so/docs/api/webhooks",
+		});
+	});
+
 	it.each([
 		["http", "http://hooks.example.com/cap"],
 		["a private address", "https://10.0.0.8/cap"],

apps/web/app/api/developer/v1/[...route]/webhooks.ts

@@ -31,8 +31,15 @@ const failureBody = (failure: WebhookFailure) =>
 const endpointFields = {
 	url: z.string().min(1).max(2048),
 	events: z
-		.array(z.enum(WEBHOOK_EVENT_TYPES))
-		.min(1)
+		.array(
+			z.enum(WEBHOOK_EVENT_TYPES, {
+				error: (issue) =>
+					`Unknown event${typeof issue.input === "string" ? ` ${JSON.stringify(issue.input.slice(0, 100))}` : ""}. Use ${WEBHOOK_EVENT_TYPES.join(" or ")}.`,
+			}),
+		)
+		.min(1, {
+			error: `events must list at least one of: ${WEBHOOK_EVENT_TYPES.join(", ")}`,
+		})
 		.refine((types) => new Set(types).size === types.length, {
 			message: "Each event type can be listed once",
 		}),

apps/web/content/docs/api/get-started.mdx

@@ -358,6 +358,8 @@ Save this as `index.html` beside `server.mjs`. It uses no library.
 
 Completing the upload starts processing. [Get Video Status](/docs/api/rest-api#get-video-status) moves from `uploading` to `processing`, then to `ready` or `errored`. The page asks your server, and your server asks Cap with the secret key.
 
+Or let Cap tell your server when it's ready: see [Webhooks](/docs/api/webhooks).
+
 Add this to `index.html`, below the upload script:
 
 ```html
@@ -468,7 +470,7 @@ Safari plays HLS by itself. Every other browser needs [hls.js](https://github.co
 | `playMp4` on a fatal error | Keep playing from the same position if HLS fails for any other reason |
 | `stopHls` at the start of `playVideo` | Stop the previous video's player before starting the next one on the same page |
 
-There is no start level to set. The master playlist lists a rendition that starts quickly first, and players begin with the first one listed.
+There is no start level to set. Players begin with the first rendition the master playlist lists, and Cap lists the largest one whose peak bitrate is at most 2.6 Mbps first, at 1080p or smaller. On most videos that is 720p. The player steps up or down from there.
 
 [Play a Video](/docs/api/rest-api#play-a-video) covers a Content Security Policy, bitrates and what happens when playback stops.
 
@@ -492,7 +494,7 @@ Every upload also returns `shareUrl` and `embedUrl`. They are Cap's hosted playe
 
 `loading="lazy"` leaves an embed further down the page unloaded until the viewer scrolls near it. The embed takes options in its URL, such as a start time, autoplay and loop. See [Embed Parameters](/docs/api/rest-api#embed-parameters).
 
-Use the hosted player when you want a video on a page with the least work. Build your own player, as above, when you need to control how it looks or react to what the viewer does. [Share and Embed](/docs/api/rest-api#share-and-embed) has the details.
+Use the hosted player for the look and behaviour too: [Embed Options](/docs/api/rest-api#embed-options) set its colour, corners and controls, [Embed Parameters](/docs/api/rest-api#embed-parameters) its start time and autoplay, and [Player Events](/docs/api/rest-api#player-events) tell your page what the viewer does. Build your own player, as above, only when you need full control of the UI. [Share and Embed](/docs/api/rest-api#share-and-embed) has the details.
 
 ## 6. Upload from your server
 

apps/web/content/docs/api/rest-api.mdx

@@ -572,7 +572,7 @@ Poll this endpoint every 3 seconds after completing an upload, and stop when `st
 
 A video is `ready` as soon as its first output can be played, so playback can start while the rest is still encoding. The playback URLs are `null` until then. `hlsUrl` is an adaptive HLS master playlist, `mp4Url` is a progressive MP4 and `posterUrl` is a poster image. They need no API key and can be loaded from any origin. [Play a Video](/docs/api/rest-api#play-a-video) shows how to use them, and [Delete Video](/docs/api/rest-api#delete-video) says when they stop working.
 
-On a ready video, one of `hlsUrl` and `mp4Url` can still be `null` while that output finishes encoding. It is filled in later without the video leaving `ready`, and `renditions` is filled in together with `hlsUrl`. Play whichever URL is present. A player that wants HLS should keep polling after `ready` until `hlsUrl` is set, and can play `mp4Url` in the meantime.
+On a ready video, one of `hlsUrl` and `mp4Url` can still be `null` while that output finishes encoding. It is filled in later without the video leaving `ready`, and `renditions` is filled in together with `hlsUrl`. Play whichever URL is present. A player that wants HLS should keep polling after `ready` until `hlsUrl` is set, and can play `mp4Url` in the meantime. A [webhook](/docs/api/webhooks)'s `video.ready` is sent once every playback URL is ready instead.
 
 `outputs` says whether a `null` URL is still coming. Each of `outputs.hls` and `outputs.mp4` is `ready` when its URL is set, `processing` while it is being encoded or will be retried, and `failed` when it will not be produced. A video can be `ready` with one output `failed`, so stop waiting for that URL and play the other. `none` only appears on videos uploaded before encoding existed. While `takenDown` is `true` the playback URLs are `null` whatever `outputs` says, and `outputs` keeps describing what was encoded.
 
@@ -1394,7 +1394,7 @@ Replace `HLS_URL`, `MP4_URL` and `POSTER_URL` with the video's `hlsUrl`, `mp4Url
 - If hls.js hits an error it cannot recover from, the page switches to the MP4 at the same position and keeps playing.
 - If the browser's own HLS player fails, the page loads the MP4 in its place. A viewer who had already pressed play may have to press it again.
 - `capLevelToPlayerSize` stops hls.js from loading a rendition larger than the player. Without it, a small player on a fast connection climbs to the largest rendition.
-- There is no start level to set. The master playlist lists a rendition that starts quickly first, and players begin with the first one listed.
+- There is no start level to set. Players begin with the first rendition the master playlist lists, and Cap lists the largest one whose peak bitrate is at most 2.6 Mbps first, at 1080p or smaller. On most videos that is 720p.
 
 ### Play in Safari
 

apps/web/content/docs/api/webhooks.mdx

@@ -18,7 +18,7 @@ Webhooks belong to an app in [Cap Infra](/dashboard/infra) and are managed with
 
 A video sends `video.ready` or `video.errored` once. Handlers should ignore a type they do not know, so a new event type never breaks them.
 
-On a ready video, one of `hlsUrl` and `mp4Url` can still be `null` while that output finishes encoding. No second event is sent when it arrives. Read it with [Get Video](/docs/api/rest-api#get-video).
+`video.ready` arrives once every playback URL is ready, so `hlsUrl` is set unless HLS failed. In the rare case that an output is still encoding 30 minutes after the video could first be played, the event is sent without it, and [Get Video](/docs/api/rest-api#get-video) returns it once it is ready.
 
 ## Create an endpoint
 
@@ -88,7 +88,7 @@ Anything else returns `400` with code `invalid_webhook_url`. An app can have 5 e
 | `POST /api/developer/v1/webhooks/:id/rotate-secret` | [Rotate the signing secret](#rotate-the-signing-secret) |
 | `POST /api/developer/v1/webhooks/:id/test` | [Send a test event](#send-a-test-event) |
 
-An ID that does not belong to the app returns `404` with code `webhook_not_found`.
+An ID that does not belong to the app returns `404` with code `webhook_not_found`. Deleting an endpoint that is already deleted returns `200`, the same as deleting a video twice, so a retried delete is safe.
 
 Every endpoint is returned with these fields:
 
@@ -143,7 +143,7 @@ Cap-Event-Type: video.ready
     "width": 1920,
     "height": 1080,
     "playbackId": "Zk3vQ9s2Lm8Rt1Xy7Bc4Df6G",
-    "hlsUrl": null,
+    "hlsUrl": "https://cap.so/api/developer/playback/Zk3vQ9s2Lm8Rt1Xy7Bc4Df6G/master.m3u8",
     "mp4Url": "https://cap.so/api/developer/playback/Zk3vQ9s2Lm8Rt1Xy7Bc4Df6G/video.mp4",
     "posterUrl": "https://cap.so/api/developer/playback/Zk3vQ9s2Lm8Rt1Xy7Bc4Df6G/poster.jpg"
   }
@@ -178,21 +178,21 @@ Sign the body exactly as it arrived, byte for byte. Parsing the JSON and seriali
 import { createHmac, timingSafeEqual } from "node:crypto";
 
 export function verifyCapWebhook(rawBody, signatureHeader, secret) {
-	const parts = Object.fromEntries(
-		String(signatureHeader ?? "")
-			.split(",")
-			.map((part) => part.split("=")),
-	);
-	const timestamp = Number(parts.t);
-	if (!Number.isInteger(timestamp) || !parts.v1) return false;
-	if (Math.abs(Date.now() / 1000 - timestamp) > 300) return false;
-	const expected = createHmac("sha256", secret)
-		.update(`${parts.t}.${rawBody}`)
-		.digest();
-	const received = Buffer.from(parts.v1, "hex");
-	return (
-		received.length === expected.length && timingSafeEqual(received, expected)
-	);
+  const parts = Object.fromEntries(
+    String(signatureHeader ?? "")
+      .split(",")
+      .map((part) => part.split("=")),
+  );
+  const timestamp = Number(parts.t);
+  if (!Number.isInteger(timestamp) || !parts.v1) return false;
+  if (Math.abs(Date.now() / 1000 - timestamp) > 300) return false;
+  const expected = createHmac("sha256", secret)
+    .update(`${parts.t}.${rawBody}`)
+    .digest();
+  const received = Buffer.from(parts.v1, "hex");
+  return (
+    received.length === expected.length && timingSafeEqual(received, expected)
+  );
 }
 ```
 
@@ -223,6 +223,57 @@ def verify_cap_webhook(raw_body: bytes, signature_header: str, secret: str) -> b
     return hmac.compare_digest(expected, received)
 ```
 
+### A complete receiver
+
+This server uses only Node.js. Save the function above as `verify-cap-webhook.mjs` next to it and start it with your signing secret in `CAP_WEBHOOK_SECRET`. It reads the raw body, refuses one over 1 MB (Cap's events are far smaller), checks the signature, answers `200` straight away, then handles the event.
+
+```js
+import { createServer } from "node:http";
+import { verifyCapWebhook } from "./verify-cap-webhook.mjs";
+
+const secret = process.env.CAP_WEBHOOK_SECRET;
+const handled = new Set();
+
+function handleEvent(event) {
+  if (handled.has(event.id)) return;
+  handled.add(event.id);
+  if (event.type === "video.ready") {
+    console.log("ready", event.data.id, event.data.hlsUrl ?? event.data.mp4Url);
+  } else if (event.type === "video.errored") {
+    console.log("errored", event.data.id, event.data.errorReason);
+  }
+}
+
+createServer((req, res) => {
+  if (req.method !== "POST" || req.url !== "/webhooks/cap") {
+    res.writeHead(404).end();
+    return;
+  }
+  const chunks = [];
+  let size = 0;
+  req.on("data", (chunk) => {
+    size += chunk.length;
+    if (size > 1_000_000) {
+      res.writeHead(413).end();
+      req.destroy();
+      return;
+    }
+    chunks.push(chunk);
+  });
+  req.on("end", () => {
+    const rawBody = Buffer.concat(chunks).toString("utf8");
+    if (!verifyCapWebhook(rawBody, req.headers["cap-signature"], secret)) {
+      res.writeHead(400).end();
+      return;
+    }
+    res.writeHead(200).end();
+    handleEvent(JSON.parse(rawBody));
+  });
+}).listen(Number(process.env.PORT ?? 3000));
+```
+
+With Express, read this route's body with `express.raw({ type: "application/json" })`, not `express.json()`: a parsed and re-serialised body no longer matches the signature. Keep the ids you have handled in your database rather than in memory, so a restart does not forget them.
+
 ## Respond
 
 Answer with any `2xx` status within 10 seconds. Do the slow work after you have answered: store the event or put it on a queue, respond, then process it.

apps/web/lib/api-errors.ts

@@ -1,4 +1,5 @@
 export const API_DOCS_URL = "https://cap.so/docs/api/rest-api";
+export const WEBHOOK_DOCS_URL = "https://cap.so/docs/api/webhooks";
 export const OPENAPI_SPEC_PATH = "/openapi.json";
 
 export type ApiErrorBody = {
@@ -18,7 +19,7 @@ export const apiErrorBody = (
 	code,
 	message,
 	hint,
-	docs: API_DOCS_URL,
+	docs: code.includes("webhook") ? WEBHOOK_DOCS_URL : API_DOCS_URL,
 });
 
 export const apiNotFoundBody = (pathname: string) =>

apps/web/lib/developer-openapi.ts

@@ -19,6 +19,7 @@ import {
 } from "@/lib/developer-transcript-errors";
 import {
 	MAX_WEBHOOK_ENDPOINTS_PER_APP,
+	READY_EVENT_MAX_WAIT_MS,
 	WEBHOOK_DISABLE_AFTER_FAILURES,
 	WEBHOOK_EVENT_TYPES,
 	WEBHOOK_MAX_ATTEMPTS,
@@ -602,6 +603,7 @@ export const developerOpenApiSpec = (baseUrl: string) => ({
 			"video.ready",
 			"A video is ready to play",
 			ref("DeveloperVideo"),
+			`Sent once every playback URL is ready, so hlsUrl is set unless HLS failed. An output still encoding ${READY_EVENT_MAX_WAIT_MS / 60_000} minutes after the video could first be played is left out, and Get Video returns it once it is ready. ${videoEventDescription("video.ready")}`,
 		),
 		"video.errored": webhookDelivery(
 			"video.errored",