fix(web): point agents at Cap Infra in llms files and serve docs as Markdown

apps/web/__tests__/unit/developer-billing-docs.test.ts

@@ -135,9 +135,11 @@ describe("the Billing page", () => {
 			compileMDX({ source: rest, components: {} }),
 		).resolves.toBeDefined();
 
+		const slugsOf = (source: string) =>
+			extractHeadings(source.replace(/^####\s/gm, "### ")).map((h) => h.slug);
 		const anchors = new Map([
-			["/docs/api/rest-api", extractHeadings(rest).map((h) => h.slug)],
-			["/docs/api/billing", extractHeadings(billing).map((h) => h.slug)],
+			["/docs/api/rest-api", slugsOf(rest)],
+			["/docs/api/billing", slugsOf(billing)],
 		]);
 		const links = [
 			...billing.matchAll(/\]\((\/docs\/api\/rest-api)#([\w-]+)\)/g),

apps/web/__tests__/unit/developer-transcript-docs.test.ts

@@ -251,7 +251,8 @@ describe("transcript docs keep up with the API", () => {
 		expect(JSON.stringify(schemas.TranscriptDetail)).not.toContain(
 			"confidence",
 		);
-		expect(transcriptDocs).toContain("steps of 0.08 seconds");
+		expect(transcriptDocs).not.toContain("steps of 0.08 seconds");
+		expect(JSON.stringify(schemas.TranscriptDetail)).not.toContain("0.08");
 	});
 
 	it("states the download limit and the upload expiry", () => {

apps/web/__tests__/unit/docs-markdown.test.ts

@@ -0,0 +1,156 @@
+import { NextRequest } from "next/server";
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("@cap/database", () => ({
+	db: () => ({
+		select: () => ({ from: () => ({ where: async () => [] }) }),
+	}),
+}));
+vi.mock("@cap/database/schema", () => ({ organizations: {} }));
+vi.mock("@cap/env", () => ({
+	buildEnv: { NEXT_PUBLIC_IS_CAP: "true" },
+	serverEnv: () => ({ WEB_URL: "https://cap.so" }),
+}));
+
+import { GET } from "@/app/docs-markdown/[...slug]/route";
+import { CAP_AGENT_PROMPT } from "@/data/agent-prompt";
+import { DOCS_MARKDOWN_ROUTE, docsMarkdownSlug } from "@/lib/agent-markdown";
+import { docToMarkdown } from "@/lib/docs-markdown";
+import { getAllDocs, getDocBySlug } from "@/utils/docs";
+import { proxy } from "../../proxy";
+
+const getMarkdown = (slug: string, method = "GET") =>
+	GET(new Request(`https://cap.so/docs/${slug}.md`, { method }), {
+		params: Promise.resolve({ slug: slug.split("/") }),
+	});
+
+const proseLines = (markdown: string) => {
+	const lines: string[] = [];
+	let fence: string | null = null;
+	for (const line of markdown.split("\n")) {
+		const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
+		if (fence) {
+			if (marker && marker[0] === fence[0] && marker.length >= fence.length)
+				fence = null;
+		} else if (marker) fence = marker;
+		else lines.push(line);
+	}
+	return lines;
+};
+
+describe("docsMarkdownSlug", () => {
+	it.each([
+		["/docs/api/rest-api.md", "text/html", "api/rest-api"],
+		["/docs/api/rest-api.mdx", null, "api/rest-api"],
+		["/docs/api/rest-api", "text/markdown", "api/rest-api"],
+		["/docs/api/rest-api/", "text/markdown", "api/rest-api"],
+		["/docs", "text/markdown", "introduction"],
+		["/docs.md", null, "introduction"],
+		["/docs/api/rest-api", "text/html,*/*;q=0.8", null],
+		["/docs/api/rest-api", null, null],
+		["/pricing.md", null, null],
+		["/docsearch", "text/markdown", null],
+	])("%s with Accept %s gives %s", (path, accept, slug) => {
+		expect(docsMarkdownSlug(path, accept)).toBe(slug);
+	});
+});
+
+describe("docToMarkdown", () => {
+	const docs = getAllDocs();
+
+	it.each(docs.map((doc) => [doc.slug, doc] as const))(
+		"turns %s into plain Markdown",
+		(_slug, doc) => {
+			const markdown = docToMarkdown(doc, "https://cap.so");
+			expect(markdown.startsWith(`# ${doc.metadata.title}\n`)).toBe(true);
+			expect(markdown).not.toMatch(/^---\n/);
+			for (const line of proseLines(markdown)) {
+				expect(line).not.toMatch(/^\s*<\/?[A-Z]/);
+				expect(line).not.toMatch(/\]\(\/[^/]/);
+			}
+		},
+	);
+
+	it("keeps every code block exactly as written", () => {
+		const doc = getDocBySlug("api/get-started");
+		if (!doc) throw new Error("missing get-started doc");
+		const markdown = docToMarkdown(doc, "https://cap.so");
+		const blocks = doc.content.match(/```[\s\S]*?\n```/g) ?? [];
+		expect(blocks.length).toBeGreaterThan(5);
+		for (const block of blocks) expect(markdown).toContain(block);
+	});
+
+	it("shows the copyable prompt and warnings as text", () => {
+		const agents = getDocBySlug("agents");
+		const selfHosting = getDocBySlug("self-hosting");
+		if (!agents || !selfHosting) throw new Error("missing docs");
+		expect(docToMarkdown(agents, "https://cap.so")).toContain(CAP_AGENT_PROMPT);
+		const hosting = docToMarkdown(selfHosting, "https://cap.so");
+		expect(hosting).toContain(
+			"> **Security Notice for Production Deployments**",
+		);
+		expect(hosting).toContain(
+			"[Watch the video](https://cap.so/embed/k310v2vf41d6ffp)",
+		);
+	});
+});
+
+describe("docs Markdown route", () => {
+	it("serves a docs page as Markdown", async () => {
+		const response = await getMarkdown("api/rest-api");
+		expect(response.status).toBe(200);
+		expect(response.headers.get("content-type")).toBe(
+			"text/markdown; charset=utf-8",
+		);
+		expect(response.headers.get("vary")).toBe("Accept");
+		const body = await response.text();
+		expect(body.startsWith("# REST API\n")).toBe(true);
+		expect(body).toContain("every 3 seconds");
+		expect(body).not.toContain("<html");
+	});
+
+	it("answers a missing page with a Markdown 404", async () => {
+		const response = await getMarkdown("api/nope");
+		expect(response.status).toBe(404);
+		expect(response.headers.get("content-type")).toBe(
+			"text/markdown; charset=utf-8",
+		);
+		expect(await response.text()).toContain("/docs/api/nope");
+	});
+
+	it("refuses a path outside the docs folder", async () => {
+		const response = await getMarkdown("../../package");
+		expect(response.status).toBe(404);
+	});
+});
+
+describe("proxy routes docs Markdown requests", () => {
+	const send = (path: string, accept: string, method = "GET") =>
+		proxy(
+			new NextRequest(`https://cap.so${path}`, {
+				method,
+				headers: { accept },
+			}),
+		);
+	const rewriteOf = (response: Response) =>
+		response.headers.get("x-middleware-rewrite");
+
+	it.each([
+		["/docs/api/get-started", "text/markdown"],
+		["/docs/api/get-started.md", "text/html,*/*;q=0.8"],
+	])("rewrites %s (Accept %s)", async (path, accept) => {
+		const response = await send(path, accept);
+		expect(rewriteOf(response)).toBe(
+			`https://cap.so${DOCS_MARKDOWN_ROUTE}/api/get-started`,
+		);
+	});
+
+	it("leaves the HTML page alone for a browser", async () => {
+		const response = await send(
+			"/docs/api/get-started",
+			"text/html,application/xhtml+xml,*/*;q=0.8",
+		);
+		expect(rewriteOf(response)).toBeNull();
+		expect(response.headers.get("x-middleware-next")).toBe("1");
+	});
+});

apps/web/__tests__/unit/infra-agent-prompt.test.ts

@@ -61,7 +61,12 @@ describe("Cap Infra agent prompt", () => {
 		]) {
 			expect(named(CAP_INFRA_AGENT_PROMPT)).toContain(call);
 		}
-		expect(CAP_INFRA_AGENT_PROMPT).toContain("every 2 seconds");
+		expect(CAP_INFRA_AGENT_PROMPT).toContain(
+			"/videos/{id}/status every 3 seconds",
+		);
+		expect(CAP_INFRA_AGENT_PROMPT).toContain(
+			"/transcripts/{id} every 5 seconds",
+		);
 		expect(CAP_INFRA_AGENT_PROMPT).toContain("Retry-After");
 		expect(CAP_INFRA_AGENT_PROMPT).toContain(
 			`${DEVELOPER_KEY_REQUESTS_PER_MINUTE} requests a minute`,

apps/web/__tests__/unit/proxy-infra-path.test.ts

@@ -22,6 +22,8 @@ vi.mock("@cap/env", () => ({
 
 vi.mock("@/lib/agent-markdown", () => ({
 	agentMarkdownResponse: async () => null,
+	docsMarkdownSlug: () => null,
+	DOCS_MARKDOWN_ROUTE: "/docs-markdown",
 }));
 
 const FORWARDED = "x-middleware-request-x-cap-infra-path";

apps/web/app/(docs)/docs/[[...slug]]/page.tsx

@@ -43,7 +43,10 @@ export async function generateMetadata(
 	return {
 		title: pageTitle,
 		description,
-		alternates: { canonical: `https://cap.so${docPath}` },
+		alternates: {
+			canonical: `https://cap.so${docPath}`,
+			types: { "text/markdown": `https://cap.so${docPath}.md` },
+		},
 		openGraph: {
 			title: pageTitle,
 			description,

apps/web/app/docs-markdown/[...slug]/route.ts

@@ -0,0 +1,23 @@
+import { markdownHeaders, markdownNotFoundBody } from "@/lib/agent-markdown";
+import { docToMarkdown } from "@/lib/docs-markdown";
+import { getDocBySlug } from "@/utils/docs";
+
+export async function GET(
+	request: Request,
+	props: RouteContext<"/docs-markdown/[...slug]">,
+) {
+	const { slug } = await props.params;
+	const { origin } = new URL(request.url);
+	const doc = getDocBySlug(slug.join("/"));
+
+	if (!doc) {
+		return new Response(
+			markdownNotFoundBody(origin, `/docs/${slug.join("/")}`),
+			{ status: 404, headers: markdownHeaders("private, no-store") },
+		);
+	}
+
+	return new Response(docToMarkdown(doc, origin), {
+		headers: markdownHeaders("public, max-age=300"),
+	});
+}

apps/web/components/pages/infra/Agent.tsx

@@ -32,7 +32,7 @@ export const Agent = () => (
 			<div className="flex flex-wrap items-center gap-x-5 gap-y-3 self-start lg:col-start-1 lg:row-start-2">
 				<StartLink className={BTN_GHOST} />
 				<Link
-					href={LINKS.docs}
+					href={LINKS.reference}
 					className={`${TEXT_LINK} text-[14px] text-[rgba(232,236,244,0.6)] hover:text-[#E8ECF4]`}
 				>
 					API reference

apps/web/components/pages/infra/Faq.tsx

@@ -9,7 +9,7 @@ export const Faq = () => (
 			<SectionHeading title="Questions">
 				<p>
 					The full reference is in the{" "}
-					<Link href={LINKS.docs} className={TEXT_LINK}>
+					<Link href={LINKS.reference} className={TEXT_LINK}>
 						docs
 					</Link>
 					.

apps/web/components/pages/infra/content.ts

@@ -92,11 +92,12 @@ export const BAA = "BAA available.";
 
 export const TRUST = [COMPLIANCE, BAA, "Open source."] as const;
 
-const DOCS = "/docs/api/rest-api";
+const REFERENCE = "/docs/api/rest-api";
 
 export const LINKS = {
-	docs: DOCS,
-	uploadDocs: `${DOCS}#multipart-upload`,
+	docs: "/docs/api/get-started",
+	reference: REFERENCE,
+	uploadDocs: `${REFERENCE}#multipart-upload`,
 	pro: "/pricing",
 	trust: "https://trust.cap.so",
 	signIn: "/login",
@@ -353,7 +354,7 @@ export const infraFaqs: readonly Faq[] = [
 	{
 		question: "How do I know when a video is ready?",
 		answer:
-			"Check the video's status from your server every few seconds until it is ready or errored. A video is ready as soon as its first output can be played, and the other output follows.",
+			"Check the video's status from your server every 3 seconds until it is ready or errored. A video is ready as soon as its first output can be played, and the other output follows.",
 	},
 	{
 		question: "How do API keys work?",

apps/web/content/docs/api/get-started.mdx

@@ -351,7 +351,7 @@ Add this to `index.html`, below the upload script:
 - **`ready`** means the video can be played. Processing has made at least one of `hlsUrl` and `mp4Url`, and the poster.
 - **`hlsUrl` can still be `null` for a moment after `ready`.** The MP4 is often finished first and the adaptive HLS renditions follow a few seconds later. The loop above keeps checking a few more times for `hlsUrl`, then plays the MP4.
 - **`errored`** is final. `errorReason` says why, for example that the file could not be read as a video. It is written for you and can mention your credit balance, so log it and show your users your own message. A failed video is not billed and cannot be processed again: create a new upload.
-- **Pace the checks.** A secret key allows 300 requests per minute, and every status check your server makes for any user counts toward it. Leave at least 3 seconds between checks and slow down from there. On a `429`, wait for the `Retry-After` seconds and ask again. See [Rate Limits](/docs/api/rest-api#rate-limits).
+- **Pace the checks.** A secret key allows 300 requests per minute, and every status check your server makes for any user counts toward it. Check every 3 seconds, which usually finds a short video ready on the first check, and slow down for longer ones as the loop above does. On a `429`, wait for the `Retry-After` seconds and ask again. See [Rate Limits](/docs/api/rest-api#rate-limits).
 
 ## 5. Play it
 
@@ -395,28 +395,7 @@ Add this to `index.html`, below the other scripts:
     if (width && height) player.style.aspectRatio = `${width} / ${height}`;
 
     if (hlsUrl && window.Hls && Hls.isSupported()) {
-      const megabits = navigator.connection?.downlink;
-      const estimate = megabits > 0 ? megabits * 800000 : 1500000;
-      hls = new Hls({
-        autoStartLoad: false,
-        abrEwmaDefaultEstimate: estimate,
-        abrEwmaDefaultEstimateMax: estimate,
-        startFragPrefetch: true,
-        capLevelToPlayerSize: true,
-      });
-      hls.on(Hls.Events.MANIFEST_PARSED, (_event, { levels }) => {
-        const wanted = player.clientWidth * window.devicePixelRatio;
-        let level = 0;
-        while (
-          levels[level + 1] &&
-          levels[level + 1].bitrate <= estimate &&
-          levels[level].width < wanted
-        ) {
-          level++;
-        }
-        hls.startLevel = level;
-        hls.startLoad();
-      });
+      hls = new Hls({ capLevelToPlayerSize: true });
       hls.on(Hls.Events.ERROR, (_event, data) => {
         if (data.fatal || data.response?.code === 403) playMp4();
       });
@@ -442,14 +421,13 @@ Safari plays HLS by itself. Every other browser needs [hls.js](https://github.co
 
 | Setting | Why |
 |---------|-----|
-| `autoStartLoad: false`, then `startLevel` and `startLoad()` | Start on a rendition the viewer's connection can carry. Without it hls.js starts on the first rendition the playlist lists, which is too large for a slow connection |
-| `abrEwmaDefaultEstimate` and `abrEwmaDefaultEstimateMax` | Give hls.js the browser's own estimate of the connection. Safari and Firefox do not give one, so the page assumes a slow connection there: the video starts on the smallest rendition and steps up once hls.js has measured a download |
-| `startFragPrefetch` | Fetch the first segment while the video element is still being set up |
 | `capLevelToPlayerSize` | Never load a rendition larger than the player on the page |
 | The `403` check | Media links inside the playlist expire after 12 hours. A `403` means one has expired, so switch to the MP4 at once instead of letting hls.js retry |
 | `playMp4` on a fatal error | Keep playing from the same position if HLS fails for any other reason |
 | `stopHls` at the start of `playVideo` | Stop the previous video's player before starting the next one on the same page |
 
+There is no start level to set. The master playlist lists a rendition that starts quickly first, and players begin with the first one listed.
+
 [Play a Video](/docs/api/rest-api#play-a-video) covers a Content Security Policy, bitrates and what happens when playback stops.
 
 ### Use the hosted player instead
@@ -807,7 +785,9 @@ The same secret key turns an audio or video file into text. It is a separate cal
 
 1. Create the transcript with `POST /api/developer/v1/transcripts/uploads`, sending the file's exact size in bytes as `contentLength`. The response holds the transcript's `id` and a signed `upload.url`.
 2. `PUT` the file to `upload.url`.
-3. Start it with `POST /api/developer/v1/transcripts/:id/start`, then check `GET /api/developer/v1/transcripts/:id` until `status` is `complete` or `error`, leaving at least 3 seconds between checks. A complete transcript has its `text`.
+3. Start it with `POST /api/developer/v1/transcripts/:id/start`, then check `GET /api/developer/v1/transcripts/:id` every 5 seconds until `status` is `complete` or `error`. A complete transcript has its `text`.
+
+To wait on several transcripts, check them as a batch with one list call per status instead of one call each: `GET /api/developer/v1/transcripts?status=queued&limit=100`, then `status=processing`. Ask for `queued` first: a transcript only moves from `queued` to `processing` to a final status, so one missing from both lists has finished. Get it for its `text`, and keep checking it if its `status` is not yet `complete` or `error`. An app has at most 20 transcripts in flight, and checking each of 20 every 5 seconds takes 240 requests a minute, most of a key's limit, while the two list calls take 24.
 
 A file that is already at a public URL needs no upload. Send it as `url` to `POST /api/developer/v1/transcripts` and check the transcript the same way. That includes a video you uploaded here: send its `mp4Url` as the `url` once the video is `ready`.