diff --git a/apps/web/__tests__/unit/agent-readiness.test.ts b/apps/web/__tests__/unit/agent-readiness.test.ts index 34aa1ab63bb..bec65c56d29 100644 --- a/apps/web/__tests__/unit/agent-readiness.test.ts +++ b/apps/web/__tests__/unit/agent-readiness.test.ts @@ -397,7 +397,7 @@ describe("developer OpenAPI spec", () => { ] as const; const implemented = new Set(); for (const [dir, base] of routeDirs) { - const source = readFileSync(join(apiRoot, dir, "route.ts"), "utf8"); + const source = readFileSync(join(apiRoot, dir, "app.ts"), "utf8"); const modules = new Map( [...source.matchAll(/import \* as (\w+) from "\.\/([^"]+)"/g)].map( ([, name, file]) => [name, file], diff --git a/apps/web/__tests__/unit/developer-get-started-docs.test.ts b/apps/web/__tests__/unit/developer-get-started-docs.test.ts index a06082e7119..07119def467 100644 --- a/apps/web/__tests__/unit/developer-get-started-docs.test.ts +++ b/apps/web/__tests__/unit/developer-get-started-docs.test.ts @@ -70,10 +70,12 @@ describe("the Get Started guide", () => { } }); - it("keeps real credentials, the CLI and the em dash out", () => { + it("keeps real credentials, the CLI and the em dash out, and points agents at the hosted MCP server", () => { expect(content).not.toContain("—"); expect(content).not.toMatch(/\bcap (guide|record|upload|caps|mcp)\b/); - expect(content).not.toMatch(/\bMCP\b/); + expect(content.match(/https:\/\/[^\s"`]*\/mcp\b/g) ?? []).toEqual( + Array(4).fill("https://cap.so/api/developer/mcp"), + ); const keys = content.match(/\bc(?:sk|pk|ut)_[A-Za-z0-9._-]+/g) ?? []; expect(keys.every((key) => key === "csk_your_secret_key")).toBe(true); }); diff --git a/apps/web/__tests__/unit/infra-agent-prompt.test.ts b/apps/web/__tests__/unit/infra-agent-prompt.test.ts index f542379374d..350d4c5618a 100644 --- a/apps/web/__tests__/unit/infra-agent-prompt.test.ts +++ b/apps/web/__tests__/unit/infra-agent-prompt.test.ts @@ -1,7 +1,10 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { CAP_INFRA_AGENT_PROMPT } from "@/data/infra-agent-prompt"; +import { + CAP_INFRA_AGENT_PROMPT, + CAP_INFRA_MCP_URL, +} from "@/data/infra-agent-prompt"; import { developerOpenApiSpec } from "@/lib/developer-openapi"; const METHODS = ["get", "post", "put", "patch", "delete"] as const; @@ -30,7 +33,9 @@ describe("Cap Infra agent prompt", () => { ); } const pathMentions = - CAP_INFRA_AGENT_PROMPT.match(/\/api\/developer\//g) ?? []; + CAP_INFRA_AGENT_PROMPT.replaceAll(CAP_INFRA_MCP_URL, "").match( + /\/api\/developer\//g, + ) ?? []; expect(pathMentions).toHaveLength(calls.length); }); @@ -60,17 +65,22 @@ describe("Cap Infra agent prompt", () => { expect(CAP_INFRA_AGENT_PROMPT).toContain("60 requests a minute"); }); - it("links only to the API reference", () => { + it("links only to the API reference and the hosted MCP server", () => { const links = CAP_INFRA_AGENT_PROMPT.match(/https?:\/\/[^\s)]+/g) ?? []; expect([...new Set(links)].sort()).toEqual([ + "https://cap.so/api/developer/mcp", "https://cap.so/docs/api/rest-api", "https://cap.so/openapi.json", ]); }); - it("never asks for the Cap CLI, an MCP server or a pasted key", () => { + it("never asks for the Cap CLI, a local MCP server or a pasted key", () => { expect(CAP_INFRA_AGENT_PROMPT).not.toMatch(/\bcap [a-z]/); - expect(CAP_INFRA_AGENT_PROMPT).not.toMatch(/\bCLI\b|\bMCP\b|curl|install/i); + expect(CAP_INFRA_AGENT_PROMPT).not.toMatch(/\bCLI\b|curl|install|npx/i); + expect(CAP_INFRA_AGENT_PROMPT.match(/\bMCP\b/g)).toEqual(["MCP", "MCP"]); + expect(CAP_INFRA_AGENT_PROMPT).toContain( + "hosted MCP server at https://cap.so/api/developer/mcp", + ); expect(CAP_INFRA_AGENT_PROMPT).not.toContain("cpk_"); expect(CAP_INFRA_AGENT_PROMPT).toContain( "Never ask me to paste the key into the chat", @@ -85,7 +95,7 @@ describe("Cap Infra agent prompt", () => { ); }); - it("is introduced in the console without the Cap CLI or an MCP server", () => { + it("is introduced in the console without the Cap CLI", () => { const source = readFileSync( join( process.cwd(), @@ -93,7 +103,7 @@ describe("Cap Infra agent prompt", () => { ), "utf8", ); - expect(source).not.toMatch(/\bCLI\b|\bMCP\b/); + expect(source).not.toMatch(/\bCLI\b/); expect(source).toContain("secret key"); }); }); diff --git a/apps/web/__tests__/unit/infra-hosted-mcp.test.ts b/apps/web/__tests__/unit/infra-hosted-mcp.test.ts new file mode 100644 index 00000000000..9ed360c45ee --- /dev/null +++ b/apps/web/__tests__/unit/infra-hosted-mcp.test.ts @@ -0,0 +1,629 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { DEVELOPER_KEY_REQUESTS_PER_MINUTE } from "@/lib/developer-rate-limits"; + +const ORIGIN = "https://cap.test"; +const ENDPOINT = `${ORIGIN}/api/developer/mcp`; + +const state = vi.hoisted(() => ({ + keys: new Map(), + gatedApps: new Set(), + rest: [] as Request[], + nextKey: 0, + presignFails: false, + completeStatus: 200, + completeBody: null as Record | null, +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const database = drizzle(async (query, params) => { + if (query.includes("from `developer_api_keys`")) { + const key = state.keys.get(String(params[0]).replace("hash:", "")); + if (!key || key.revoked || params[1] !== "secret") return { rows: [] }; + return { rows: [[key.appId]] }; + } + return { rows: [{ affectedRows: 1 }] }; + }); + return { db: () => database }; +}); +vi.mock("@cap/database/auth/session", () => ({ getCurrentUser: vi.fn() })); +vi.mock("@cap/database/auth/domain-utils", () => ({ + isBlockedAccountEmail: () => false, +})); +vi.mock("@cap/database/auth/request-site", () => ({ + isCrossSiteCookieWrite: () => false, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_WEB_URL: ORIGIN }, + serverEnv: () => ({ WEB_URL: ORIGIN }), +})); +vi.mock("@/lib/developer-key-hash", () => ({ + hashKey: async (key: string) => `hash:${key}`, +})); +vi.mock("@/lib/rate-limit", async (importOriginal) => ({ + ...(await importOriginal()), + sharedRateLimitDecision: async () => undefined, +})); + +const SIGNED_PART = + "https://storage.test/developer/app1/video-1/video?partNumber=1&X-Amz-Credential=AKIAEXAMPLE&X-Amz-Signature=deadbeefsignature"; + +function appOf(request: Request) { + const key = request.headers.get("authorization")?.split(" ")[1] ?? ""; + return state.keys.get(key)?.appId ?? null; +} + +async function restAnswer(request: Request): Promise { + state.rest.push(request); + const url = new URL(request.url); + const route = `${request.method} ${url.pathname}`; + const appId = appOf(request); + switch (route) { + case "GET /api/developer/v1/usage": + return Response.json({ + data: { + balanceMicroCredits: 5_000_000, + standing: { state: "active" }, + upload: { allowed: true }, + }, + }); + case "GET /api/developer/v1/videos": + return Response.json({ + data: [{ id: "video-1", name: "Demo", status: "ready", duration: 12 }], + total: 1, + }); + case "POST /api/developer/v1/uploads": + return Response.json({ + data: { + id: "video-1", + videoId: "video-1", + token: "cut_uploadtoken123.signature", + shareUrl: `${ORIGIN}/infra/s/video-1`, + embedUrl: `${ORIGIN}/infra/e/video-1`, + }, + }); + case "POST /api/developer/v1/uploads/video-1/token": + return Response.json({ + data: { videoId: "video-1", token: "cut_renewedtoken123.signature" }, + }); + case "POST /api/developer/sdk/v1/upload/multipart/initiate": + return Response.json({ uploadId: "upload-1" }); + case "POST /api/developer/sdk/v1/upload/multipart/presign-part": + if (state.presignFails) { + return Response.json( + { code: "storage_unavailable", message: "Storage failed" }, + { status: 400 }, + ); + } + return Response.json({ + parts: [{ partNumber: 1, presignedUrl: SIGNED_PART }], + }); + case "POST /api/developer/sdk/v1/upload/multipart/complete": + if (state.completeStatus !== 200) { + return Response.json( + state.completeBody ?? { + code: "internal_error", + message: "Something went wrong", + }, + { status: state.completeStatus }, + ); + } + return Response.json({ success: true }); + case "GET /api/developer/v1/videos/video-1": + return Response.json({ + data: { id: "video-1", status: "uploading", shareUrl: null }, + }); + case "POST /api/developer/sdk/v1/upload/multipart/abort": + return Response.json({ success: true }); + case "POST /api/developer/v1/transcripts": + if (appId !== null && state.gatedApps.has(appId)) { + return Response.json( + { + code: "infra_not_available", + message: "Cap Infra is not available for this account yet", + hint: "Cap Infra is in preview. Contact hello@cap.so for access.", + }, + { status: 403 }, + ); + } + return Response.json({ + data: { id: "transcript-1", status: "queued", name: "Talk" }, + }); + case "GET /api/developer/v1/transcripts/transcript-1": + return Response.json({ + data: { id: "transcript-1", status: "complete", text: "Hello there" }, + }); + case "GET /api/developer/v1/videos/video-slow": + return new Promise(() => {}); + default: + return Response.json( + { code: "not_found", message: route }, + { status: 404 }, + ); + } +} + +vi.mock("@/app/api/developer/v1/[...route]/app", () => ({ + app: { fetch: restAnswer }, +})); +vi.mock("@/app/api/developer/sdk/v1/[...route]/app", () => ({ + app: { fetch: restAnswer }, +})); + +type Route = typeof import("@/app/api/developer/mcp/route"); +let route: Route; + +beforeAll(async () => { + route = await import("@/app/api/developer/mcp/route"); +}); + +beforeEach(() => { + state.rest.length = 0; + state.gatedApps.clear(); + state.presignFails = false; + state.completeStatus = 200; + state.completeBody = null; +}); + +const newKey = (appId = "app1", revoked = false) => { + const key = `csk_${appId}_unitsecret${++state.nextKey}`; + state.keys.set(key, { appId, revoked }); + return key; +}; + +function mcp( + credential: string | null, + body: unknown, + headers: Record = {}, +) { + return route.POST( + new Request(ENDPOINT, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + "X-Forwarded-For": "203.0.113.7", + ...(credential ? { Authorization: `Bearer ${credential}` } : {}), + ...headers, + }, + body: typeof body === "string" ? body : JSON.stringify(body), + }), + ); +} + +async function rpc( + credential: string, + method: string, + params: Record = {}, +) { + const response = await mcp(credential, { + jsonrpc: "2.0", + id: 1, + method, + params, + }); + expect(response.status).toBe(200); + const text = await response.text(); + const line = text + .split("\n") + .find((entry) => entry.startsWith("data: ") && entry.includes('"id":1')); + return { + raw: text, + message: JSON.parse(line ? line.slice(6) : text) as { + result?: Record; + error?: unknown; + }, + }; +} + +async function callTool( + credential: string, + name: string, + args: Record = {}, +) { + const { raw, message } = await rpc(credential, "tools/call", { + name, + arguments: args, + }); + const result = message.result as { + content: { text: string }[]; + isError?: boolean; + }; + return { + raw, + isError: result.isError === true, + body: JSON.parse(result.content[0]?.text ?? "null"), + }; +} + +describe("hosted Cap Infra MCP auth", () => { + const initialize = { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "test", version: "1.0.0" }, + }, + }; + + it.each([ + [ + "no Authorization header", + null, + 401, + "invalid_secret_key", + "Missing secret key", + ], + [ + "a public key", + "cpk_app1_copied_from_a_page", + 403, + "public_key_not_allowed", + "Public keys cannot call the REST API", + ], + [ + "an upload token", + "cut_uploadtoken123.signature", + 401, + "invalid_secret_key", + "Invalid secret key", + ], + [ + "a secret key that does not exist", + "csk_app1_neverissued", + 401, + "invalid_secret_key", + "Invalid or revoked secret key", + ], + ])( + "refuses %s like the REST API", + async (_label, credential, status, code, message) => { + const response = await mcp(credential, initialize); + expect(response.status).toBe(status); + if (status === 401) { + expect(response.headers.get("www-authenticate")).toContain("Bearer"); + } else { + expect(response.headers.get("www-authenticate")).toBeNull(); + } + expect(await response.json()).toMatchObject({ code, message }); + expect(state.rest).toHaveLength(0); + }, + ); + + it("sends the REST API's response headers on refusals and answers", async () => { + const refused = await mcp(null, initialize, { + "x-vercel-id": "iad1::req-1", + }); + expect(refused.status).toBe(401); + expect(refused.headers.get("cache-control")).toBe("private, no-store"); + expect(refused.headers.get("x-request-id")).toBe("iad1::req-1"); + expect(refused.headers.get("www-authenticate")).toBe( + 'Bearer realm="cap-infra"', + ); + + const answered = await mcp(newKey(), initialize, { + "x-vercel-id": "iad1::req-2", + }); + expect(answered.status).toBe(200); + expect(answered.headers.get("cache-control")).toBe("private, no-store"); + expect(answered.headers.get("x-request-id")).toBe("iad1::req-2"); + expect(answered.headers.get("www-authenticate")).toBeNull(); + }); + + it("refuses a revoked secret key", async () => { + const response = await mcp(newKey("app1", true), initialize); + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject({ + code: "invalid_secret_key", + message: "Invalid or revoked secret key", + }); + }); + + it("passes the REST preview refusal through for an owner outside the preview", async () => { + const key = newKey("gated"); + state.gatedApps.add("gated"); + const created = await callTool(key, "transcripts_create", { + url: "https://example.com/talk.mp3", + }); + expect(created.isError).toBe(true); + expect(created.body.error).toMatchObject({ + code: "infra_not_available", + status: 403, + }); + }); + + it("answers initialize for a valid secret key without touching REST", async () => { + const response = await mcp(newKey(), initialize); + expect(response.status).toBe(200); + expect(await response.text()).toContain("cap-infra"); + expect(state.rest).toHaveLength(0); + }); +}); + +describe("hosted Cap Infra MCP tools", () => { + it("lists the hosted tools with no local file access", async () => { + const { message } = await rpc(newKey(), "tools/list"); + const tools = ( + message.result as { + tools: { + name: string; + inputSchema: { properties: Record }; + }[]; + } + ).tools; + expect(tools.map((tool) => tool.name).sort()).toEqual([ + "account_transactions", + "account_usage", + "transcripts_create", + "transcripts_delete", + "transcripts_get", + "transcripts_list", + "transcripts_text", + "transcripts_wait", + "videos_complete_upload", + "videos_create_upload", + "videos_delete", + "videos_get", + "videos_list", + "videos_wait", + ]); + const create = tools.find((tool) => tool.name === "transcripts_create"); + expect(Object.keys(create?.inputSchema.properties ?? {})).not.toContain( + "path", + ); + for (const name of ["videos_wait", "transcripts_wait"]) { + const wait = tools.find((tool) => tool.name === name); + expect(wait?.inputSchema.properties.timeoutSeconds?.maximum).toBe(45); + } + }); + + it("reads usage and videos through the in-process REST API with the caller's key and address", async () => { + const key = newKey(); + const usage = await callTool(key, "account_usage"); + expect(usage.isError).toBe(false); + expect(usage.body).toMatchObject({ balanceMicroCredits: 5_000_000 }); + + const videos = await callTool(key, "videos_list", { limit: 5 }); + expect(videos.body.videos).toEqual([ + expect.objectContaining({ id: "video-1", status: "ready" }), + ]); + + const [first, second] = state.rest; + expect(new URL(first?.url ?? "").pathname).toBe("/api/developer/v1/usage"); + expect(first?.headers.get("authorization")).toBe(`Bearer ${key}`); + expect(first?.headers.get("x-forwarded-for")).toBe("203.0.113.7"); + expect(new URL(second?.url ?? "").search).toBe("?limit=5&offset=0"); + expect(usage.raw).not.toContain(key); + }); + + it("creates an upload with signed part links and completes it", async () => { + const key = newKey(); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + const created = await callTool(key, "videos_create_upload", { + contentLength: 1_000_000, + fileName: "demo.mp4", + }); + expect(created.isError).toBe(false); + expect(created.body).toMatchObject({ + videoId: "video-1", + uploadId: "upload-1", + status: "uploading", + parts: [{ partNumber: 1, bytes: 1_000_000, url: SIGNED_PART }], + shareUrl: `${ORIGIN}/infra/s/video-1`, + }); + expect(created.body.steps.join(" ")).toContain("videos_complete_upload"); + expect(created.raw).not.toContain("cut_"); + expect(created.raw).not.toContain(key); + + const initiate = state.rest.find((request) => + request.url.endsWith("/multipart/initiate"), + ); + expect(initiate?.headers.get("authorization")).toBe( + "Bearer cut_uploadtoken123.signature", + ); + expect(await initiate?.json()).toEqual({ + videoId: "video-1", + contentType: "video/mp4", + contentLength: 1_000_000, + }); + + const completed = await callTool(key, "videos_complete_upload", { + videoId: "video-1", + uploadId: "upload-1", + parts: [{ partNumber: 1, etag: '"etag-1"' }], + }); + expect(completed.body).toMatchObject({ + videoId: "video-1", + status: "processing", + }); + const complete = state.rest.find((request) => + request.url.endsWith("/multipart/complete"), + ); + expect(complete?.headers.get("authorization")).toBe( + "Bearer cut_renewedtoken123.signature", + ); + + const logged = info.mock.calls.flat().join("\n"); + expect(logged).toContain("video_created"); + expect(logged).not.toContain(key); + expect(logged).not.toContain("deadbeefsignature"); + expect(logged).not.toContain("cut_uploadtoken123"); + info.mockRestore(); + }); + + it("creates a transcript from a url and reads its text", async () => { + const key = newKey(); + const created = await callTool(key, "transcripts_create", { + url: "https://example.com/talk.mp3", + }); + expect(created.body).toMatchObject({ id: "transcript-1" }); + const sent = state.rest.at(-1); + expect(sent?.method).toBe("POST"); + expect(await sent?.json()).toEqual({ url: "https://example.com/talk.mp3" }); + + const text = await callTool(key, "transcripts_text", { + transcriptId: "transcript-1", + }); + expect(text.body).toEqual({ + id: "transcript-1", + status: "complete", + text: "Hello there", + }); + }); + + it("transcribes a stored video by id through the REST videoId field", async () => { + const created = await callTool(newKey(), "transcripts_create", { + videoId: "video-1", + }); + expect(created.isError).toBe(false); + expect(state.rest).toHaveLength(1); + expect(await state.rest[0]?.json()).toEqual({ videoId: "video-1" }); + }); + + it("aborts the upload it opened for a new video when signing fails", async () => { + state.presignFails = true; + const created = await callTool(newKey(), "videos_create_upload", { + contentLength: 1_000_000, + }); + expect(created.isError).toBe(true); + expect(created.body).toMatchObject({ + videoId: "video-1", + videoStatus: "uploading", + }); + expect( + state.rest.some((request) => request.url.endsWith("/multipart/abort")), + ).toBe(true); + }); + + it("keeps a reused upload and its stored parts when signing fails on retry", async () => { + state.presignFails = true; + const retried = await callTool(newKey(), "videos_create_upload", { + contentLength: 1_000_000, + videoId: "video-1", + }); + expect(retried.isError).toBe(true); + expect( + state.rest.some((request) => request.url.endsWith("/multipart/abort")), + ).toBe(false); + }); + + it("says the outcome is unknown when completing fails on the server", async () => { + state.completeStatus = 500; + const completed = await callTool(newKey(), "videos_complete_upload", { + videoId: "video-1", + uploadId: "upload-1", + parts: [{ partNumber: 1, etag: "etag-1" }], + }); + expect(completed.isError).toBe(true); + expect(completed.body).toMatchObject({ + videoId: "video-1", + videoStatus: "unknown", + next: expect.stringContaining("videos_get"), + }); + }); + + it("tells the caller to upload again when completing removed the file", async () => { + state.completeStatus = 402; + state.completeBody = { + code: "insufficient_credits", + message: "Not enough credit to store this video", + fileRemoved: true, + }; + const completed = await callTool(newKey(), "videos_complete_upload", { + videoId: "video-1", + uploadId: "upload-1", + parts: [{ partNumber: 1, etag: "etag-1" }], + }); + expect(completed.isError).toBe(true); + expect(completed.body.error).toMatchObject({ fileRemoved: true }); + expect(completed.body.next).toContain("upload the file again"); + expect(completed.body.next).not.toContain("parts are kept"); + }); + + it("refuses a local path for a transcript", async () => { + const result = await callTool(newKey(), "transcripts_create", { + path: "/etc/passwd", + }); + expect(result.isError).toBe(true); + expect(result.body.error.code).toBe("invalid_source"); + expect(state.rest).toHaveLength(0); + }); +}); + +describe("hosted Cap Infra MCP limits", () => { + it("refuses a body over 64 KB before any MCP work", async () => { + const response = await mcp(newKey(), { + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: "videos_list", + arguments: { userId: "x".repeat(70_000) }, + }, + }); + expect(response.status).toBe(413); + expect(await response.json()).toMatchObject({ code: "request_too_large" }); + expect(state.rest).toHaveLength(0); + }); + + it("refuses a body that is not JSON", async () => { + const response = await mcp(newKey(), "hello", { + "Content-Type": "text/plain", + }); + expect(response.status).toBe(415); + }); + + it("answers GET with 405 since the endpoint holds no sessions", async () => { + const response = await route.GET( + new Request(ENDPOINT, { + headers: { Authorization: `Bearer ${newKey()}` }, + }), + ); + expect(response.status).toBe(405); + expect(response.headers.get("allow")).toBe("POST"); + }); + + it("applies the REST API's per-key limit to MCP requests", async () => { + const key = newKey(); + const list = { jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }; + for (let index = 0; index < DEVELOPER_KEY_REQUESTS_PER_MINUTE; index++) { + expect((await mcp(key, list)).status).toBe(200); + } + const limited = await mcp(key, list); + expect(limited.status).toBe(429); + expect(limited.headers.get("retry-after")).not.toBeNull(); + expect(await limited.json()).toMatchObject({ code: "rate_limited" }); + expect((await mcp(newKey(), list)).status).toBe(200); + }); + + it("stops a tool that outlives its budget and says so", async () => { + const { inProcessDeveloperFetch } = await import("@/lib/infra-mcp-rest"); + const { createHostedTools } = await import("@cap/infra-mcp/hosted"); + const tools = createHostedTools({ + apiUrl: ORIGIN, + secretKey: newKey(), + fetch: inProcessDeveloperFetch(new Headers()), + log: () => {}, + budgetMs: 200, + }); + const get = tools.find((tool) => tool.name === "videos_get"); + const startedAt = Date.now(); + const result = await get?.run( + { videoId: "video-slow" }, + { + signal: new AbortController().signal, + progressToken: undefined, + sendProgress: async () => {}, + sendLog: async () => {}, + }, + ); + expect(Date.now() - startedAt).toBeLessThan(2_000); + expect(result?.isError).toBe(true); + expect(JSON.parse(result?.content[0]?.text ?? "{}").error).toMatchObject({ + code: "timeout", + }); + }); +}); diff --git a/apps/web/app/api/developer/mcp/route.ts b/apps/web/app/api/developer/mcp/route.ts new file mode 100644 index 00000000000..da3750cef6e --- /dev/null +++ b/apps/web/app/api/developer/mcp/route.ts @@ -0,0 +1,37 @@ +import { + HttpApi, + HttpApiBuilder, + HttpApiEndpoint, + HttpApiGroup, +} from "@effect/platform"; +import { Layer } from "effect"; +import { serveInfraMcp } from "@/lib/infra-mcp-endpoint"; +import { mcpApiToHandler, mcpRequest } from "@/lib/mcp-effect"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +class Api extends HttpApi.make("CapInfraMcpApi").add( + HttpApiGroup.make("root") + .add(HttpApiEndpoint.post("post")`/api/developer/mcp`) + .add(HttpApiEndpoint.get("get")`/api/developer/mcp`) + .add(HttpApiEndpoint.del("delete")`/api/developer/mcp`), +) {} + +const ApiLive = HttpApiBuilder.api(Api).pipe( + Layer.provide( + HttpApiBuilder.group(Api, "root", (handlers) => + handlers + .handle("post", () => mcpRequest(serveInfraMcp)) + .handle("get", () => mcpRequest(serveInfraMcp)) + .handle("delete", () => mcpRequest(serveInfraMcp)), + ), + ), +); + +const handler = mcpApiToHandler(ApiLive); + +export const POST = handler; +export const GET = handler; +export const DELETE = handler; diff --git a/apps/web/app/api/developer/sdk/v1/[...route]/app.ts b/apps/web/app/api/developer/sdk/v1/[...route]/app.ts new file mode 100644 index 00000000000..3a0511e919d --- /dev/null +++ b/apps/web/app/api/developer/sdk/v1/[...route]/app.ts @@ -0,0 +1,21 @@ +import { Hono } from "hono"; +import { + developerApiError, + developerApiNotFound, + developerResponseHeaders, + developerSdkCors, + developerUploadRateLimiter, + refuseDeveloperPublicKey, +} from "../../../../utils"; +import * as upload from "./upload"; + +export const app = new Hono() + .basePath("/api/developer/sdk/v1") + .use(developerResponseHeaders) + .use(developerSdkCors) + .use(developerUploadRateLimiter) + .use(refuseDeveloperPublicKey) + .route("/upload/multipart", upload.app) + .onError(developerApiError); + +app.notFound((c) => developerApiNotFound(c, app.routes)); diff --git a/apps/web/app/api/developer/sdk/v1/[...route]/route.ts b/apps/web/app/api/developer/sdk/v1/[...route]/route.ts index b9fc9928035..402e6ac2dd1 100644 --- a/apps/web/app/api/developer/sdk/v1/[...route]/route.ts +++ b/apps/web/app/api/developer/sdk/v1/[...route]/route.ts @@ -1,25 +1,5 @@ -import { Hono } from "hono"; import { handle } from "hono/vercel"; -import { - developerApiError, - developerApiNotFound, - developerResponseHeaders, - developerSdkCors, - developerUploadRateLimiter, - refuseDeveloperPublicKey, -} from "../../../../utils"; -import * as upload from "./upload"; - -const app = new Hono() - .basePath("/api/developer/sdk/v1") - .use(developerResponseHeaders) - .use(developerSdkCors) - .use(developerUploadRateLimiter) - .use(refuseDeveloperPublicKey) - .route("/upload/multipart", upload.app) - .onError(developerApiError); - -app.notFound((c) => developerApiNotFound(c, app.routes)); +import { app } from "./app"; export const GET = handle(app); export const POST = handle(app); diff --git a/apps/web/app/api/developer/v1/[...route]/app.ts b/apps/web/app/api/developer/v1/[...route]/app.ts new file mode 100644 index 00000000000..ae4eeeae394 --- /dev/null +++ b/apps/web/app/api/developer/v1/[...route]/app.ts @@ -0,0 +1,26 @@ +import { Hono } from "hono"; +import { + developerApiError, + developerApiNotFound, + developerRateLimiter, + developerResponseHeaders, +} from "../../../utils"; + +import * as transactions from "./transactions"; +import * as transcripts from "./transcripts"; +import * as uploads from "./uploads"; +import * as usage from "./usage"; +import * as videos from "./videos"; + +export const app = new Hono() + .basePath("/api/developer/v1") + .use(developerResponseHeaders) + .use(developerRateLimiter) + .route("/videos", videos.app) + .route("/uploads", uploads.app) + .route("/transcripts", transcripts.app) + .route("/usage", usage.app) + .route("/transactions", transactions.app) + .onError(developerApiError); + +app.notFound((c) => developerApiNotFound(c, app.routes)); diff --git a/apps/web/app/api/developer/v1/[...route]/route.ts b/apps/web/app/api/developer/v1/[...route]/route.ts index 305273842fa..402e6ac2dd1 100644 --- a/apps/web/app/api/developer/v1/[...route]/route.ts +++ b/apps/web/app/api/developer/v1/[...route]/route.ts @@ -1,30 +1,5 @@ -import { Hono } from "hono"; import { handle } from "hono/vercel"; -import { - developerApiError, - developerApiNotFound, - developerRateLimiter, - developerResponseHeaders, -} from "../../../utils"; - -import * as transactions from "./transactions"; -import * as transcripts from "./transcripts"; -import * as uploads from "./uploads"; -import * as usage from "./usage"; -import * as videos from "./videos"; - -const app = new Hono() - .basePath("/api/developer/v1") - .use(developerResponseHeaders) - .use(developerRateLimiter) - .route("/videos", videos.app) - .route("/uploads", uploads.app) - .route("/transcripts", transcripts.app) - .route("/usage", usage.app) - .route("/transactions", transactions.app) - .onError(developerApiError); - -app.notFound((c) => developerApiNotFound(c, app.routes)); +import { app } from "./app"; export const GET = handle(app); export const POST = handle(app); diff --git a/apps/web/content/docs/api/get-started.mdx b/apps/web/content/docs/api/get-started.mdx index 313a8b7e2a7..1c3e5fcfc26 100644 --- a/apps/web/content/docs/api/get-started.mdx +++ b/apps/web/content/docs/api/get-started.mdx @@ -820,3 +820,46 @@ Uploading and encoding are free. Stored video and transcription are paid for fro - The app's Billing page in the console shows the balance and every charge. - [Get Usage](/docs/api/rest-api#get-usage) returns the balance, whether the app can take an upload, and the current prices. - [List Transactions](/docs/api/rest-api#list-transactions) returns every credit and charge. + +## 10. Use with agents (MCP) + +Cap Infra has a hosted MCP server, so an agent such as Claude Code, Cursor or Codex can upload, check and transcribe video with the app's secret key. + +- URL: `https://cap.so/api/developer/mcp` +- Header: `Authorization: Bearer` followed by the secret key + +Claude Code, with the key in `CAP_SECRET_KEY`: + +```bash +claude mcp add --transport http cap-infra https://cap.so/api/developer/mcp --header "Authorization: Bearer $CAP_SECRET_KEY" +``` + +Cursor, in `.cursor/mcp.json`: + +```json +{ + "mcpServers": { + "cap-infra": { + "url": "https://cap.so/api/developer/mcp", + "headers": { "Authorization": "Bearer csk_your_secret_key" } + } + } +} +``` + +Codex, in `~/.codex/config.toml`: + +```toml +[mcp_servers.cap-infra] +url = "https://cap.so/api/developer/mcp" +bearer_token_env_var = "CAP_SECRET_KEY" +``` + +The tools: + +- `videos_create_upload` and `videos_complete_upload`: create a video and get signed links that the agent uploads the file to from your machine, then start encoding. +- `videos_wait`, `videos_get`, `videos_list` and `videos_delete`: follow a video until it plays, read its links, list and delete. +- `transcripts_create`: transcribe a public URL or a ready video. `transcripts_wait`, `transcripts_text`, `transcripts_get`, `transcripts_list` and `transcripts_delete` follow it and read the text. +- `account_usage` and `account_transactions`: the balance, prices and every charge. + +Each MCP request, and each API call a tool makes, counts toward the key's per-minute limit. Wait tools return within 45 seconds; call them again to keep waiting. diff --git a/apps/web/data/infra-agent-prompt.ts b/apps/web/data/infra-agent-prompt.ts index 90e370b8436..4a23a7f1f27 100644 --- a/apps/web/data/infra-agent-prompt.ts +++ b/apps/web/data/infra-agent-prompt.ts @@ -1,6 +1,10 @@ +import { INFRA_MCP_PATH } from "@/lib/infra-mcp-path"; + export const CAP_INFRA_POINTER_PROMPT = "Read https://cap.so/infra and set up Cap Infra for this project."; +export const CAP_INFRA_MCP_URL = `https://cap.so${INFRA_MCP_PATH}`; + const api = "/api/developer/v1"; const sdk = "/api/developer/sdk/v1/upload/multipart"; @@ -32,5 +36,6 @@ Cap Infra is Cap's video and speech API. All you need is the app's secret key an 7. Transcribe when I ask. Transcripts are billed per audio hour, so confirm with me first. For a public URL call ${calls.transcribeUrl}. For a file call ${calls.transcribeFile}, PUT the file to the signed URL it returns, then call ${calls.startTranscript}. Poll ${calls.transcript} at the same pace until status is complete or error. 8. Stay inside the limit. A secret key allows 60 requests a minute across every endpoint. After a 429, wait for the number of seconds in the Retry-After header before you send the request again. 9. Watch the credits. ${calls.usage} shows the balance and whether uploads are allowed. A 402 means the app needs credit, so tell me and I will add it in the console. +10. Use the MCP server if you want tools. Cap Infra has a hosted MCP server at ${CAP_INFRA_MCP_URL} that takes the same secret key as Authorization: Bearer. Its requests count toward the same limit. Finish by uploading one short test video through the code you wrote. Ask me first, because stored video is billed daily. Then tell me what you changed and give me the video's shareUrl, and ask what I want to build next.`; diff --git a/apps/web/lib/developer-openapi.ts b/apps/web/lib/developer-openapi.ts index 36f197b7302..08a4bdf5859 100644 --- a/apps/web/lib/developer-openapi.ts +++ b/apps/web/lib/developer-openapi.ts @@ -14,6 +14,7 @@ import { transcriptFailureCodesText, transcriptRequestErrorsText, } from "@/lib/developer-transcript-errors"; +import { INFRA_MCP_PATH } from "@/lib/infra-mcp-path"; import { INFRA_RESOLUTION_TIERS } from "@/lib/infra-pricing"; const ref = (name: string) => ({ $ref: `#/components/schemas/${name}` }); @@ -451,9 +452,18 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ `A secret key is limited to ${DEVELOPER_KEY_REQUESTS_PER_MINUTE} requests per minute across the REST operations, and each upload token to ${DEVELOPER_UPLOAD_TOKEN_REQUESTS_PER_MINUTE} requests per minute across the SDK operations. Responses carry X-RateLimit-Limit, X-RateLimit-Remaining and X-RateLimit-Reset, and a 429 also carries Retry-After. Remaining and Reset are what the server that answered has counted, so they are a guide and Remaining can read higher than what is left. A 429 is authoritative: wait for its Retry-After seconds, at most ${DEVELOPER_RATE_LIMIT_WINDOW_SECONDS}, then retry.`, "", "The REST operations take a secret key and send no CORS headers, so a page cannot call them. Call them from your server. The SDK operations and the playback URLs can be called from any origin.", + "", + `Agents can use the same operations as tools through the hosted MCP server at ${baseUrl}${INFRA_MCP_PATH} (Streamable HTTP). Send the secret key as Authorization: Bearer. Its requests count toward the same per-key limit. Setup for Claude Code, Cursor and Codex is at https://cap.so/docs/api/get-started#10-use-with-agents-mcp.`, ].join("\n"), termsOfService: "https://cap.so/terms", contact: { name: "Cap", email: "hello@cap.so", url: "https://cap.so" }, + "x-mcp": { + url: `${baseUrl}${INFRA_MCP_PATH}`, + transport: "streamable-http", + authorization: "Authorization: Bearer ", + documentation: + "https://cap.so/docs/api/get-started#10-use-with-agents-mcp", + }, }, externalDocs: { description: "REST API guide", url: API_DOCS_URL }, servers: [{ url: baseUrl, description: "Cap" }], diff --git a/apps/web/lib/infra-mcp-endpoint.ts b/apps/web/lib/infra-mcp-endpoint.ts new file mode 100644 index 00000000000..41f6b603422 --- /dev/null +++ b/apps/web/lib/infra-mcp-endpoint.ts @@ -0,0 +1,74 @@ +import { Hono } from "hono"; +import { + developerApiError, + developerApiNotFound, + developerRateLimiter, + developerResponseHeaders, + withDeveloperSecretAuth, +} from "@/app/api/utils"; +import { apiErrorBody } from "@/lib/api-errors"; +import { + INFRA_MCP_MAX_BODY_BYTES, + INFRA_MCP_PATH, + infraMcpHandler, +} from "@/lib/infra-mcp-server"; +import { readMcpBody } from "@/lib/mcp-http"; + +const app = new Hono<{ + Variables: { developerAppId: string; developerKeyType: "secret" }; +}>() + .basePath(INFRA_MCP_PATH) + .use(developerResponseHeaders) + .use(async (c, next) => { + await next(); + if (c.res.status === 401) { + c.header("WWW-Authenticate", 'Bearer realm="cap-infra"'); + } + }) + .use(developerRateLimiter) + .use(withDeveloperSecretAuth) + .post("/", async (c) => { + const contentType = c.req.header("content-type")?.split(";")[0]?.trim(); + if (contentType !== "application/json") { + return c.json( + apiErrorBody( + "unsupported_media_type", + "Send MCP messages as application/json", + "Use an MCP client with the Streamable HTTP transport.", + ), + 415, + ); + } + const body = await readMcpBody(c.req.raw, INFRA_MCP_MAX_BODY_BYTES); + if (!body) { + return c.json( + apiErrorBody( + "request_too_large", + `The request body is larger than ${INFRA_MCP_MAX_BODY_BYTES} bytes`, + "Send smaller tool arguments.", + ), + 413, + ); + } + const secretKey = c.req.header("authorization")?.split(" ")[1] ?? ""; + return infraMcpHandler.fetch( + new Request(c.req.url, { + method: "POST", + headers: c.req.raw.headers, + body, + }), + { + authInfo: { + token: secretKey, + clientId: c.get("developerAppId"), + scopes: [], + }, + }, + ); + }) + .onError(developerApiError); + +app.notFound((c) => developerApiNotFound(c, app.routes)); + +export const serveInfraMcp = (request: Request) => + Promise.resolve(app.fetch(request)); diff --git a/apps/web/lib/infra-mcp-path.ts b/apps/web/lib/infra-mcp-path.ts new file mode 100644 index 00000000000..6d0468dccc6 --- /dev/null +++ b/apps/web/lib/infra-mcp-path.ts @@ -0,0 +1 @@ +export const INFRA_MCP_PATH = "/api/developer/mcp"; diff --git a/apps/web/lib/infra-mcp-rest.ts b/apps/web/lib/infra-mcp-rest.ts new file mode 100644 index 00000000000..4c0aede2905 --- /dev/null +++ b/apps/web/lib/infra-mcp-rest.ts @@ -0,0 +1,49 @@ +import { app as developerSdk } from "@/app/api/developer/sdk/v1/[...route]/app"; +import { app as developerApi } from "@/app/api/developer/v1/[...route]/app"; +import { apiNotFoundBody } from "@/lib/api-errors"; +import { firewallRequestHeaders } from "@/lib/rate-limit"; + +const SDK_PREFIX = "/api/developer/sdk/v1/"; +const API_PREFIX = "/api/developer/v1/"; + +const aborted = (signal: AbortSignal) => + signal.reason instanceof Error + ? signal.reason + : new DOMException("The request was aborted", "AbortError"); + +function untilAborted(work: Promise, signal: AbortSignal | null) { + if (!signal) return work; + if (signal.aborted) return Promise.reject(aborted(signal)); + let onAbort = () => {}; + const stopped = new Promise((_, reject) => { + onAbort = () => reject(aborted(signal)); + signal.addEventListener("abort", onAbort, { once: true }); + }); + return Promise.race([work, stopped]).finally(() => + signal.removeEventListener("abort", onAbort), + ); +} + +// Serves the developer REST API from this process instead of over the +// network, so a tool call runs the same handlers, key checks and per-key rate +// limit as a direct REST request. The caller's address headers ride along so +// the shared firewall limit sees the same client. +export function inProcessDeveloperFetch(incoming: Headers): typeof fetch { + const forwarded = firewallRequestHeaders(incoming); + return async (input, init) => { + const headers = new Headers(init?.headers); + for (const [name, value] of forwarded) headers.set(name, value); + const request = new Request(input, { ...init, headers }); + const path = new URL(request.url).pathname; + const target = path.startsWith(SDK_PREFIX) + ? developerSdk + : path.startsWith(API_PREFIX) + ? developerApi + : null; + if (!target) return Response.json(apiNotFoundBody(path), { status: 404 }); + return untilAborted( + Promise.resolve(target.fetch(request)), + init?.signal ?? null, + ); + }; +} diff --git a/apps/web/lib/infra-mcp-server.ts b/apps/web/lib/infra-mcp-server.ts new file mode 100644 index 00000000000..5a7e5306c14 --- /dev/null +++ b/apps/web/lib/infra-mcp-server.ts @@ -0,0 +1,61 @@ +import { serverEnv } from "@cap/env"; +import { + createHostedTools, + HOSTED_INSTRUCTIONS, + SERVER_NAME, + SERVER_VERSION, +} from "@cap/infra-mcp/hosted"; +import { createMcpHandler, McpServer } from "@modelcontextprotocol/server"; +import { inProcessDeveloperFetch } from "@/lib/infra-mcp-rest"; + +export { INFRA_MCP_PATH } from "@/lib/infra-mcp-path"; +export const INFRA_MCP_MAX_BODY_BYTES = 65_536; + +export function createInfraMcpServer(options: { + secretKey: string; + fetch: typeof fetch; + budgetMs?: number; +}) { + const server = new McpServer( + { name: SERVER_NAME, version: SERVER_VERSION }, + { capabilities: { logging: {} }, instructions: HOSTED_INSTRUCTIONS }, + ); + const tools = createHostedTools({ + apiUrl: serverEnv().WEB_URL, + secretKey: options.secretKey, + fetch: options.fetch, + log: (line) => console.info(line), + budgetMs: options.budgetMs, + }); + for (const tool of tools) { + server.registerTool( + tool.name, + { + title: tool.title, + description: tool.description, + inputSchema: tool.inputSchema, + annotations: tool.annotations, + }, + (args, ctx) => + tool.run(args, { + signal: ctx.mcpReq.signal, + progressToken: ctx.mcpReq._meta?.progressToken, + sendProgress: (params) => + ctx.mcpReq.notify({ method: "notifications/progress", params }), + sendLog: (data) => ctx.mcpReq.log("info", data, SERVER_NAME), + }), + ); + } + return server; +} + +export const infraMcpHandler = createMcpHandler( + ({ authInfo, requestInfo }) => { + if (!authInfo?.token) throw new Error("Missing Cap Infra secret key"); + return createInfraMcpServer({ + secretKey: authInfo.token, + fetch: inProcessDeveloperFetch(requestInfo?.headers ?? new Headers()), + }); + }, + { maxSubscriptions: 0, maxRequestBodySize: INFRA_MCP_MAX_BODY_BYTES }, +); diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 3acae75b012..a981d61155d 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -46,6 +46,7 @@ const nextConfig = { "@cap/web-api-contract", "@cap/web-domain", "@cap/env", + "@cap/infra-mcp", "@cap/database", "@cap/recorder-core", "next-mdx-remote", diff --git a/apps/web/package.json b/apps/web/package.json index 4cf91c10076..a350f50db82 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -30,6 +30,7 @@ "@aws-sdk/s3-request-presigner": "^3.485.0", "@cap/database": "workspace:*", "@cap/env": "workspace:*", + "@cap/infra-mcp": "workspace:*", "@cap/recorder-core": "workspace:*", "@cap/ui": "workspace:*", "@cap/utils": "workspace:*", diff --git a/bun.lock b/bun.lock index 6f66684ac8c..45a0f913148 100644 --- a/bun.lock +++ b/bun.lock @@ -129,6 +129,7 @@ "@aws-sdk/s3-request-presigner": "^3.485.0", "@cap/database": "workspace:*", "@cap/env": "workspace:*", + "@cap/infra-mcp": "workspace:*", "@cap/recorder-core": "workspace:*", "@cap/ui": "workspace:*", "@cap/utils": "workspace:*", diff --git a/packages/infra-mcp/README.md b/packages/infra-mcp/README.md index 997f6d1dba3..09bf30499d2 100644 --- a/packages/infra-mcp/README.md +++ b/packages/infra-mcp/README.md @@ -70,7 +70,7 @@ There is no tool that adds credit. Buying credit needs a signed-in browser, so w - Results are one compact JSON object. Lists carry only the fields an agent needs, plus `nextOffset` when there is another page. - A failed call has `isError` set and an `error` object with the API's `code`, `message` and `hint`. -- The API allows 60 requests a minute for each secret key. Waits ask no more often than every 3 seconds and slow down from there, and every request waits for `Retry-After` when it is told to. +- The API limits how many requests each secret key can send a minute. Waits ask no more often than every 3 seconds and slow down from there, and every request waits for `Retry-After` when it is told to. - A wait runs for up to 15 minutes. Pass `timeoutSeconds` to make it shorter. When the time runs out, the result has `timedOut` set with the id and the last known status, so the same call picks up where it stopped. - Some clients end a tool call after a minute. Either raise that limit, as in the Codex example above, or pass a `timeoutSeconds` below it and call again. - A video is `ready` when its first playback link exists, and the other link usually follows within seconds. `videos_wait` returns once both exist, or the one named in `output`. It allows the other link 30 seconds after `ready`, never past the deadline, and then returns the links that exist with `missing` naming the rest. That result is final and not a timeout: the video plays, and the missing link may not be coming. `transcripts_create` with a `videoId` that has no MP4 after 30 seconds answers `video_has_no_mp4` and says to use `path` or `url`. diff --git a/packages/infra-mcp/__tests__/harness.ts b/packages/infra-mcp/__tests__/harness.ts index 34170202020..99f6cbd9fe8 100644 --- a/packages/infra-mcp/__tests__/harness.ts +++ b/packages/infra-mcp/__tests__/harness.ts @@ -98,12 +98,17 @@ export function expectNoSecrets(stub: StubApi, output: readonly string[]) { expect(everything).not.toMatch(/cut_[0-9a-f]{12}/); } +export const MP4_HEADER = new Uint8Array([ + 0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d, +]); + export async function tempFiles() { const directory = await mkdtemp(join(tmpdir(), "infra-mcp-")); return { - async write(name: string, bytes: number) { + async write(name: string, bytes: number, header: Uint8Array = MP4_HEADER) { const path = join(directory, name); const content = randomBytes(bytes); + content.set(header.subarray(0, bytes)); await writeFile(path, content); return { path, content }; }, diff --git a/packages/infra-mcp/__tests__/limits.test.ts b/packages/infra-mcp/__tests__/limits.test.ts index 29fc0067132..b64138e62a3 100644 --- a/packages/infra-mcp/__tests__/limits.test.ts +++ b/packages/infra-mcp/__tests__/limits.test.ts @@ -86,8 +86,8 @@ describe("rate limits", () => { status: 429, retryAfterSeconds: 31, }); - expect((refused.body.error as { hint: string }).hint).toMatch( - /60 requests per minute/, + expect((refused.body.error as { hint: string }).hint).toBe( + "Wait for the Retry-After seconds, then retry.", ); const again = await session.call("videos_list"); diff --git a/packages/infra-mcp/__tests__/media-sniff.test.ts b/packages/infra-mcp/__tests__/media-sniff.test.ts new file mode 100644 index 00000000000..75f9d56b490 --- /dev/null +++ b/packages/infra-mcp/__tests__/media-sniff.test.ts @@ -0,0 +1,105 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { inspectFile } from "../src/files"; +import { connect, expectNoSecrets, type Session, tempFiles } from "./harness"; +import { type StubApi, startStubApi } from "./stub-api"; + +const ascii = (text: string) => new TextEncoder().encode(text); +const bytes = (...values: number[]) => new Uint8Array(values); +const join = (...parts: Uint8Array[]) => { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +}; +const tsPackets = () => { + const out = new Uint8Array(400); + out[0] = 0x47; + out[188] = 0x47; + return out; +}; + +const LIMIT = { maxBytes: 1e9, what: "a video" }; + +let files: Awaited>; + +beforeEach(async () => { + files = await tempFiles(); +}); + +afterEach(async () => { + await files.remove(); +}); + +describe("local media check", () => { + it.each([ + ["MP4", "a.mp4", join(bytes(0, 0, 0, 0x18), ascii("ftypisom"))], + ["MOV", "a.mov", join(bytes(0, 0, 0, 0x14), ascii("ftypqt "))], + ["WebM or MKV", "a.webm", bytes(0x1a, 0x45, 0xdf, 0xa3)], + ["MPEG-TS", "a.ts", tsPackets()], + ["AVI", "a.avi", join(ascii("RIFF"), bytes(0, 0, 0, 0), ascii("AVI "))], + ["WAV", "a.wav", join(ascii("RIFF"), bytes(0, 0, 0, 0), ascii("WAVE"))], + ["MP3", "a.mp3", bytes(0xff, 0xfb, 0x90, 0x64)], + ["Ogg", "a.ogg", ascii("OggS")], + ])("accepts %s", async (_label, name, header) => { + const file = await files.write(name, 1024, header); + await expect(inspectFile(file.path, LIMIT)).resolves.toMatchObject({ + size: 1024, + }); + }); + + it.each([ + ["text", "notes.mp4", ascii("These are my notes, not a video.")], + [ + "a PNG", + "frame.mp4", + bytes(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a), + ], + ["an HLS playlist", "stream.mp4", ascii("#EXTM3U\n#EXT-X-VERSION:3\n")], + ["a PDF", "deck.mov", ascii("%PDF-1.7\n")], + ])("refuses %s whatever its name", async (_label, name, header) => { + const file = await files.write(name, 1024, header); + await expect(inspectFile(file.path, LIMIT)).rejects.toMatchObject({ + code: "not_media", + }); + }); +}); + +describe("videos_upload with a file that is not media", () => { + let stub: StubApi; + let session: Session; + + beforeEach(async () => { + stub = await startStubApi(); + session = await connect(stub); + }); + + afterEach(async () => { + expectNoSecrets(stub, session.output); + await session.close(); + await stub.close(); + }); + + it("refuses it before any API call", async () => { + const file = await files.write( + "report.mp4", + 4096, + ascii("Quarterly report\nRevenue went up.\n"), + ); + const uploaded = await session.call("videos_upload", { path: file.path }); + expect(uploaded.isError).toBe(true); + expect(uploaded.body.error).toMatchObject({ code: "not_media" }); + expect(stub.requests).toHaveLength(0); + }); + + it("uploads a real container as before", async () => { + const file = await files.write("clip.mp4", 4096); + const uploaded = await session.call("videos_upload", { path: file.path }); + expect(uploaded.isError).toBe(false); + expect(stub.videos.get(String(uploaded.body.videoId))?.file).toEqual( + file.content, + ); + }); +}); diff --git a/packages/infra-mcp/__tests__/stub-api.ts b/packages/infra-mcp/__tests__/stub-api.ts index 0ba868e803d..f416a658810 100644 --- a/packages/infra-mcp/__tests__/stub-api.ts +++ b/packages/infra-mcp/__tests__/stub-api.ts @@ -1010,7 +1010,7 @@ export async function startStubApi() { ? apiError( "rate_limited", "Rate limit exceeded", - "Wait for the Retry-After seconds, then retry. Each key allows 60 requests per minute.", + "Wait for the Retry-After seconds, then retry.", ) : apiError( "internal_error", @@ -1033,7 +1033,7 @@ export async function startStubApi() { apiError( "rate_limited", "Rate limit exceeded", - "Wait for the Retry-After seconds, then retry. Each key allows 60 requests per minute.", + "Wait for the Retry-After seconds, then retry.", ), { "Retry-After": String(state.retryAfterSeconds) }, ); diff --git a/packages/infra-mcp/package.json b/packages/infra-mcp/package.json index 63c1a8584de..f71a68f4987 100644 --- a/packages/infra-mcp/package.json +++ b/packages/infra-mcp/package.json @@ -4,6 +4,9 @@ "private": true, "description": "MCP server for Cap Infra, the video and speech API", "type": "module", + "exports": { + "./hosted": "./src/hosted.ts" + }, "bin": { "cap-infra-mcp": "./dist/bin.js" }, diff --git a/packages/infra-mcp/skill/SKILL.md b/packages/infra-mcp/skill/SKILL.md index e24204f80e7..1a8d3998e22 100644 --- a/packages/infra-mcp/skill/SKILL.md +++ b/packages/infra-mcp/skill/SKILL.md @@ -65,6 +65,6 @@ Tell the user before you do something that keeps costing money, such as uploadin ## Limits to respect -- The API allows 60 requests a minute for each key. The wait tools already pace themselves, so call a wait tool and let it run, and do not poll `videos_get` or `transcripts_get` in a loop. +- The API limits how many requests each key can send a minute. The wait tools already pace themselves, so call a wait tool and let it run, and do not poll `videos_get` or `transcripts_get` in a loop. - A video can be up to 36 GB and 4 hours. A transcript source can be up to 2 GB and 4 hours. - Lists return 50 items unless you set `limit` (up to 100). When `nextOffset` is not null, pass it as `offset` for the next page. diff --git a/packages/infra-mcp/src/files.ts b/packages/infra-mcp/src/files.ts index aa252619ce9..f6ddf2a0316 100644 --- a/packages/infra-mcp/src/files.ts +++ b/packages/infra-mcp/src/files.ts @@ -1,7 +1,8 @@ -import { stat } from "node:fs/promises"; +import { open, stat } from "node:fs/promises"; import { homedir } from "node:os"; import { basename, extname, isAbsolute, join } from "node:path"; import { LocalError } from "./errors"; +import { isSelfContainedMedia, MEDIA_SNIFF_BYTES } from "./media-sniff"; export type LocalFile = { path: string; @@ -11,7 +12,7 @@ export type LocalFile = { contentType: string; }; -const CONTENT_TYPES: Record = { +export const CONTENT_TYPES: Record = { ".mp4": "video/mp4", ".m4v": "video/mp4", ".webm": "video/webm", @@ -22,6 +23,17 @@ const CONTENT_TYPES: Record = { const GIGABYTE = 1e9; +async function firstBytes(path: string) { + const handle = await open(path, "r"); + try { + const buffer = Buffer.alloc(MEDIA_SNIFF_BYTES); + const { bytesRead } = await handle.read(buffer, 0, MEDIA_SNIFF_BYTES, 0); + return buffer.subarray(0, bytesRead); + } finally { + await handle.close().catch(() => {}); + } +} + export async function inspectFile( rawPath: string, limit: { maxBytes: number; what: string }, @@ -58,6 +70,21 @@ export async function inspectFile( "Export it at a lower bitrate or split it, then try again.", ); } + const head = await firstBytes(path).catch(() => null); + if (!head) { + throw new LocalError( + "file_not_found", + "No file exists at that path, or it cannot be read", + "Check the path and that this server runs on the machine that holds the file.", + ); + } + if (!isSelfContainedMedia(head)) { + throw new LocalError( + "not_media", + "The file is not a video or audio file", + "Send a video or audio file such as MP4, MOV, WebM, MKV, AVI, MPEG-TS, MP3, M4A, WAV, FLAC or Ogg. A playlist, image or document cannot be uploaded.", + ); + } const fileName = basename(path); const extension = extname(fileName); const stem = fileName.slice(0, fileName.length - extension.length); diff --git a/packages/infra-mcp/src/hosted-upload.ts b/packages/infra-mcp/src/hosted-upload.ts new file mode 100644 index 00000000000..b95f6d8eb2e --- /dev/null +++ b/packages/infra-mcp/src/hosted-upload.ts @@ -0,0 +1,360 @@ +import { extname } from "node:path"; +import { z } from "zod"; +import { ApiError, isUnanswered, LocalError, withContext } from "./errors"; +import { CONTENT_TYPES } from "./files"; +import { asRecord, confirmed, dataOf } from "./http"; +import { Disclosed } from "./redact"; +import { videoLinks } from "./shape"; +import { RESOLUTION_TIERS, type Tool, type ToolDeps, tool } from "./tool"; +import { MAX_PARTS, MAX_VIDEO_BYTES, PART_URL_BATCH } from "./upload"; + +const MEBIBYTE = 1024 * 1024; +export const HOSTED_PART_MIN_BYTES = 64 * MEBIBYTE; +export const HOSTED_MAX_PARTS = 100; +export const PART_LINK_SECONDS = 3600; +const ABORT_BUDGET_MS = 5_000; + +export function planParts(size: number) { + const partBytes = Math.max( + HOSTED_PART_MIN_BYTES, + Math.ceil(size / HOSTED_MAX_PARTS / MEBIBYTE) * MEBIBYTE, + ); + return { partBytes, count: Math.ceil(size / partBytes) }; +} + +function required(value: unknown) { + if (typeof value === "string" && value.length > 0) return value; + throw new LocalError( + "unexpected_response", + "Cap answered in a shape this server does not understand", + null, + ); +} + +const fileTitle = (fileName: string | undefined) => { + if (fileName === undefined) return undefined; + const stem = fileName.slice(0, fileName.length - extname(fileName).length); + return (stem || fileName).slice(0, 255); +}; + +function uploadSteps(partBytes: number, count: number) { + const put = (file: string, part: string) => + `curl -sS --fail -X PUT --upload-file ${file} -D - -o /dev/null "<${part}>"`; + const finish = + "Call videos_complete_upload with videoId, uploadId and parts as a list of { partNumber, etag }, using the ETag header each upload answered with."; + if (count === 1) { + return [ + `Upload the whole file to the url of part 1 in one request, from the machine that holds it: ${put("", "url of part 1")}. Keep the ETag response header.`, + finish, + ]; + } + return [ + `Split the file into ${count} parts of ${partBytes} bytes, in order: split -b ${partBytes} cap-part- (cap-part-aa is part 1, cap-part-ab is part 2, and so on).`, + `Upload each part to its own url: ${put("cap-part-aa", "url of part 1")}, then the same for every other part. Keep the ETag response header of each.`, + finish, + ]; +} + +export function createHostedUploadTools(deps: ToolDeps): Tool[] { + const { client } = deps; + + return [ + tool({ + name: "videos_create_upload", + title: "Create a video upload", + description: `Create a video and get signed links to upload its file directly to storage, for when this server cannot read your files. Send the file size in bytes. Returns the video id, an uploadId and one link per part with the exact upload steps; links work for ${PART_LINK_SECONDS / 60} minutes. Then call videos_complete_upload. Uploading and encoding are free, and stored video is billed daily by its length and resolution tier until it is deleted. Pass videoId to start a failed upload again in the same video.`, + kind: "write", + input: { + contentLength: z + .number() + .int() + .min(1) + .max(MAX_VIDEO_BYTES) + .describe("Size of the video file in bytes"), + fileName: z + .string() + .min(1) + .max(255) + .optional() + .describe( + "Name of the file, used for its type and as the default video name", + ), + contentType: z + .string() + .min(1) + .max(255) + .optional() + .describe("MIME type of the file, such as video/mp4"), + name: z + .string() + .max(255) + .optional() + .describe("Video name (defaults to the file name)"), + userId: z + .string() + .min(1) + .max(255) + .optional() + .describe("Your own user id for this video, used to filter lists"), + maxResolutionTier: z + .enum(RESOLUTION_TIERS) + .optional() + .describe( + "Highest resolution to encode and bill (default 1080p, the cheapest)", + ), + metadata: z + .record(z.string(), z.unknown()) + .optional() + .describe("Your own JSON object to store with it, under 8 KB"), + videoId: z + .string() + .min(1) + .optional() + .describe( + "Start again in this existing video whose upload never completed, instead of creating a new one", + ), + }, + run: async (args, call) => { + const plan = planParts(args.contentLength); + const name = args.name ?? fileTitle(args.fileName); + const contentType = + args.contentType ?? + (args.fileName === undefined + ? undefined + : CONTENT_TYPES[extname(args.fileName).toLowerCase()]) ?? + "application/octet-stream"; + + let pages: Record; + let token: string; + if (args.videoId === undefined) { + pages = dataOf( + await client.call("createUpload", { + body: { + ...(name === undefined ? {} : { name }), + ...(args.userId === undefined ? {} : { userId: args.userId }), + ...(args.maxResolutionTier === undefined + ? {} + : { maxResolutionTier: args.maxResolutionTier }), + ...(args.metadata === undefined + ? {} + : { metadata: args.metadata }), + }, + signal: call.signal, + }), + ); + token = required(pages.token); + call.announce("video_created", { + videoId: required(pages.videoId ?? pages.id), + }); + } else { + pages = dataOf( + await client.call("getVideo", { + id: args.videoId, + signal: call.signal, + }), + ); + token = required( + dataOf( + await client.call("renewUploadToken", { + id: args.videoId, + signal: call.signal, + }), + ).token, + ); + } + const videoId = args.videoId ?? required(pages.videoId ?? pages.id); + + let uploadId: string | null = null; + try { + uploadId = required( + asRecord( + await client.call("initiateUpload", { + body: { + videoId, + contentType, + contentLength: args.contentLength, + }, + token, + signal: call.signal, + }), + ).uploadId, + ); + const parts: { partNumber: number; bytes: number; url: Disclosed }[] = + []; + for (let first = 1; first <= plan.count; first += PART_URL_BATCH) { + const partNumbers = Array.from( + { length: Math.min(PART_URL_BATCH, plan.count - first + 1) }, + (_, offset) => first + offset, + ); + const reply = asRecord( + await client.call("partUrls", { + body: { videoId, uploadId, partNumbers }, + token, + signal: call.signal, + }), + ); + const signed = new Map(); + for (const part of Array.isArray(reply.parts) ? reply.parts : []) { + const { partNumber, presignedUrl } = asRecord(part); + if ( + typeof partNumber === "number" && + typeof presignedUrl === "string" + ) { + signed.set(partNumber, presignedUrl); + } + } + for (const partNumber of partNumbers) { + const offset = (partNumber - 1) * plan.partBytes; + parts.push({ + partNumber, + bytes: Math.min(plan.partBytes, args.contentLength - offset), + url: new Disclosed(required(signed.get(partNumber))), + }); + } + } + return { + videoId, + uploadId, + status: "uploading", + bytes: args.contentLength, + partBytes: plan.partBytes, + parts, + linksExpireInSeconds: PART_LINK_SECONDS, + ...videoLinks(pages), + steps: uploadSteps(plan.partBytes, plan.count), + next: "Upload the parts as the steps say, then call videos_complete_upload. If the links expire first, call videos_create_upload again with this videoId: it continues the same upload, so parts already stored keep their ETags.", + }; + } catch (error) { + // A retry with videoId gets back the open upload and the parts + // already stored in it, so only an upload opened for a new video + // is safe to abort. + if (uploadId !== null && args.videoId === undefined) { + await client + .call("abortUpload", { + body: { videoId, uploadId }, + token, + deadlineAt: Date.now() + ABORT_BUDGET_MS, + }) + .catch(() => {}); + } + throw withContext(error, { + videoId, + videoStatus: "uploading", + next: "The video exists with no file. Call videos_create_upload again with this videoId to retry, or remove it with videos_delete.", + }); + } + }, + }), + tool({ + name: "videos_complete_upload", + title: "Complete a video upload", + description: + "Finish an upload started with videos_create_upload once every part is stored, and start encoding. Send the ETag that storage answered with for each part. Then call videos_wait for the playback links.", + kind: "write", + input: { + videoId: z.string().min(1).describe("The video id"), + uploadId: z + .string() + .min(1) + .max(1024) + .describe("The uploadId from videos_create_upload"), + parts: z + .array( + z.object({ + partNumber: z.number().int().min(1).max(MAX_PARTS), + etag: z + .string() + .min(1) + .max(256) + .describe("The ETag response header of that part's upload"), + }), + ) + .min(1) + .max(MAX_PARTS) + .describe("Every uploaded part"), + durationSeconds: z + .number() + .positive() + .max(4 * 60 * 60) + .optional() + .describe( + "Length of the video if known. Cap measures it after the upload either way", + ), + }, + run: async (args, call) => { + let token: string; + try { + token = required( + dataOf( + await client.call("renewUploadToken", { + id: args.videoId, + signal: call.signal, + }), + ).token, + ); + } catch (error) { + if ( + error instanceof ApiError && + error.code === "upload_already_completed" + ) { + return { + videoId: args.videoId, + status: "processing", + next: "This upload was already completed. Call videos_wait with this videoId to get the playback links.", + }; + } + throw error; + } + try { + confirmed( + await client.call("completeUpload", { + body: { + videoId: args.videoId, + uploadId: args.uploadId, + // Complete measures every part from storage and ignores size. + parts: args.parts.map((part) => ({ ...part, size: 0 })), + durationInSecs: args.durationSeconds ?? 1, + }, + token, + signal: call.signal, + }), + ); + } catch (error) { + const unclear = + isUnanswered(error) || + (error instanceof ApiError && + (error.status >= 500 || error.code === "completion_in_progress")); + const discarded = + error instanceof ApiError && + (error.details.fileRemoved === true || + error.code === "file_too_large"); + throw withContext( + error, + unclear + ? { + videoId: args.videoId, + videoStatus: "unknown", + next: "The upload may have completed. Check it with videos_get before uploading again, and call videos_complete_upload again with the same parts if it is still uploading.", + } + : discarded + ? { + videoId: args.videoId, + videoStatus: "uploading", + next: "Cap removed the uploaded file, so its parts cannot be completed. Fix what the error says, then upload the file again: call videos_create_upload with this videoId for new links.", + } + : { + videoId: args.videoId, + videoStatus: "uploading", + next: "The video still has no file and its stored parts are kept. Fix what the error says and call videos_complete_upload again, or call videos_create_upload with this videoId for new links.", + }, + ); + } + return { + videoId: args.videoId, + status: "processing", + next: "Call videos_wait with this videoId to get the playback links when encoding finishes.", + }; + }, + }), + ]; +} diff --git a/packages/infra-mcp/src/hosted.ts b/packages/infra-mcp/src/hosted.ts new file mode 100644 index 00000000000..36b99459485 --- /dev/null +++ b/packages/infra-mcp/src/hosted.ts @@ -0,0 +1,99 @@ +import { z } from "zod"; +import { CONSOLE_PATH } from "./config"; +import { type ClientTuning, InfraClient } from "./http"; +import { createRedactor } from "./redact"; +import { + ANNOTATIONS, + createToolRunner, + type RunHooks, + type ToolAnnotations, + type ToolResult, +} from "./runner"; +import { createTools } from "./tools"; +import { DEFAULT_UPLOAD_TUNING } from "./upload"; +import { SERVER_NAME, SERVER_VERSION } from "./version"; +import { DEFAULT_WAIT_TUNING, type WaitTuning } from "./wait"; + +export { SERVER_NAME, SERVER_VERSION }; +export type { RunHooks, ToolResult }; + +export const HOSTED_MAX_WAIT_SECONDS = 45; +export const HOSTED_TOOL_BUDGET_MS = 50_000; + +export const HOSTED_CLIENT_TUNING: ClientTuning = { + requestTimeoutMs: 20_000, + callTimeoutMs: 25_000, + rateLimitRetries: 1, + transientRetries: 1, +}; + +export const HOSTED_WAIT_TUNING: WaitTuning = { + ...DEFAULT_WAIT_TUNING, + mp4WaitMs: 20_000, + outputGraceMs: 20_000, +}; + +export const HOSTED_INSTRUCTIONS = [ + "Cap Infra hosts video and turns speech into text, billed from the app's prepaid credit.", + "Video: videos_create_upload with the file size, upload the parts to the signed links it returns from the machine that holds the file, videos_complete_upload, then videos_wait until it is ready and use the hlsUrl, mp4Url, posterUrl, shareUrl or embedUrl it returns.", + "Speech: transcripts_create from a public url or a ready videoId, transcripts_wait, then transcripts_text.", + `Wait tools return within ${HOSTED_MAX_WAIT_SECONDS} seconds; when a result has timedOut, call the same tool again.`, + "Stored video is billed daily until deleted and transcripts are billed per audio hour. Check account_usage before large jobs.", + "Credit can only be added in the console by a signed-in person. When a tool reports out of credit, give them the addCreditAt link.", +].join(" "); + +export type HostedTool = { + name: string; + title: string; + description: string; + inputSchema: z.ZodObject; + annotations: ToolAnnotations & { title: string }; + run: (args: unknown, hooks: RunHooks) => Promise; +}; + +export type HostedOptions = { + apiUrl: string; + secretKey: string; + fetch: typeof fetch; + log: (line: string) => void; + client?: Partial; + wait?: Partial; + budgetMs?: number; +}; + +export function createHostedTools(options: HostedOptions): HostedTool[] { + let apiUrl = options.apiUrl; + while (apiUrl.endsWith("/")) apiUrl = apiUrl.slice(0, -1); + const config = { + secretKey: options.secretKey, + apiUrl, + consoleUrl: `${apiUrl}${CONSOLE_PATH}`, + }; + const redact = createRedactor(config.secretKey); + const run = createToolRunner({ + consoleUrl: config.consoleUrl, + redact, + log: (line) => options.log(redact(`${SERVER_NAME}: ${line}`)), + budgetMs: options.budgetMs ?? HOSTED_TOOL_BUDGET_MS, + }); + const tools = createTools({ + config, + client: new InfraClient( + config, + { ...HOSTED_CLIENT_TUNING, ...options.client }, + options.fetch, + ), + upload: DEFAULT_UPLOAD_TUNING, + wait: { ...HOSTED_WAIT_TUNING, ...options.wait }, + surface: "hosted", + maxWaitSeconds: HOSTED_MAX_WAIT_SECONDS, + }); + return tools.map((tool) => ({ + name: tool.name, + title: tool.title, + description: tool.description, + inputSchema: z.object(tool.input), + annotations: { title: tool.title, ...ANNOTATIONS[tool.kind] }, + run: (args, hooks) => run(tool, args, hooks), + })); +} diff --git a/packages/infra-mcp/src/http.ts b/packages/infra-mcp/src/http.ts index 3130fed0665..d9200e7a8fa 100644 --- a/packages/infra-mcp/src/http.ts +++ b/packages/infra-mcp/src/http.ts @@ -118,6 +118,8 @@ export class InfraClient { constructor( private readonly config: Pick, private readonly tuning: ClientTuning = DEFAULT_CLIENT_TUNING, + private readonly transport: typeof fetch = (input, init) => + fetch(input, init), ) {} async call(name: RouteName, input: CallInput = {}): Promise { @@ -225,7 +227,7 @@ export class InfraClient { 429, "rate_limited", "Rate limit exceeded", - "Wait for retryAfterSeconds, then run the tool again. Each key allows 60 requests per minute.", + "Wait for retryAfterSeconds, then run the tool again.", {}, Math.ceil(wait / 1000), ); @@ -263,7 +265,7 @@ export class InfraClient { const limit = timeLimit(budget, input.signal); try { - const response = await fetch(url, { + const response = await this.transport(url, { method: route.method, headers: { Accept: "application/json", diff --git a/packages/infra-mcp/src/media-sniff.ts b/packages/infra-mcp/src/media-sniff.ts new file mode 100644 index 00000000000..c7c8470c967 --- /dev/null +++ b/packages/infra-mcp/src/media-sniff.ts @@ -0,0 +1,144 @@ +// Mirrors isSelfContainedMedia in apps/web/lib/media-container.ts, which the +// API runs on every upload, so a file this refuses would be refused there too. +const ascii = (bytes: Uint8Array, offset: number, text: string) => { + if (bytes.length < offset + text.length) return false; + for (let index = 0; index < text.length; index++) { + if (bytes[offset + index] !== text.charCodeAt(index)) return false; + } + return true; +}; + +const startsWith = (bytes: Uint8Array, signature: readonly number[]) => + bytes.length >= signature.length && + signature.every((value, index) => bytes[index] === value); + +const ISO_BOX_TYPES = ["ftyp", "moov", "mdat", "wide", "free", "skip"]; +const WAVE64_RIFF_GUID = [ + 0x72, 0x69, 0x66, 0x66, 0x2e, 0x91, 0xcf, 0x11, 0xa5, 0xd6, 0x28, 0xdb, 0x04, + 0xc1, 0x00, 0x00, +]; +const TS_PACKET_BYTES = 188; +const M2TS_PACKET_BYTES = 192; +const M2TS_SYNC_OFFSET = 4; + +export const MEDIA_SNIFF_BYTES = 4096; + +const ID3_HEADER_BYTES = 10; +const ID3_FOOTER_FLAG = 0x10; +const MAX_ID3_TAGS = 4; + +function id3TagBytes(bytes: Uint8Array, offset: number) { + if ( + bytes.length < offset + ID3_HEADER_BYTES || + !ascii(bytes, offset, "ID3") || + bytes[offset + 3] === 0xff || + bytes[offset + 4] === 0xff + ) { + return null; + } + let size = 0; + for (let index = offset + 6; index < offset + ID3_HEADER_BYTES; index++) { + const value = bytes[index] ?? 0; + if (value & 0x80) return null; + size = size * 128 + value; + } + const footer = + (bytes[offset + 5] ?? 0) & ID3_FOOTER_FLAG ? ID3_HEADER_BYTES : 0; + return ID3_HEADER_BYTES + size + footer; +} + +function id3TagsEnd(bytes: Uint8Array) { + let end = 0; + for (let tags = 0; tags < MAX_ID3_TAGS && end < bytes.length; tags++) { + const tagBytes = id3TagBytes(bytes, end); + if (tagBytes === null) break; + end += tagBytes; + } + return end; +} + +export function isSelfContainedMedia(bytes: Uint8Array) { + if (bytes.length < 4) return false; + const tagsEnd = id3TagsEnd(bytes); + if (tagsEnd > 0) { + return ( + tagsEnd >= bytes.length || !looksLikePlaylist(bytes.subarray(tagsEnd)) + ); + } + if (ISO_BOX_TYPES.some((type) => ascii(bytes, 4, type))) return true; + if (startsWith(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return true; + if (ascii(bytes, 0, "RIFF")) { + return ascii(bytes, 8, "WAVE") || ascii(bytes, 8, "AVI "); + } + if (ascii(bytes, 0, "FORM")) { + return ascii(bytes, 8, "AIFF") || ascii(bytes, 8, "AIFC"); + } + if (ascii(bytes, 0, "OggS")) return true; + if (ascii(bytes, 0, "fLaC")) return true; + if (ascii(bytes, 0, "caff")) return true; + if (ascii(bytes, 0, "FLV")) return true; + if (ascii(bytes, 0, "#!AMR")) return true; + if (ascii(bytes, 0, "ID3")) return true; + if (startsWith(bytes, [0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11])) { + return true; + } + if (bytes[0] === 0xff && ((bytes[1] ?? 0) & 0xe0) === 0xe0) return true; + if ( + bytes[0] === 0x47 && + bytes.length > TS_PACKET_BYTES && + bytes[TS_PACKET_BYTES] === 0x47 + ) { + return true; + } + if ( + bytes.length > M2TS_PACKET_BYTES + M2TS_SYNC_OFFSET && + bytes[M2TS_SYNC_OFFSET] === 0x47 && + bytes[M2TS_PACKET_BYTES + M2TS_SYNC_OFFSET] === 0x47 + ) { + return true; + } + if (startsWith(bytes, [0x00, 0x00, 0x01, 0xba])) return true; + if (startsWith(bytes, WAVE64_RIFF_GUID)) return true; + return false; +} + +const PLAYLIST_MARKERS = [ + "#extm3u", + "#ext-x-", + " 0) { + return tagsEnd < bytes.length && looksLikePlaylist(bytes.subarray(tagsEnd)); + } + let start = startsWith(bytes, UTF8_BOM) ? UTF8_BOM.length : 0; + while ( + start < bytes.length && + (bytes[start] === 0x20 || + bytes[start] === 0x09 || + bytes[start] === 0x0a || + bytes[start] === 0x0d) + ) { + start++; + } + let text = ""; + const end = Math.min(bytes.length, start + PLAYLIST_MARKER_BYTES); + for (let index = start; index < end; index++) { + text += String.fromCharCode(bytes[index] ?? 0); + } + const lowered = text.toLowerCase(); + return PLAYLIST_MARKERS.some((marker) => lowered.startsWith(marker)); +} diff --git a/packages/infra-mcp/src/redact.ts b/packages/infra-mcp/src/redact.ts index 4bf3c4e5256..21e63e6ffe2 100644 --- a/packages/infra-mcp/src/redact.ts +++ b/packages/infra-mcp/src/redact.ts @@ -13,19 +13,27 @@ export function createRedactor(secretKey: string) { .replace(URL_CREDENTIAL, "$1[redacted]"); } +export class Disclosed { + constructor(readonly value: string) {} +} + export function redactValue( redact: (text: string) => string, value: unknown, + options: { disclose?: boolean } = {}, ): unknown { + if (value instanceof Disclosed) { + return options.disclose ? value.value : redact(value.value); + } if (typeof value === "string") return redact(value); if (Array.isArray(value)) { - return value.map((entry) => redactValue(redact, entry)); + return value.map((entry) => redactValue(redact, entry, options)); } if (typeof value === "object" && value !== null) { return Object.fromEntries( Object.entries(value).map(([name, entry]) => [ redact(name), - redactValue(redact, entry), + redactValue(redact, entry, options), ]), ); } diff --git a/packages/infra-mcp/src/routes.ts b/packages/infra-mcp/src/routes.ts index 02df0ad1379..634b5b74460 100644 --- a/packages/infra-mcp/src/routes.ts +++ b/packages/infra-mcp/src/routes.ts @@ -63,7 +63,7 @@ export const ROUTES = { body: ["videoId", "uploadId"], }), createTranscript: route("POST", `${REST}/transcripts`, "secret", { - body: ["url", ...TRANSCRIPT_OPTIONS], + body: ["url", "videoId", ...TRANSCRIPT_OPTIONS], headers: IDEMPOTENCY, }), createTranscriptUpload: route( diff --git a/packages/infra-mcp/src/runner.ts b/packages/infra-mcp/src/runner.ts new file mode 100644 index 00000000000..e8bdcacf317 --- /dev/null +++ b/packages/infra-mcp/src/runner.ts @@ -0,0 +1,139 @@ +import { + ApiError, + creditNotice, + isCreditError, + LocalError, + type ToolError, + toToolError, + withContext, +} from "./errors"; +import { redactValue } from "./redact"; +import type { Tool, ToolCall, ToolKind } from "./tool"; + +export type ToolAnnotations = { + readOnlyHint: boolean; + destructiveHint: boolean; + idempotentHint: boolean; +}; + +export const ANNOTATIONS: Record = { + read: { readOnlyHint: true, destructiveHint: false, idempotentHint: true }, + write: { readOnlyHint: false, destructiveHint: false, idempotentHint: false }, + delete: { readOnlyHint: false, destructiveHint: true, idempotentHint: true }, +}; + +export type ToolResult = { + content: { type: "text"; text: string }[]; + isError?: true; +}; + +export type RunHooks = { + signal: AbortSignal; + progressToken: string | number | undefined; + sendProgress: (params: { + progressToken: string | number; + progress: number; + total?: number; + message: string; + }) => Promise; + sendLog: (data: unknown) => Promise; +}; + +function errorBody(error: ToolError, consoleUrl: string) { + const api = error instanceof ApiError ? error : null; + return { + error: { + code: error.code, + message: error.message, + ...(error.hint ? { hint: error.hint } : {}), + ...(api ? { status: api.status, ...api.details } : {}), + ...(api?.retryAfterSeconds + ? { retryAfterSeconds: api.retryAfterSeconds } + : {}), + }, + ...(isCreditError(error) + ? { credit: creditNotice(consoleUrl), addCreditAt: consoleUrl } + : {}), + ...error.context, + }; +} + +export function createToolRunner(options: { + consoleUrl: string; + redact: (text: string) => string; + log: (line: string) => void; + budgetMs?: number; +}) { + const { consoleUrl, redact, log, budgetMs } = options; + + const result = (body: unknown, isError: boolean): ToolResult => ({ + content: [ + { + type: "text", + text: JSON.stringify(redactValue(redact, body, { disclose: !isError })), + }, + ], + ...(isError ? { isError: true as const } : {}), + }); + + return async ( + tool: Tool, + args: unknown, + hooks: RunHooks, + ): Promise => { + const { progressToken } = hooks; + const budget = + budgetMs === undefined ? null : AbortSignal.timeout(budgetMs); + const signal = + budget === null ? hooks.signal : AbortSignal.any([hooks.signal, budget]); + const overBudget = () => budget?.aborted === true && !hooks.signal.aborted; + const announce: ToolCall["announce"] = (event, fields) => { + log(`${event} ${JSON.stringify(redactValue(redact, fields))}`); + hooks.sendLog(redactValue(redact, { event, ...fields })).catch(() => {}); + }; + const call: ToolCall = { + signal, + wantsProgress: progressToken !== undefined, + announce, + progress: (done, total, message) => { + if (progressToken === undefined) return; + hooks + .sendProgress({ + progressToken, + progress: done, + ...(total === undefined ? {} : { total }), + message, + }) + .catch(() => {}); + }, + }; + + try { + return result(await tool.run(args, call), false); + } catch (thrown) { + const error = overBudget() + ? withContext( + new LocalError( + "timeout", + `The call ran longer than ${Math.round((budgetMs ?? 0) / 1000)} seconds and was stopped`, + "Run the tool again. Wait tools return what they have before this limit, so call them again to keep waiting.", + ), + toToolError(thrown).context, + ) + : toToolError(thrown); + const body = errorBody(error, consoleUrl); + if (error.code === "internal_error" && thrown instanceof Error) { + log(`${tool.name} failed unexpectedly: ${thrown.message}`); + } + if (Object.keys(error.context).length > 0) { + announce( + hooks.signal.aborted + ? `${tool.name}_cancelled` + : `${tool.name}_failed`, + { code: error.code, ...error.context }, + ); + } + return result(body, true); + } + }; +} diff --git a/packages/infra-mcp/src/server.ts b/packages/infra-mcp/src/server.ts index d22267d14c6..6892b475d3d 100644 --- a/packages/infra-mcp/src/server.ts +++ b/packages/infra-mcp/src/server.ts @@ -1,22 +1,14 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js"; import type { - CallToolResult, ServerNotification, ServerRequest, - ToolAnnotations, } from "@modelcontextprotocol/sdk/types.js"; import type { Config } from "./config"; -import { - ApiError, - creditNotice, - isCreditError, - type ToolError, - toToolError, -} from "./errors"; import { type ClientTuning, DEFAULT_CLIENT_TUNING, InfraClient } from "./http"; -import { createRedactor, redactValue } from "./redact"; -import { createTools, type Tool, type ToolCall, type ToolKind } from "./tools"; +import { createRedactor } from "./redact"; +import { ANNOTATIONS, createToolRunner } from "./runner"; +import { createTools } from "./tools"; import { DEFAULT_UPLOAD_TUNING, type UploadTuning } from "./upload"; import { SERVER_NAME, SERVER_VERSION } from "./version"; import { DEFAULT_WAIT_TUNING, type WaitTuning } from "./wait"; @@ -30,12 +22,6 @@ export type ServerOptions = { type Extra = RequestHandlerExtra; -const ANNOTATIONS: Record = { - read: { readOnlyHint: true, destructiveHint: false, idempotentHint: true }, - write: { readOnlyHint: false, destructiveHint: false, idempotentHint: false }, - delete: { readOnlyHint: false, destructiveHint: true, idempotentHint: true }, -}; - export const INSTRUCTIONS = [ "Cap Infra hosts video and turns speech into text, billed from the app's prepaid credit.", "Video: videos_upload a local file, videos_wait until it is ready, then use the hlsUrl, mp4Url, posterUrl, shareUrl or embedUrl it returns.", @@ -44,25 +30,6 @@ export const INSTRUCTIONS = [ "Credit can only be added in the console by a signed-in person. When a tool reports out of credit, give them the addCreditAt link.", ].join(" "); -function errorBody(error: ToolError, consoleUrl: string) { - const api = error instanceof ApiError ? error : null; - return { - error: { - code: error.code, - message: error.message, - ...(error.hint ? { hint: error.hint } : {}), - ...(api ? { status: api.status, ...api.details } : {}), - ...(api?.retryAfterSeconds - ? { retryAfterSeconds: api.retryAfterSeconds } - : {}), - }, - ...(isCreditError(error) - ? { credit: creditNotice(consoleUrl), addCreditAt: consoleUrl } - : {}), - ...error.context, - }; -} - export function createInfraServer(config: Config, options: ServerOptions = {}) { const redact = createRedactor(config.secretKey); const write = @@ -84,65 +51,12 @@ export function createInfraServer(config: Config, options: ServerOptions = {}) { wait: { ...DEFAULT_WAIT_TUNING, ...options.wait }, }); - const result = (body: unknown, isError: boolean): CallToolResult => ({ - content: [ - { type: "text", text: JSON.stringify(redactValue(redact, body)) }, - ], - ...(isError ? { isError: true } : {}), + const run = createToolRunner({ + consoleUrl: config.consoleUrl, + redact, + log, }); - const run = async (tool: Tool, args: unknown, extra: Extra) => { - const progressToken = extra._meta?.progressToken; - const announce: ToolCall["announce"] = (event, fields) => { - log(`${event} ${JSON.stringify(redactValue(redact, fields))}`); - server - .sendLoggingMessage({ - level: "info", - logger: SERVER_NAME, - data: redactValue(redact, { event, ...fields }), - }) - .catch(() => {}); - }; - const call: ToolCall = { - signal: extra.signal, - wantsProgress: progressToken !== undefined, - announce, - progress: (done, total, message) => { - if (progressToken === undefined) return; - extra - .sendNotification({ - method: "notifications/progress", - params: { - progressToken, - progress: done, - ...(total === undefined ? {} : { total }), - message, - }, - }) - .catch(() => {}); - }, - }; - - try { - return result(await tool.run(args, call), false); - } catch (thrown) { - const error = toToolError(thrown); - const body = errorBody(error, config.consoleUrl); - if (error.code === "internal_error" && thrown instanceof Error) { - log(`${tool.name} failed unexpectedly: ${thrown.message}`); - } - if (Object.keys(error.context).length > 0) { - announce( - extra.signal.aborted - ? `${tool.name}_cancelled` - : `${tool.name}_failed`, - { code: error.code, ...error.context }, - ); - } - return result(body, true); - } - }; - for (const tool of tools) { server.registerTool( tool.name, @@ -152,7 +66,22 @@ export function createInfraServer(config: Config, options: ServerOptions = {}) { inputSchema: tool.input, annotations: { title: tool.title, ...ANNOTATIONS[tool.kind] }, }, - (args: unknown, extra: Extra) => run(tool, args, extra), + (args: unknown, extra: Extra) => + run(tool, args, { + signal: extra.signal, + progressToken: extra._meta?.progressToken, + sendProgress: (params) => + extra.sendNotification({ + method: "notifications/progress", + params, + }), + sendLog: (data) => + server.sendLoggingMessage({ + level: "info", + logger: SERVER_NAME, + data, + }), + }), ); } diff --git a/packages/infra-mcp/src/tool.ts b/packages/infra-mcp/src/tool.ts new file mode 100644 index 00000000000..254e1ead5e6 --- /dev/null +++ b/packages/infra-mcp/src/tool.ts @@ -0,0 +1,51 @@ +import { z } from "zod"; +import type { Config } from "./config"; +import type { InfraClient } from "./http"; +import type { UploadTuning } from "./upload"; +import type { WaitTuning } from "./wait"; + +export const RESOLUTION_TIERS = ["1080p", "1440p", "2160p"] as const; + +export type ToolKind = "read" | "write" | "delete"; + +export type ToolCall = { + signal: AbortSignal; + wantsProgress: boolean; + progress: (done: number, total: number | undefined, message: string) => void; + announce: (event: string, fields: Record) => void; +}; + +export type ToolSurface = "local" | "hosted"; + +export type ToolDeps = { + config: Config; + client: InfraClient; + upload: UploadTuning; + wait: WaitTuning; + surface?: ToolSurface; + maxWaitSeconds?: number; +}; + +export type Tool = { + name: string; + title: string; + description: string; + kind: ToolKind; + input: z.ZodRawShape; + run: (args: unknown, call: ToolCall) => Promise; +}; + +export function tool(definition: { + name: string; + title: string; + description: string; + kind: ToolKind; + input: Shape; + run: (args: z.infer>, call: ToolCall) => Promise; +}): Tool { + const schema = z.object(definition.input); + return { + ...definition, + run: (args, call) => definition.run(schema.parse(args), call), + }; +} diff --git a/packages/infra-mcp/src/tools.ts b/packages/infra-mcp/src/tools.ts index 8958fc29a23..1191f91f582 100644 --- a/packages/infra-mcp/src/tools.ts +++ b/packages/infra-mcp/src/tools.ts @@ -1,17 +1,10 @@ import { createReadStream } from "node:fs"; import { Readable } from "node:stream"; import { z } from "zod"; -import type { Config } from "./config"; import { ApiError, isUnanswered, LocalError, withContext } from "./errors"; import { inspectFile, storageTimeoutMs } from "./files"; -import { - asRecord, - confirmed, - dataOf, - type InfraClient, - listOf, - storagePut, -} from "./http"; +import { createHostedUploadTools } from "./hosted-upload"; +import { asRecord, confirmed, dataOf, listOf, storagePut } from "./http"; import { page, transactionRow, @@ -20,51 +13,26 @@ import { videoLinks, videoRow, } from "./shape"; -import { type UploadTuning, uploadVideo } from "./upload"; -import { type WaitTuning, waitFor } from "./wait"; - -export type ToolKind = "read" | "write" | "delete"; - -export type ToolCall = { - signal: AbortSignal; - wantsProgress: boolean; - progress: (done: number, total: number | undefined, message: string) => void; - announce: (event: string, fields: Record) => void; -}; - -export type ToolDeps = { - config: Config; - client: InfraClient; - upload: UploadTuning; - wait: WaitTuning; -}; - -export type Tool = { - name: string; - title: string; - description: string; - kind: ToolKind; - input: z.ZodRawShape; - run: (args: unknown, call: ToolCall) => Promise; -}; +import { + RESOLUTION_TIERS, + type Tool, + type ToolCall, + type ToolDeps, + tool, +} from "./tool"; +import { uploadVideo } from "./upload"; +import { waitFor } from "./wait"; -function tool(definition: { - name: string; - title: string; - description: string; - kind: ToolKind; - input: Shape; - run: (args: z.infer>, call: ToolCall) => Promise; -}): Tool { - const schema = z.object(definition.input); - return { - ...definition, - run: (args, call) => definition.run(schema.parse(args), call), - }; -} +export type { + Tool, + ToolCall, + ToolDeps, + ToolKind, + ToolSurface, +} from "./tool"; +export { RESOLUTION_TIERS } from "./tool"; export const TRANSCRIPT_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024; -export const RESOLUTION_TIERS = ["1080p", "1440p", "2160p"] as const; export const TRANSCRIPT_STATUSES = [ "uploading", "queued", @@ -73,7 +41,7 @@ export const TRANSCRIPT_STATUSES = [ "error", ] as const; export const VIDEO_OUTPUTS = ["all", "hls", "mp4", "any"] as const; -const MAX_WAIT_SECONDS = 900; +export const MAX_WAIT_SECONDS = 900; const START_ATTEMPTS = 2; const MEGABYTE = 1_000_000; @@ -88,11 +56,23 @@ const OUTPUT_LINKS: Record = { const megabytes = (bytes: number) => (bytes / MEGABYTE).toFixed(1); -const MISSING_LINK_ADVICE: Record<"hlsUrl" | "mp4Url", string> = { - mp4Url: - "The MP4 link has not arrived since the video became ready and may not be coming. The video plays with hlsUrl, shareUrl and embedUrl. An MP4 is needed only for download and for transcripts_create with videoId. If you need one of those, read the video later with videos_get, or transcribe the original file with transcripts_create and path.", - hlsUrl: - "The HLS link has not arrived since the video became ready and may not be coming. The video plays with mp4Url, shareUrl and embedUrl. If you need adaptive streaming, read the video later with videos_get.", +const HLS_ADVICE = + "The HLS link has not arrived since the video became ready and may not be coming. The video plays with mp4Url, shareUrl and embedUrl. If you need adaptive streaming, read the video later with videos_get."; + +const MISSING_LINK_ADVICE: Record< + "local" | "hosted", + Record<"hlsUrl" | "mp4Url", string> +> = { + local: { + mp4Url: + "The MP4 link has not arrived since the video became ready and may not be coming. The video plays with hlsUrl, shareUrl and embedUrl. An MP4 is needed only for download and for transcripts_create with videoId. If you need one of those, read the video later with videos_get, or transcribe the original file with transcripts_create and path.", + hlsUrl: HLS_ADVICE, + }, + hosted: { + mp4Url: + "The MP4 link has not arrived since the video became ready and may not be coming. The video plays with hlsUrl, shareUrl and embedUrl. An MP4 is needed only for download and for transcripts_create with videoId. If you need one of those, read the video later with videos_get, or transcribe a public link to the original file with transcripts_create and url.", + hlsUrl: HLS_ADVICE, + }, }; const videoCanPlay = (video: Record) => @@ -104,6 +84,11 @@ const videoIsFinal = (video: Record) => (videoCanPlay(video) && video.playbackId === null); const videoId = z.string().min(1).describe("The video id"); +const localPath = z + .string() + .min(1) + .optional() + .describe("Absolute path of an audio or video file on this machine"); const transcriptId = z.string().min(1).describe("The transcript id"); const userScope = z .string() @@ -124,15 +109,14 @@ const offset = z .min(0) .optional() .describe("How many to skip (default 0)"); -const timeoutSeconds = z - .number() - .int() - .min(1) - .max(MAX_WAIT_SECONDS) - .optional() - .describe( - `Longest time to wait, in seconds (default and maximum ${MAX_WAIT_SECONDS})`, - ); +const waitSeconds = (max: number) => + z + .number() + .int() + .min(1) + .max(max) + .optional() + .describe(`Longest time to wait, in seconds (default and maximum ${max})`); const metadata = z .record(z.string(), z.unknown()) .optional() @@ -141,8 +125,19 @@ const metadata = z const waitAgain = (toolName: string) => `Still in progress when the wait ended. Call ${toolName} again to keep waiting.`; +const UPLOADING_NEXT = { + local: + "This video has no file yet. Upload one with videos_upload and this videoId, or remove it with videos_delete.", + hosted: + "This video has no file yet. Upload one with videos_create_upload and this videoId, or remove it with videos_delete.", +} as const; + export function createTools(deps: ToolDeps): Tool[] { const { client, config } = deps; + const surface = deps.surface ?? "local"; + const local = surface === "local"; + const maxWaitSeconds = deps.maxWaitSeconds ?? MAX_WAIT_SECONDS; + const timeoutSeconds = waitSeconds(maxWaitSeconds); const transcript = (fields: Record, text = false) => transcriptView(fields, config.consoleUrl, { text }); @@ -303,7 +298,7 @@ export function createTools(deps: ToolDeps): Tool[] { }; }; - return [ + const tools = [ tool({ name: "videos_upload", title: "Upload a video", @@ -437,7 +432,7 @@ export function createTools(deps: ToolDeps): Tool[] { return Date.now() - playableAt >= deps.wait.outputGraceMs; }, tuning: deps.wait, - timeoutMs: (args.timeoutSeconds ?? MAX_WAIT_SECONDS) * 1000, + timeoutMs: (args.timeoutSeconds ?? maxWaitSeconds) * 1000, signal: call.signal, onPoll: (video, polls) => call.progress( @@ -461,11 +456,11 @@ export function createTools(deps: ToolDeps): Tool[] { : { missing: absent, waitedSeconds: outcome.waitedSeconds, - next: MISSING_LINK_ADVICE[firstAbsent], + next: MISSING_LINK_ADVICE[surface][firstAbsent], }), ...(last.status === "uploading" ? { - next: "This video has no file yet. Upload one with videos_upload and this videoId, or remove it with videos_delete.", + next: UPLOADING_NEXT[surface], } : {}), }; @@ -519,8 +514,9 @@ export function createTools(deps: ToolDeps): Tool[] { tool({ name: "transcripts_create", title: "Create a transcript", - description: - "Turn speech into text from exactly one source: url (a public audio or video link), path (a local file up to 2 GB) or videoId (a stored video that is ready). Returns the transcript id; call transcripts_wait for the result. Transcription is billed per audio hour when it completes, and a transcript that fails is not charged.", + description: local + ? "Turn speech into text from exactly one source: url (a public audio or video link), path (a local file up to 2 GB) or videoId (a stored video that is ready). Returns the transcript id; call transcripts_wait for the result. Transcription is billed per audio hour when it completes, and a transcript that fails is not charged." + : "Turn speech into text from exactly one source: url (a public audio or video link) or videoId (a stored video that is ready, which also gets captions when it completes). Returns the transcript id; call transcripts_wait for the result. Transcription is billed per audio hour when it completes, and a transcript that fails is not charged.", kind: "write", input: { url: z @@ -529,11 +525,7 @@ export function createTools(deps: ToolDeps): Tool[] { .max(2048) .optional() .describe("Public http or https link to an audio or video file"), - path: z - .string() - .min(1) - .optional() - .describe("Absolute path of an audio or video file on this machine"), + ...(local ? { path: localPath } : {}), videoId: z .string() .min(1) @@ -567,13 +559,16 @@ export function createTools(deps: ToolDeps): Tool[] { ), }, run: async (args, call) => { - const sources = [args.url, args.path, args.videoId].filter( + const path = typeof args.path === "string" ? args.path : undefined; + const sources = [args.url, path, args.videoId].filter( (source) => source !== undefined, ); if (sources.length !== 1) { throw new LocalError( "invalid_source", - "Send exactly one of url, path or videoId", + local + ? "Send exactly one of url, path or videoId" + : "Send exactly one of url or videoId", null, ); } @@ -589,10 +584,10 @@ export function createTools(deps: ToolDeps): Tool[] { const next = "Call transcripts_wait with this id, then transcripts_text for the text."; - if (args.path !== undefined) { + if (path !== undefined) { return { ...(await transcriptFromFile( - args.path, + path, { ...options, ...(args.metadata === undefined @@ -606,6 +601,23 @@ export function createTools(deps: ToolDeps): Tool[] { }; } + if (!local && args.videoId !== undefined) { + const created = dataOf( + await client.call("createTranscript", { + body: { + videoId: args.videoId, + ...options, + ...(args.metadata === undefined + ? {} + : { metadata: args.metadata }), + }, + headers, + signal: call.signal, + }), + ); + return { ...transcript(created), next }; + } + const fromVideo = args.videoId === undefined ? null @@ -693,7 +705,7 @@ export function createTools(deps: ToolDeps): Tool[] { found.status === "error" || found.status === "uploading", tuning: deps.wait, - timeoutMs: (args.timeoutSeconds ?? MAX_WAIT_SECONDS) * 1000, + timeoutMs: (args.timeoutSeconds ?? maxWaitSeconds) * 1000, signal: call.signal, onPoll: (found, polls) => call.progress( @@ -837,4 +849,10 @@ export function createTools(deps: ToolDeps): Tool[] { }, }), ]; + + if (local) return tools; + return [ + ...createHostedUploadTools(deps), + ...tools.filter((entry) => entry.name !== "videos_upload"), + ]; }