diff --git a/apps/web/__tests__/integration/processing-retry-database.test.ts b/apps/web/__tests__/integration/processing-retry-database.test.ts new file mode 100644 index 00000000000..bca8899e786 --- /dev/null +++ b/apps/web/__tests__/integration/processing-retry-database.test.ts @@ -0,0 +1,386 @@ +import { createHash, randomUUID } from "node:crypto"; +import * as Db from "@cap/database/schema"; +import { eq } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +// Drives the real agent processCap handler and the real share page retry +// routes against a real migrated MySQL database. Only the session lookup, the +// transcription provider call and the AI workflow start are stubbed, so the +// idempotency store, the rerun charge, the rate limiter and startAiGeneration's +// own state checks all run for real. + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + sessionUserId: "", + transcribe: vi.fn(), + startWorkflow: vi.fn(), +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", async (importOriginal) => ({ + ...(await importOriginal()), + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => { + const [user] = await (fixture.database as MySql2Database) + .select() + .from(Db.users) + .where(eq(Db.users.id, fixture.sessionUserId as never)); + return user ?? null; + }, +})); +vi.mock("@cap/database/auth/request-site", () => ({ + isCrossSiteCookieWrite: () => false, +})); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/lib/transcribe", () => ({ transcribeVideo: fixture.transcribe })); +vi.mock("@/lib/ai/provider", () => ({ isAiConfigured: () => true })); +vi.mock("workflow/api", () => ({ start: fixture.startWorkflow })); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); +const database = () => fixture.database as MySql2Database; + +type Owner = { userId: string; orgId: string; token: string }; + +const createOwner = async (): Promise => { + const userId = id(); + const orgId = id(); + const token = `cap_cli_${randomUUID().replaceAll("-", "")}${"a".repeat(11)}`; + await database() + .insert(Db.users) + .values({ + id: userId as never, + email: `${userId}@example.com`, + name: "Owner", + activeOrganizationId: orgId as never, + defaultOrgId: orgId as never, + stripeSubscriptionStatus: "active", + }); + await database() + .insert(Db.organizations) + .values({ id: orgId as never, name: "Team", ownerId: userId as never }); + await database() + .insert(Db.organizationMembers) + .values({ + id: id() as never, + userId: userId as never, + organizationId: orgId as never, + role: "owner", + }); + await database() + .insert(Db.agentApiKeys) + .values({ + id: id() as never, + userId: userId as never, + name: "Test agent", + tokenHash: createHash("sha256").update(token).digest("hex"), + scopes: ["caps:read", "caps:process"], + expiresAt: new Date(Date.now() + 60 * 60 * 1000), + } as never); + return { userId, orgId, token }; +}; + +const createVideo = async ( + owner: Owner, + state: { + transcriptionStatus: "COMPLETE" | "ERROR" | null; + aiGenerationStatus?: string; + }, +) => { + const videoId = id(); + await database() + .insert(Db.videos) + .values({ + id: videoId as never, + ownerId: owner.userId as never, + orgId: owner.orgId as never, + name: "Recording", + transcriptionStatus: state.transcriptionStatus, + metadata: state.aiGenerationStatus + ? { aiGenerationStatus: state.aiGenerationStatus } + : {}, + } as never); + return videoId; +}; + +const videoState = async (videoId: string) => { + const [video] = await database() + .select({ + transcriptionStatus: Db.videos.transcriptionStatus, + metadata: Db.videos.metadata, + }) + .from(Db.videos) + .where(eq(Db.videos.id, videoId as never)); + return { + transcriptionStatus: video?.transcriptionStatus ?? null, + aiGenerationStatus: + (video?.metadata as { aiGenerationStatus?: string } | null) + ?.aiGenerationStatus ?? null, + }; +}; + +describe.runIf(Boolean(databaseUrl))( + "processing retries against a real database", + () => { + let pool: Pool; + let v1: typeof import("@/app/api/v1/[...route]/route").POST; + let retryAi: typeof import("@/app/api/videos/[videoId]/retry-ai/route").POST; + let retryTranscription: typeof import("@/app/api/videos/[videoId]/retry-transcription/route").POST; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_infra_[a-z0-9_]+$/.test(url.pathname) + ) + throw new Error("These tests require a session-scoped local database."); + pool = createPool({ uri: databaseUrl, connectionLimit: 8 }); + fixture.database = drizzle(pool); + Object.assign(process.env, { + NEXT_PUBLIC_WEB_URL: "http://localhost", + WEB_URL: "http://localhost", + DATABASE_URL: databaseUrl, + NEXTAUTH_SECRET: "synthetic-test-secret-that-is-long-enough", + NEXTAUTH_URL: "http://localhost", + CAP_AWS_BUCKET: "synthetic", + CAP_AWS_REGION: "us-east-1", + CAP_AWS_ACCESS_KEY: "synthetic", + CAP_AWS_SECRET_KEY: "synthetic", + NODE_ENV: "test", + }); + ({ POST: v1 } = await import("@/app/api/v1/[...route]/route")); + ({ POST: retryAi } = await import( + "@/app/api/videos/[videoId]/retry-ai/route" + )); + ({ POST: retryTranscription } = await import( + "@/app/api/videos/[videoId]/retry-transcription/route" + )); + }); + + beforeEach(() => { + fixture.transcribe.mockResolvedValue({ success: true, message: "ok" }); + fixture.startWorkflow.mockResolvedValue(undefined); + }); + + afterAll(async () => { + await pool?.end(); + }); + + const processCap = ( + owner: Owner, + videoId: string, + payload: { target: "all" | "transcript" | "ai"; retry?: boolean }, + key = randomUUID(), + ) => + v1( + new Request(`http://localhost/api/v1/caps/${videoId}/process`, { + method: "POST", + headers: { + Authorization: `Bearer ${owner.token}`, + "Content-Type": "application/json", + "Idempotency-Key": key, + "X-Cap-Confirmation": "user", + }, + body: JSON.stringify(payload), + }), + ); + + const sharePageRetry = ( + route: (request: Request, props: never) => Promise, + owner: Owner, + videoId: string, + ) => { + fixture.sessionUserId = owner.userId; + return route( + new Request(`http://localhost/api/videos/${videoId}/retry`, { + method: "POST", + }), + { params: Promise.resolve({ videoId }) } as never, + ); + }; + + it("processes a fresh video through the agent API without charging a retry", async () => { + const owner = await createOwner(); + for (let index = 0; index < 35; index++) { + const videoId = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + }); + fixture.startWorkflow.mockClear(); + const response = await processCap(owner, videoId, { target: "all" }); + expect(response.status, await response.clone().text()).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + expect((await videoState(videoId)).aiGenerationStatus).toBe("QUEUED"); + } + + const pending = await createVideo(owner, { transcriptionStatus: null }); + fixture.transcribe.mockClear(); + const transcript = await processCap(owner, pending, { + target: "transcript", + }); + expect(transcript.status).toBe(200); + expect(fixture.transcribe).toHaveBeenCalledOnce(); + }); + + it("retries a failed transcription and a failed AI run once through the agent API", async () => { + const owner = await createOwner(); + const failedTranscript = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + fixture.transcribe.mockClear(); + const transcript = await processCap(owner, failedTranscript, { + target: "transcript", + retry: true, + }); + expect(transcript.status).toBe(200); + expect(fixture.transcribe).toHaveBeenCalledOnce(); + expect((await videoState(failedTranscript)).transcriptionStatus).toBe( + null, + ); + + const failedAi = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + aiGenerationStatus: "ERROR", + }); + fixture.startWorkflow.mockClear(); + const ai = await processCap(owner, failedAi, { + target: "ai", + retry: true, + }); + expect(ai.status).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + expect((await videoState(failedAi)).aiGenerationStatus).toBe("QUEUED"); + }); + + it("retries a failed transcription and a failed AI run once from the share page", async () => { + const owner = await createOwner(); + const failedTranscript = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + const transcript = await sharePageRetry( + retryTranscription, + owner, + failedTranscript, + ); + expect(transcript.status).toBe(200); + expect((await videoState(failedTranscript)).transcriptionStatus).toBe( + null, + ); + + const failedAi = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + aiGenerationStatus: "ERROR", + }); + fixture.startWorkflow.mockClear(); + const ai = await sharePageRetry(retryAi, owner, failedAi); + expect(ai.status).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + expect((await videoState(failedAi)).aiGenerationStatus).toBe("QUEUED"); + }); + + it("only refuses retries past 30 an hour, shared across the share page and the agent API", async () => { + const owner = await createOwner(); + for (let index = 0; index < 15; index++) { + const videoId = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + expect( + (await sharePageRetry(retryTranscription, owner, videoId)).status, + ).toBe(200); + } + for (let index = 0; index < 15; index++) { + const videoId = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + expect( + ( + await processCap(owner, videoId, { + target: "transcript", + retry: true, + }) + ).status, + ).toBe(200); + } + + const refusedAgent = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + const limited = await processCap(owner, refusedAgent, { + target: "transcript", + retry: true, + }); + expect(limited.status).toBe(429); + expect((await videoState(refusedAgent)).transcriptionStatus).toBe( + "ERROR", + ); + + const refusedSharePage = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + expect( + (await sharePageRetry(retryTranscription, owner, refusedSharePage)) + .status, + ).toBe(429); + expect((await videoState(refusedSharePage)).transcriptionStatus).toBe( + "ERROR", + ); + + const other = await createOwner(); + const otherVideo = await createVideo(other, { + transcriptionStatus: "ERROR", + }); + expect( + (await sharePageRetry(retryTranscription, other, otherVideo)).status, + ).toBe(200); + + const fresh = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + }); + expect((await processCap(owner, fresh, { target: "all" })).status).toBe( + 200, + ); + }); + + it("replays a saved request without charging or starting AI again", async () => { + const owner = await createOwner(); + const failedAi = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + aiGenerationStatus: "ERROR", + }); + const key = randomUUID(); + fixture.startWorkflow.mockClear(); + expect( + (await processCap(owner, failedAi, { target: "ai", retry: true }, key)) + .status, + ).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + + await database() + .update(Db.videos) + .set({ metadata: { aiGenerationStatus: "ERROR" } } as never) + .where(eq(Db.videos.id, failedAi as never)); + fixture.startWorkflow.mockClear(); + expect( + (await processCap(owner, failedAi, { target: "ai", retry: true }, key)) + .status, + ).toBe(200); + expect(fixture.startWorkflow).not.toHaveBeenCalled(); + }); + }, +); diff --git a/apps/web/__tests__/unit/ai-generation-rerun.test.ts b/apps/web/__tests__/unit/ai-generation-rerun.test.ts new file mode 100644 index 00000000000..a10316e86ba --- /dev/null +++ b/apps/web/__tests__/unit/ai-generation-rerun.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/database", () => ({ db: vi.fn() })); +vi.mock("workflow/api", () => ({ start: vi.fn() })); +vi.mock("@/lib/ai/provider", () => ({ isAiConfigured: () => true })); +vi.mock("@/workflows/generate-ai", () => ({ generateAiWorkflow: vi.fn() })); + +import { chargesProcessingRerun, isAiGenerationRerun } from "@/lib/generate-ai"; + +const legacySummary = + "The AI was unable to generate a proper summary for this content."; + +describe("AI generation rerun", () => { + it("treats a video that never ran AI generation as a first run", () => { + expect(isAiGenerationRerun({})).toBe(false); + }); + + it("does not count work that is already queued or running", () => { + expect(isAiGenerationRerun({ aiGenerationStatus: "QUEUED" })).toBe(false); + expect(isAiGenerationRerun({ aiGenerationStatus: "PROCESSING" })).toBe( + false, + ); + }); + + it("does not count a finished result, which generation leaves alone", () => { + expect( + isAiGenerationRerun({ + aiGenerationStatus: "COMPLETE", + summary: "A summary", + chapters: [{ title: "Intro", start: 0 }], + }), + ).toBe(false); + }); + + it("counts every state that starts generation again", () => { + expect(isAiGenerationRerun({ aiGenerationStatus: "ERROR" })).toBe(true); + expect(isAiGenerationRerun({ aiGenerationStatus: "SKIPPED" })).toBe(true); + expect( + isAiGenerationRerun({ + aiGenerationStatus: "COMPLETE", + summary: legacySummary, + chapters: [{ title: "Intro", start: 0 }], + }), + ).toBe(true); + expect( + isAiGenerationRerun({ + aiGenerationStatus: "COMPLETE", + summary: "A summary", + }), + ).toBe(true); + }); +}); + +describe("agent processing rerun charge", () => { + const complete = { + aiGenerationStatus: "COMPLETE" as const, + summary: "A summary", + chapters: [{ title: "Intro", start: 0 }], + }; + + it("does not charge first-time processing", () => { + for (const target of ["all", "transcript", "ai"] as const) { + expect( + chargesProcessingRerun({ + target, + retry: undefined, + transcriptionStatus: target === "ai" ? "COMPLETE" : null, + metadata: {}, + }), + ).toBe(false); + } + expect( + chargesProcessingRerun({ + target: "ai", + retry: true, + transcriptionStatus: "COMPLETE", + metadata: complete, + }), + ).toBe(false); + }); + + it("charges a transcript retry after a failure", () => { + for (const target of ["all", "transcript"] as const) { + expect( + chargesProcessingRerun({ + target, + retry: true, + transcriptionStatus: "ERROR", + metadata: {}, + }), + ).toBe(true); + } + expect( + chargesProcessingRerun({ + target: "transcript", + retry: undefined, + transcriptionStatus: "ERROR", + metadata: {}, + }), + ).toBe(false); + }); + + it("charges an AI retry after a failure or skip", () => { + for (const aiGenerationStatus of ["ERROR", "SKIPPED"] as const) { + for (const target of ["all", "ai"] as const) { + expect( + chargesProcessingRerun({ + target, + retry: true, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus }, + }), + ).toBe(true); + } + } + }); + + it("does not charge a request the mutation refuses as a conflict", () => { + for (const aiGenerationStatus of ["ERROR", "SKIPPED"] as const) { + expect( + chargesProcessingRerun({ + target: "ai", + retry: undefined, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus }, + }), + ).toBe(false); + } + }); + + it("charges a rerun of an incomplete result even without retry", () => { + expect( + chargesProcessingRerun({ + target: "ai", + retry: undefined, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus: "COMPLETE", summary: "A summary" }, + }), + ).toBe(true); + }); + + it("does not charge AI work that waits on transcription", () => { + for (const target of ["all", "ai"] as const) { + for (const transcriptionStatus of [ + null, + "PROCESSING", + "ERROR", + "SKIPPED", + "NO_AUDIO", + ]) { + expect( + chargesProcessingRerun({ + target, + retry: target === "ai" ? true : undefined, + transcriptionStatus, + metadata: { aiGenerationStatus: "ERROR" }, + }), + ).toBe(false); + } + } + }); +}); diff --git a/apps/web/__tests__/unit/processing-retry-limit.test.ts b/apps/web/__tests__/unit/processing-retry-limit.test.ts new file mode 100644 index 00000000000..0c6276b4aaa --- /dev/null +++ b/apps/web/__tests__/unit/processing-retry-limit.test.ts @@ -0,0 +1,168 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + user: vi.fn(), + video: vi.fn(), + update: vi.fn(), + startAiGeneration: vi.fn(), + aiEnabled: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ + where: () => ({ limit: async () => mocks.video() }), + }), + }), + update: () => ({ + set: (values: unknown) => ({ + where: async () => mocks.update(values), + }), + }), + }), +})); +vi.mock("@cap/database/auth/request-site", () => ({ + isCrossSiteCookieWrite: () => false, +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: mocks.user, +})); +vi.mock("@cap/database/schema", () => ({ users: {}, videos: {} })); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/lib/generate-ai", () => ({ + startAiGeneration: mocks.startAiGeneration, +})); +vi.mock("@/utils/flags", () => ({ isAiGenerationEnabled: mocks.aiEnabled })); + +import { POST as retryAi } from "@/app/api/videos/[videoId]/retry-ai/route"; +import { POST as retryTranscription } from "@/app/api/videos/[videoId]/retry-transcription/route"; +import { + PROCESSING_RETRY_RATE_MESSAGE, + processingRetryLimited, +} from "@/lib/processing-retry-limit"; + +const RETRIES_PER_HOUR = 30; + +let userCounter = 0; +let userId = ""; + +const request = () => + new Request("http://localhost/api/videos/video-1/retry", { method: "POST" }); +const params = { params: Promise.resolve({ videoId: "video-1" }) } as never; + +beforeEach(() => { + vi.clearAllMocks(); + userCounter++; + userId = `owner-${userCounter}`; + mocks.user.mockResolvedValue({ id: userId }); + mocks.video.mockImplementation(() => [ + { + id: "video-1", + ownerId: userId, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus: "ERROR" }, + email: "owner@example.com", + }, + ]); + mocks.aiEnabled.mockResolvedValue(true); + mocks.startAiGeneration.mockResolvedValue({ success: true, message: "ok" }); +}); + +describe("processing retry limit", () => { + it("allows normal retries and refuses a loop of them", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + await expect(processingRetryLimited(userId)).resolves.toBe(false); + } + await expect(processingRetryLimited(userId)).resolves.toBe(true); + await expect(processingRetryLimited(`${userId}-other`)).resolves.toBe( + false, + ); + }); + + it("retries AI generation for the owner", async () => { + const response = await retryAi(request(), params); + + expect(response.status).toBe(200); + expect(mocks.startAiGeneration).toHaveBeenCalledOnce(); + }); + + it("stops AI generation retries past the limit with a clear error", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + expect((await retryAi(request(), params)).status).toBe(200); + } + + const response = await retryAi(request(), params); + + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: PROCESSING_RETRY_RATE_MESSAGE, + }); + expect(mocks.startAiGeneration).toHaveBeenCalledTimes(RETRIES_PER_HOUR); + }); + + it("retries transcription for the owner", async () => { + const response = await retryTranscription(request(), params); + + expect(response.status).toBe(200); + expect(mocks.update).toHaveBeenCalledExactlyOnceWith({ + transcriptionStatus: null, + }); + }); + + it("stops transcription retries past the limit without resetting the status", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + expect((await retryTranscription(request(), params)).status).toBe(200); + } + + const response = await retryTranscription(request(), params); + + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: PROCESSING_RETRY_RATE_MESSAGE, + }); + expect(mocks.update).toHaveBeenCalledTimes(RETRIES_PER_HOUR); + }); + + it("shares one budget between both share page retries", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + await retryTranscription(request(), params); + } + + expect((await retryAi(request(), params)).status).toBe(429); + expect(mocks.startAiGeneration).not.toHaveBeenCalled(); + }); + + it("charges new agent reruns before the mutation and never reruns on a replay", () => { + const source = readFileSync( + join(process.cwd(), "app/api/v1/[...route]/route.ts"), + "utf8", + ); + const start = source.indexOf('.handle("processCap"'); + const handler = source.slice( + start, + source.indexOf('.handle("replaceTranscript"', start), + ); + const at = (text: string) => { + const index = handler.indexOf(text); + expect(index, text).toBeGreaterThan(-1); + return index; + }; + + expect(handler).toMatch( + /const chargesRerun = chargesProcessingRerun\(\{\s*target: payload\.target,\s*retry: payload\.retry,\s*transcriptionStatus: row\.transcriptionStatus,\s*metadata,\s*\}\);/, + ); + expect(handler).toMatch( + /runAgentMutation\(\{\s*admit: Effect\.gen\(function\* \(\) \{\s*newRequest = true;\s*if \(\s*chargesRerun &&\s*\(yield\* Effect\.promise\(\(\) =>\s*processingRetryLimited\(principal\.id, headers\),\s*\)\)\s*\)\s*\{\s*return yield\* rateLimited\(\s*requestId,\s*PROCESSING_RETRY_WINDOW_MS,\s*\);/, + ); + expect(handler).toMatch( + /state\.target !== "transcript" &&\s*\(newRequest \|\| !isAiGenerationRerun\(metadata\)\)\s*\)\s*\{\s*const result = yield\* Effect\.tryPromise\(\(\) =>\s*startAiGeneration\(/, + ); + expect(at("const chargesRerun")).toBeGreaterThan( + at("isAiGenerationEnabled("), + ); + expect(at("admit:")).toBeLessThan(at("transcribeVideo(")); + }); +}); diff --git a/apps/web/app/api/v1/[...route]/route.ts b/apps/web/app/api/v1/[...route]/route.ts index 4ec2a3cc0b7..2949fb5135b 100644 --- a/apps/web/app/api/v1/[...route]/route.ts +++ b/apps/web/app/api/v1/[...route]/route.ts @@ -161,7 +161,11 @@ import { completeDeveloperVideoUpload, createDeveloperVideoUpload, } from "@/lib/developer-video-upload"; -import { startAiGeneration } from "@/lib/generate-ai"; +import { + chargesProcessingRerun, + isAiGenerationRerun, + startAiGeneration, +} from "@/lib/generate-ai"; import { canUseInfra } from "@/lib/infra-access"; import { autoTopUpSettingsError } from "@/lib/infra-auto-top-up"; import { @@ -189,6 +193,10 @@ import { getEffectiveOrganizationRole, } from "@/lib/permissions/roles"; import { normalizePlaybackSpeed } from "@/lib/playback-speed"; +import { + PROCESSING_RETRY_WINDOW_MS, + processingRetryLimited, +} from "@/lib/processing-retry-limit"; import { isRateLimited, RATE_LIMIT_IDS } from "@/lib/rate-limit"; import { blockedS3EndpointMessage, @@ -436,12 +444,12 @@ const temporarilyUnavailable = ( requestId, }); -const rateLimited = (requestId: string) => +const rateLimited = (requestId: string, retryAfterMs = 60_000) => new Agent.AgentRateLimitedError({ message: "Too many requests. Try again later", code: "RATE_LIMITED", retryable: true, - retryAfterMs: 60_000, + retryAfterMs, requestId, }); @@ -7930,7 +7938,34 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( ); } } + // Reruns are judged from the video's current state. A new request is + // charged before the mutation, mirroring its conflict rules, so a + // refused request stores nothing. A replay of a saved request is not + // charged and never starts generation again on a failed video. + const metadata = normalizeAgentMetadata(row.metadata); + const chargesRerun = chargesProcessingRerun({ + target: payload.target, + retry: payload.retry, + transcriptionStatus: row.transcriptionStatus, + metadata, + }); + const headers = yield* requestHeaders; + let newRequest = false; const state = yield* runAgentMutation({ + admit: Effect.gen(function* () { + newRequest = true; + if ( + chargesRerun && + (yield* Effect.promise(() => + processingRetryLimited(principal.id, headers), + )) + ) { + return yield* rateLimited( + requestId, + PROCESSING_RETRY_WINDOW_MS, + ); + } + }), principal, operation: "process_cap", idempotencyKey: yield* requestIdempotencyKey, @@ -8028,7 +8063,10 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( if (!result.success) { return yield* temporarilyUnavailable(requestId, result.message); } - } else if (state.target !== "transcript") { + } else if ( + state.target !== "transcript" && + (newRequest || !isAiGenerationRerun(metadata)) + ) { const result = yield* Effect.tryPromise(() => startAiGeneration(state.id, state.ownerId), ); diff --git a/apps/web/app/api/videos/[videoId]/retry-ai/route.ts b/apps/web/app/api/videos/[videoId]/retry-ai/route.ts index eb26330ae7a..6c796c330a5 100644 --- a/apps/web/app/api/videos/[videoId]/retry-ai/route.ts +++ b/apps/web/app/api/videos/[videoId]/retry-ai/route.ts @@ -7,6 +7,10 @@ import type { Video } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { startAiGeneration } from "@/lib/generate-ai"; +import { + PROCESSING_RETRY_RATE_MESSAGE, + processingRetryLimited, +} from "@/lib/processing-retry-limit"; import { isAiGenerationEnabled } from "@/utils/flags"; const LEGACY_AI_SUMMARY_FALLBACK = @@ -97,6 +101,13 @@ export async function POST( ); } + if (await processingRetryLimited(user.id, request.headers)) { + return Response.json( + { error: PROCESSING_RETRY_RATE_MESSAGE }, + { status: 429 }, + ); + } + const result = await startAiGeneration(videoId, video.ownerId); if (!result.success) { diff --git a/apps/web/app/api/videos/[videoId]/retry-transcription/route.ts b/apps/web/app/api/videos/[videoId]/retry-transcription/route.ts index 00b03f36c0b..3bc331eb178 100644 --- a/apps/web/app/api/videos/[videoId]/retry-transcription/route.ts +++ b/apps/web/app/api/videos/[videoId]/retry-transcription/route.ts @@ -5,6 +5,10 @@ import { videos } from "@cap/database/schema"; import type { Video } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; +import { + PROCESSING_RETRY_RATE_MESSAGE, + processingRetryLimited, +} from "@/lib/processing-retry-limit"; export async function POST( request: Request, @@ -39,6 +43,13 @@ export async function POST( return Response.json({ error: "Unauthorized" }, { status: 403 }); } + if (await processingRetryLimited(user.id, request.headers)) { + return Response.json( + { error: PROCESSING_RETRY_RATE_MESSAGE }, + { status: 429 }, + ); + } + // Reset status to null - this will trigger automatic retry via get-status.ts await db() .update(videos) diff --git a/apps/web/lib/generate-ai.ts b/apps/web/lib/generate-ai.ts index d7d5aeaebb1..228aafe9ffe 100644 --- a/apps/web/lib/generate-ai.ts +++ b/apps/web/lib/generate-ai.ts @@ -15,6 +15,36 @@ type GenerateAiResult = { const LEGACY_AI_SUMMARY_FALLBACK = "The AI was unable to generate a proper summary for this content."; +export function isAiGenerationRerun(metadata: VideoMetadata) { + const status = metadata.aiGenerationStatus; + if (!status || status === "PROCESSING" || status === "QUEUED") return false; + return !( + status === "COMPLETE" && + metadata.summary && + metadata.summary !== LEGACY_AI_SUMMARY_FALLBACK && + metadata.chapters + ); +} + +export function chargesProcessingRerun(input: { + target: "all" | "transcript" | "ai"; + retry: boolean | undefined; + transcriptionStatus: string | null; + metadata: VideoMetadata; +}) { + if ( + input.target !== "ai" && + input.retry === true && + input.transcriptionStatus === "ERROR" + ) + return true; + if (input.target === "transcript") return false; + if (input.transcriptionStatus !== "COMPLETE") return false; + if (!isAiGenerationRerun(input.metadata)) return false; + const status = input.metadata.aiGenerationStatus; + return input.retry === true || (status !== "ERROR" && status !== "SKIPPED"); +} + const getAffectedRows = (result: unknown) => { if (Array.isArray(result)) { return ( diff --git a/apps/web/lib/processing-retry-limit.ts b/apps/web/lib/processing-retry-limit.ts new file mode 100644 index 00000000000..7b2a40122a3 --- /dev/null +++ b/apps/web/lib/processing-retry-limit.ts @@ -0,0 +1,13 @@ +import { isRateLimited, RATE_LIMIT_IDS } from "@/lib/rate-limit"; + +export const PROCESSING_RETRY_WINDOW_MS = 60 * 60 * 1000; + +export const PROCESSING_RETRY_RATE_MESSAGE = + "Too many processing retries. Try again in an hour."; + +export function processingRetryLimited(userId: string, headers?: Headers) { + return isRateLimited(RATE_LIMIT_IDS.PROCESSING_RETRY, { + key: `processing-retry:${userId}`, + ...(headers ? { headers } : {}), + }); +} diff --git a/apps/web/lib/rate-limit.ts b/apps/web/lib/rate-limit.ts index 4e361a9c50c..8eb5fc31d1f 100644 --- a/apps/web/lib/rate-limit.ts +++ b/apps/web/lib/rate-limit.ts @@ -210,6 +210,8 @@ export const RATE_LIMIT_IDS = { COMMENT_CREATE: "rl_comment_create", /** Comment media upload starts per viewer. Suggested: 60 / 1h per key (user). */ COMMENT_MEDIA_UPLOAD: "rl_comment_media_upload", + /** Transcription and AI generation retries per user, from the share page and the agent API. Suggested: 30 / 1h per key (user). */ + PROCESSING_RETRY: "rl_processing_retry", /** Organizations created from the dashboard per user. Suggested: 20 / 1h per key (user). */ ORGANIZATION_CREATE: "rl_organization_create", /** Spaces created from the dashboard per user. Suggested: 300 / 1h per key (user). */ @@ -284,6 +286,7 @@ const FALLBACK_RATE_LIMITS: Record = { limit: 60, windowMs: 60 * MINUTE_MS, }, + [RATE_LIMIT_IDS.PROCESSING_RETRY]: { limit: 30, windowMs: 60 * MINUTE_MS }, [RATE_LIMIT_IDS.ORGANIZATION_CREATE]: { limit: 20, windowMs: 60 * MINUTE_MS,