diff --git a/apps/web/__tests__/unit/developer-api-contract.test.ts b/apps/web/__tests__/unit/developer-api-contract.test.ts index 06223292ed..3a379e3e49 100644 --- a/apps/web/__tests__/unit/developer-api-contract.test.ts +++ b/apps/web/__tests__/unit/developer-api-contract.test.ts @@ -641,11 +641,32 @@ describe("browser access", () => { "access-control-allow-headers": "Content-Type,Authorization", "access-control-expose-headers": "Retry-After,X-RateLimit-Limit,X-RateLimit-Remaining,X-RateLimit-Reset,X-Request-Id", + "access-control-max-age": "7200", "cache-control": "private, no-store", vary: "Access-Control-Request-Headers", }); }); + it.each([ + "/api/developer/sdk/v1/upload/multipart/initiate", + "/api/developer/sdk/v1/upload/multipart/presign-part", + "/api/developer/sdk/v1/upload/multipart/complete", + "/api/developer/sdk/v1/upload/multipart/abort", + ])( + "lets a browser cache the preflight for %s for two hours", + async (path) => { + const preflight = await call(sdk, "OPTIONS", path, { + headers: { + ...PREFLIGHT_HEADERS, + "Access-Control-Request-Method": "POST", + }, + }); + + expect(preflight.status).toBe(204); + expect(preflight.headers.get("Access-Control-Max-Age")).toBe("7200"); + }, + ); + it("still lets a page read an upload route's answer", async () => { const response = await call( sdk, @@ -663,6 +684,48 @@ describe("browser access", () => { }); }); +describe("the credential sent to the REST API", () => { + it("says the key is missing when no Authorization header is sent", async () => { + const response = await getUsage(""); + + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject({ + code: "invalid_secret_key", + error: "Missing secret key", + hint: expect.stringContaining("No Authorization header was sent"), + }); + }); + + it("refuses a public key with its own code", async () => { + const response = await getUsage(freshKey("cpk")); + + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ + code: "public_key_not_allowed", + error: "Public keys cannot call the REST API", + hint: expect.stringContaining("csk_"), + }); + expect(mocks.billing).not.toHaveBeenCalled(); + }); + + it("keeps the answers for any other credential and for a wrong secret key", async () => { + const other = await getUsage("not_a_key"); + mocks.keyRows = []; + const wrong = await getUsage(freshKey("csk")); + + expect(other.status).toBe(401); + expect(await other.json()).toMatchObject({ + code: "invalid_secret_key", + error: "Invalid secret key", + }); + expect(wrong.status).toBe(401); + expect(await wrong.json()).toMatchObject({ + code: "invalid_secret_key", + error: "Invalid or revoked secret key", + }); + }); +}); + describe("wrong method and unknown path", () => { it.each([ [rest, "PATCH", "/api/developer/v1/videos/video-1", "csk", "GET, DELETE"], diff --git a/apps/web/__tests__/unit/developer-sdk-upload.test.ts b/apps/web/__tests__/unit/developer-sdk-upload.test.ts index 7b075eb140..1341d3f7a6 100644 --- a/apps/web/__tests__/unit/developer-sdk-upload.test.ts +++ b/apps/web/__tests__/unit/developer-sdk-upload.test.ts @@ -677,17 +677,19 @@ describe("client mistakes on complete and abort", () => { }, ); - it("answers 404 when aborting an upload that does not exist", async () => { + it("answers a repeated abort with success, whether storage accepts it again or no longer knows the upload", async () => { mocks.video = uploaded; - mocks.abort.mockImplementation(fails("NoSuchUpload", 404)); - const response = await post("/abort", { uploadId: "not-an-upload" }); + const first = await post("/abort", { uploadId: "upload-1" }); + const repeated = await post("/abort", { uploadId: "upload-1" }); + mocks.abort.mockImplementation(fails("NoSuchUpload", 404)); + const gone = await post("/abort", { uploadId: "upload-1" }); - expect(await refusal(response)).toMatchObject({ - status: 404, - code: "upload_not_found", - hint: expect.stringContaining("Nothing was aborted"), - }); + for (const response of [first, repeated, gone]) { + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ success: true }); + } + expect(mocks.abort).toHaveBeenCalledTimes(3); expect(console.error).not.toHaveBeenCalled(); }); diff --git a/apps/web/__tests__/unit/developer-transactions-api.test.ts b/apps/web/__tests__/unit/developer-transactions-api.test.ts index a4e29961c0..aa8e2591f7 100644 --- a/apps/web/__tests__/unit/developer-transactions-api.test.ts +++ b/apps/web/__tests__/unit/developer-transactions-api.test.ts @@ -55,14 +55,22 @@ vi.mock("@cap/database", async () => { b.createdAt.getTime() - a.createdAt.getTime() || (a.id < b.id ? 1 : -1), ); - if (cursor.length > 0) { - const time = asTime(cursor[0]); + const at = (index: number) => asTime(cursor[index]); + if (cursor.length === 3) { const id = String(cursor.at(-1)); rows = rows.filter( (row) => - row.createdAt.getTime() < time || - (row.createdAt.getTime() === time && row.id < id), + row.createdAt.getTime() < at(0) || + (row.createdAt.getTime() === at(0) && row.id < id), ); + } else if (rendered.sql.includes(">= ?")) { + rows = rows.filter( + (row) => + row.createdAt.getTime() >= at(0) && + row.createdAt.getTime() <= at(1), + ); + } else if (cursor.length === 1) { + rows = rows.filter((row) => row.createdAt.getTime() === at(0)); } if (fields && "total" in fields) return [{ total: rows.length }]; return rows.slice(offset, offset + limit); @@ -186,10 +194,14 @@ describe("GET /transactions", () => { const publicKey = await get("/transactions", "cpk_public_key"); const unknown = await get("/transactions", "csk_not_a_key"); - for (const response of [missing, publicKey, unknown]) { - expect(response.status).toBe(401); + for (const [response, status, code] of [ + [missing, 401, "invalid_secret_key"], + [publicKey, 403, "public_key_not_allowed"], + [unknown, 401, "invalid_secret_key"], + ] as const) { + expect(response.status).toBe(status); expect(await response.json()).toMatchObject({ - code: "invalid_secret_key", + code, hint: expect.any(String), docs: expect.any(String), }); diff --git a/apps/web/__tests__/unit/developer-transcribe-download.test.ts b/apps/web/__tests__/unit/developer-transcribe-download.test.ts index 13b18bd84e..7d4cbf10e3 100644 --- a/apps/web/__tests__/unit/developer-transcribe-download.test.ts +++ b/apps/web/__tests__/unit/developer-transcribe-download.test.ts @@ -431,25 +431,33 @@ describe("downloading a URL source from a slow server", () => { head, }); + const WEB_PAGE = "The URL returned a web page, not a media file"; + const NOT_MEDIA = "The file is not a supported audio or video format"; + const bytes = (text: string) => new TextEncoder().encode(text); + it.each([ - ["an HTML page", new TextEncoder().encode("")], - ["a playlist", new TextEncoder().encode("#EXTM3U\n#EXT-X-VERSION:3\n")], - ["random bytes", new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8])], + ["an HTML page", bytes(""), WEB_PAGE], + [ + "an HTML page after a BOM and blank lines", + new Uint8Array([0xef, 0xbb, 0xbf, ...bytes("\r\n\t ")]), + WEB_PAGE, + ], + ["an HTML page with no doctype", bytes("\n"), WEB_PAGE], + ["an XML document", bytes(''), NOT_MEDIA], + ["a playlist", bytes("#EXTM3U\n#EXT-X-VERSION:3\n"), NOT_MEDIA], + ["random bytes", new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]), NOT_MEDIA], ])( "is refused from its first bytes when it is %s, with nothing stored", - async (_label, head) => { + async (_label, head, message) => { const server = await notMedia(head); const result = await run(); - expect(result).toEqual({ - success: false, - message: "The file is not a supported audio or video format", - }); + expect(result).toEqual({ success: false, message }); expect(transcript()).toMatchObject({ status: "error", errorCode: "unsupported_format", - errorMessage: "The file is not a supported audio or video format", + errorMessage: message, billedMicroCredits: 0, reservedMicroCredits: 0, }); diff --git a/apps/web/__tests__/unit/developer-transcripts-api.test.ts b/apps/web/__tests__/unit/developer-transcripts-api.test.ts index 692e3be103..33981d2a0b 100644 --- a/apps/web/__tests__/unit/developer-transcripts-api.test.ts +++ b/apps/web/__tests__/unit/developer-transcripts-api.test.ts @@ -101,29 +101,49 @@ describe("developer transcripts API", () => { describe("authentication", () => { it.each([ - ["no key", {}], - ["a public key", { Authorization: "Bearer cpk_public_key" }], - ["an agent token", { Authorization: "Bearer cap_cli_token" }], - ["a malformed header", { Authorization: "csk_valid_key" }], - ])("rejects %s before touching transcripts", async (_label, headers) => { - for (const [method, path] of [ - ["GET", "/"], - ["GET", "/tr-1"], - ["DELETE", "/tr-1"], - ["POST", "/tr-1/start"], - ] as const) { - const response = await app.request(path, { method, headers }); - expect(response.status).toBe(401); - expect(await response.json()).toMatchObject({ - code: "invalid_secret_key", - }); - } - const created = await post("/", { url: "https://a.example/a" }, headers); - expect(created.status).toBe(401); - for (const mock of Object.values(mocks.store)) { - expect(mock).not.toHaveBeenCalled(); - } - }); + ["no key", {}, 401, "invalid_secret_key"], + [ + "a public key", + { Authorization: "Bearer cpk_public_key" }, + 403, + "public_key_not_allowed", + ], + [ + "an agent token", + { Authorization: "Bearer cap_cli_token" }, + 401, + "invalid_secret_key", + ], + [ + "a malformed header", + { Authorization: "csk_valid_key" }, + 401, + "invalid_secret_key", + ], + ])( + "rejects %s before touching transcripts", + async (_label, headers, status, code) => { + for (const [method, path] of [ + ["GET", "/"], + ["GET", "/tr-1"], + ["DELETE", "/tr-1"], + ["POST", "/tr-1/start"], + ] as const) { + const response = await app.request(path, { method, headers }); + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); + } + const created = await post( + "/", + { url: "https://a.example/a" }, + headers, + ); + expect(created.status).toBe(status); + for (const mock of Object.values(mocks.store)) { + expect(mock).not.toHaveBeenCalled(); + } + }, + ); it("rejects a revoked or unknown secret key", async () => { mocks.keyRows = []; diff --git a/apps/web/__tests__/unit/developer-upload-token-flow.test.ts b/apps/web/__tests__/unit/developer-upload-token-flow.test.ts index 06a7ffce8f..f725addd9a 100644 --- a/apps/web/__tests__/unit/developer-upload-token-flow.test.ts +++ b/apps/web/__tests__/unit/developer-upload-token-flow.test.ts @@ -274,21 +274,24 @@ describe("creating an upload with the secret key", () => { }); it.each([ - ["a public key", "cpk_app1_page"], - ["an upload token", "cut_abc.def"], - ["no key", null], - ])("refuses %s and inserts nothing", async (_label, credential) => { - const response = await post( - rest, - "/api/developer/v1/uploads", - credential, - {}, - ); + ["a public key", "cpk_app1_page", 403, "public_key_not_allowed"], + ["an upload token", "cut_abc.def", 401, "invalid_secret_key"], + ["no key", null, 401, "invalid_secret_key"], + ])( + "refuses %s and inserts nothing", + async (_label, credential, status, code) => { + const response = await post( + rest, + "/api/developer/v1/uploads", + credential, + {}, + ); - expect(response.status).toBe(401); - expect(await response.json()).toMatchObject({ code: "invalid_secret_key" }); - expect(inserts()).toHaveLength(0); - }); + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); + expect(inserts()).toHaveLength(0); + }, + ); it.each([ [60, 60], @@ -794,15 +797,15 @@ describe("renewing the token of an open upload", () => { }); it.each([ - ["a public key", "cpk_app1_page"], - ["no key", null], - ])("refuses %s", async (_label, credential) => { + ["a public key", "cpk_app1_page", 403, "public_key_not_allowed"], + ["no key", null, 401, "invalid_secret_key"], + ])("refuses %s", async (_label, credential, status, code) => { const first = await created(); const response = await renew(first.videoId, {}, credential); - expect(response.status).toBe(401); - expect(await response.json()).toMatchObject({ code: "invalid_secret_key" }); + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); }); it("cannot be renewed with the upload token itself", async () => { diff --git a/apps/web/__tests__/unit/infra-ledger-paging.test.ts b/apps/web/__tests__/unit/infra-ledger-paging.test.ts index a24dac2170..1f73fec6a7 100644 --- a/apps/web/__tests__/unit/infra-ledger-paging.test.ts +++ b/apps/web/__tests__/unit/infra-ledger-paging.test.ts @@ -1,6 +1,11 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -type Row = { id: string; createdAt: Date; amountMicroCredits: number }; +type Row = { + id: string; + createdAt: Date; + amountMicroCredits: number; + balanceAfterMicroCredits?: number; +}; const state = vi.hoisted(() => ({ rows: [] as Row[], @@ -26,7 +31,18 @@ vi.mock("@cap/database", async () => { if (!ascending) sorted.reverse(); let rest = params.slice(1); let matching = sorted; - if (sql.includes("`createdAt` < ?") || sql.includes("`createdAt` > ?")) { + if (sql.includes("`createdAt` >= ?")) { + const from = toDate(rest[0]).getTime(); + const to = toDate(rest[1]).getTime(); + matching = sorted.filter( + (row) => + row.createdAt.getTime() >= from && row.createdAt.getTime() <= to, + ); + rest = [matching.length]; + } else if ( + sql.includes("`createdAt` < ?") || + sql.includes("`createdAt` > ?") + ) { const at = toDate(rest[0]).getTime(); const id = String(rest[2]); rest = rest.slice(3); @@ -36,6 +52,10 @@ vi.mock("@cap/database", async () => { ? time > at || (time === at && row.id > id) : time < at || (time === at && row.id < id); }); + } else if (sql.includes("`createdAt` = ?")) { + const at = toDate(rest[0]).getTime(); + matching = sorted.filter((row) => row.createdAt.getTime() === at); + rest = [matching.length]; } const limit = Number(rest[0]); const offset = sql.includes("offset ?") ? Number(rest[1]) : 0; @@ -46,7 +66,7 @@ vi.mock("@cap/database", async () => { row.id, "transcription", row.amountMicroCredits, - 0, + row.balanceAfterMicroCredits ?? 0, null, null, null, @@ -57,12 +77,13 @@ vi.mock("@cap/database", async () => { return { db: () => client }; }); -import { LEDGER_PAGE_SIZE } from "@/lib/infra-ledger"; +import { LEDGER_PAGE_SIZE, type LedgerRow } from "@/lib/infra-ledger"; import { formatLedgerCursor, listAppLedger, listAppLedgerPage, parseLedgerCursor, + readAppLedger, } from "@/lib/infra-ledger-store"; const START = Date.parse("2026-11-01T00:00:00Z"); @@ -190,7 +211,7 @@ describe("ledger pages by cursor", () => { }); describe("the transactions API keeps limit and offset", () => { - it("lists with limit and offset in one query and never counts", async () => { + it("lists with limit and offset and never counts", async () => { const result = await listAppLedger("app-1", { limit: 50, offset: 100 }); expect(result.rows).toHaveLength(20); @@ -198,8 +219,84 @@ describe("the transactions API keeps limit and offset", () => { expect( (await listAppLedger("app-1", { limit: 50, offset: 0 })).hasMore, ).toBe(true); - expect(state.queries).toHaveLength(2); + expect(state.queries).toHaveLength(4); expect(state.queries[0]).toContain("limit ? offset ?"); expect(state.queries.join("\n")).not.toMatch(/count\(/i); }); }); + +describe("rows written in the same second across a page edge", () => { + const SECOND = (offset: number) => new Date(START + offset * 1000); + const seedChain = () => { + const groups = [ + { at: SECOND(0), ids: ["q1", "c7"] }, + { at: SECOND(1), ids: ["m4", "z9", "a2", "t6", "f3"] }, + { at: SECOND(2), ids: ["k8", "b5"] }, + ]; + let balance = 1_000_000; + let amount = 11; + state.rows = groups.flatMap(({ at, ids }) => + ids.map((id) => { + amount += 7; + balance -= amount; + return { + id, + createdAt: at, + amountMicroCredits: -amount, + balanceAfterMicroCredits: balance, + }; + }), + ); + }; + const chainBreaks = ( + rows: { amountMicroCredits: number; balanceAfterMicroCredits: number }[], + ) => + rows.slice(0, -1).filter((row, index) => { + const older = rows[index + 1]; + return ( + !older || + row.balanceAfterMicroCredits - row.amountMicroCredits !== + older.balanceAfterMicroCredits + ); + }).length; + const walk = async (limit: number) => { + const rows: LedgerRow[] = []; + for (let offset = 0; ; offset += limit) { + const page = await listAppLedger("app-1", { limit, offset }); + rows.push(...page.rows); + if (!page.hasMore) return rows; + } + }; + + beforeEach(seedChain); + + it("gives the same chain whatever the page size", async () => { + const whole = (await listAppLedger("app-1", { limit: 100, offset: 0 })) + .rows; + + expect(whole).toHaveLength(9); + expect(chainBreaks(whole)).toBe(0); + for (const limit of [1, 2, 3, 4, 5, 8]) { + const paged = await walk(limit); + expect( + paged.map((row) => row.id), + `limit ${limit}`, + ).toEqual(whole.map((row) => row.id)); + } + }); + + it("gives the same chain in an export read in small batches", async () => { + const whole = (await listAppLedger("app-1", { limit: 100, offset: 0 })) + .rows; + for (const batchSize of [2, 3, 4]) { + const exported: LedgerRow[] = []; + for await (const batch of readAppLedger("app-1", batchSize)) { + exported.push(...batch); + } + expect( + exported.map((row) => row.id), + `batch ${batchSize}`, + ).toEqual(whole.map((row) => row.id)); + } + }); +}); diff --git a/apps/web/app/api/utils.ts b/apps/web/app/api/utils.ts index 3b31ac9cce..401853552e 100644 --- a/apps/web/app/api/utils.ts +++ b/apps/web/app/api/utils.ts @@ -243,6 +243,7 @@ export const developerSdkCors = cors({ credentials: false, allowMethods: ["GET", "POST", "OPTIONS"], allowHeaders: ["Content-Type", "Authorization"], + maxAge: 7200, exposeHeaders: [ "Retry-After", "X-RateLimit-Limit", @@ -365,7 +366,28 @@ export const withDeveloperSecretAuth = createMiddleware<{ developerKeyType: "secret"; }; }>(async (c, next) => { - const authHeader = c.req.header("authorization")?.split(" ")[1]; + const authorization = c.req.header("authorization"); + const authHeader = authorization?.split(" ")[1]; + if (!authorization?.trim()) { + return c.json( + apiErrorBody( + "invalid_secret_key", + "Missing secret key", + "No Authorization header was sent. Send Authorization: Bearer csk_... with a secret key from https://cap.so/dashboard/infra.", + ), + 401, + ); + } + if (isDeveloperPublicKey(bearerCredential(c))) { + return c.json( + apiErrorBody( + "public_key_not_allowed", + "Public keys cannot call the REST API", + "A public key (cpk_) was sent. The REST API takes the app's secret key: send Authorization: Bearer csk_... from your server, with the secret key from https://cap.so/dashboard/infra.", + ), + 403, + ); + } if (!authHeader?.startsWith("csk_")) { return c.json( apiErrorBody( diff --git a/apps/web/content/docs/api/rest-api.mdx b/apps/web/content/docs/api/rest-api.mdx index 974cb3997c..69ff598394 100644 --- a/apps/web/content/docs/api/rest-api.mdx +++ b/apps/web/content/docs/api/rest-api.mdx @@ -49,7 +49,7 @@ The SDK API takes an upload token instead. Your server gets one from [Create Upl Authorization: Bearer cut_your_upload_token_here ``` -An upload token works for one video and for a limited time, so it is safe to send to the browser. A public key on its own cannot start an upload: the SDK API answers `403` with code `public_key_not_allowed`. +An upload token works for one video and for a limited time, so it is safe to send to the browser. A public key on its own cannot start an upload or call the REST API: both answer `403` with code `public_key_not_allowed`. A REST request with no `Authorization` header answers `401` with code `invalid_secret_key` and the message `Missing secret key`. Keep your secret key private. Never expose it in client-side code, commit it to version control, or share it publicly. The REST API sends no CORS headers, so a page cannot call it. Call it from your server, and give the page an upload token for the SDK API. @@ -117,7 +117,7 @@ Send request bodies as JSON with the header `Content-Type: application/json`. Ma | `400` | Bad request. The `code` says what was wrong, for example `invalid_request`, `invalid_json`, `invalid_limit` or `unknown_parameter` | | `401` | Unauthorized. The API key or upload token is missing, invalid, revoked or expired | | `402` | Payment required. The app is out of credit, its balance does not cover 7 days of storage, or it is below the minimum to start a transcript | -| `403` | Forbidden. On the SDK API, a public key was sent (`public_key_not_allowed`), the `Origin` differs from the upload token's `corsOrigin` (`origin_not_allowed`) or the body names another video (`video_mismatch`). Elsewhere, the account cannot use this feature yet | +| `403` | Forbidden. A public key was sent where a secret key or upload token is required (`public_key_not_allowed`). On the SDK API, the `Origin` differs from the upload token's `corsOrigin` (`origin_not_allowed`) or the body names another video (`video_mismatch`). Elsewhere, the account cannot use this feature yet | | `404` | Not found. The video, transcript or multipart upload does not exist (`video_not_found`, `transcript_not_found`, `upload_not_found` for an `uploadId`), or no route matches the path (`not_found`) | | `405` | Method not allowed. The path exists but not with this method (`method_not_allowed`). The `Allow` header lists the methods it takes | | `409` | Conflict with the current state, for example an upload that already completed, or a transcript started before its file was uploaded (`upload_not_found`). The `code` says which | @@ -874,7 +874,7 @@ A failed transcript has the status `error` and an `error` object with one of the | `source_too_large` | The file is larger than 2 GB | | `source_missing` | The uploaded file was not found | | `no_audio_track` | The file has no audio | -| `unsupported_format` | The file is not a self-contained audio or video file. Playlists and manifests are not accepted | +| `unsupported_format` | The file is not a self-contained audio or video file. Playlists and manifests are not accepted. When a URL returns a web page, the message says so | | `no_speech` | No longer returned. Audio with no speech now completes with empty text. Transcripts that failed this way before the change keep the code | | `duration_too_long` | The audio is longer than 4 hours | | `upload_expired` | The file was not uploaded within 24 hours | @@ -979,7 +979,7 @@ curl https://cap.so/api/developer/v1/usage \ Credits are tracked in micro-credits (1 dollar = 100,000 micro-credits). `balanceDollars` is the same balance in dollars, exact and never rounded up: two decimals when it is a whole number of cents and five otherwise, so 2,999,901 micro-credits is `"29.99901"`. [Billing](/docs/api/billing) explains every charge. -**Speech.** `speech` is what speech to text has cost in the current calendar month, counted in UTC from `periodStart`: the measured audio seconds of completed transcripts, the micro-credits charged for them and how many there were. Deleted transcripts stay counted, so the figure reconciles with the balance. `heldMicroCredits` is credit held by transcripts that are running now, and `availableMicroCredits` is the balance minus those holds, which is what a new transcript or upload is checked against. +**Speech.** `speech` is what speech to text has cost in the current calendar month, counted in UTC from `periodStart`: the measured audio seconds of completed transcripts, the micro-credits charged for them and how many there were. Each transcript is charged on its length rounded to a whole second and then down to a whole micro-credit, so `seconds` times the rate can differ slightly from `microCredits`: for the exact charge use `microCredits`, or each transcript's `costMicroCredits`. Deleted transcripts stay counted, so the figure reconciles with the balance. `heldMicroCredits` is credit held by transcripts that are running now, and `availableMicroCredits` is the balance minus those holds, which is what a new transcript or upload is checked against. **Totals.** `totalVideos` counts videos that are stored and ready to play, and `totalDurationMinutes` is their combined length, equal to the sum of `storage.minutesByTier`. `uploadingVideos` counts videos that were created and have not completed an upload. A video that is still encoding, or that failed or was refused, is in neither count. [List Videos](/docs/api/rest-api#list-videos) returns it with its status. @@ -1085,7 +1085,7 @@ curl "https://cap.so/api/developer/v1/transactions?limit=2" \ A storage row is described as "Storage, part paid" or "Storage, unpaid" when the balance could not cover that day. `note` holds a sentence of explanation for the rows that need one. -Rows written in the same second have the same `createdAt`. Within a page they are ordered so each balance follows from the row beneath it. +Rows written in the same second have the same `createdAt`. They are ordered so each balance follows from the row beneath it, within a page and across page edges, so walking the list with any `limit` returns the same order. The console's Billing page shows the same rows and exports them all as CSV. @@ -1188,7 +1188,7 @@ curl -X POST https://cap.so/api/developer/sdk/v1/upload/multipart/presign-part \ } ``` -A request with `partNumber` returns one URL as `{ "presignedUrl": "https://..." }`. More than 50 part numbers, or an empty list, returns `400` with the code `invalid_part_count`. +A request with `partNumber` returns one URL as `{ "presignedUrl": "https://..." }`. More than 50 part numbers, or an empty list, returns `400` with the code `invalid_part_count`. Signing does not contact storage, so an `uploadId` that is wrong or was aborted is not refused here: the `PUT` to its URL fails, and Complete Upload answers `404` with the code `upload_not_found`. A token signs part numbers up to the last one its size limit allows: `maxBytes` divided by 5,242,880 and rounded up, or 6,867 for a token with no `maxBytes`. A higher part number returns `400` with the code `part_number_too_large`, and no part in that request is signed. @@ -1274,7 +1274,7 @@ Abort also returns `409` with `completion_in_progress` while a completion holds #### Abort Upload -Cancel an in-progress multipart upload and discard its parts. If no open upload matches the `uploadId`, abort returns `404` with the code `upload_not_found`, which means there is nothing left to abort. +Cancel an in-progress multipart upload and discard its parts. Abort is safe to repeat: when no open upload matches the `uploadId`, for example because it was already aborted, it answers `200` with `{ "success": true }` again. ``` POST /api/developer/sdk/v1/upload/multipart/abort diff --git a/apps/web/lib/developer-openapi.ts b/apps/web/lib/developer-openapi.ts index 084546754a..36f197b730 100644 --- a/apps/web/lib/developer-openapi.ts +++ b/apps/web/lib/developer-openapi.ts @@ -234,6 +234,7 @@ const uploadTokenAuth = [{ uploadToken: [] }]; const restErrors = { "401": response("Unauthorized"), + "403": response("Forbidden"), "429": response("RateLimited"), "500": response("InternalError"), }; @@ -638,7 +639,6 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ ), "400": response("BadRequest"), "402": response("PaymentRequired"), - "403": response("Forbidden"), "404": response("NotFound"), "409": response("Conflict"), "415": response("UnsupportedMediaType"), @@ -664,7 +664,6 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ ), "400": response("BadRequest"), "402": response("PaymentRequired"), - "403": response("Forbidden"), "409": response("Conflict"), "415": response("UnsupportedMediaType"), "503": response("Unavailable"), @@ -688,7 +687,6 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ ), "400": response("BadRequest"), "402": response("PaymentRequired"), - "403": response("Forbidden"), "404": response("NotFound"), "409": response("Conflict"), "503": response("Unavailable"), @@ -846,7 +844,7 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ operationId: "presignSdkUploadPart", summary: "Presign upload parts", description: - "Returns presigned URLs, each valid for one hour. Send partNumbers to sign up to 50 parts in one request and get parts back, or partNumber to sign one part and get presignedUrl back. PUT the bytes for each part to its URL and keep the ETag response header for completeSdkUpload. Every part except the last must be at least 5 MB (5,242,880 bytes). More than 50 part numbers, or none, returns 400 with code invalid_part_count. A part number above the last one the upload's size limit allows (maxBytes divided by 5,242,880 and rounded up, or 6,867 with no maxBytes) returns 400 with code part_number_too_large and signs nothing, so number the parts from 1 with no gaps.", + "Returns presigned URLs, each valid for one hour. Send partNumbers to sign up to 50 parts in one request and get parts back, or partNumber to sign one part and get presignedUrl back. PUT the bytes for each part to its URL and keep the ETag response header for completeSdkUpload. Every part except the last must be at least 5 MB (5,242,880 bytes). More than 50 part numbers, or none, returns 400 with code invalid_part_count. A part number above the last one the upload's size limit allows (maxBytes divided by 5,242,880 and rounded up, or 6,867 with no maxBytes) returns 400 with code part_number_too_large and signs nothing, so number the parts from 1 with no gaps. Signing does not contact storage, so an uploadId that is wrong or was aborted is not refused here: the PUT to its URL fails, and completeSdkUpload answers 404 with code upload_not_found.", tags: ["SDK"], security: uploadTokenAuth, requestBody: jsonBody("PresignPartRequest"), @@ -882,7 +880,7 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ operationId: "abortSdkUpload", summary: "Abort a multipart upload", description: - "Cancels an unfinished upload and discards its parts. Returns 404 with code upload_not_found when no open upload matches the uploadId, which means there is nothing left to abort.", + "Cancels an unfinished upload and discards its parts. Abort is safe to repeat: when no open upload matches the uploadId, for example because it was already aborted, it answers 200 with success true again.", tags: ["SDK"], security: uploadTokenAuth, requestBody: jsonBody("AbortUploadRequest"), @@ -1754,7 +1752,7 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ ), Forbidden: json( ref("Error"), - "The request is not allowed. On the upload operations the code is public_key_not_allowed when a public key was sent, origin_not_allowed when the Origin header differs from the token's corsOrigin, and video_mismatch when the body names a video other than the token's. Elsewhere, the account cannot use this feature yet.", + "The request is not allowed. The code is public_key_not_allowed when a public key was sent where a secret key or upload token is required. On the upload operations the code is origin_not_allowed when the Origin header differs from the token's corsOrigin, and video_mismatch when the body names a video other than the token's. Elsewhere, the account cannot use this feature yet.", ), NotFound: json( ref("Error"), diff --git a/apps/web/lib/developer-transcript-errors.ts b/apps/web/lib/developer-transcript-errors.ts index 88e1ff9aac..dabb3dadef 100644 --- a/apps/web/lib/developer-transcript-errors.ts +++ b/apps/web/lib/developer-transcript-errors.ts @@ -160,7 +160,7 @@ export const TRANSCRIPT_FAILURE_CODES = [ { code: "unsupported_format", meaning: - "The file is not a self-contained audio or video file. Playlists and manifests are not accepted", + "The file is not a self-contained audio or video file. Playlists and manifests are not accepted. When a URL returns a web page, the message says so", }, { code: "no_speech", diff --git a/apps/web/lib/infra-ledger-store.ts b/apps/web/lib/infra-ledger-store.ts index 7a972f93e9..f86bac9b4b 100644 --- a/apps/web/lib/infra-ledger-store.ts +++ b/apps/web/lib/infra-ledger-store.ts @@ -3,7 +3,7 @@ import { developerCreditAccounts, developerCreditTransactions, } from "@cap/database/schema"; -import { and, asc, desc, eq, gt, lt, or } from "drizzle-orm"; +import { and, asc, desc, eq, gt, gte, lt, lte, or } from "drizzle-orm"; import { type LedgerRow, orderLedgerRows } from "@/lib/infra-ledger"; const EXPORT_BATCH_SIZE = 1_000; @@ -85,9 +85,30 @@ export async function listAppLedger( .orderBy(...newestFirst) .limit(limit + 1) .offset(offset); + const hasMore = rows.length > limit; + const page = rows.slice(0, limit); + const newest = page[0]; + const oldest = page.at(-1); + if (!newest || !oldest || (offset === 0 && !hasMore)) { + return { rows: orderLedgerRows(page), hasMore }; + } + + const seconds: LedgerRow[] = await appLedger() + .where( + and( + ofApp(appId), + gte(developerCreditTransactions.createdAt, oldest.createdAt), + lte(developerCreditTransactions.createdAt, newest.createdAt), + ), + ) + .orderBy(...newestFirst); + const skipped = Math.max( + 0, + seconds.findIndex((row) => row.id === newest.id), + ); return { - rows: orderLedgerRows(rows.slice(0, limit)), - hasMore: rows.length > limit, + rows: orderLedgerRows(seconds).slice(skipped, skipped + limit), + hasMore, }; } @@ -153,8 +174,23 @@ export async function* readAppLedger( .limit(batchSize); const last = rows.at(-1); if (!last) return; - yield orderLedgerRows(rows); - if (rows.length < batchSize) return; - cursor = { createdAt: last.createdAt, id: last.id }; + if (rows.length < batchSize) { + yield orderLedgerRows(rows); + return; + } + const lastSecond = new Date(last.createdAt).getTime(); + const restOfSecond: LedgerRow[] = await appLedger() + .where( + and( + ofApp(appId), + eq(developerCreditTransactions.createdAt, last.createdAt), + ), + ) + .orderBy(...newestFirst); + yield orderLedgerRows([ + ...rows.filter((row) => new Date(row.createdAt).getTime() !== lastSecond), + ...restOfSecond, + ]); + cursor = { createdAt: last.createdAt, id: "" }; } } diff --git a/apps/web/lib/infra-video-upload.ts b/apps/web/lib/infra-video-upload.ts index 0e0476a842..9a6b5d199d 100644 --- a/apps/web/lib/infra-video-upload.ts +++ b/apps/web/lib/infra-video-upload.ts @@ -1072,11 +1072,8 @@ export async function abortInfraUpload({ return ok({ success: true }); } catch (error) { - if (storageErrorCode(error) === "NoSuchUpload") { - return uploadNotFound( - "Nothing was aborted. The upload was already aborted or completed, or the uploadId is wrong.", - ); - } + if (storageErrorCode(error) === "NoSuchUpload") + return ok({ success: true }); console.error("Error aborting multipart upload:", error); return storageFailure("Failed to abort upload"); } diff --git a/apps/web/lib/media-container.ts b/apps/web/lib/media-container.ts index 59ba3f6de9..9527163cd3 100644 --- a/apps/web/lib/media-container.ts +++ b/apps/web/lib/media-container.ts @@ -120,11 +120,7 @@ const PLAYLIST_MARKERS = [ const UTF8_BOM = [0xef, 0xbb, 0xbf]; const PLAYLIST_MARKER_BYTES = 64; -export function looksLikePlaylist(bytes: Uint8Array): boolean { - const tagsEnd = id3TagsEnd(bytes); - if (tagsEnd > 0) { - return tagsEnd < bytes.length && looksLikePlaylist(bytes.subarray(tagsEnd)); - } +function leadingText(bytes: Uint8Array) { let start = startsWith(bytes, UTF8_BOM) ? UTF8_BOM.length : 0; while ( start < bytes.length && @@ -140,10 +136,23 @@ export function looksLikePlaylist(bytes: Uint8Array): boolean { for (let index = start; index < end; index++) { text += String.fromCharCode(bytes[index] ?? 0); } - const lowered = text.toLowerCase(); + return text.toLowerCase(); +} + +export function looksLikePlaylist(bytes: Uint8Array): boolean { + const tagsEnd = id3TagsEnd(bytes); + if (tagsEnd > 0) { + return tagsEnd < bytes.length && looksLikePlaylist(bytes.subarray(tagsEnd)); + } + const lowered = leadingText(bytes); return PLAYLIST_MARKERS.some((marker) => lowered.startsWith(marker)); } +export function looksLikeHtml(bytes: Uint8Array): boolean { + const lowered = leadingText(bytes); + return lowered.startsWith(" 0 && sniffMedia(source.head) !== "media") { await source.cancel(); return stop( - failure("unsupported_format", FAILURE_MESSAGES.unsupported_format), + failure( + "unsupported_format", + looksLikeHtml(source.head) + ? FAILURE_MESSAGES.web_page + : FAILURE_MESSAGES.unsupported_format, + ), ); } const contentType =