diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts new file mode 100644 index 00000000000..2a2b75aa112 --- /dev/null +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -0,0 +1,944 @@ +import { randomUUID } from "node:crypto"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +type FakeIntent = { + id: string; + status: "succeeded"; + amount: number; + amount_received: number; + currency: string; + metadata: Record; +}; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + keys: new Map< + string, + | { state: "pending"; params: string } + | { state: "done"; params: string; intent: FakeIntent } + >(), + created: [] as FakeIntent[], + createdKeys: [] as string[], + sent: [] as Record[], + duringCreate: null as null | (() => Promise), + loseNextResponse: false, + failNextCall: false, + duringSearch: null as null | (() => Promise), + delayMs: 40, + calledAt: [] as number[], + stored: new Map(), + retrieved: [] as string[], +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ WEB_URL: "https://web" }), +})); +vi.mock("@/lib/developer-credits-product", () => ({ + developerCreditsProductId: () => "prod_test", +})); +vi.mock("@/lib/infra-billing-emails", () => ({ + sendInfraTopUpFailedEmail: vi.fn(), +})); +vi.mock("@cap/utils", () => ({ + stripe: () => ({ + paymentIntents: { + search: async () => { + const during = fixture.duringSearch; + fixture.duringSearch = null; + await during?.(); + return { data: [] }; + }, + retrieve: async (id: string) => { + fixture.retrieved.push(id); + const intent = fixture.stored.get(id); + if (!intent) throw new Error(`No such payment_intent: ${id}`); + return intent; + }, + create: async ( + params: { amount: number; metadata: Record }, + options: { idempotencyKey: string }, + ) => { + const key = options.idempotencyKey; + if (fixture.failNextCall) { + fixture.failNextCall = false; + throw Object.assign(new Error("socket hang up"), { + type: "StripeConnectionError", + }); + } + const entry = fixture.keys.get(key); + const sent = JSON.stringify(params); + if (entry && entry.params !== sent) { + throw Object.assign(new Error("Keys must match parameters"), { + type: "StripeIdempotencyError", + statusCode: 400, + }); + } + if (entry?.state === "pending") { + throw Object.assign(new Error("Key in use"), { + type: "StripeInvalidRequestError", + code: "idempotency_key_in_use", + statusCode: 409, + }); + } + if (entry?.state === "done") return entry.intent; + fixture.keys.set(key, { state: "pending", params: sent }); + fixture.sent.push(params as unknown as Record); + const during = fixture.duringCreate; + fixture.duringCreate = null; + await during?.(); + fixture.calledAt.push(Date.now()); + await new Promise((resolve) => setTimeout(resolve, fixture.delayMs)); + const intent: FakeIntent = { + id: `pi_${fixture.created.length + 1}`, + status: "succeeded", + amount: params.amount, + amount_received: params.amount, + currency: "usd", + metadata: params.metadata, + }; + fixture.created.push(intent); + fixture.createdKeys.push(key); + fixture.keys.set(key, { state: "done", params: sent, intent }); + if (fixture.loseNextResponse) { + fixture.loseNextResponse = false; + throw Object.assign(new Error("Request aborted due to timeout"), { + type: "StripeConnectionError", + }); + } + return intent; + }, + }, + }), +})); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); + +describe.runIf(Boolean(databaseUrl))( + "Cap Infra auto top-up against a real database", + () => { + let pool: Pool; + const accounts: string[] = []; + const owners: string[] = []; + + const makeOwner = async () => { + const userId = id(); + owners.push(userId); + await pool.query( + "INSERT INTO users (id, email, activeOrganizationId) VALUES (?, ?, '')", + [userId, `${userId}@example.com`], + ); + return userId; + }; + + const makeAccount = async ( + balanceMicroCredits: number, + { + ownerId, + amountCents = 500, + }: { ownerId?: string; amountCents?: number } = {}, + ) => { + const userId = ownerId ?? (await makeOwner()); + const appId = id(); + const accountId = id(); + accounts.push(accountId); + await pool.query( + "INSERT INTO developer_apps (id, ownerId, name, environment) VALUES (?, ?, 'Top-up test', 'production')", + [appId, userId], + ); + await pool.query( + `INSERT INTO developer_credit_accounts + (id, appId, ownerId, balanceMicroCredits, stripeCustomerId, stripePaymentMethodId, + autoTopUpEnabled, autoTopUpThresholdMicroCredits, autoTopUpAmountCents) + VALUES (?, ?, ?, ?, 'cus_test', 'pm_test', 1, 500000, ?)`, + [accountId, appId, userId, balanceMicroCredits, amountCents], + ); + return { accountId, appId, ownerId: userId }; + }; + + const balanceOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT balanceMicroCredits AS balance FROM developer_credit_accounts WHERE id = ?", + [accountId], + ); + return Number((rows as { balance: number }[])[0]?.balance); + }; + + const topUpsOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT referenceId FROM developer_credit_transactions WHERE accountId = ? AND referenceType = 'auto_top_up' AND type = 'topup'", + [accountId], + ); + return (rows as { referenceId: string }[]).map((row) => row.referenceId); + }; + + const setBalance = (accountId: string, balanceMicroCredits: number) => + pool.query( + "UPDATE developer_credit_accounts SET balanceMicroCredits = ? WHERE id = ?", + [balanceMicroCredits, accountId], + ); + + beforeAll(() => { + pool = createPool({ uri: databaseUrl, connectionLimit: 12 }); + fixture.database = drizzle(pool, { + mode: "default", + }) as MySql2Database; + }); + + afterAll(async () => { + for (const accountId of accounts) { + await pool.query( + "DELETE FROM developer_credit_transactions WHERE accountId = ?", + [accountId], + ); + await pool.query( + "DELETE a, c FROM developer_credit_accounts c JOIN developer_apps a ON a.id = c.appId WHERE c.id = ?", + [accountId], + ); + } + for (const ownerId of owners) { + await pool.query("DELETE FROM users WHERE id = ?", [ownerId]); + } + await pool.end(); + }); + + it("creates one charge when several triggers race for one account", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + const early = Array.from({ length: 6 }, () => runAutoTopUp(accountId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + const late = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + const results = await Promise.all([...early, ...late]); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + expect( + results.filter((result) => result.state === "succeeded").length, + ).toBeGreaterThanOrEqual(1); + for (const result of results) { + expect(["succeeded", "skipped"]).toContain(result.state); + } + }); + + it("creates one charge when the amount changes while triggers race", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + const first = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + await new Promise((resolve) => setTimeout(resolve, 10)); + await pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = 2500 WHERE id = ?", + [accountId], + ); + const changed = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + await Promise.all([...first, ...changed]); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + }); + + it("makes no second payment when the amount changes after a lost response", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + expect(await topUpsOf(accountId)).toHaveLength(0); + + await pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = 2500 WHERE id = ?", + [accountId], + ); + expect(await runAutoTopUp(accountId)).toMatchObject({ + state: "succeeded", + }); + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + + await setBalance(accountId, 10); + expect(await runAutoTopUp(accountId)).toMatchObject({ + state: "succeeded", + }); + expect( + fixture.created.slice(before).map((intent) => intent.amount), + ).toEqual([500, 2500]); + errors.mockRestore(); + }); + + const addRow = ( + accountId: string, + type: string, + referenceType: string, + referenceId: string, + hoursAgo: number, + ) => + pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, ?, 0, 30000, ?, ?, '{}', UTC_TIMESTAMP() - INTERVAL ? HOUR)`, + [id(), accountId, type, referenceId, referenceType, hoursAgo], + ); + + it("makes the locked check wait on the same rules the sweep reads", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const declined = await makeAccount(30_000); + await addRow( + declined.accountId, + "topup_failed", + "auto_top_up_failed", + `declined-${declined.accountId}`, + 2, + ); + const young = await makeAccount(30_000); + await addRow( + young.accountId, + "topup_pending", + "auto_top_up_pending", + `pi_young_${young.accountId}`, + 25, + ); + const old = await makeAccount(30_000); + const oldPayment = `pi_old_${old.accountId}`; + await addRow( + old.accountId, + "topup_pending", + "auto_top_up_pending", + oldPayment, + 73, + ); + fixture.stored.set(oldPayment, { + id: oldPayment, + status: "succeeded", + amount: 500, + amount_received: 500, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: old.accountId }, + }); + const before = fixture.created.length; + + expect(await runAutoTopUp(declined.accountId)).toEqual({ + state: "skipped", + reason: "recent_decline", + }); + expect(await runAutoTopUp(young.accountId)).toEqual({ + state: "skipped", + reason: "awaiting_payment", + }); + expect(await runAutoTopUp(old.accountId)).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(await topUpsOf(old.accountId)).toEqual([oldPayment]); + expect(fixture.created.length).toBe(before); + }); + + it("lets the sweep reconcile a payment pending for over 72 hours, and only that", async () => { + const { runDueAutoTopUps } = await import("@/lib/infra-auto-top-up"); + + const stale = await makeAccount(30_000); + const stalePayment = `pi_stale_${stale.accountId}`; + await addRow( + stale.accountId, + "topup_pending", + "auto_top_up_pending", + stalePayment, + 73, + ); + fixture.stored.set(stalePayment, { + id: stalePayment, + status: "succeeded", + amount: 500, + amount_received: 500, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: stale.accountId }, + }); + const young = await makeAccount(30_000); + await addRow( + young.accountId, + "topup_pending", + "auto_top_up_pending", + `pi_young_${young.accountId}`, + 25, + ); + const declined = await makeAccount(30_000); + await addRow( + declined.accountId, + "topup_failed", + "auto_top_up_failed", + `declined-${declined.accountId}`, + 2, + ); + const free = await makeAccount(30_000); + const before = fixture.created.length; + + await runDueAutoTopUps(new Date()); + + expect(fixture.retrieved).not.toContain(`pi_young_${young.accountId}`); + expect(fixture.retrieved.at(-1)).toBe(stalePayment); + expect(await topUpsOf(stale.accountId)).toEqual([stalePayment]); + expect(await topUpsOf(young.accountId)).toEqual([]); + expect(await topUpsOf(declined.accountId)).toEqual([]); + expect(await topUpsOf(free.accountId)).toHaveLength(1); + const charged = fixture.created + .slice(before) + .map((intent) => intent.metadata.accountId); + expect(charged).toContain(free.accountId); + expect(charged).not.toContain(stale.accountId); + expect(charged).not.toContain(young.accountId); + expect(charged).not.toContain(declined.accountId); + }); + + it("stops an owner's automatic top-ups at $1,000 a day across all their apps", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + expect((await runAutoTopUp(first.accountId)).state).toBe("succeeded"); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + expect(fixture.created.length - before).toBe(1); + const [[enabled]] = (await pool.query( + "SELECT autoTopUpEnabled AS on_ FROM developer_credit_accounts WHERE id = ?", + [second.accountId], + )) as unknown as [[{ on_: number }]]; + expect(enabled.on_).toBe(1); + expect(warn).toHaveBeenCalledWith( + "[infra-billing] Auto top-up waits for its daily limit", + { accountId: second.accountId, reason: "owner_daily_limit" }, + ); + warn.mockRestore(); + }); + + it("lets two apps of one owner racing near the ceiling make one charge", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + const results = await Promise.all([ + runAutoTopUp(first.accountId), + runAutoTopUp(second.accountId), + ]); + + expect(fixture.created.length - before).toBe(1); + expect(results.map((result) => result.state).sort()).toEqual([ + "skipped", + "succeeded", + ]); + expect(results).toContainEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + warn.mockRestore(); + }); + + it("leaves credit bought by hand out of the owner's ceiling", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const ownerId = await makeOwner(); + const bought = await makeAccount(30_000, { ownerId }); + const automatic = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 90000000, 90030000, ?, 'stripe_payment_intent', '{}')`, + [id(), bought.accountId, `pi_manual_${bought.accountId}`], + ); + + expect((await runAutoTopUp(automatic.accountId)).state).toBe("succeeded"); + }); + + it("holds the owner lock only while reserving, not through a slow Stripe call", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { lockInfraAccount } = await import("@/lib/infra-balance"); + const ownerId = await makeOwner(); + const slow = await makeAccount(30_000, { ownerId }); + const other = await makeAccount(30_000, { ownerId }); + fixture.delayMs = 2_000; + try { + const startedAt = Date.now(); + const before = fixture.calledAt.length; + const topUp = runAutoTopUp(slow.accountId); + while (fixture.calledAt.length === before) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + const lockHeldMs = (fixture.calledAt.at(-1) ?? 0) - startedAt; + + const waitStarted = Date.now(); + await (fixture.database as MySql2Database).transaction(async (tx) => { + await lockInfraAccount(tx, other.appId); + }); + const otherAppWaitedMs = Date.now() - waitStarted; + + expect((await topUp).state).toBe("succeeded"); + const stripeCallMs = Date.now() - (fixture.calledAt.at(-1) ?? 0); + console.log( + JSON.stringify({ lockHeldMs, otherAppWaitedMs, stripeCallMs }), + ); + expect(stripeCallMs).toBeGreaterThanOrEqual(1_900); + expect(otherAppWaitedMs).toBeLessThan(200); + } finally { + fixture.delayMs = 40; + } + }); + + it("recovers a reservation left by a crash before Stripe was called, with one charge", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const { accountId } = await makeAccount(30_000); + const key = autoTopUpIdempotencyKey(accountId, { + topUps: 0, + declines: 0, + }); + const [[owner]] = (await pool.query( + "SELECT u.email FROM users u JOIN developer_credit_accounts c ON c.ownerId = u.id WHERE c.id = ?", + [accountId], + )) as unknown as [[{ email: string }]]; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', ?)`, + [ + id(), + accountId, + key, + JSON.stringify({ + amountCents: 500, + customerId: "cus_test", + paymentMethodId: "pm_test", + receiptEmail: owner.email, + description: "Cap Infra credits for Top-up test, auto top-up", + period: "2026-11-13", + }), + ], + ); + const before = fixture.createdKeys.length; + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect((await runAutoTopUp(accountId)).state).toBe("skipped"); + + expect(fixture.createdKeys.slice(before)).toEqual([key]); + expect(await topUpsOf(accountId)).toHaveLength(1); + const [rows] = await pool.query( + "SELECT COUNT(*) AS n FROM developer_credit_transactions WHERE accountId = ? AND type = 'topup_pending'", + [accountId], + ); + expect(Number((rows as { n: number }[])[0]?.n)).toBe(0); + }); + + it("recovers a reservation whose payment Stripe made before a crash, with one charge", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + expect(await topUpsOf(accountId)).toHaveLength(0); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + errors.mockRestore(); + }); + + it("checks and charges one amount, read under the lock, when the setting changes mid-attempt", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const app = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const other = await makeAccount(30_000, { ownerId }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 50000000, 50030000, ?, 'auto_top_up', '{}')`, + [id(), other.accountId, `pi_today_${other.accountId}`], + ); + const setAmount = (cents: number) => + pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = ? WHERE id = ?", + [cents, app.accountId], + ); + + expect(await runAutoTopUp(app.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + await setAmount(500); + fixture.duringCreate = () => setAmount(60_000).then(() => undefined); + expect((await runAutoTopUp(app.accountId)).state).toBe("succeeded"); + + expect(fixture.sent.at(-1)?.amount).toBe(500); + const [[total]] = (await pool.query( + `SELECT SUM(t.amountMicroCredits) AS micro FROM developer_credit_transactions t + JOIN developer_credit_accounts c ON c.id = t.accountId + WHERE c.ownerId = ? AND t.type = 'topup' AND t.referenceType = 'auto_top_up'`, + [ownerId], + )) as unknown as [[{ micro: number }]]; + expect(Number(total.micro)).toBe(50_500_000); + warn.mockRestore(); + }); + + it("retries a $600 attempt whose Stripe call failed, once, while it still holds back the owner's other app", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + fixture.failNextCall = true; + expect(await runAutoTopUp(first.accountId)).toEqual({ state: "error" }); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + expect(await runAutoTopUp(first.accountId)).toMatchObject({ + state: "succeeded", + }); + expect(fixture.created.length - before).toBe(1); + expect(fixture.created.at(-1)?.amount).toBe(60_000); + expect(await topUpsOf(first.accountId)).toHaveLength(1); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + errors.mockRestore(); + warn.mockRestore(); + }); + + it("lets the sweep retry a $600 reservation whose only blocker is itself", async () => { + const { runAutoTopUp, runDueAutoTopUps } = await import( + "@/lib/infra-auto-top-up" + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000, { amountCents: 60_000 }); + const before = fixture.created.length; + + fixture.failNextCall = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + await runDueAutoTopUps(new Date()); + + expect(await topUpsOf(accountId)).toHaveLength(1); + expect( + fixture.created + .slice(before) + .filter((intent) => intent.metadata.accountId === accountId), + ).toHaveLength(1); + errors.mockRestore(); + }); + + it("voids only the stale reservation, never a new one made under the same key", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const { accountId } = await makeAccount(30_000); + const key = autoTopUpIdempotencyKey(accountId, { + topUps: 0, + declines: 0, + }); + const reserve = (rowId: string, hoursAgo: number) => + pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":500}', UTC_TIMESTAMP() - INTERVAL ? HOUR)`, + [rowId, accountId, key, hoursAgo], + ); + const stale = id(); + const fresh = id(); + await reserve(stale, 73); + fixture.duringSearch = () => reserve(fresh, 0).then(() => undefined); + + expect(await runAutoTopUp(accountId)).toEqual({ + state: "skipped", + reason: "reconciled", + }); + + const [rows] = await pool.query( + "SELECT id FROM developer_credit_transactions WHERE accountId = ? AND referenceId = ?", + [accountId, key], + ); + expect((rows as { id: string }[]).map((row) => row.id)).toEqual([fresh]); + }); + + it("lets the sweep void a deleted app's lost $600 attempt, then tops up the owner's other app and never charges the deleted one", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp, runDueAutoTopUps } = + await import("@/lib/infra-auto-top-up"); + const ownerId = await makeOwner(); + const deleted = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const other = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const key = autoTopUpIdempotencyKey(deleted.accountId, { + topUps: 0, + declines: 0, + }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":60000}', UTC_TIMESTAMP() - INTERVAL 73 HOUR)`, + [id(), deleted.accountId, key], + ); + await pool.query( + "UPDATE developer_apps SET deletedAt = UTC_TIMESTAMP() WHERE id = ?", + [deleted.appId], + ); + const before = fixture.created.length; + + expect(await runAutoTopUp(other.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + let searched = 0; + fixture.duringSearch = async () => { + searched += 1; + }; + await runDueAutoTopUps(new Date()); + + expect(searched).toBe(1); + const [rows] = await pool.query( + "SELECT id FROM developer_credit_transactions WHERE accountId = ?", + [deleted.accountId], + ); + expect(rows).toEqual([]); + + await runDueAutoTopUps(new Date()); + + const charged = fixture.created.slice(before); + expect(charged.map((intent) => intent.metadata.accountId)).toEqual([ + other.accountId, + ]); + expect(charged[0]?.amount).toBe(60_000); + expect(await topUpsOf(other.accountId)).toHaveLength(1); + + await setBalance(deleted.accountId, 30_000); + expect(await runAutoTopUp(deleted.accountId)).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + expect( + fixture.created.filter( + (intent) => intent.metadata.accountId === deleted.accountId, + ), + ).toEqual([]); + }); + + it("counts an unresolved payment of any age against the owner's ceiling until it resolves", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const processing = `pi_processing_${first.accountId}`; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":60000}', UTC_TIMESTAMP() - INTERVAL 25 HOUR)`, + [id(), first.accountId, processing], + ); + const before = fixture.created.length; + + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(fixture.created.length).toBe(before); + + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_failed', 0, 30000, ?, 'auto_top_up_failed', '{}')`, + [id(), first.accountId, processing], + ); + expect((await runAutoTopUp(second.accountId)).state).toBe("succeeded"); + expect(fixture.created.length - before).toBe(1); + warn.mockRestore(); + }); + + it("keeps counting a charge whose answer was lost against the owner's ceiling", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(first.accountId)).toEqual({ state: "error" }); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(second.accountId)).toEqual([]); + errors.mockRestore(); + warn.mockRestore(); + }); + + it("lets the sweep reconcile a 72 hour old payment even when every limit applies", async () => { + const { runDueAutoTopUps } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const stale = await makeAccount(30_000, { + ownerId, + amountCents: 100_000, + }); + const other = await makeAccount(30_000, { ownerId }); + const stalePayment = `pi_stale1000_${stale.accountId}`; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":100000}', UTC_TIMESTAMP() - INTERVAL 73 HOUR)`, + [id(), stale.accountId, stalePayment], + ); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 500000, 530000, ?, 'auto_top_up', '{}')`, + [id(), other.accountId, `pi_today_${other.accountId}`], + ); + fixture.stored.set(stalePayment, { + id: stalePayment, + status: "succeeded", + amount: 100_000, + amount_received: 100_000, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: stale.accountId }, + }); + await pool.query( + "UPDATE developer_credit_accounts SET balanceMicroCredits = 99000000, autoTopUpEnabled = 0 WHERE id = ?", + [stale.accountId], + ); + + await runDueAutoTopUps(new Date()); + + expect(await topUpsOf(stale.accountId)).toEqual([stalePayment]); + expect(await balanceOf(stale.accountId)).toBe(99_000_000 + 100_000_000); + warn.mockRestore(); + }); + + it("never shows a pending row in the customer's ledger", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { listAppLedgerPage, readAppLedger } = await import( + "@/lib/infra-ledger-store" + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId, appId } = await makeAccount(30_000); + fixture.loseNextResponse = true; + await runAutoTopUp(accountId); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_pending', 0, 30000, 'pi_processing', 'auto_top_up_pending', '{}'), + (?, ?, 'storage_daily', -100, 29900, '2026-11-12', 'storage_day', '{}')`, + [id(), accountId, id(), accountId], + ); + + const page = await listAppLedgerPage(appId, { limit: 50 }); + const exported = []; + for await (const rows of readAppLedger(appId)) exported.push(...rows); + + expect(page.rows.map((row) => row.type)).toEqual(["storage_daily"]); + expect(exported.map((row) => row.type)).toEqual(["storage_daily"]); + errors.mockRestore(); + }); + + it("names the app in the charge description", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect(fixture.sent.at(-1)?.description).toBe( + "Cap Infra credits for Top-up test, auto top-up", + ); + }); + + it("tops up again the same day with a new key once the balance is low again", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect((await runAutoTopUp(accountId)).state).toBe("skipped"); + await setBalance(accountId, 10); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + + const keys = fixture.createdKeys.slice(before); + expect(keys).toHaveLength(2); + expect(new Set(keys).size).toBe(2); + expect(await topUpsOf(accountId)).toHaveLength(2); + }); + + it("stops at the daily limit of automatic top-ups", async () => { + const { AUTO_TOP_UP_DAILY_LIMIT, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + + for (let i = 0; i < AUTO_TOP_UP_DAILY_LIMIT; i++) { + await setBalance(accountId, 10); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + } + await setBalance(accountId, 10); + expect(await runAutoTopUp(accountId)).toEqual({ + state: "skipped", + reason: "daily_limit", + }); + expect(await topUpsOf(accountId)).toHaveLength(AUTO_TOP_UP_DAILY_LIMIT); + warn.mockRestore(); + }); + }, +); diff --git a/apps/web/__tests__/unit/developer-billing-docs.test.ts b/apps/web/__tests__/unit/developer-billing-docs.test.ts index d9ce25b1e9f..e028d5cfaf7 100644 --- a/apps/web/__tests__/unit/developer-billing-docs.test.ts +++ b/apps/web/__tests__/unit/developer-billing-docs.test.ts @@ -43,10 +43,12 @@ import { } from "@/lib/developer-transcript-hold"; import { TRANSCRIPT_MIN_START_BALANCE_MICRO_CREDITS } from "@/lib/developer-transcripts"; import { + AUTO_TOP_UP_DAILY_LIMIT, AUTO_TOP_UP_MAX_AMOUNT_CENTS, AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS, AUTO_TOP_UP_MIN_AMOUNT_CENTS, AUTO_TOP_UP_MIN_THRESHOLD_MICRO_CREDITS, + AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS, } from "@/lib/infra-auto-top-up"; import { LEDGER_PAGE_SIZE } from "@/lib/infra-ledger"; import { @@ -278,6 +280,12 @@ describe("the Billing page", () => { expect(billing).toContain( `${INFRA_GRACE_DAYS} days after the app first fell short`, ); + expect(billing).toContain( + "- **When.** As soon as the available balance falls below your threshold.", + ); + expect(billing).toContain( + `up to ${AUTO_TOP_UP_DAILY_LIMIT} top-ups in any 24 hours, and at most ${wholeDollars(AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS)} a day across all your apps.`, + ); expect(billing).toContain(`${LEDGER_PAGE_SIZE} to a page`); expect(billing).toContain( `reads ${formatCredits(2_999_901)} and never $30.00`, diff --git a/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts b/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts index 9fe21d07388..b9f3153ffb4 100644 --- a/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts +++ b/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts @@ -31,6 +31,11 @@ const mocks = vi.hoisted(() => ({ claim: vi.fn(), release: vi.fn(), startProcessing: vi.fn(), + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: mocks.scheduleAutoTopUp, })); vi.mock("@/lib/developer-video-start", () => ({ @@ -286,6 +291,7 @@ const refused = { describe("developer SDK multipart completion", () => { beforeEach(() => { + mocks.scheduleAutoTopUp.mockReset(); mocks.balance = 10_000_000; mocks.accountExists = true; mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 1_000 }]; @@ -339,6 +345,13 @@ describe("developer SDK multipart completion", () => { mocks.unrecorded = []; }); + it("asks for no top-up when an upload is admitted", async () => { + mocks.admission.mockResolvedValue(admitted); + + expect((await complete({ durationInSecs: 60 })).status).toBe(200); + expect(mocks.scheduleAutoTopUp).not.toHaveBeenCalled(); + }); + it("charges nothing when an upload completes", async () => { mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 10_000_000 }]; @@ -443,6 +456,7 @@ describe("developer SDK multipart completion", () => { expect(mocks.startProcessing).not.toHaveBeenCalled(); expect(mocks.videoUpdates).toEqual([]); expect(mocks.storedDuration).toBe(null); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("keeps the file when the upload is refused before storage assembles it", async () => { @@ -454,6 +468,7 @@ describe("developer SDK multipart completion", () => { await expect(response.json()).resolves.not.toHaveProperty("fileRemoved"); expect(mocks.complete).not.toHaveBeenCalled(); expect(mocks.deleteObject).not.toHaveBeenCalled(); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("still answers 402 when the refused file cannot be removed", async () => { diff --git a/apps/web/__tests__/unit/developer-transcript-settle.test.ts b/apps/web/__tests__/unit/developer-transcript-settle.test.ts index 5c8ebf0340b..5fad3070f58 100644 --- a/apps/web/__tests__/unit/developer-transcript-settle.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-settle.test.ts @@ -12,6 +12,11 @@ const context = vi.hoisted(() => ({ reset: () => void; }, ledgerIds: 0, + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: context.scheduleAutoTopUp, })); vi.mock("@cap/database", async () => { @@ -99,6 +104,7 @@ const settle = (durationSeconds = ONE_HOUR, transcriptId = "tr-1") => beforeEach(() => { fake().reset(); context.ledgerIds = 0; + context.scheduleAutoTopUp.mockReset(); fake().seed(developerCreditAccounts, { id: "account-1", appId: "app-1", @@ -403,6 +409,48 @@ describe("transcript settlement", () => { }); }); +describe("auto top-up after a transcript charge or hold", () => { + const scheduled = () => context.scheduleAutoTopUp.mock.calls; + + it("asks for a top-up once after a settle that charged, not on a retry", async () => { + await settle(); + await settle(); + + expect(scheduled()).toEqual([[{ appId: "app-1" }]]); + }); + + it("asks for a top-up when the balance could not cover the transcript", async () => { + const current = account(); + if (current) current.balanceMicroCredits = 9_999; + + await settle(); + + expect(scheduled()).toEqual([[{ appId: "app-1" }]]); + }); + + it("asks for a top-up after a hold or a reservation, and not for skipped work", async () => { + addTranscript("tr-2"); + await holdTranscriptCredits({ transcriptId: "tr-1", microCredits: 1_000 }); + await reserveTranscriptCredits({ + transcriptId: "tr-2", + durationSeconds: ONE_HOUR, + }); + expect(scheduled()).toEqual([[{ appId: "app-1" }], [{ appId: "app-1" }]]); + + addTranscript("tr-failed", { status: "error" }); + await holdTranscriptCredits({ + transcriptId: "tr-failed", + microCredits: 1_000, + }); + await reserveTranscriptCredits({ + transcriptId: "tr-failed", + durationSeconds: ONE_HOUR, + }); + await settle(ONE_HOUR, "tr-failed"); + expect(scheduled()).toHaveLength(2); + }); +}); + describe("transcript holds", () => { const reserve = (transcriptId: string, durationSeconds: number) => reserveTranscriptCredits({ transcriptId, durationSeconds }); diff --git a/apps/web/__tests__/unit/developer-transcript-store.test.ts b/apps/web/__tests__/unit/developer-transcript-store.test.ts index b7244ea2fc4..a84af17354f 100644 --- a/apps/web/__tests__/unit/developer-transcript-store.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-store.test.ts @@ -19,6 +19,11 @@ const context = vi.hoisted(() => ({ presigned: [] as { key: string; contentLength: number }[], failSigning: false, onDelete: null as null | (() => Promise), + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: context.scheduleAutoTopUp, })); vi.mock("@cap/database", async () => { @@ -163,6 +168,7 @@ const fromUrl = (url = URL_A, idempotencyKey?: string, appId = "app-1") => beforeEach(() => { fake().reset(); context.ids = 0; + context.scheduleAutoTopUp.mockReset(); context.speechConfigured = true; context.hosted = true; context.start.mockReset(); @@ -305,6 +311,9 @@ describe("creating a transcript from a URL", () => { }); expect(transcripts()).toHaveLength(0); expect(context.start).not.toHaveBeenCalled(); + expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({ + appId: "app-1", + }); }); it("says how much to add below the minimum balance", async () => { @@ -953,12 +962,16 @@ describe("starting an uploaded transcript", () => { const id = await created(); context.objectSize = 1024; setBalance(0); + context.scheduleAutoTopUp.mockReset(); expect(await startTranscriptUpload("app-1", id)).toMatchObject({ ok: false, status: 402, }); expect(context.start).not.toHaveBeenCalled(); + expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({ + appId: "app-1", + }); }); it("can be retried after the job could not be started", async () => { diff --git a/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts index c92274936df..551c6d50d96 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts @@ -1,9 +1,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -type Account = { id: string; lastAttemptAt: Date | null }; +type Account = { id: string; lastAttemptAt: Date | null; waiting?: boolean }; const state = vi.hoisted(() => ({ - accounts: [] as { id: string; lastAttemptAt: Date | null }[], + accounts: [] as { + id: string; + lastAttemptAt: Date | null; + waiting?: boolean; + }[], pages: [] as { sql: string; params: unknown[]; @@ -25,21 +29,14 @@ vi.mock("@cap/database", async () => { const client = drizzle(async (sql, params) => { if (sql.startsWith("select `developer_credit_accounts`.`id`, cast(")) { const limit = Number(params.at(-1)); - const keyset = sql.includes(" or ("); + const keyset = sql.includes("and `developer_credit_accounts`.`id` > ?))"); const after = keyset ? { key: String(params.at(-4)), id: String(params.at(-2)) } : null; - const cutoff = String( - params.find( - (param) => - typeof param === "string" && - /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/.test(param), - ), - ); state.pages.push({ sql, params, after, limit }); const rows = state.accounts + .filter((account) => !account.waiting) .map((account) => ({ id: account.id, key: keyOf(account) })) - .filter((row) => row.key <= cutoff) .filter( (row) => !after || @@ -68,7 +65,7 @@ vi.mock("@cap/database", async () => { .map((account) => [account.id]), }; } - if (sql.includes("`autoTopUpAmountCents`")) { + if (sql.includes("`developer_apps`.`name`")) { state.reads.push(String(params[0])); vi.setSystemTime(Date.now() + state.msPerAccount); return { rows: [] }; @@ -92,7 +89,7 @@ vi.mock("@/lib/infra-billing-emails", () => ({ })); import { - autoTopUpRetryCutoff, + AUTO_TOP_UP_DAILY_LIMIT, getAutoTopUpFailure, runDueAutoTopUps, } from "@/lib/infra-auto-top-up"; @@ -104,12 +101,17 @@ const accounts = ( count: number, from = 0, lastAttemptAt: Date | null = null, + waiting = false, ): Account[] => Array.from({ length: count }, (_, index) => ({ id: `account-${String(from + index).padStart(4, "0")}`, lastAttemptAt, + waiting, })); +const sqlTime = (date: Date) => + date.toISOString().slice(0, 23).replace("T", " "); + beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(NOW); @@ -136,7 +138,7 @@ describe("auto top-up sweep selection", () => { expect(state.reads).toHaveLength(450); }); - it("asks the database for enabled accounts under their threshold after held credit, not tried since the retry cutoff", async () => { + it("asks the database for enabled accounts under their threshold after held credit that nothing makes wait", async () => { state.accounts = []; await runDueAutoTopUps(NOW); @@ -154,14 +156,27 @@ describe("auto top-up sweep selection", () => { expect(sql).toContain( "(select max(`developer_credit_transactions`.`createdAt`) from `developer_credit_transactions` where `developer_credit_transactions`.`accountId` = `developer_credit_accounts`.`id` and `developer_credit_transactions`.`type` in (?, ?, ?) and `developer_credit_transactions`.`referenceType` in (?, ?, ?))", ); + expect(sql).toContain( + "not (exists (select 1 from `developer_credit_transactions` `pending_attempt` where `pending_attempt`.`accountId` = `developer_credit_accounts`.`id` and `pending_attempt`.`type` = ? and `pending_attempt`.`referenceType` = ? and `pending_attempt`.`createdAt` > ? and `pending_attempt`.`referenceId` not like ? and not exists (select 1 from `developer_credit_transactions` `settled_attempt` where `settled_attempt`.`accountId` = `pending_attempt`.`accountId` and `settled_attempt`.`referenceId` = `pending_attempt`.`referenceId`", + ); + expect(sql).toContain( + "exists (select 1 from `developer_credit_transactions` `declined_attempt` where `declined_attempt`.`accountId` = `developer_credit_accounts`.`id` and `declined_attempt`.`type` = ? and `declined_attempt`.`referenceType` = ? and `declined_attempt`.`createdAt` > ? and not exists (select 1 from `developer_credit_transactions` `later_top_up` where `later_top_up`.`accountId` = `declined_attempt`.`accountId` and `later_top_up`.`type` = ?", + ); + expect(sql).toContain( + "(select count(*) from `developer_credit_transactions` `recent_top_up` where `recent_top_up`.`accountId` = `developer_credit_accounts`.`id` and `recent_top_up`.`type` = ? and `recent_top_up`.`referenceType` = ? and `recent_top_up`.`createdAt` > ?) >= ?", + ); expect(sql).toMatch( /order by attemptKey, `developer_credit_accounts`\.`id` asc limit \?$/, ); - expect(params).toContain( - autoTopUpRetryCutoff(NOW).toISOString().slice(0, 23).replace("T", " "), + expect(params).toContain(sqlTime(new Date(NOW.getTime() - 72 * HOUR_MS))); + expect(params).toContain(sqlTime(new Date(NOW.getTime() - 24 * HOUR_MS))); + expect(params).toContain(AUTO_TOP_UP_DAILY_LIMIT); + expect(params).toContain("infra-auto-top-up:%"); + expect(sql).toContain( + "or exists (select 1 from `developer_credit_transactions` `stale_payment` where `stale_payment`.`accountId` = `developer_credit_accounts`.`id` and `stale_payment`.`type` = ? and `stale_payment`.`referenceType` = ? and `stale_payment`.`createdAt` <= ?", ); - expect(autoTopUpRetryCutoff(NOW)).toEqual( - new Date(NOW.getTime() - 24 * HOUR_MS), + expect(sql).toContain( + "from `developer_credit_accounts` `owner_account` join `developer_credit_transactions` `owner_entry` on `owner_entry`.`accountId` = `owner_account`.`id` where `owner_account`.`ownerId` = `developer_credit_accounts`.`ownerId`", ); }); @@ -190,11 +205,12 @@ describe("auto top-up sweep selection", () => { "account-0005", "account-0003", "account-0001", + "account-0004", ]); }); - it("reaches 50 due accounts in one run behind 300 accounts tried inside the retry window", async () => { - const tried = accounts(300, 0, new Date(NOW.getTime() - HOUR_MS)); + it("reaches 50 due accounts in one run behind 300 accounts that must wait", async () => { + const tried = accounts(300, 0, new Date(NOW.getTime() - HOUR_MS), true); const due = accounts(50, 300); state.accounts = [...tried, ...due]; state.msPerAccount = 250; diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index c4f416818db..73352c57394 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -5,12 +5,17 @@ const mocks = vi.hoisted(() => ({ ids: 0, emailsEnabled: false, createPaymentIntent: vi.fn(), + retrievePaymentIntent: vi.fn(), + searchPaymentIntents: vi.fn(), retrieveSetupIntent: vi.fn(), retrievePaymentMethod: vi.fn(), detachPaymentMethod: vi.fn(), sendEmail: vi.fn(), + after: vi.fn(), })); +vi.mock("next/server", () => ({ after: mocks.after })); + vi.mock("@cap/database", () => ({ db: () => mocks.db })); vi.mock("@cap/database/helpers", () => ({ nanoId: () => `row-${++mocks.ids}`, @@ -25,7 +30,11 @@ vi.mock("@cap/env", () => ({ })); vi.mock("@cap/utils", () => ({ stripe: () => ({ - paymentIntents: { create: mocks.createPaymentIntent }, + paymentIntents: { + create: mocks.createPaymentIntent, + retrieve: mocks.retrievePaymentIntent, + search: mocks.searchPaymentIntents, + }, setupIntents: { retrieve: mocks.retrieveSetupIntent }, paymentMethods: { detach: mocks.detachPaymentMethod, @@ -46,6 +55,9 @@ vi.mock("@/lib/developer-credits-product", () => ({ import type Stripe from "stripe"; import { + AUTO_TOP_UP_DAILY_LIMIT, + AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + autoTopUpDescription, autoTopUpFailureText, autoTopUpIdempotencyKey, autoTopUpSettingsError, @@ -53,6 +65,7 @@ import { runAutoTopUp, runDueAutoTopUps, saveCardFromCheckout, + scheduleAutoTopUp, settleAutoTopUpPaymentIntent, } from "@/lib/infra-auto-top-up"; import { @@ -84,6 +97,7 @@ const intent = ( type: "developer_auto_top_up", accountId: "account-1", period: "2026-11-11", + attempt: "0:0", }, last_payment_error: null, ...extra, @@ -110,6 +124,12 @@ const arm = () => { const topUps = () => ledger.rows.filter((row) => row.referenceType === "auto_top_up"); +const reservations = () => + ledger.rows.filter( + (row) => + row.referenceType === "auto_top_up_pending" && + String(row.referenceId).startsWith("infra-auto-top-up:"), + ); const failures = () => ledger.rows.filter((row) => row.referenceType === "auto_top_up_failed"); @@ -119,6 +139,9 @@ beforeEach(() => { mocks.emailsEnabled = false; mocks.createPaymentIntent.mockReset(); mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "succeeded")); + mocks.retrievePaymentIntent.mockReset(); + mocks.searchPaymentIntents.mockReset(); + mocks.searchPaymentIntents.mockResolvedValue({ data: [] }); mocks.sendEmail.mockReset(); mocks.sendEmail.mockResolvedValue({ data: { id: "email-1" }, error: null }); mocks.retrieveSetupIntent.mockReset(); @@ -181,12 +204,28 @@ describe("auto top-up settings", () => { ).toBe(null); }); - it("keys a charge by account and UTC day", () => { - expect(autoTopUpIdempotencyKey("account-1", "2026-11-11")).toBe( - "infra-auto-top-up:account-1:2026-11-11", + it("keys each charge by the account's recorded top-ups and declines, whatever the settings", () => { + expect( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 1 }), + ).toBe("infra-auto-top-up:account-1:3:1"); + expect( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 2 }), + ).not.toBe( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 1 }), ); }); + it("names the app in the charge description, within Stripe's limit", () => { + expect(autoTopUpDescription(" Acme Video ")).toBe( + "Cap Infra credits for Acme Video, auto top-up", + ); + expect(autoTopUpDescription("")).toBe("Cap Infra credits, auto top-up"); + expect(autoTopUpDescription(null)).toBe("Cap Infra credits, auto top-up"); + const long = autoTopUpDescription("x".repeat(5_000)); + expect(long.length).toBeLessThanOrEqual(1_000); + expect(long.endsWith(", auto top-up")).toBe(true); + }); + it("explains a decline in plain words", () => { expect(autoTopUpFailureText("authentication_required")).toContain("bank"); expect(autoTopUpFailureText("insufficient_funds")).toContain( @@ -242,6 +281,25 @@ describe("auto top-up charge", () => { expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); }); + it("never charges the card of a deleted app", async () => { + arm(); + ledger.account.appDeletedAt = new Date("2026-11-10T00:00:00Z"); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + + ledger.account.appDeletedAt = null; + ledger.appDeletedBeforeLock = true; + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(ledger.rows).toHaveLength(0); + }); + it("does not charge the card of an app whose payment was disputed", async () => { arm(); ledger.rows.push({ @@ -281,11 +339,16 @@ describe("auto top-up charge", () => { metadata: { type: "developer_auto_top_up", accountId: "account-1", - period: "2026-11-11", + attempt: "0:0", }, }); expect(options).toEqual({ - idempotencyKey: "infra-auto-top-up:account-1:2026-11-11", + idempotencyKey: autoTopUpIdempotencyKey("account-1", { + topUps: 0, + declines: 0, + }), + timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + maxNetworkRetries: 0, }); expect(ledger.account.balanceMicroCredits).toBe(530_000); expect(topUps()).toEqual([ @@ -323,31 +386,239 @@ describe("auto top-up charge", () => { expect(ledger.account.balanceMicroCredits).toBe(530_000); }); - it("makes at most one attempt per account in 24 hours", async () => { + it("tops up again the same day once the balance falls below the threshold again", async () => { await runAutoTopUp("account-1", now()); + later(0.1); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "above_threshold", + }); + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + + const [first, second] = mocks.createPaymentIntent.mock.calls; + expect(second?.[0].metadata.attempt).toBe("1:0"); + expect(second?.[1].idempotencyKey).not.toBe(first?.[1].idempotencyKey); + expect(topUps()).toHaveLength(2); + expect(now().toISOString().slice(0, 10)).toBe("2026-11-11"); + }); + + it("reserves the attempt before calling Stripe and removes the reservation with the grant", async () => { + mocks.createPaymentIntent.mockImplementation(async () => { + expect(reservations()).toEqual([ + expect.objectContaining({ + type: "topup_pending", + amountMicroCredits: 0, + referenceId: "infra-auto-top-up:account-1:0:0", + }), + ]); + return intent("pi_1", "succeeded"); + }); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + expect(reservations()).toHaveLength(0); + expect(ledger.rows.map((row) => row.type)).toEqual(["topup"]); + }); + + it("voids the reservation when the request never reached Stripe", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + new Error("STRIPE_SECRET_KEY is not set"), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(ledger.rows).toHaveLength(0); + errors.mockRestore(); + }); + + it("lets the webhook settle a reservation whose response was lost", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("timeout"), { type: "StripeConnectionError" }), + ); + await runAutoTopUp("account-1", now()); + expect(reservations()).toHaveLength(1); + + await settleAutoTopUpPaymentIntent( + intent("pi_1", "succeeded"), + "succeeded", + ); + + expect(reservations()).toHaveLength(0); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + errors.mockRestore(); + }); + + it("repeats the same key when retried before the attempt settles", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("network"), { type: "StripeConnectionError" }), + ); + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + + const [first, retry] = mocks.createPaymentIntent.mock.calls; + expect(retry?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); + expect(topUps()).toHaveLength(1); + }); + + it("gives up on a payment request that times out, writes nothing and retries with the same key", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign( + new Error("Request aborted due to timeout being reached (15000ms)"), + { type: "StripeConnectionError" }, + ), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(errors).toHaveBeenCalledWith( + "[infra-billing] Auto top-up could not be attempted", + expect.objectContaining({ accountId: "account-1" }), + ); + expect(ledger.rows).toEqual([ + expect.objectContaining({ + type: "topup_pending", + amountMicroCredits: 0, + referenceId: "infra-auto-top-up:account-1:0:0", + metadata: expect.objectContaining({ + amountCents: 500, + customerId: "cus_1", + paymentMethodId: "pm_1", + period: "2026-11-11", + }), + }), + ]); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + const [timedOut, retry] = mocks.createPaymentIntent.mock.calls; + expect(timedOut?.[1].timeout).toBe(15_000); + expect(retry?.[1].idempotencyKey).toBe(timedOut?.[1].idempotencyKey); + errors.mockRestore(); + }); + + it("treats a payment already being made under the same key as in progress", async () => { + mocks.createPaymentIntent.mockRejectedValue( + Object.assign(new Error("in use"), { + type: "StripeInvalidRequestError", + code: "idempotency_key_in_use", + statusCode: 409, + }), + ); - later(1); expect(await runAutoTopUp("account-1", now())).toEqual({ state: "skipped", - reason: "recent_attempt", + reason: "in_progress", }); - later(22.9); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(reservations()).toHaveLength(1); + expect(ledger.rows).toHaveLength(1); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + }); + + it("retries an unresolved attempt with the reserved amount and card, not changed settings", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("Request aborted due to timeout"), { + type: "StripeConnectionError", + }), + ); + vi.spyOn(console, "error").mockImplementationOnce(() => {}); + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + + ledger.account.autoTopUpAmountCents = 2_500; + ledger.account.stripePaymentMethodId = "pm_new"; + mocks.createPaymentIntent.mockResolvedValueOnce( + intent("pi_1", "succeeded"), + ); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + state: "succeeded", + paymentIntentId: "pi_1", }); - expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); - later(0.2); - mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + const [first, retry] = mocks.createPaymentIntent.mock.calls; + expect(retry?.[0]).toEqual(first?.[0]); + expect(retry?.[0]).toMatchObject({ amount: 500, payment_method: "pm_1" }); + expect(retry?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); + expect(reservations()).toHaveLength(0); + expect(topUps()).toHaveLength(1); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValueOnce( + intent("pi_2", "succeeded", { amount_received: 2_500 }), + ); + await runAutoTopUp("account-1", now()); + const next = mocks.createPaymentIntent.mock.calls[2]; + expect(next?.[0]).toMatchObject({ + amount: 2_500, + payment_method: "pm_new", + }); + expect(next?.[1].idempotencyKey).not.toBe(first?.[1].idempotencyKey); + }); + + it("treats Stripe refusing a changed request under the same key as in progress", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign( + new Error( + "Keys for idempotent requests can only be used with the same parameters they were first used with.", + ), + { type: "StripeIdempotencyError", statusCode: 400 }, + ), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "in_progress", + }); + expect(reservations()).toHaveLength(1); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + }); + + it(`stops at ${AUTO_TOP_UP_DAILY_LIMIT} automatic top-ups in any 24 hours, then waits`, async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + for (let i = 1; i <= AUTO_TOP_UP_DAILY_LIMIT; i++) { + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue( + intent(`pi_${i}`, "succeeded"), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + later(1); + } + + ledger.account.balanceMicroCredits = 10; + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "daily_limit", + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes( + AUTO_TOP_UP_DAILY_LIMIT, + ); + + later(14.5); + mocks.createPaymentIntent.mockResolvedValue(intent("pi_next", "succeeded")); expect(await runAutoTopUp("account-1", now())).toMatchObject({ state: "succeeded", - paymentIntentId: "pi_2", - }); - expect(mocks.createPaymentIntent.mock.calls[1]?.[1]).toEqual({ - idempotencyKey: "infra-auto-top-up:account-1:2026-11-12", + paymentIntentId: "pi_next", }); - expect(topUps()).toHaveLength(2); + warn.mockRestore(); }); it("grants nothing for a processing payment, then once when it succeeds", async () => { @@ -379,9 +650,286 @@ describe("auto top-up charge", () => { expect(topUps()).toHaveLength(1); expect(ledger.account.balanceMicroCredits).toBe(530_000); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + }); + + describe("the owner's daily ceiling", () => { + it("waits once automatic top-ups would pass $1,000 in 24 hours, without switching anything off", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + ledger.account.autoTopUpAmountCents = 60_000; + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded", { amount_received: 60_000 }), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + + ledger.account.balanceMicroCredits = 10; + later(1); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(failures()).toHaveLength(0); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + "[infra-billing] Auto top-up waits for its daily limit", + { accountId: "account-1", reason: "owner_daily_limit" }, + ); + + later(23.1); + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_2", "succeeded", { amount_received: 60_000 }), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + warn.mockRestore(); + }); + + it("does not count credit bought by hand", async () => { + ledger.rows.push({ + id: "manual", + accountId: "account-1", + type: "topup", + amountMicroCredits: 90_000_000, + balanceAfterMicroCredits: 90_030_000, + referenceId: "pi_manual", + referenceType: "stripe_payment_intent", + createdAt: new Date(ledger.now), + }); + ledger.account.autoTopUpAmountCents = 60_000; + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded", { amount_received: 60_000 }), + ); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + }); }); - it("does not start a second payment after a processing one, even on the next UTC day", async () => { + describe("recovering an attempt whose answer was lost", () => { + const loseAnswer = async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("timeout"), { type: "StripeConnectionError" }), + ); + await runAutoTopUp("account-1", now()); + errors.mockRestore(); + mocks.createPaymentIntent.mockClear(); + }; + + it("waits instead of replaying once the reservation is a day old", async () => { + await loseAnswer(); + later(25); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "awaiting_payment", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(mocks.searchPaymentIntents).not.toHaveBeenCalled(); + }); + + it("after 72 hours, finds the payment in Stripe and grants it once", async () => { + await loseAnswer(); + later(72.1); + mocks.searchPaymentIntents.mockResolvedValue({ + data: [intent("pi_found", "succeeded")], + }); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.searchPaymentIntents.mock.calls[0]?.[0]).toEqual({ + query: + "metadata['accountId']:'account-1' AND metadata['attempt']:'0:0'", + limit: 1, + }); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_found" }), + ]); + expect(reservations()).toHaveLength(0); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + }); + + it("recovers a deleted app's lost attempt, then never charges it again", async () => { + await loseAnswer(); + later(72.1); + ledger.account.appDeletedAt = now(); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.searchPaymentIntents).toHaveBeenCalledTimes(1); + expect(ledger.rows).toHaveLength(0); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(ledger.rows).toHaveLength(0); + }); + + it("after 72 hours, voids a reservation Stripe never charged", async () => { + await loseAnswer(); + later(72.1); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(ledger.rows).toHaveLength(0); + expect(topUps()).toHaveLength(0); + }); + + it("recovers before the switch, the decline wait or any limit can stop it", async () => { + await loseAnswer(); + ledger.account.autoTopUpEnabled = false; + ledger.account.balanceMicroCredits = 99_000_000; + later(72.1); + mocks.searchPaymentIntents.mockResolvedValue({ + data: [intent("pi_found", "succeeded")], + }); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "reconciled", + }); + expect(topUps()).toHaveLength(1); + }); + }); + + describe("a payment still pending after 72 hours", () => { + const leavePending = async () => { + mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); + await runAutoTopUp("account-1", now()); + mocks.createPaymentIntent.mockReset(); + }; + + it("asks Stripe once and records a payment that succeeded", async () => { + await leavePending(); + later(71.9); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + + later(0.2); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded"), + ); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + expect(mocks.retrievePaymentIntent.mock.calls[0]?.[0]).toBe("pi_1"); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + + await settleAutoTopUpPaymentIntent( + intent("pi_1", "succeeded"), + "succeeded", + ); + expect(topUps()).toHaveLength(1); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + }); + + it("is picked up by the storage sweep once it passes 72 hours, with no other trigger", async () => { + await leavePending(); + later(71); + expect(await runDueAutoTopUps(now())).toEqual({ + attempted: 0, + succeeded: 0, + failed: 0, + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + + later(1.1); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded"), + ); + await runDueAutoTopUps(now()); + + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + }); + + it("records a payment that failed and switches auto top-up off", async () => { + await leavePending(); + later(72.1); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "requires_payment_method", { + last_payment_error: { + decline_code: "insufficient_funds", + } as Stripe.PaymentIntent.LastPaymentError, + }), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(failures()).toEqual([ + expect.objectContaining({ + referenceId: "pi_1", + metadata: expect.objectContaining({ reason: "insufficient_funds" }), + }), + ]); + expect(topUps()).toHaveLength(0); + expect(ledger.account.autoTopUpEnabled).toBe(false); + expect(ledger.account.balanceMicroCredits).toBe(30_000); + }); + + it("leaves a payment Stripe still reports as processing", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await leavePending(); + later(80); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "processing"), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(topUps()).toHaveLength(0); + expect(failures()).toHaveLength(0); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "reconciled", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + }); + + it("does not start a second payment while one is still processing, however long it takes", async () => { ledger.now = new Date("2026-11-11T23:50:00Z"); mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); await runAutoTopUp("account-1", now()); @@ -391,12 +939,21 @@ describe("auto top-up charge", () => { expect(now().toISOString().slice(0, 10)).toBe("2026-11-12"); expect(await runAutoTopUp("account-1", now())).toEqual({ state: "skipped", - reason: "recent_attempt", + reason: "awaiting_payment", }); later(23); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + reason: "awaiting_payment", + }); + later(1.5); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", }); + later(46.5); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); expect(ledger.rows).toHaveLength(1); @@ -507,9 +1064,25 @@ describe("auto top-up charge", () => { ledger.account.autoTopUpEnabled = true; later(2); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + reason: "recent_decline", + }); + later(21.9); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "recent_decline", }); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + + later(0.2); + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + const [declined, next] = mocks.createPaymentIntent.mock.calls; + expect(declined?.[1].idempotencyKey).toBe( + "infra-auto-top-up:account-1:0:0", + ); + expect(next?.[1].idempotencyKey).toBe("infra-auto-top-up:account-1:0:1"); }); it("treats a charge that needs the customer as a failure", async () => { @@ -532,7 +1105,8 @@ describe("auto top-up charge", () => { expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); expect(ledger.account.autoTopUpEnabled).toBe(true); - expect(ledger.rows).toHaveLength(0); + expect(reservations()).toHaveLength(1); + expect(ledger.rows).toHaveLength(1); }); it("ignores payments that are not auto top-ups or did not settle in dollars", async () => { @@ -569,6 +1143,88 @@ describe("auto top-up charge", () => { }); }); +describe("auto top-up after a charge", () => { + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + const outsideRequest = () => { + throw new Error("`after` was called outside a request scope"); + }; + + beforeEach(() => { + mocks.after.mockReset(); + }); + + it("waits until the response is sent, then tops up the account", async () => { + const tasks: (() => Promise)[] = []; + mocks.after.mockImplementation((task: () => Promise) => { + tasks.push(task); + }); + + await scheduleAutoTopUp({ accountId: "account-1" }); + await settle(); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + + await tasks[0]?.(); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(topUps()).toHaveLength(1); + }); + + it("finds the account from the app", async () => { + const tasks: (() => Promise)[] = []; + mocks.after.mockImplementation((task: () => Promise) => { + tasks.push(task); + }); + + await scheduleAutoTopUp({ appId: "app-1" }); + await tasks[0]?.(); + + expect(mocks.createPaymentIntent.mock.calls[0]?.[0].metadata).toMatchObject( + { accountId: "account-1", appId: "app-1" }, + ); + }); + + it("outside a request, finishes the top-up before it returns", async () => { + mocks.after.mockImplementation(outsideRequest); + + await scheduleAutoTopUp({ accountId: "account-1" }); + + expect(mocks.after).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(topUps()).toHaveLength(1); + }); + + it("inside a request, returns at once and leaves the top-up to after()", async () => { + mocks.after.mockImplementation(() => {}); + + await scheduleAutoTopUp({ accountId: "account-1" }); + + expect(mocks.after).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + }); + + it("never throws into the caller and logs what went wrong", async () => { + mocks.after.mockImplementation(outsideRequest); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const broken = { + ...fakeDb, + select: () => { + throw new Error("database unavailable"); + }, + }; + mocks.db = broken; + + await expect( + scheduleAutoTopUp({ accountId: "account-1" }), + ).resolves.toBeUndefined(); + expect(errors).toHaveBeenCalledWith( + "[infra-billing] Auto top-up after a charge failed", + expect.objectContaining({ target: { accountId: "account-1" } }), + ); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + mocks.db = fakeDb; + errors.mockRestore(); + }); +}); + describe("saving a card from checkout", () => { const session = (extra: Partial = {}) => ({ diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index aa334f5b97f..495b7a66990 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -1,4 +1,5 @@ import { + developerApps, developerCreditAccounts, developerCreditTransactions, developerDailyStorageSnapshots, @@ -52,6 +53,7 @@ const freshAccount = () => ({ autoTopUpThresholdMicroCredits: 0, autoTopUpAmountCents: 0, email: "richie@cap.so", + appDeletedAt: null as Date | null, }); export const ledger = { @@ -61,6 +63,7 @@ export const ledger = { snapshots: [] as SnapshotRow[], videos: [] as VideoGroup[], heldMicroCredits: 0, + appDeletedBeforeLock: false, }; export function resetLedger() { @@ -68,6 +71,7 @@ export function resetLedger() { ledger.account = freshAccount(); ledger.rows = []; ledger.snapshots = []; + ledger.appDeletedBeforeLock = false; ledger.videos = []; ledger.heldMicroCredits = 0; } @@ -121,6 +125,71 @@ const latestStorageDayRows = () => { return days.filter((row) => row.referenceId === latestDay).sort(byId); }; +const HOUR_MS = 60 * 60 * 1000; + +function autoTopUpWaits() { + const now = ledger.now.getTime(); + const of = (type: string, referenceType: string) => + ledger.rows.filter( + (row) => row.type === type && row.referenceType === referenceType, + ); + const settledIds = new Set( + ledger.rows + .filter( + (row) => + row.referenceType === "auto_top_up" || + row.referenceType === "auto_top_up_failed", + ) + .map((row) => row.referenceId), + ); + const topUps = of("topup", "auto_top_up"); + return { + pending: of("topup_pending", "auto_top_up_pending").some( + (row) => + row.createdAt.getTime() > now - 72 * HOUR_MS && + !String(row.referenceId).startsWith("infra-auto-top-up:") && + !settledIds.has(row.referenceId), + ), + decline: of("topup_failed", "auto_top_up_failed").some( + (row) => + row.createdAt.getTime() > now - 24 * HOUR_MS && + !topUps.some((topUp) => topUp.createdAt > row.createdAt), + ), + limit: + topUps.filter((row) => row.createdAt.getTime() > now - 24 * HOUR_MS) + .length >= 10, + owner: + ledger.rows + .filter( + (row) => + !String(row.referenceId).startsWith("infra-auto-top-up:") && + ((row.type === "topup" && + row.referenceType === "auto_top_up" && + row.createdAt.getTime() > now - 24 * HOUR_MS) || + (row.type === "topup_pending" && + row.referenceType === "auto_top_up_pending" && + !settledIds.has(row.referenceId))), + ) + .reduce( + (total, row) => + total + + (row.type === "topup" + ? row.amountMicroCredits + : Number(row.metadata?.amountCents ?? 0) * 1000), + 0, + ) + + Number( + ledger.rows.find( + (row) => + row.type === "topup_pending" && + String(row.referenceId).startsWith("infra-auto-top-up:"), + )?.metadata?.amountCents ?? ledger.account.autoTopUpAmountCents, + ) * + 1000 > + 100_000_000, + }; +} + function runSelect(query: Query): Record[] { const keys = Object.keys(query.fields).sort().join(","); const rendered = query.where @@ -130,12 +199,30 @@ function runSelect(query: Query): Record[] { if (query.table === developerCreditAccounts) { const { exists, ...account } = ledger.account; if (!exists) return []; + if ( + keys === + "amountCents,balanceMicroCredits,customerId,declineWait,enabled,id,limitWait,ownerWait,paymentMethodId,pendingWait,thresholdMicroCredits" + ) { + if (rendered.params.at(-1) !== account.id) return []; + const waits = autoTopUpWaits(); + return [ + { + id: account.id, + enabled: account.autoTopUpEnabled, + thresholdMicroCredits: account.autoTopUpThresholdMicroCredits, + amountCents: account.autoTopUpAmountCents, + customerId: account.stripeCustomerId, + paymentMethodId: account.stripePaymentMethodId, + balanceMicroCredits: account.balanceMicroCredits, + pendingWait: Number(waits.pending), + declineWait: Number(waits.decline), + limitWait: Number(waits.limit), + ownerWait: Number(waits.owner), + }, + ]; + } if (keys === "attemptKey,id") { - const cutoff = rendered.params.find( - (param) => - typeof param === "string" && - /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/.test(param), - ); + const waits = autoTopUpWaits(); const attempts = ledger.rows .filter((row) => ["auto_top_up", "auto_top_up_failed", "auto_top_up_pending"].includes( @@ -149,7 +236,10 @@ function runSelect(query: Query): Record[] { account.stripePaymentMethodId && account.balanceMicroCredits < account.autoTopUpThresholdMicroCredits + ledger.heldMicroCredits && - (lastAttempt === null || lastAttempt <= asTime(cutoff)); + !waits.pending && + !waits.decline && + !waits.limit && + !waits.owner; return due ? [ { @@ -257,6 +347,54 @@ function runSelect(query: Query): Record[] { ), ).slice(0, 1); } + if (keys === "pendingId,pendingRow,pendingSince") { + return ledger.rows + .filter( + (row) => + row.type === "topup_pending" && + row.referenceType === "auto_top_up_pending", + ) + .map((row) => ({ + pendingRow: row.id, + pendingId: row.referenceId, + pendingSince: row.createdAt, + })); + } + if (keys === "reservation,reservedAt") { + const [accountId, referenceId] = rendered.params; + return ledger.rows + .filter( + (row) => + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === "auto_top_up_pending", + ) + .map((row) => ({ + reservation: row.metadata, + reservedAt: row.createdAt, + })); + } + if (keys === "settledId") { + const ids = rendered.params.slice(3); + return ledger.rows + .filter( + (row) => + (row.referenceType === "auto_top_up" || + row.referenceType === "auto_top_up_failed") && + ids.includes(row.referenceId), + ) + .map((row) => ({ settledId: row.referenceId })); + } + if (keys === "declines,topUps") { + const count = (type: string) => + ledger.rows.filter((row) => row.referenceType === type).length; + return [ + { + topUps: count("auto_top_up"), + declines: count("auto_top_up_failed"), + }, + ]; + } if (keys === "createdAt,metadata,type") { const types = rendered.params.slice(1); return newestFirst( @@ -290,6 +428,13 @@ function runSelect(query: Query): Record[] { } } + if (query.table === developerApps && keys === "appDeletedAt") { + const deletedAt = ledger.appDeletedBeforeLock + ? ledger.now + : ledger.account.appDeletedAt; + return [{ appDeletedAt: deletedAt }]; + } + if (query.table === developerTranscripts) { return [{ total: ledger.heldMicroCredits }]; } @@ -345,7 +490,20 @@ function update(table: unknown) { return { set(values: Record) { return { - async where() { + async where(condition?: SQL) { + if (table === developerCreditTransactions && condition) { + const [accountId, referenceId, referenceType, rowId] = + dialect.sqlToQuery(condition).params; + const rows = ledger.rows.filter( + (row) => + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === referenceType && + (rowId === undefined || row.id === rowId), + ); + for (const row of rows) Object.assign(row, values); + return [{ affectedRows: rows.length }]; + } if (table !== developerCreditAccounts) return [{ affectedRows: 0 }]; const { balanceMicroCredits, ...rest } = values; if (balanceMicroCredits !== undefined) { @@ -396,10 +554,32 @@ function insert(table: unknown) { }; } +function remove(table: unknown) { + return { + async where(condition: SQL) { + if (table !== developerCreditTransactions) return [{ affectedRows: 0 }]; + const [accountId, referenceId, referenceType, rowId] = + dialect.sqlToQuery(condition).params; + const before = ledger.rows.length; + ledger.rows = ledger.rows.filter( + (row) => + !( + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === referenceType && + (rowId === undefined || row.id === rowId) + ), + ); + return [{ affectedRows: before - ledger.rows.length }]; + }, + }; +} + type FakeDb = { select: typeof select; update: typeof update; insert: typeof insert; + delete: typeof remove; transaction: (run: (tx: FakeDb) => Promise) => Promise; }; @@ -407,5 +587,6 @@ export const fakeDb: FakeDb = { select, update, insert, + delete: remove, transaction: (run) => run(fakeDb), }; diff --git a/apps/web/__tests__/unit/infra-billing-gates.test.ts b/apps/web/__tests__/unit/infra-billing-gates.test.ts index cabf48c095d..7a2ab864394 100644 --- a/apps/web/__tests__/unit/infra-billing-gates.test.ts +++ b/apps/web/__tests__/unit/infra-billing-gates.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ createSession: vi.fn(), detach: vi.fn(), ensureCustomer: vi.fn(), + scheduleAutoTopUp: vi.fn(), })); vi.mock("@cap/database", () => ({ @@ -94,7 +95,10 @@ vi.mock("@/lib/infra-auto-top-up", async () => { const settings = await vi.importActual< typeof import("@/lib/infra-auto-top-up") >("@/lib/infra-auto-top-up"); - return { autoTopUpSettingsError: settings.autoTopUpSettingsError }; + return { + autoTopUpSettingsError: settings.autoTopUpSettingsError, + scheduleAutoTopUp: mocks.scheduleAutoTopUp, + }; }); vi.mock("@/app/api/utils", async () => { const { createMiddleware } = await import("hono/factory"); @@ -141,6 +145,7 @@ const cardCheckout = () => }); beforeEach(() => { + mocks.scheduleAutoTopUp.mockReset(); mocks.user = preview; mocks.selects = []; mocks.sets = []; @@ -179,6 +184,7 @@ describe("creating an upload", () => { }); expect(mocks.admission).toHaveBeenCalledWith("app-1"); expect(mocks.inserts).toHaveLength(0); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("creates the video for free when the app is admitted", async () => { @@ -189,6 +195,7 @@ describe("creating an upload", () => { expect(response.status).toBe(200); const { data } = await response.json(); expect(data).toMatchObject({ id: "video-1", videoId: "video-1" }); + expect(mocks.scheduleAutoTopUp).not.toHaveBeenCalled(); expect(Object.keys(data).sort()).toEqual([ "corsOrigin", "embedUrl", diff --git a/apps/web/__tests__/unit/infra-top-up-nudge.test.ts b/apps/web/__tests__/unit/infra-top-up-nudge.test.ts new file mode 100644 index 00000000000..504edf10cea --- /dev/null +++ b/apps/web/__tests__/unit/infra-top-up-nudge.test.ts @@ -0,0 +1,42 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const navigation = vi.hoisted(() => ({ query: "" })); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn(), refresh: vi.fn() }), + useSearchParams: () => new URLSearchParams(navigation.query), +})); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); +vi.mock("@tanstack/react-query", () => ({ + useMutation: () => ({ mutate: vi.fn(), isPending: false }), +})); + +import { TopUp } from "@/app/(infra)/dashboard/infra/[appId]/billing/top-up"; + +const NUDGE = + "to add credit as soon as the balance falls below a threshold you set."; + +const render = (autoTopUpHref: string | null) => + renderToStaticMarkup(createElement(TopUp, { appId: "app-1", autoTopUpHref })); + +describe("the nudge after a purchase", () => { + beforeEach(() => { + navigation.query = ""; + }); + + it("points to auto top-up after a purchase while it is off", () => { + navigation.query = "purchase=success"; + const html = render("#auto-top-up"); + + expect(html).toContain(NUDGE); + expect(html).toContain('href="#auto-top-up"'); + }); + + it("stays out of the way before a purchase and when auto top-up is on", () => { + expect(render("#auto-top-up")).not.toContain(NUDGE); + navigation.query = "purchase=success"; + expect(render(null)).not.toContain(NUDGE); + }); +}); diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx index 3ad9dc0b54b..d16fa649792 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx @@ -143,11 +143,11 @@ export function AutoTopUp({

Saving a card charges nothing and leaves auto top-up off. Once you - switch it on, Cap adds ${initialAmount} from that card whenever this + switch it on, Cap adds ${initialAmount} from that card as soon as this app's balance falls below ${initialThreshold}. You can change both - amounts, the card is charged at most once a day, and auto top-up - switches itself off if a charge is declined. Remove the card at any - time. + amounts. The card is charged at most ten times in any 24 hours, and at + most $1,000 a day across all your apps. Auto top-up switches itself + off if a charge is declined. Remove the card at any time.

); @@ -263,8 +263,9 @@ export function AutoTopUp({

- A card is saved for this app. It is charged at most once a day, and auto - top-up switches itself off if a charge is declined. + A card is saved for this app. It is charged at most ten times in any 24 + hours, and at most $1,000 a day across all your apps. Auto top-up + switches itself off if a charge is declined.

); diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx index 581a37e2555..55d42f61fca 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx @@ -49,6 +49,7 @@ const RATE_PRICE = clsx( ); const LEDGER_ANCHOR = "transactions"; +const AUTO_TOP_UP_ANCHOR = "auto-top-up"; export type BillingAccount = { balanceMicroCredits: number; @@ -111,7 +112,12 @@ export function BillingView({
{account ? ( <> - +
@@ -186,19 +192,21 @@ export function BillingView({
{account && ( - - - +
+ + + +
)}
diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx index 6a0c40eb2ad..39fb4da76b8 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx @@ -13,9 +13,15 @@ import { TOP_UP_PRESETS_CENTS, topUpButtonLabel, } from "@/lib/infra-top-up"; -import { Button, FOCUS, Input, MONO } from "../../_components/ui"; +import { Button, FOCUS, Input, MONO, TEXT_LINK } from "../../_components/ui"; -export function TopUp({ appId }: { appId: string }) { +export function TopUp({ + appId, + autoTopUpHref, +}: { + appId: string; + autoTopUpHref: string | null; +}) { const router = useRouter(); const searchParams = useSearchParams(); const amountId = useId(); @@ -26,6 +32,9 @@ export function TopUp({ appId }: { appId: string }) { const codeNoteId = useId(); const [codeOpen, setCodeOpen] = useState(false); const [code, setCode] = useState(""); + const [purchased] = useState( + () => searchParams.get("purchase") === "success", + ); useEffect(() => { if (searchParams.get("purchase") === "success") { @@ -173,6 +182,18 @@ export function TopUp({ appId }: { appId: string }) { )}
+ {purchased && autoTopUpHref && ( +

+ Switch on{" "} + + auto top-up + {" "} + to add credit as soon as the balance falls below a threshold you set. +

+ )} ); } diff --git a/apps/web/content/docs/api/billing.mdx b/apps/web/content/docs/api/billing.mdx index a18c88b1462..5a136b4b995 100644 --- a/apps/web/content/docs/api/billing.mdx +++ b/apps/web/content/docs/api/billing.mdx @@ -119,9 +119,9 @@ Cap Pro includes $5 of credit every month. It is for any account with Cap Pro, w Auto top-up adds credit from a saved card when an app's balance runs low. It is off until you switch it on, and saving a card charges nothing. -- **When.** Cap checks at 00:41, 06:41, 12:41 and 18:41 UTC. If the available balance is under your threshold, the card is charged. +- **When.** As soon as the available balance falls below your threshold. Cap also checks at 00:41, 06:41, 12:41 and 18:41 UTC. - **How much.** The amount you set, from $5 to $1,000. The threshold is from $1 to $1,000. The console suggests $25 when the balance is under $5. -- **How often.** At most one attempt in any 24 hours. +- **How often.** Each time the balance falls below the threshold, up to 10 top-ups in any 24 hours, and at most $1,000 a day across all your apps. Credit you buy yourself does not count toward that. After a declined charge, the next attempt waits 24 hours. - **How to stop.** Switch it off or remove the card on the Billing page. If a charge is declined, auto top-up switches itself off and no credit is added. ## When credit runs out diff --git a/apps/web/content/docs/api/rest-api.mdx b/apps/web/content/docs/api/rest-api.mdx index 12a2628b518..3c98db57936 100644 --- a/apps/web/content/docs/api/rest-api.mdx +++ b/apps/web/content/docs/api/rest-api.mdx @@ -1079,7 +1079,6 @@ curl "https://cap.so/api/developer/v1/transactions?limit=2" \ | `topup` | Code redeemed | `stripe_checkout_session` | The checkout a code covered in full | | `topup` | Auto top-up | `auto_top_up` | The payment | | `topup_failed` | Auto top-up declined | `auto_top_up_failed` | The payment, or `declined-` and the UTC day when there was none | -| `topup_pending` | Auto top-up pending | `auto_top_up_pending` | The payment | | `pro_credit` | Cap Pro credit | `pro_monthly_credit` | The month, as `YYYY-MM` | | `storage_daily` | Storage | `storage_day` | The UTC day, as `YYYY-MM-DD` | | `transcription` | Transcription | `developer_transcript` | The transcript ID | diff --git a/apps/web/lib/developer-credits.ts b/apps/web/lib/developer-credits.ts index de32295489d..523929b02f2 100644 --- a/apps/web/lib/developer-credits.ts +++ b/apps/web/lib/developer-credits.ts @@ -16,18 +16,20 @@ export async function addCreditsToAccount({ referenceId, referenceType, metadata, + executor, }: { accountId: string; amountCents: number; referenceId?: string; referenceType?: DeveloperCreditReferenceType; metadata?: Record; + executor?: DeveloperCreditTransaction; }): Promise { const microCreditsToAdd = Math.floor( (amountCents / 100) * MICRO_CREDITS_PER_DOLLAR, ); - const newBalance = await db().transaction(async (tx) => { + const grant = async (tx: DeveloperCreditTransaction) => { if (referenceId && referenceType) { const [account] = await tx .select({ @@ -86,9 +88,9 @@ export async function addCreditsToAccount({ }); return updated.balanceMicroCredits; - }); + }; - return newBalance; + return executor ? grant(executor) : db().transaction(grant); } type DeveloperCreditTransaction = Parameters< diff --git a/apps/web/lib/developer-openapi.ts b/apps/web/lib/developer-openapi.ts index f19dd6b727e..190bc12993a 100644 --- a/apps/web/lib/developer-openapi.ts +++ b/apps/web/lib/developer-openapi.ts @@ -1453,7 +1453,7 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ type: { type: "string", description: - "The kind of row: topup, topup_failed, topup_pending, pro_credit, storage_daily, transcription or adjustment. Treat a value you do not know as a row to show as it is.", + "The kind of row: topup, topup_failed, pro_credit, storage_daily, transcription or adjustment. Treat a value you do not know as a row to show as it is.", }, description: { type: "string", diff --git a/apps/web/lib/developer-transcript-billing.ts b/apps/web/lib/developer-transcript-billing.ts index d542c45cbaf..86e223a9578 100644 --- a/apps/web/lib/developer-transcript-billing.ts +++ b/apps/web/lib/developer-transcript-billing.ts @@ -14,6 +14,7 @@ import { TRANSCRIPT_MIN_START_BALANCE_MICRO_CREDITS, type TranscriptErrorCode, } from "@/lib/developer-transcripts"; +import { scheduleAutoTopUp } from "@/lib/infra-auto-top-up"; import { availableBalance, heldTranscriptMicroCredits, @@ -127,7 +128,7 @@ export async function holdTranscriptCredits(input: { if (!appId) return { state: "skipped" }; const needed = Math.max(0, Math.ceil(input.microCredits)); - return db().transaction(async (tx) => { + const hold = await db().transaction(async (tx) => { const [account] = await tx .select({ balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, @@ -188,6 +189,8 @@ export async function holdTranscriptCredits(input: { return { state: "held", reservedMicroCredits: needed }; }); + if (hold.state !== "skipped") await scheduleAutoTopUp({ appId }); + return hold; } export async function reserveTranscriptCredits(input: { @@ -201,7 +204,7 @@ export async function reserveTranscriptCredits(input: { TRANSCRIPT_FULL_HOLD_MICRO_CREDITS, ); - return db().transaction(async (tx) => { + const outcome = await db().transaction(async (tx) => { const [account] = await tx .select({ balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, @@ -248,6 +251,8 @@ export async function reserveTranscriptCredits(input: { return { state: "reserved", reservedMicroCredits: cost }; }); + if (outcome.state !== "skipped") await scheduleAutoTopUp({ appId }); + return outcome; } const measurableSeconds = (seconds: number | null | undefined) => @@ -266,7 +271,7 @@ export async function settleTranscript(input: { const appId = await findTranscriptApp(input.transcriptId); if (!appId) return { state: "skipped", reason: "missing" }; - return db().transaction(async (tx) => { + const settled = await db().transaction(async (tx) => { await tx .select({ id: developerCreditAccounts.id }) .from(developerCreditAccounts) @@ -374,4 +379,11 @@ export async function settleTranscript(input: { alreadySettled: false, }; }); + if ( + settled.state === "insufficient_credits" || + (settled.state === "complete" && settled.chargedMicroCredits > 0) + ) { + await scheduleAutoTopUp({ appId }); + } + return settled; } diff --git a/apps/web/lib/developer-transcript-store.ts b/apps/web/lib/developer-transcript-store.ts index dcc0f81fe3b..698111a056f 100644 --- a/apps/web/lib/developer-transcript-store.ts +++ b/apps/web/lib/developer-transcript-store.ts @@ -55,6 +55,7 @@ import { syncVideoCaptions, } from "@/lib/developer-video-captions"; import { canUseInfra } from "@/lib/infra-access"; +import { scheduleAutoTopUp } from "@/lib/infra-auto-top-up"; import type { InfraDbExecutor } from "@/lib/infra-standing"; import { assertPublicMediaUrl, @@ -441,9 +442,9 @@ async function startGate(appId: string): Promise { if (!(await appOwnerCanUseInfra(appId))) return NOT_AVAILABLE; if (!isSpeechConfigured()) return SPEECH_UNAVAILABLE; const refusal = await transcriptStartRefusal(appId); - return refusal - ? fail(402, refusal.code, refusal.message, ADD_CREDITS_HINT) - : null; + if (!refusal) return null; + await scheduleAutoTopUp({ appId }); + return fail(402, refusal.code, refusal.message, ADD_CREDITS_HINT); } export async function createTranscriptFromUrl( diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 47839d7f95c..7408b71da56 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -12,11 +12,15 @@ import { asc, desc, eq, + getTableName, inArray, isNotNull, isNull, + or, + type SQL, sql, } from "drizzle-orm"; +import { after } from "next/server"; import type Stripe from "stripe"; import { addCreditsToAccount } from "@/lib/developer-credits"; import { developerCreditsProductId } from "@/lib/developer-credits-product"; @@ -24,7 +28,7 @@ import { MICRO_CREDITS_PER_DOLLAR } from "@/lib/developer-pricing"; import { canUseInfra } from "@/lib/infra-access"; import { availableBalance, heldTranscriptsByApp } from "@/lib/infra-balance"; import { sendInfraTopUpFailedEmail } from "@/lib/infra-billing-emails"; -import { disputedSince } from "@/lib/infra-standing"; +import { disputedSince, type InfraDbExecutor } from "@/lib/infra-standing"; import { utcDateKey } from "@/lib/infra-usage"; export const AUTO_TOP_UP_REFERENCE_TYPE = "auto_top_up" as const; @@ -38,9 +42,29 @@ export const AUTO_TOP_UP_MIN_THRESHOLD_MICRO_CREDITS = MICRO_CREDITS_PER_DOLLAR; export const AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS = 1_000 * MICRO_CREDITS_PER_DOLLAR; -const ATTEMPT_INTERVAL_MS = 24 * 60 * 60 * 1000; +export const AUTO_TOP_UP_DAILY_LIMIT = 10; +export const AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS = 100_000; +const MICRO_CREDITS_PER_CENT = MICRO_CREDITS_PER_DOLLAR / 100; +export const AUTO_TOP_UP_STRIPE_TIMEOUT_MS = 15_000; +export const AUTO_TOP_UP_PENDING_RECONCILE_MS = 72 * 60 * 60 * 1000; +const DAY_MS = 24 * 60 * 60 * 1000; const DUE_PAGE_SIZE = 200; +type AutoTopUpTransaction = Parameters< + Parameters["transaction"]>[0] +>[0]; + +type SkipReason = Extract["reason"]; + +type LockedAttempt = + | { kind: "skipped"; reason: SkipReason } + | { + kind: "reserved"; + key: string; + attempt: AutoTopUpAttempt; + request: ReservedRequest; + }; + export type AutoTopUpResult = | { state: "skipped"; @@ -52,7 +76,13 @@ export type AutoTopUpResult = | "invalid_settings" | "above_threshold" | "disputed" - | "recent_attempt"; + | "awaiting_payment" + | "reconciled" + | "app_deleted" + | "recent_decline" + | "daily_limit" + | "owner_daily_limit" + | "in_progress"; } | { state: "succeeded"; paymentIntentId: string; balanceMicroCredits: number } | { state: "pending"; paymentIntentId: string } @@ -89,8 +119,62 @@ export function autoTopUpSettingsError({ return null; } -export function autoTopUpIdempotencyKey(accountId: string, period: string) { - return `infra-auto-top-up:${accountId}:${period}`; +export type AutoTopUpAttempt = { topUps: number; declines: number }; + +const RESERVATION_PREFIX = "infra-auto-top-up:"; + +export function autoTopUpIdempotencyKey( + accountId: string, + { topUps, declines }: AutoTopUpAttempt, +) { + return `${RESERVATION_PREFIX}${accountId}:${topUps}:${declines}`; +} + +function reservationKeyOf(paymentIntent: Stripe.PaymentIntent) { + const { accountId, attempt } = paymentIntent.metadata ?? {}; + const [topUps, declines] = (attempt ?? "").split(":").map(Number); + if (!accountId || !Number.isInteger(topUps) || !Number.isInteger(declines)) { + return null; + } + return autoTopUpIdempotencyKey(accountId, { + topUps: topUps ?? 0, + declines: declines ?? 0, + }); +} + +const reservationOf = (accountId: string, key: string, rowId?: string) => + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.referenceId, key), + eq( + developerCreditTransactions.referenceType, + AUTO_TOP_UP_PENDING_REFERENCE_TYPE, + ), + rowId ? eq(developerCreditTransactions.id, rowId) : undefined, + ); + +async function dropReservation( + executor: AutoTopUpTransaction | ReturnType, + accountId: string, + key: string | null, + rowId?: string, +) { + if (!key) return; + await executor + .delete(developerCreditTransactions) + .where(reservationOf(accountId, key, rowId)); +} + +async function linkReservation( + accountId: string, + key: string, + paymentIntentId: string, + rowId?: string, +) { + await db() + .update(developerCreditTransactions) + .set({ referenceId: paymentIntentId }) + .where(reservationOf(accountId, key, rowId)); } export function autoTopUpFailureText(reason: string | null | undefined) { @@ -106,6 +190,38 @@ export function autoTopUpFailureText(reason: string | null | undefined) { } } +function isUnresolvedAttempt(error: unknown) { + if (typeof error !== "object" || error === null) return false; + const fields = error as { + type?: unknown; + code?: unknown; + statusCode?: unknown; + }; + return ( + fields.code === "idempotency_key_in_use" || + fields.statusCode === 409 || + fields.type === "StripeIdempotencyError" + ); +} + +const UNSENT_STRIPE_ERRORS = new Set([ + "StripeRateLimitError", + "StripeAuthenticationError", + "StripePermissionError", +]); + +function neverReachedStripe(error: unknown) { + const type = + typeof error === "object" && error !== null + ? (error as { type?: unknown }).type + : undefined; + return ( + typeof type !== "string" || + !type.startsWith("Stripe") || + UNSENT_STRIPE_ERRORS.has(type) + ); +} + function declineFrom(error: unknown) { if (typeof error !== "object" || error === null) return null; const fields = error as { @@ -153,8 +269,70 @@ const NEVER_ATTEMPTED = "1970-01-01 00:00:00"; const sqlTime = (date: Date) => date.toISOString().slice(0, 23).replace("T", " "); -export function autoTopUpRetryCutoff(now: Date) { - return new Date(now.getTime() - ATTEMPT_INTERVAL_MS); +function ledgerAlias(name: string) { + const column = (field: { name: string }) => + sql`${sql.identifier(name)}.${sql.identifier(field.name)}`; + return { + from: sql`${sql.identifier(getTableName(developerCreditTransactions))} ${sql.identifier(name)}`, + accountId: column(developerCreditTransactions.accountId), + type: column(developerCreditTransactions.type), + referenceId: column(developerCreditTransactions.referenceId), + referenceType: column(developerCreditTransactions.referenceType), + createdAt: column(developerCreditTransactions.createdAt), + amountMicroCredits: column(developerCreditTransactions.amountMicroCredits), + metadata: column(developerCreditTransactions.metadata), + }; +} + +const inList = (values: readonly string[]) => + sql.join( + values.map((value) => sql`${value}`), + sql`, `, + ); + +export function autoTopUpWaits(now: Date) { + const accounts = sql.identifier(getTableName(developerCreditAccounts)); + const accountColumn = (field: { name: string }) => + sql`${accounts}.${sql.identifier(field.name)}`; + const accountRow = accountColumn(developerCreditAccounts.id); + const ownerAccount = (field: { name: string }) => + sql`${sql.identifier("owner_account")}.${sql.identifier(field.name)}`; + const ownerEntry = ledgerAlias("owner_entry"); + const ownerSettled = ledgerAlias("owner_settled"); + const ownReservation = ledgerAlias("own_reservation"); + const reservedLike = `${RESERVATION_PREFIX}%`; + const jsonCents = (column: SQL) => + sql`cast(json_unquote(json_extract(${column}, '$.amountCents')) as unsigned)`; + const pendingAttempt = ledgerAlias("pending_attempt"); + const settledAttempt = ledgerAlias("settled_attempt"); + const declinedAttempt = ledgerAlias("declined_attempt"); + const laterTopUp = ledgerAlias("later_top_up"); + const recentTopUp = ledgerAlias("recent_top_up"); + const dayAgo = sqlTime(new Date(now.getTime() - DAY_MS)); + const reconcileFrom = sqlTime( + new Date(now.getTime() - AUTO_TOP_UP_PENDING_RECONCILE_MS), + ); + return { + pending: sql`exists (select 1 from ${pendingAttempt.from} where ${pendingAttempt.accountId} = ${accountRow} and ${pendingAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${pendingAttempt.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${pendingAttempt.createdAt} > ${reconcileFrom} and ${pendingAttempt.referenceId} not like ${`${RESERVATION_PREFIX}%`} and not exists (select 1 from ${settledAttempt.from} where ${settledAttempt.accountId} = ${pendingAttempt.accountId} and ${settledAttempt.referenceId} = ${pendingAttempt.referenceId} and ${settledAttempt.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`, + decline: sql`exists (select 1 from ${declinedAttempt.from} where ${declinedAttempt.accountId} = ${accountRow} and ${declinedAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_FAILED_REFERENCE_TYPE]} and ${declinedAttempt.referenceType} = ${AUTO_TOP_UP_FAILED_REFERENCE_TYPE} and ${declinedAttempt.createdAt} > ${dayAgo} and not exists (select 1 from ${laterTopUp.from} where ${laterTopUp.accountId} = ${declinedAttempt.accountId} and ${laterTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${laterTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${laterTopUp.createdAt} > ${declinedAttempt.createdAt}))`, + owner: sql`(select coalesce(sum(case when ${ownerEntry.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} then ${ownerEntry.amountMicroCredits} else ${jsonCents(ownerEntry.metadata)} * ${MICRO_CREDITS_PER_CENT} end), 0) from ${accounts} ${sql.identifier("owner_account")} join ${ownerEntry.from} on ${ownerEntry.accountId} = ${ownerAccount(developerCreditAccounts.id)} where ${ownerAccount(developerCreditAccounts.ownerId)} = ${accountColumn(developerCreditAccounts.ownerId)} and ${ownerEntry.type} in (${inList([ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE], ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]])}) and not (${ownerEntry.accountId} = ${accountRow} and ${ownerEntry.referenceId} like ${reservedLike}) and ((${ownerEntry.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${ownerEntry.createdAt} > ${dayAgo}) or (${ownerEntry.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and not exists (select 1 from ${ownerSettled.from} where ${ownerSettled.accountId} = ${ownerEntry.accountId} and ${ownerSettled.referenceId} = ${ownerEntry.referenceId} and ${ownerSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)}))))) + coalesce((select ${jsonCents(ownReservation.metadata)} from ${ownReservation.from} where ${ownReservation.accountId} = ${accountRow} and ${ownReservation.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${ownReservation.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${ownReservation.referenceId} like ${reservedLike} limit 1), ${accountColumn(developerCreditAccounts.autoTopUpAmountCents)}) * ${MICRO_CREDITS_PER_CENT} > ${AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS * MICRO_CREDITS_PER_CENT}`, + limit: sql`(select count(*) from ${recentTopUp.from} where ${recentTopUp.accountId} = ${accountRow} and ${recentTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${recentTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${recentTopUp.createdAt} > ${dayAgo}) >= ${AUTO_TOP_UP_DAILY_LIMIT}`, + }; +} + +export function autoTopUpNeedsReconcile(now: Date) { + const stale = ledgerAlias("stale_payment"); + const staleSettled = ledgerAlias("stale_settled"); + const accountRow = sql`${sql.identifier(getTableName(developerCreditAccounts))}.${sql.identifier(developerCreditAccounts.id.name)}`; + const reconcileFrom = sqlTime( + new Date(now.getTime() - AUTO_TOP_UP_PENDING_RECONCILE_MS), + ); + return sql`exists (select 1 from ${stale.from} where ${stale.accountId} = ${accountRow} and ${stale.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${stale.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${stale.createdAt} <= ${reconcileFrom} and not exists (select 1 from ${staleSettled.from} where ${staleSettled.accountId} = ${stale.accountId} and ${staleSettled.referenceId} = ${stale.referenceId} and ${staleSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`; +} + +export function autoTopUpReleased(now: Date) { + const waits = autoTopUpWaits(now); + return sql`not (${waits.pending} or ${waits.decline} or ${waits.limit} or ${waits.owner})`; } export function autoTopUpLastAttemptAt() { @@ -193,6 +371,126 @@ async function lastAttempt( return row ?? null; } +async function attemptHistory(executor: InfraDbExecutor, accountId: string) { + const [settled] = await executor + .select({ + topUps: sql< + number | string | null + >`SUM(${developerCreditTransactions.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE})`, + declines: sql< + number | string | null + >`SUM(${developerCreditTransactions.referenceType} = ${AUTO_TOP_UP_FAILED_REFERENCE_TYPE})`, + }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + inArray( + developerCreditTransactions.referenceType, + SETTLED_REFERENCE_TYPES, + ), + ), + ); + return { + topUps: Number(settled?.topUps ?? 0), + declines: Number(settled?.declines ?? 0), + }; +} + +async function unresolvedPayments(accountId: string) { + const pending = await db() + .select({ + pendingRow: developerCreditTransactions.id, + pendingId: developerCreditTransactions.referenceId, + pendingSince: developerCreditTransactions.createdAt, + }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "topup_pending"), + eq( + developerCreditTransactions.referenceType, + AUTO_TOP_UP_PENDING_REFERENCE_TYPE, + ), + ), + ); + const pendingIds = pending.flatMap((row) => + row.pendingId ? [row.pendingId] : [], + ); + const resolved = pendingIds.length + ? await db() + .select({ settledId: developerCreditTransactions.referenceId }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + inArray( + developerCreditTransactions.referenceType, + SETTLED_REFERENCE_TYPES, + ), + inArray(developerCreditTransactions.referenceId, pendingIds), + ), + ) + : []; + const resolvedIds = new Set(resolved.map((row) => row.settledId)); + return pending.flatMap((row) => + row.pendingId && !resolvedIds.has(row.pendingId) + ? [ + { + rowId: row.pendingRow, + referenceId: row.pendingId, + since: new Date(row.pendingSince), + }, + ] + : [], + ); +} + +async function reconcileReservation( + accountId: string, + key: string, + rowId: string, +) { + const attempt = key.slice(`${RESERVATION_PREFIX}${accountId}:`.length); + const found = await stripe().paymentIntents.search( + { + query: `metadata['accountId']:'${accountId}' AND metadata['attempt']:'${attempt}'`, + limit: 1, + }, + { timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS }, + ); + const [paymentIntent] = found.data; + if (!paymentIntent) { + await dropReservation(db(), accountId, key, rowId); + return; + } + if (paymentIntent.status === "processing") { + await linkReservation(accountId, key, paymentIntent.id, rowId); + return; + } + await settleAutoTopUpPaymentIntent( + paymentIntent, + paymentIntent.status === "succeeded" ? "succeeded" : "failed", + ); +} + +async function reconcileStalePayments(accountId: string, now: Date) { + const stale = (await unresolvedPayments(accountId)).filter( + (payment) => + now.getTime() - payment.since.getTime() >= + AUTO_TOP_UP_PENDING_RECONCILE_MS, + ); + for (const payment of stale) { + if (payment.referenceId.startsWith(RESERVATION_PREFIX)) { + await reconcileReservation(accountId, payment.referenceId, payment.rowId); + } else { + await reconcilePendingAutoTopUp(accountId, payment.referenceId); + } + } + return stale.length > 0; +} + export async function getAutoTopUpFailure(accountId: string) { const attempt = await lastAttempt(accountId, SETTLED_REFERENCE_TYPES); if (attempt?.type !== "topup_failed") return null; @@ -212,12 +510,14 @@ async function failAutoTopUp({ reason, amountCents, period, + reservationKey = null, }: { accountId: string; paymentIntentId: string | null; reason: string; amountCents: number; period: string; + reservationKey?: string | null; }) { const referenceId = paymentIntentId ?? `declined-${period}`; const recorded = await db().transaction(async (tx) => { @@ -230,6 +530,7 @@ async function failAutoTopUp({ .limit(1) .for("update"); if (!account) return false; + await dropReservation(tx, accountId, reservationKey); const [existing] = await tx .select({ id: developerCreditTransactions.id }) @@ -271,67 +572,18 @@ async function failAutoTopUp({ return recorded; } -async function recordPendingAutoTopUp({ - accountId, - paymentIntentId, - amountCents, - period, -}: { - accountId: string; - paymentIntentId: string; - amountCents: number; - period: string; -}) { - await db().transaction(async (tx) => { - const [account] = await tx - .select({ - balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, - }) - .from(developerCreditAccounts) - .where(eq(developerCreditAccounts.id, accountId)) - .limit(1) - .for("update"); - if (!account) return; - - const [existing] = await tx - .select({ id: developerCreditTransactions.id }) - .from(developerCreditTransactions) - .where( - and( - eq(developerCreditTransactions.accountId, accountId), - eq(developerCreditTransactions.referenceId, paymentIntentId), - inArray( - developerCreditTransactions.referenceType, - ATTEMPT_REFERENCE_TYPES, - ), - ), - ) - .limit(1); - if (existing) return; - - await tx.insert(developerCreditTransactions).values({ - id: nanoId(), - accountId, - type: "topup_pending", - amountMicroCredits: 0, - balanceAfterMicroCredits: account.balanceMicroCredits, - referenceId: paymentIntentId, - referenceType: AUTO_TOP_UP_PENDING_REFERENCE_TYPE, - metadata: { amountCents, period }, - }); - }); -} - function grantAutoTopUp({ accountId, paymentIntentId, amountCents, period, + executor, }: { accountId: string; paymentIntentId: string; amountCents: number; period: string | null; + executor?: AutoTopUpTransaction; }) { return addCreditsToAccount({ accountId, @@ -339,9 +591,53 @@ function grantAutoTopUp({ referenceId: paymentIntentId, referenceType: AUTO_TOP_UP_REFERENCE_TYPE, metadata: { amountCents, autoTopUp: true, period }, + executor, }); } +async function reconcilePendingAutoTopUp( + accountId: string, + paymentIntentId: string, +) { + const paymentIntent = await stripe().paymentIntents.retrieve( + paymentIntentId, + {}, + { timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS }, + ); + if (paymentIntent.metadata?.accountId !== accountId) return; + if (paymentIntent.status === "processing") { + console.warn("[infra-billing] Auto top-up payment still processing", { + accountId, + paymentIntentId, + }); + return; + } + await settleAutoTopUpPaymentIntent( + paymentIntent, + paymentIntent.status === "succeeded" ? "succeeded" : "failed", + ); +} + +const RESERVATION_REPLAY_MS = DAY_MS; +const STRIPE_DESCRIPTION_LIMIT = 1_000; + +export function autoTopUpDescription(appName: string | null | undefined) { + const suffix = ", auto top-up"; + const prefix = "Cap Infra credits for "; + const name = (appName ?? "").replace(/\s+/g, " ").trim(); + if (!name) return `Cap Infra credits${suffix}`; + const room = STRIPE_DESCRIPTION_LIMIT - prefix.length - suffix.length; + return `${prefix}${name.slice(0, room).trim()}${suffix}`; +} + +type ReservedRequest = { + amountCents: number; + customerId: string; + paymentMethodId: string; + receiptEmail: string; + description: string; +}; + export async function runAutoTopUp( accountId: string, now: Date = new Date(), @@ -349,13 +645,9 @@ export async function runAutoTopUp( const [account] = await db() .select({ appId: developerCreditAccounts.appId, - balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, - stripeCustomerId: developerCreditAccounts.stripeCustomerId, - stripePaymentMethodId: developerCreditAccounts.stripePaymentMethodId, - autoTopUpEnabled: developerCreditAccounts.autoTopUpEnabled, - autoTopUpThresholdMicroCredits: - developerCreditAccounts.autoTopUpThresholdMicroCredits, - autoTopUpAmountCents: developerCreditAccounts.autoTopUpAmountCents, + appName: developerApps.name, + appDeletedAt: developerApps.deletedAt, + ownerId: developerCreditAccounts.ownerId, email: users.email, }) .from(developerCreditAccounts) @@ -364,96 +656,205 @@ export async function runAutoTopUp( eq(developerCreditAccounts.appId, developerApps.id), ) .innerJoin(users, eq(developerCreditAccounts.ownerId, users.id)) - .where( - and( - eq(developerCreditAccounts.id, accountId), - isNull(developerApps.deletedAt), - ), - ) + .where(eq(developerCreditAccounts.id, accountId)) .limit(1); if (!account) return { state: "skipped", reason: "not_found" }; - if (!account.autoTopUpEnabled) - return { state: "skipped", reason: "disabled" }; + if (await reconcileStalePayments(accountId, now)) { + return { state: "skipped", reason: "reconciled" }; + } + if (account.appDeletedAt) return { state: "skipped", reason: "app_deleted" }; if (!canUseInfra(account.email)) { return { state: "skipped", reason: "not_allowed" }; } - if (!account.stripeCustomerId || !account.stripePaymentMethodId) { - return { state: "skipped", reason: "no_card" }; - } - const amountCents = account.autoTopUpAmountCents; - if ( - autoTopUpSettingsError({ - enabled: true, - thresholdMicroCredits: account.autoTopUpThresholdMicroCredits, - amountCents, - hasCard: true, - }) - ) { - return { state: "skipped", reason: "invalid_settings" }; - } - const available = (await availableBalance(account.appId)) ?? 0; - if (available >= account.autoTopUpThresholdMicroCredits) { - return { state: "skipped", reason: "above_threshold" }; - } - - if (await disputedSince(db(), accountId)) { - return { state: "skipped", reason: "disputed" }; - } + const period = utcDateKey(now); + const outcome = await db().transaction(async (tx): Promise => { + await tx + .select({ id: developerCreditAccounts.id }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.ownerId, account.ownerId)) + .orderBy(asc(developerCreditAccounts.id)) + .for("update"); + const [app] = await tx + .select({ appDeletedAt: developerApps.deletedAt }) + .from(developerApps) + .where(eq(developerApps.id, account.appId)) + .limit(1); + if (!app || app.appDeletedAt) { + return { kind: "skipped", reason: "app_deleted" }; + } + const waits = autoTopUpWaits(now); + const [current] = await tx + .select({ + id: developerCreditAccounts.id, + enabled: developerCreditAccounts.autoTopUpEnabled, + thresholdMicroCredits: + developerCreditAccounts.autoTopUpThresholdMicroCredits, + amountCents: developerCreditAccounts.autoTopUpAmountCents, + customerId: developerCreditAccounts.stripeCustomerId, + paymentMethodId: developerCreditAccounts.stripePaymentMethodId, + balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, + pendingWait: waits.pending, + declineWait: waits.decline, + limitWait: waits.limit, + ownerWait: waits.owner, + }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.id, accountId)) + .limit(1); + if (!current) return { kind: "skipped", reason: "not_found" }; + if (!current.enabled) return { kind: "skipped", reason: "disabled" }; + if (!current.customerId || !current.paymentMethodId) { + return { kind: "skipped", reason: "no_card" }; + } + if ( + autoTopUpSettingsError({ + enabled: true, + thresholdMicroCredits: current.thresholdMicroCredits, + amountCents: current.amountCents, + hasCard: true, + }) + ) { + return { kind: "skipped", reason: "invalid_settings" }; + } + const available = (await availableBalance(account.appId, tx)) ?? 0; + if (available >= current.thresholdMicroCredits) { + return { kind: "skipped", reason: "above_threshold" }; + } + if (await disputedSince(tx, accountId)) { + return { kind: "skipped", reason: "disputed" }; + } + if (Number(current.pendingWait)) { + return { kind: "skipped", reason: "awaiting_payment" }; + } + if (Number(current.declineWait)) { + return { kind: "skipped", reason: "recent_decline" }; + } + if (Number(current.limitWait)) { + return { kind: "skipped", reason: "daily_limit" }; + } + if (Number(current.ownerWait)) { + return { kind: "skipped", reason: "owner_daily_limit" }; + } + const attempt = await attemptHistory(tx, accountId); + const key = autoTopUpIdempotencyKey(accountId, attempt); + const [reserved] = await tx + .select({ + reservedAt: developerCreditTransactions.createdAt, + reservation: developerCreditTransactions.metadata, + }) + .from(developerCreditTransactions) + .where(reservationOf(accountId, key)) + .limit(1); + if (reserved) { + if ( + now.getTime() - new Date(reserved.reservedAt).getTime() >= + RESERVATION_REPLAY_MS + ) { + return { kind: "skipped", reason: "awaiting_payment" }; + } + const request = reservedRequestOf(reserved.reservation); + if (!request) return { kind: "skipped", reason: "in_progress" }; + return { kind: "reserved", key, attempt, request }; + } + const request: ReservedRequest = { + amountCents: current.amountCents, + customerId: current.customerId, + paymentMethodId: current.paymentMethodId, + receiptEmail: account.email, + description: autoTopUpDescription(account.appName), + }; + await tx.insert(developerCreditTransactions).values({ + id: nanoId(), + accountId, + type: "topup_pending", + amountMicroCredits: 0, + balanceAfterMicroCredits: current.balanceMicroCredits, + referenceId: key, + referenceType: AUTO_TOP_UP_PENDING_REFERENCE_TYPE, + metadata: { ...request, period }, + }); + return { kind: "reserved", key, attempt, request }; + }); - const previous = await lastAttempt(accountId); - if ( - previous && - new Date(previous.createdAt).getTime() > autoTopUpRetryCutoff(now).getTime() - ) { - return { state: "skipped", reason: "recent_attempt" }; + if (outcome.kind === "skipped") { + if ( + outcome.reason === "daily_limit" || + outcome.reason === "owner_daily_limit" + ) { + console.warn("[infra-billing] Auto top-up waits for its daily limit", { + accountId, + reason: outcome.reason, + }); + } + return { state: "skipped", reason: outcome.reason }; } - const period = utcDateKey(now); + const { key, attempt, request: reserved } = outcome; + const amountCents = reserved.amountCents; const productId = developerCreditsProductId(); + const request = { + amount: amountCents, + currency: "usd", + customer: reserved.customerId, + payment_method: reserved.paymentMethodId, + payment_method_types: ["card"], + off_session: true, + confirm: true, + description: reserved.description, + receipt_email: reserved.receiptEmail, + metadata: { + type: AUTO_TOP_UP_METADATA_TYPE, + accountId, + appId: account.appId, + amountCents: String(amountCents), + attempt: `${attempt.topUps}:${attempt.declines}`, + ...(productId ? { productId } : {}), + }, + } satisfies Stripe.PaymentIntentCreateParams; let paymentIntent: Stripe.PaymentIntent; try { - paymentIntent = await stripe().paymentIntents.create( - { - amount: amountCents, - currency: "usd", - customer: account.stripeCustomerId, - payment_method: account.stripePaymentMethodId, - payment_method_types: ["card"], - off_session: true, - confirm: true, - description: "Cap Infra credits, auto top-up", - receipt_email: account.email, - metadata: { - type: AUTO_TOP_UP_METADATA_TYPE, - accountId, - appId: account.appId, - amountCents: String(amountCents), - period, - ...(productId ? { productId } : {}), - }, - }, - { idempotencyKey: autoTopUpIdempotencyKey(accountId, period) }, - ); + paymentIntent = await stripe().paymentIntents.create(request, { + idempotencyKey: key, + timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + maxNetworkRetries: 0, + }); } catch (error) { + if (isUnresolvedAttempt(error)) { + return { state: "skipped", reason: "in_progress" }; + } const decline = declineFrom(error); - if (!decline) { - console.error("[infra-billing] Auto top-up could not be attempted", { + if (decline) { + await failAutoTopUp({ accountId, - error, + amountCents, + period, + reservationKey: key, + ...decline, }); - return { state: "error" }; + return { state: "failed", reason: decline.reason }; + } + if (neverReachedStripe(error)) { + await dropReservation(db(), accountId, key); } - await failAutoTopUp({ accountId, amountCents, period, ...decline }); - return { state: "failed", reason: decline.reason }; + console.error("[infra-billing] Auto top-up could not be attempted", { + accountId, + error, + }); + return { state: "error" }; } if (paymentIntent.status === "succeeded") { - const balanceMicroCredits = await grantAutoTopUp({ - accountId, - paymentIntentId: paymentIntent.id, - amountCents: paymentIntent.amount_received, - period, + const balanceMicroCredits = await db().transaction(async (tx) => { + const balance = await grantAutoTopUp({ + accountId, + paymentIntentId: paymentIntent.id, + amountCents: paymentIntent.amount_received, + period, + executor: tx, + }); + await dropReservation(tx, accountId, key); + return balance; }); return { state: "succeeded", @@ -461,27 +862,57 @@ export async function runAutoTopUp( balanceMicroCredits, }; } - if (paymentIntent.status === "processing") { - await recordPendingAutoTopUp({ - accountId, - paymentIntentId: paymentIntent.id, - amountCents, - period, - }); + await linkReservation(accountId, key, paymentIntent.id); return { state: "pending", paymentIntentId: paymentIntent.id }; } - await failAutoTopUp({ accountId, paymentIntentId: paymentIntent.id, reason: paymentIntent.status, amountCents, period, + reservationKey: key, }); return { state: "failed", reason: paymentIntent.status }; } +function reservedRequestOf(value: unknown): ReservedRequest | null { + let metadata: unknown = value; + if (typeof metadata === "string") { + try { + metadata = JSON.parse(metadata); + } catch { + return null; + } + } + if (typeof metadata !== "object" || metadata === null) return null; + const fields = metadata as Record; + const { + amountCents, + customerId, + paymentMethodId, + receiptEmail, + description, + } = fields; + if ( + typeof amountCents !== "number" || + typeof customerId !== "string" || + typeof paymentMethodId !== "string" || + typeof receiptEmail !== "string" || + typeof description !== "string" + ) { + return null; + } + return { + amountCents, + customerId, + paymentMethodId, + receiptEmail, + description, + }; +} + type DueCursor = { attemptKey: string; id: string }; function dueAutoTopUpsAfter(now: Date, cursor: DueCursor | null) { @@ -500,11 +931,16 @@ function dueAutoTopUpsAfter(now: Date, cursor: DueCursor | null) { .leftJoin(held, eq(held.appId, developerCreditAccounts.appId)) .where( and( - eq(developerCreditAccounts.autoTopUpEnabled, true), - isNotNull(developerCreditAccounts.stripePaymentMethodId), - isNull(developerApps.deletedAt), - sql`${developerCreditAccounts.balanceMicroCredits} < ${developerCreditAccounts.autoTopUpThresholdMicroCredits} + coalesce(${held.heldMicroCredits}, 0)`, - sql`${attemptKey} <= ${sqlTime(autoTopUpRetryCutoff(now))}`, + or( + and( + isNull(developerApps.deletedAt), + eq(developerCreditAccounts.autoTopUpEnabled, true), + isNotNull(developerCreditAccounts.stripePaymentMethodId), + sql`${developerCreditAccounts.balanceMicroCredits} < ${developerCreditAccounts.autoTopUpThresholdMicroCredits} + coalesce(${held.heldMicroCredits}, 0)`, + autoTopUpReleased(now), + ), + autoTopUpNeedsReconcile(now), + ), cursor ? sql`(${attemptKey} > ${cursor.attemptKey} or (${attemptKey} = ${cursor.attemptKey} and ${developerCreditAccounts.id} > ${cursor.id}))` : undefined, @@ -543,6 +979,46 @@ export async function runDueAutoTopUps( } } +async function creditAccountOf(target: AutoTopUpTarget) { + if ("accountId" in target) return target.accountId; + const [account] = await db() + .select({ id: developerCreditAccounts.id }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.appId, target.appId)) + .limit(1); + return account?.id ?? null; +} + +export type AutoTopUpTarget = { accountId: string } | { appId: string }; + +export async function scheduleAutoTopUp( + target: AutoTopUpTarget, +): Promise { + const attempt = async () => { + try { + const accountId = await creditAccountOf(target); + if (!accountId) return; + const result = await runAutoTopUp(accountId); + if (result.state !== "skipped") { + console.log("[infra-billing] Auto top-up after a charge", { + accountId, + state: result.state, + }); + } + } catch (error) { + console.error("[infra-billing] Auto top-up after a charge failed", { + target, + error, + }); + } + }; + try { + after(attempt); + } catch { + await attempt(); + } +} + export async function settleAutoTopUpPaymentIntent( paymentIntent: Stripe.PaymentIntent, outcome: "succeeded" | "failed", @@ -559,16 +1035,21 @@ export async function settleAutoTopUpPaymentIntent( ) { return false; } - await grantAutoTopUp({ - accountId, - paymentIntentId: paymentIntent.id, - amountCents: paymentIntent.amount_received, - period: period ?? null, + await db().transaction(async (tx) => { + await grantAutoTopUp({ + accountId, + paymentIntentId: paymentIntent.id, + amountCents: paymentIntent.amount_received, + period: period ?? null, + executor: tx, + }); + await dropReservation(tx, accountId, reservationKeyOf(paymentIntent)); }); return true; } await failAutoTopUp({ + reservationKey: reservationKeyOf(paymentIntent), accountId, paymentIntentId: paymentIntent.id, reason: diff --git a/apps/web/lib/infra-ledger-store.ts b/apps/web/lib/infra-ledger-store.ts index f86bac9b4bf..73c04ef00d3 100644 --- a/apps/web/lib/infra-ledger-store.ts +++ b/apps/web/lib/infra-ledger-store.ts @@ -3,7 +3,7 @@ import { developerCreditAccounts, developerCreditTransactions, } from "@cap/database/schema"; -import { and, asc, desc, eq, gt, gte, lt, lte, or } from "drizzle-orm"; +import { and, asc, desc, eq, gt, gte, lt, lte, or, sql } from "drizzle-orm"; import { type LedgerRow, orderLedgerRows } from "@/lib/infra-ledger"; const EXPORT_BATCH_SIZE = 1_000; @@ -20,7 +20,11 @@ const columns = { createdAt: developerCreditTransactions.createdAt, }; -const ofApp = (appId: string) => eq(developerCreditAccounts.appId, appId); +const ofApp = (appId: string) => + and( + eq(developerCreditAccounts.appId, appId), + sql`${developerCreditTransactions.type} <> 'topup_pending'`, + ); const appLedger = () => db() diff --git a/apps/web/lib/infra-video-upload.ts b/apps/web/lib/infra-video-upload.ts index 2dd6966ab88..f4388267281 100644 --- a/apps/web/lib/infra-video-upload.ts +++ b/apps/web/lib/infra-video-upload.ts @@ -29,6 +29,7 @@ import { withoutReservedMetadata, } from "@/lib/developer-video-takedown"; import { canUseInfra } from "@/lib/infra-access"; +import { scheduleAutoTopUp } from "@/lib/infra-auto-top-up"; import { lockInfraAccount } from "@/lib/infra-balance"; import { getInfraUploadAdmission, @@ -443,6 +444,7 @@ export async function createInfraVideo({ > { const admission = await getInfraUploadAdmission(appId); if (!admission.allowed) { + await scheduleAutoTopUp({ appId }); return { status: 402, body: uploadAdmissionError(admission) }; } @@ -765,6 +767,7 @@ export async function completeInfraUpload({ if (await videoIsRecorded(db(), videoId)) return ok({ success: true }); const admission = await admitDeveloperVideoUpload(upload); if (!admission.allowed) { + await scheduleAutoTopUp({ appId }); return { status: 402, body: uploadAdmissionError(admission) }; } } @@ -830,6 +833,7 @@ export async function completeInfraUpload({ await removeStoredFile(s3Key); await releaseDeveloperVideoUpload(videoId, claim); claim = null; + await scheduleAutoTopUp({ appId }); const refusal = uploadAdmissionError(admission); return { status: 402,