From 5ec0285e7b1fde487a25a620619f5ea3dae08b15 Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:33:11 -0500 Subject: [PATCH 01/26] fix: allow several Cap Infra auto top-ups a day A top-up now waits only after a decline, while a payment is still processing, or at ten automatic top-ups in any 24 hours. Each attempt is keyed by the account's count of settled top-ups and the exact request, read under the account lock, so a retried or racing trigger makes one payment. Adds scheduleAutoTopUp, which runs a top-up after the response is sent and never throws into the caller. --- .../infra-auto-top-up-database.test.ts | 209 ++++++++++++++++ .../__tests__/unit/infra-auto-top-up.test.ts | 199 +++++++++++++-- .../__tests__/unit/infra-billing-fake-db.ts | 19 ++ apps/web/lib/infra-auto-top-up.ts | 229 +++++++++++++++--- 4 files changed, 593 insertions(+), 63 deletions(-) create mode 100644 apps/web/__tests__/integration/infra-auto-top-up-database.test.ts diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts new file mode 100644 index 00000000000..ed762e2a53a --- /dev/null +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -0,0 +1,209 @@ +import { randomUUID } from "node:crypto"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +type FakeIntent = { + id: string; + status: "succeeded"; + amount: number; + amount_received: number; + currency: string; + metadata: Record; +}; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + keys: new Map< + string, + { state: "pending" } | { state: "done"; intent: FakeIntent } + >(), + created: [] as FakeIntent[], + createdKeys: [] as string[], +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ WEB_URL: "https://web" }), +})); +vi.mock("@/lib/developer-credits-product", () => ({ + developerCreditsProductId: () => "prod_test", +})); +vi.mock("@/lib/infra-billing-emails", () => ({ + sendInfraTopUpFailedEmail: vi.fn(), +})); +vi.mock("@cap/utils", () => ({ + stripe: () => ({ + paymentIntents: { + create: async ( + params: { amount: number; metadata: Record }, + options: { idempotencyKey: string }, + ) => { + const key = options.idempotencyKey; + const entry = fixture.keys.get(key); + if (entry?.state === "pending") { + throw Object.assign(new Error("Key in use"), { + type: "StripeInvalidRequestError", + code: "idempotency_key_in_use", + statusCode: 409, + }); + } + if (entry?.state === "done") return entry.intent; + fixture.keys.set(key, { state: "pending" }); + await new Promise((resolve) => setTimeout(resolve, 40)); + const intent: FakeIntent = { + id: `pi_${fixture.created.length + 1}`, + status: "succeeded", + amount: params.amount, + amount_received: params.amount, + currency: "usd", + metadata: params.metadata, + }; + fixture.created.push(intent); + fixture.createdKeys.push(key); + fixture.keys.set(key, { state: "done", intent }); + return intent; + }, + }, + }), +})); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); + +describe.runIf(Boolean(databaseUrl))( + "Cap Infra auto top-up against a real database", + () => { + let pool: Pool; + const accounts: string[] = []; + + const makeAccount = async (balanceMicroCredits: number) => { + const userId = id(); + const appId = id(); + const accountId = id(); + accounts.push(accountId); + await pool.query( + "INSERT INTO users (id, email, activeOrganizationId) VALUES (?, ?, '')", + [userId, `${userId}@example.com`], + ); + await pool.query( + "INSERT INTO developer_apps (id, ownerId, name, environment) VALUES (?, ?, 'Top-up test', 'production')", + [appId, userId], + ); + await pool.query( + `INSERT INTO developer_credit_accounts + (id, appId, ownerId, balanceMicroCredits, stripeCustomerId, stripePaymentMethodId, + autoTopUpEnabled, autoTopUpThresholdMicroCredits, autoTopUpAmountCents) + VALUES (?, ?, ?, ?, 'cus_test', 'pm_test', 1, 500000, 500)`, + [accountId, appId, userId, balanceMicroCredits], + ); + return { accountId, appId }; + }; + + const balanceOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT balanceMicroCredits AS balance FROM developer_credit_accounts WHERE id = ?", + [accountId], + ); + return Number((rows as { balance: number }[])[0]?.balance); + }; + + const topUpsOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT referenceId FROM developer_credit_transactions WHERE accountId = ? AND referenceType = 'auto_top_up' AND type = 'topup'", + [accountId], + ); + return (rows as { referenceId: string }[]).map((row) => row.referenceId); + }; + + const setBalance = (accountId: string, balanceMicroCredits: number) => + pool.query( + "UPDATE developer_credit_accounts SET balanceMicroCredits = ? WHERE id = ?", + [balanceMicroCredits, accountId], + ); + + beforeAll(() => { + pool = createPool({ uri: databaseUrl, connectionLimit: 12 }); + fixture.database = drizzle(pool, { + mode: "default", + }) as MySql2Database; + }); + + afterAll(async () => { + for (const accountId of accounts) { + await pool.query( + "DELETE FROM developer_credit_transactions WHERE accountId = ?", + [accountId], + ); + await pool.query( + "DELETE a, c, u FROM developer_credit_accounts c JOIN developer_apps a ON a.id = c.appId JOIN users u ON u.id = c.ownerId WHERE c.id = ?", + [accountId], + ); + } + await pool.end(); + }); + + it("creates one charge when several triggers race for one account", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + const early = Array.from({ length: 6 }, () => runAutoTopUp(accountId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + const late = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + const results = await Promise.all([...early, ...late]); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + expect( + results.filter((result) => result.state === "succeeded").length, + ).toBeGreaterThanOrEqual(1); + for (const result of results) { + expect(["succeeded", "skipped"]).toContain(result.state); + } + }); + + it("tops up again the same day with a new key once the balance is low again", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect((await runAutoTopUp(accountId)).state).toBe("skipped"); + await setBalance(accountId, 10); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + + const keys = fixture.createdKeys.slice(before); + expect(keys).toHaveLength(2); + expect(new Set(keys).size).toBe(2); + expect(await topUpsOf(accountId)).toHaveLength(2); + }); + + it("stops at the daily limit of automatic top-ups", async () => { + const { AUTO_TOP_UP_DAILY_LIMIT, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + + for (let i = 0; i < AUTO_TOP_UP_DAILY_LIMIT; i++) { + await setBalance(accountId, 10); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + } + await setBalance(accountId, 10); + expect(await runAutoTopUp(accountId)).toEqual({ + state: "skipped", + reason: "daily_limit", + }); + expect(await topUpsOf(accountId)).toHaveLength(AUTO_TOP_UP_DAILY_LIMIT); + warn.mockRestore(); + }); + }, +); diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index 151022dd457..7fa45975451 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -9,8 +9,11 @@ const mocks = vi.hoisted(() => ({ retrievePaymentMethod: vi.fn(), detachPaymentMethod: vi.fn(), sendEmail: vi.fn(), + after: vi.fn(), })); +vi.mock("next/server", () => ({ after: mocks.after })); + vi.mock("@cap/database", () => ({ db: () => mocks.db })); vi.mock("@cap/database/helpers", () => ({ nanoId: () => `row-${++mocks.ids}`, @@ -46,6 +49,7 @@ vi.mock("@/lib/developer-credits-product", () => ({ import type Stripe from "stripe"; import { + AUTO_TOP_UP_DAILY_LIMIT, autoTopUpFailureText, autoTopUpIdempotencyKey, autoTopUpSettingsError, @@ -53,6 +57,7 @@ import { runAutoTopUp, runDueAutoTopUps, saveCardFromCheckout, + scheduleAutoTopUp, settleAutoTopUpPaymentIntent, } from "@/lib/infra-auto-top-up"; import { @@ -181,10 +186,16 @@ describe("auto top-up settings", () => { ).toBe(null); }); - it("keys a charge by account and UTC day", () => { - expect(autoTopUpIdempotencyKey("account-1", "2026-11-11")).toBe( - "infra-auto-top-up:account-1:2026-11-11", - ); + it("keys each charge by account, attempt and the exact request", () => { + const request = { amount: 500, customer: "cus_1" }; + const key = autoTopUpIdempotencyKey("account-1", 3, request); + + expect(key).toMatch(/^infra-auto-top-up:account-1:3:[0-9a-f]{24}$/); + expect(autoTopUpIdempotencyKey("account-1", 3, { ...request })).toBe(key); + expect(autoTopUpIdempotencyKey("account-1", 4, request)).not.toBe(key); + expect( + autoTopUpIdempotencyKey("account-1", 3, { ...request, amount: 600 }), + ).not.toBe(key); }); it("explains a decline in plain words", () => { @@ -281,11 +292,11 @@ describe("auto top-up charge", () => { metadata: { type: "developer_auto_top_up", accountId: "account-1", - period: "2026-11-11", + sequence: "0", }, }); expect(options).toEqual({ - idempotencyKey: "infra-auto-top-up:account-1:2026-11-11", + idempotencyKey: autoTopUpIdempotencyKey("account-1", 0, params), }); expect(ledger.account.balanceMicroCredits).toBe(530_000); expect(topUps()).toEqual([ @@ -323,31 +334,90 @@ describe("auto top-up charge", () => { expect(ledger.account.balanceMicroCredits).toBe(530_000); }); - it("makes at most one attempt per account in 24 hours", async () => { + it("tops up again the same day once the balance falls below the threshold again", async () => { await runAutoTopUp("account-1", now()); - ledger.account.balanceMicroCredits = 10; - - later(1); + later(0.1); expect(await runAutoTopUp("account-1", now())).toEqual({ state: "skipped", - reason: "recent_attempt", - }); - later(22.9); - expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + reason: "above_threshold", }); - expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); - later(0.2); + ledger.account.balanceMicroCredits = 10; mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); expect(await runAutoTopUp("account-1", now())).toMatchObject({ state: "succeeded", paymentIntentId: "pi_2", }); - expect(mocks.createPaymentIntent.mock.calls[1]?.[1]).toEqual({ - idempotencyKey: "infra-auto-top-up:account-1:2026-11-12", - }); + + const [first, second] = mocks.createPaymentIntent.mock.calls; + expect(second?.[0].metadata.sequence).toBe("1"); + expect(second?.[1].idempotencyKey).not.toBe(first?.[1].idempotencyKey); expect(topUps()).toHaveLength(2); + expect(now().toISOString().slice(0, 10)).toBe("2026-11-11"); + }); + + it("repeats the same key when retried before the attempt settles", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("network"), { type: "StripeConnectionError" }), + ); + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + + const [first, retry] = mocks.createPaymentIntent.mock.calls; + expect(retry?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); + expect(topUps()).toHaveLength(1); + }); + + it("treats a payment already being made under the same key as in progress", async () => { + mocks.createPaymentIntent.mockRejectedValue( + Object.assign(new Error("in use"), { + type: "StripeInvalidRequestError", + code: "idempotency_key_in_use", + statusCode: 409, + }), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "in_progress", + }); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(ledger.rows).toHaveLength(0); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + }); + + it(`stops at ${AUTO_TOP_UP_DAILY_LIMIT} automatic top-ups in any 24 hours, then waits`, async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + for (let i = 1; i <= AUTO_TOP_UP_DAILY_LIMIT; i++) { + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue( + intent(`pi_${i}`, "succeeded"), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + later(1); + } + + ledger.account.balanceMicroCredits = 10; + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "daily_limit", + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes( + AUTO_TOP_UP_DAILY_LIMIT, + ); + + later(14.5); + mocks.createPaymentIntent.mockResolvedValue(intent("pi_next", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_next", + }); + warn.mockRestore(); }); it("grants nothing for a processing payment, then once when it succeeds", async () => { @@ -391,11 +461,11 @@ describe("auto top-up charge", () => { expect(now().toISOString().slice(0, 10)).toBe("2026-11-12"); expect(await runAutoTopUp("account-1", now())).toEqual({ state: "skipped", - reason: "recent_attempt", + reason: "awaiting_payment", }); later(23); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + reason: "awaiting_payment", }); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); @@ -507,9 +577,22 @@ describe("auto top-up charge", () => { ledger.account.autoTopUpEnabled = true; later(2); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + reason: "recent_decline", + }); + later(21.9); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "recent_decline", }); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + + later(0.2); + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + const [declined, next] = mocks.createPaymentIntent.mock.calls; + expect(next?.[1].idempotencyKey).not.toBe(declined?.[1].idempotencyKey); }); it("treats a charge that needs the customer as a failure", async () => { @@ -569,6 +652,76 @@ describe("auto top-up charge", () => { }); }); +describe("auto top-up after a charge", () => { + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + const outsideRequest = () => { + throw new Error("`after` was called outside a request scope"); + }; + + beforeEach(() => { + mocks.after.mockReset(); + }); + + it("waits until the response is sent, then tops up the account", async () => { + const tasks: (() => Promise)[] = []; + mocks.after.mockImplementation((task: () => Promise) => { + tasks.push(task); + }); + + scheduleAutoTopUp({ accountId: "account-1" }); + await settle(); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + + await tasks[0]?.(); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(topUps()).toHaveLength(1); + }); + + it("finds the account from the app", async () => { + const tasks: (() => Promise)[] = []; + mocks.after.mockImplementation((task: () => Promise) => { + tasks.push(task); + }); + + scheduleAutoTopUp({ appId: "app-1" }); + await tasks[0]?.(); + + expect(mocks.createPaymentIntent.mock.calls[0]?.[0].metadata).toMatchObject( + { accountId: "account-1", appId: "app-1" }, + ); + }); + + it("still tops up outside a request", async () => { + mocks.after.mockImplementation(outsideRequest); + + scheduleAutoTopUp({ accountId: "account-1" }); + await vi.waitFor(() => expect(topUps()).toHaveLength(1)); + }); + + it("never throws into the caller and logs what went wrong", async () => { + mocks.after.mockImplementation(outsideRequest); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const broken = { + ...fakeDb, + select: () => { + throw new Error("database unavailable"); + }, + }; + mocks.db = broken; + + expect(() => scheduleAutoTopUp({ accountId: "account-1" })).not.toThrow(); + await vi.waitFor(() => + expect(errors).toHaveBeenCalledWith( + "[infra-billing] Auto top-up after a charge failed", + expect.objectContaining({ target: { accountId: "account-1" } }), + ), + ); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + mocks.db = fakeDb; + errors.mockRestore(); + }); +}); + describe("saving a card from checkout", () => { const session = (extra: Partial = {}) => ({ diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index 69b9a4766f8..ce7bfc21ff9 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -166,6 +166,25 @@ function runSelect(query: Query): Record[] { ), ).slice(0, 1); } + if (keys === "createdAt,referenceId,type") { + const [, ...rest] = rendered.params; + const since = asTime(rest.at(-1)); + const types = rest.slice(0, -1); + return ledger.rows.filter( + (row) => + types.includes(row.referenceType) && row.createdAt.getTime() >= since, + ); + } + if (keys === "settledAttempts") { + const types = rendered.params.slice(1); + return [ + { + settledAttempts: ledger.rows.filter((row) => + types.includes(row.referenceType), + ).length, + }, + ]; + } if (keys === "createdAt,metadata,type") { const types = rendered.params.slice(1); return newestFirst( diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index bceba7b7c11..031f786adfa 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { db } from "@cap/database"; import { nanoId } from "@cap/database/helpers"; import { @@ -7,7 +8,17 @@ import { users, } from "@cap/database/schema"; import { stripe } from "@cap/utils"; -import { and, desc, eq, inArray, isNotNull, isNull } from "drizzle-orm"; +import { + and, + desc, + eq, + gte, + inArray, + isNotNull, + isNull, + sql, +} from "drizzle-orm"; +import { after } from "next/server"; import type Stripe from "stripe"; import { addCreditsToAccount } from "@/lib/developer-credits"; import { developerCreditsProductId } from "@/lib/developer-credits-product"; @@ -15,7 +26,7 @@ import { MICRO_CREDITS_PER_DOLLAR } from "@/lib/developer-pricing"; import { canUseInfra } from "@/lib/infra-access"; import { availableBalance } from "@/lib/infra-balance"; import { sendInfraTopUpFailedEmail } from "@/lib/infra-billing-emails"; -import { disputedSince } from "@/lib/infra-standing"; +import { disputedSince, type InfraDbExecutor } from "@/lib/infra-standing"; import { utcDateKey } from "@/lib/infra-usage"; export const AUTO_TOP_UP_REFERENCE_TYPE = "auto_top_up" as const; @@ -29,7 +40,8 @@ export const AUTO_TOP_UP_MIN_THRESHOLD_MICRO_CREDITS = MICRO_CREDITS_PER_DOLLAR; export const AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS = 1_000 * MICRO_CREDITS_PER_DOLLAR; -const ATTEMPT_INTERVAL_MS = 24 * 60 * 60 * 1000; +export const AUTO_TOP_UP_DAILY_LIMIT = 10; +const DAY_MS = 24 * 60 * 60 * 1000; export type AutoTopUpResult = | { @@ -42,7 +54,10 @@ export type AutoTopUpResult = | "invalid_settings" | "above_threshold" | "disputed" - | "recent_attempt"; + | "awaiting_payment" + | "recent_decline" + | "daily_limit" + | "in_progress"; } | { state: "succeeded"; paymentIntentId: string; balanceMicroCredits: number } | { state: "pending"; paymentIntentId: string } @@ -79,8 +94,16 @@ export function autoTopUpSettingsError({ return null; } -export function autoTopUpIdempotencyKey(accountId: string, period: string) { - return `infra-auto-top-up:${accountId}:${period}`; +export function autoTopUpIdempotencyKey( + accountId: string, + sequence: number, + request: unknown, +) { + const digest = createHash("sha256") + .update(JSON.stringify(request)) + .digest("hex") + .slice(0, 24); + return `infra-auto-top-up:${accountId}:${sequence}:${digest}`; } export function autoTopUpFailureText(reason: string | null | undefined) { @@ -96,6 +119,12 @@ export function autoTopUpFailureText(reason: string | null | undefined) { } } +function isKeyInUse(error: unknown) { + if (typeof error !== "object" || error === null) return false; + const fields = error as { code?: unknown; statusCode?: unknown }; + return fields.code === "idempotency_key_in_use" || fields.statusCode === 409; +} + function declineFrom(error: unknown) { if (typeof error !== "object" || error === null) return null; const fields = error as { @@ -154,6 +183,67 @@ async function lastAttempt( return row ?? null; } +async function attemptHistory( + executor: InfraDbExecutor, + accountId: string, + now: Date, +) { + const recent = await executor + .select({ + type: developerCreditTransactions.type, + referenceId: developerCreditTransactions.referenceId, + createdAt: developerCreditTransactions.createdAt, + }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + inArray( + developerCreditTransactions.referenceType, + ATTEMPT_REFERENCE_TYPES, + ), + gte( + developerCreditTransactions.createdAt, + new Date(now.getTime() - DAY_MS), + ), + ), + ); + const [settled] = await executor + .select({ settledAttempts: sql`COUNT(*)` }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + inArray( + developerCreditTransactions.referenceType, + SETTLED_REFERENCE_TYPES, + ), + ), + ); + + const settledIds = new Set( + recent + .filter((row) => row.type !== "topup_pending") + .map((row) => row.referenceId), + ); + const latest = (type: string) => + Math.max( + 0, + ...recent + .filter((row) => row.type === type) + .map((row) => new Date(row.createdAt).getTime()), + ); + const lastDecline = latest("topup_failed"); + return { + sequence: Number(settled?.settledAttempts ?? 0), + awaitingPayment: recent.some( + (row) => row.type === "topup_pending" && !settledIds.has(row.referenceId), + ), + declinedRecently: lastDecline > 0 && lastDecline >= latest("topup"), + recentTopUps: recent.filter((row) => row.type === "topup").length, + }; +} + export async function getAutoTopUpFailure(accountId: string) { const attempt = await lastAttempt(accountId, SETTLED_REFERENCE_TYPES); if (attempt?.type !== "topup_failed") return null; @@ -353,50 +443,71 @@ export async function runAutoTopUp( ) { return { state: "skipped", reason: "invalid_settings" }; } - const available = (await availableBalance(account.appId)) ?? 0; - if (available >= account.autoTopUpThresholdMicroCredits) { + const snapshot = await db().transaction(async (tx) => { + const [locked] = await tx + .select({ id: developerCreditAccounts.id }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.id, accountId)) + .limit(1) + .for("update"); + if (!locked) return null; + return { + available: (await availableBalance(account.appId, tx)) ?? 0, + disputed: Boolean(await disputedSince(tx, accountId)), + ...(await attemptHistory(tx, accountId, now)), + }; + }); + if (!snapshot) return { state: "skipped", reason: "not_found" }; + if (snapshot.available >= account.autoTopUpThresholdMicroCredits) { return { state: "skipped", reason: "above_threshold" }; } - - if (await disputedSince(db(), accountId)) { - return { state: "skipped", reason: "disputed" }; + if (snapshot.disputed) return { state: "skipped", reason: "disputed" }; + if (snapshot.awaitingPayment) { + return { state: "skipped", reason: "awaiting_payment" }; } - - const previous = await lastAttempt(accountId); - if ( - previous && - now.getTime() - new Date(previous.createdAt).getTime() < ATTEMPT_INTERVAL_MS - ) { - return { state: "skipped", reason: "recent_attempt" }; + if (snapshot.declinedRecently) { + return { state: "skipped", reason: "recent_decline" }; + } + if (snapshot.recentTopUps >= AUTO_TOP_UP_DAILY_LIMIT) { + console.warn( + "[infra-billing] Auto top-up waits: the daily limit of automatic top-ups is reached", + { accountId, limit: AUTO_TOP_UP_DAILY_LIMIT }, + ); + return { state: "skipped", reason: "daily_limit" }; } const period = utcDateKey(now); const productId = developerCreditsProductId(); + const request = { + amount: amountCents, + currency: "usd", + customer: account.stripeCustomerId, + payment_method: account.stripePaymentMethodId, + payment_method_types: ["card"], + off_session: true, + confirm: true, + description: "Cap Infra credits, auto top-up", + receipt_email: account.email, + metadata: { + type: AUTO_TOP_UP_METADATA_TYPE, + accountId, + appId: account.appId, + amountCents: String(amountCents), + sequence: String(snapshot.sequence), + ...(productId ? { productId } : {}), + }, + } satisfies Stripe.PaymentIntentCreateParams; let paymentIntent: Stripe.PaymentIntent; try { - paymentIntent = await stripe().paymentIntents.create( - { - amount: amountCents, - currency: "usd", - customer: account.stripeCustomerId, - payment_method: account.stripePaymentMethodId, - payment_method_types: ["card"], - off_session: true, - confirm: true, - description: "Cap Infra credits, auto top-up", - receipt_email: account.email, - metadata: { - type: AUTO_TOP_UP_METADATA_TYPE, - accountId, - appId: account.appId, - amountCents: String(amountCents), - period, - ...(productId ? { productId } : {}), - }, - }, - { idempotencyKey: autoTopUpIdempotencyKey(accountId, period) }, - ); + paymentIntent = await stripe().paymentIntents.create(request, { + idempotencyKey: autoTopUpIdempotencyKey( + accountId, + snapshot.sequence, + request, + ), + }); } catch (error) { + if (isKeyInUse(error)) return { state: "skipped", reason: "in_progress" }; const decline = declineFrom(error); if (!decline) { console.error("[infra-billing] Auto top-up could not be attempted", { @@ -477,6 +588,44 @@ export async function runDueAutoTopUps(now: Date = new Date()) { return summary; } +async function creditAccountOf(target: AutoTopUpTarget) { + if ("accountId" in target) return target.accountId; + const [account] = await db() + .select({ id: developerCreditAccounts.id }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.appId, target.appId)) + .limit(1); + return account?.id ?? null; +} + +export type AutoTopUpTarget = { accountId: string } | { appId: string }; + +export function scheduleAutoTopUp(target: AutoTopUpTarget) { + const task = async () => { + try { + const accountId = await creditAccountOf(target); + if (!accountId) return; + const result = await runAutoTopUp(accountId); + if (result.state !== "skipped") { + console.log("[infra-billing] Auto top-up after a charge", { + accountId, + state: result.state, + }); + } + } catch (error) { + console.error("[infra-billing] Auto top-up after a charge failed", { + target, + error, + }); + } + }; + try { + after(task); + } catch { + void task(); + } +} + export async function settleAutoTopUpPaymentIntent( paymentIntent: Stripe.PaymentIntent, outcome: "succeeded" | "failed", From 39071b3c8ea3d3d1804a0454a0834327abd9a05f Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:33:11 -0500 Subject: [PATCH 02/26] fix: top up as soon as a charge or hold drops the balance Transcript holds, reservations and settlements, and low-balance refusals at upload create, upload complete and transcript create or start, now ask for an auto top-up straight away instead of waiting for the next storage run. --- .../unit/developer-sdk-upload-billing.test.ts | 15 ++++++ .../unit/developer-transcript-settle.test.ts | 48 +++++++++++++++++++ .../unit/developer-transcript-store.test.ts | 13 +++++ .../unit/infra-billing-gates.test.ts | 9 +++- apps/web/lib/developer-transcript-billing.ts | 18 +++++-- apps/web/lib/developer-transcript-store.ts | 7 +-- apps/web/lib/infra-video-upload.ts | 4 ++ 7 files changed, 107 insertions(+), 7 deletions(-) diff --git a/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts b/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts index 9243e844102..6c7fe87c119 100644 --- a/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts +++ b/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts @@ -29,6 +29,11 @@ const mocks = vi.hoisted(() => ({ claim: vi.fn(), release: vi.fn(), startProcessing: vi.fn(), + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: mocks.scheduleAutoTopUp, })); vi.mock("@/lib/developer-video-start", () => ({ @@ -280,6 +285,7 @@ const refused = { describe("developer SDK multipart completion", () => { beforeEach(() => { + mocks.scheduleAutoTopUp.mockReset(); mocks.balance = 10_000_000; mocks.accountExists = true; mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 1_000 }]; @@ -333,6 +339,13 @@ describe("developer SDK multipart completion", () => { mocks.unrecorded = []; }); + it("asks for no top-up when an upload is admitted", async () => { + mocks.admission.mockResolvedValue(admitted); + + expect((await complete({ durationInSecs: 60 })).status).toBe(200); + expect(mocks.scheduleAutoTopUp).not.toHaveBeenCalled(); + }); + it("charges nothing when an upload completes", async () => { mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 10_000_000 }]; @@ -437,6 +450,7 @@ describe("developer SDK multipart completion", () => { expect(mocks.startProcessing).not.toHaveBeenCalled(); expect(mocks.videoUpdates).toEqual([]); expect(mocks.storedDuration).toBe(null); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("keeps the file when the upload is refused before storage assembles it", async () => { @@ -448,6 +462,7 @@ describe("developer SDK multipart completion", () => { await expect(response.json()).resolves.not.toHaveProperty("fileRemoved"); expect(mocks.complete).not.toHaveBeenCalled(); expect(mocks.deleteObject).not.toHaveBeenCalled(); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("still answers 402 when the refused file cannot be removed", async () => { diff --git a/apps/web/__tests__/unit/developer-transcript-settle.test.ts b/apps/web/__tests__/unit/developer-transcript-settle.test.ts index dc1c69fc27f..3e6af0218ee 100644 --- a/apps/web/__tests__/unit/developer-transcript-settle.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-settle.test.ts @@ -12,6 +12,11 @@ const context = vi.hoisted(() => ({ reset: () => void; }, ledgerIds: 0, + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: context.scheduleAutoTopUp, })); vi.mock("@cap/database", async () => { @@ -99,6 +104,7 @@ const settle = (durationSeconds = ONE_HOUR, transcriptId = "tr-1") => beforeEach(() => { fake().reset(); context.ledgerIds = 0; + context.scheduleAutoTopUp.mockReset(); fake().seed(developerCreditAccounts, { id: "account-1", appId: "app-1", @@ -284,6 +290,48 @@ describe("transcript settlement", () => { }); }); +describe("auto top-up after a transcript charge or hold", () => { + const scheduled = () => context.scheduleAutoTopUp.mock.calls; + + it("asks for a top-up once after a settle that charged, not on a retry", async () => { + await settle(); + await settle(); + + expect(scheduled()).toEqual([[{ appId: "app-1" }]]); + }); + + it("asks for a top-up when the balance could not cover the transcript", async () => { + const current = account(); + if (current) current.balanceMicroCredits = 9_999; + + await settle(); + + expect(scheduled()).toEqual([[{ appId: "app-1" }]]); + }); + + it("asks for a top-up after a hold or a reservation, and not for skipped work", async () => { + addTranscript("tr-2"); + await holdTranscriptCredits({ transcriptId: "tr-1", microCredits: 1_000 }); + await reserveTranscriptCredits({ + transcriptId: "tr-2", + durationSeconds: ONE_HOUR, + }); + expect(scheduled()).toEqual([[{ appId: "app-1" }], [{ appId: "app-1" }]]); + + addTranscript("tr-gone", { deletedAt: new Date() }); + await holdTranscriptCredits({ + transcriptId: "tr-gone", + microCredits: 1_000, + }); + await reserveTranscriptCredits({ + transcriptId: "tr-gone", + durationSeconds: ONE_HOUR, + }); + await settle(ONE_HOUR, "tr-gone"); + expect(scheduled()).toHaveLength(2); + }); +}); + describe("transcript holds", () => { const reserve = (transcriptId: string, durationSeconds: number) => reserveTranscriptCredits({ transcriptId, durationSeconds }); diff --git a/apps/web/__tests__/unit/developer-transcript-store.test.ts b/apps/web/__tests__/unit/developer-transcript-store.test.ts index d18dda13969..446314b1060 100644 --- a/apps/web/__tests__/unit/developer-transcript-store.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-store.test.ts @@ -19,6 +19,11 @@ const context = vi.hoisted(() => ({ presigned: [] as { key: string; contentLength: number }[], failSigning: false, onDelete: null as null | (() => Promise), + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: context.scheduleAutoTopUp, })); vi.mock("@cap/database", async () => { @@ -163,6 +168,7 @@ const fromUrl = (url = URL_A, idempotencyKey?: string, appId = "app-1") => beforeEach(() => { fake().reset(); context.ids = 0; + context.scheduleAutoTopUp.mockReset(); context.speechConfigured = true; context.hosted = true; context.start.mockReset(); @@ -305,6 +311,9 @@ describe("creating a transcript from a URL", () => { }); expect(transcripts()).toHaveLength(0); expect(context.start).not.toHaveBeenCalled(); + expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({ + appId: "app-1", + }); }); it("says how much to add below the minimum balance", async () => { @@ -953,12 +962,16 @@ describe("starting an uploaded transcript", () => { const id = await created(); context.objectSize = 1024; setBalance(0); + context.scheduleAutoTopUp.mockReset(); expect(await startTranscriptUpload("app-1", id)).toMatchObject({ ok: false, status: 402, }); expect(context.start).not.toHaveBeenCalled(); + expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({ + appId: "app-1", + }); }); it("can be retried after the job could not be started", async () => { diff --git a/apps/web/__tests__/unit/infra-billing-gates.test.ts b/apps/web/__tests__/unit/infra-billing-gates.test.ts index cabf48c095d..7a2ab864394 100644 --- a/apps/web/__tests__/unit/infra-billing-gates.test.ts +++ b/apps/web/__tests__/unit/infra-billing-gates.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ createSession: vi.fn(), detach: vi.fn(), ensureCustomer: vi.fn(), + scheduleAutoTopUp: vi.fn(), })); vi.mock("@cap/database", () => ({ @@ -94,7 +95,10 @@ vi.mock("@/lib/infra-auto-top-up", async () => { const settings = await vi.importActual< typeof import("@/lib/infra-auto-top-up") >("@/lib/infra-auto-top-up"); - return { autoTopUpSettingsError: settings.autoTopUpSettingsError }; + return { + autoTopUpSettingsError: settings.autoTopUpSettingsError, + scheduleAutoTopUp: mocks.scheduleAutoTopUp, + }; }); vi.mock("@/app/api/utils", async () => { const { createMiddleware } = await import("hono/factory"); @@ -141,6 +145,7 @@ const cardCheckout = () => }); beforeEach(() => { + mocks.scheduleAutoTopUp.mockReset(); mocks.user = preview; mocks.selects = []; mocks.sets = []; @@ -179,6 +184,7 @@ describe("creating an upload", () => { }); expect(mocks.admission).toHaveBeenCalledWith("app-1"); expect(mocks.inserts).toHaveLength(0); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("creates the video for free when the app is admitted", async () => { @@ -189,6 +195,7 @@ describe("creating an upload", () => { expect(response.status).toBe(200); const { data } = await response.json(); expect(data).toMatchObject({ id: "video-1", videoId: "video-1" }); + expect(mocks.scheduleAutoTopUp).not.toHaveBeenCalled(); expect(Object.keys(data).sort()).toEqual([ "corsOrigin", "embedUrl", diff --git a/apps/web/lib/developer-transcript-billing.ts b/apps/web/lib/developer-transcript-billing.ts index d5eeeae4654..d45065c8043 100644 --- a/apps/web/lib/developer-transcript-billing.ts +++ b/apps/web/lib/developer-transcript-billing.ts @@ -14,6 +14,7 @@ import { TRANSCRIPT_MIN_START_BALANCE_MICRO_CREDITS, type TranscriptErrorCode, } from "@/lib/developer-transcripts"; +import { scheduleAutoTopUp } from "@/lib/infra-auto-top-up"; import { availableBalance, heldTranscriptMicroCredits, @@ -125,7 +126,7 @@ export async function holdTranscriptCredits(input: { if (!appId) return { state: "skipped" }; const needed = Math.max(0, Math.ceil(input.microCredits)); - return db().transaction(async (tx) => { + const hold = await db().transaction(async (tx) => { const [account] = await tx .select({ balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, @@ -186,6 +187,8 @@ export async function holdTranscriptCredits(input: { return { state: "held", reservedMicroCredits: needed }; }); + if (hold.state !== "skipped") scheduleAutoTopUp({ appId }); + return hold; } export async function reserveTranscriptCredits(input: { @@ -199,7 +202,7 @@ export async function reserveTranscriptCredits(input: { TRANSCRIPT_FULL_HOLD_MICRO_CREDITS, ); - return db().transaction(async (tx) => { + const outcome = await db().transaction(async (tx) => { const [account] = await tx .select({ balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, @@ -246,6 +249,8 @@ export async function reserveTranscriptCredits(input: { return { state: "reserved", reservedMicroCredits: cost }; }); + if (outcome.state !== "skipped") scheduleAutoTopUp({ appId }); + return outcome; } export async function settleTranscript(input: { @@ -260,7 +265,7 @@ export async function settleTranscript(input: { if (!appId) return { state: "skipped", reason: "missing" }; const cost = transcriptionMicroCredits(input.durationSeconds); - return db().transaction(async (tx) => { + const settled = await db().transaction(async (tx) => { await tx .select({ id: developerCreditAccounts.id }) .from(developerCreditAccounts) @@ -353,4 +358,11 @@ export async function settleTranscript(input: { alreadySettled: false, }; }); + if ( + settled.state === "insufficient_credits" || + (settled.state === "complete" && settled.chargedMicroCredits > 0) + ) { + scheduleAutoTopUp({ appId }); + } + return settled; } diff --git a/apps/web/lib/developer-transcript-store.ts b/apps/web/lib/developer-transcript-store.ts index 7fd551bf5f6..ba7a5f0df2f 100644 --- a/apps/web/lib/developer-transcript-store.ts +++ b/apps/web/lib/developer-transcript-store.ts @@ -33,6 +33,7 @@ import { transcriptUploadExpiry, } from "@/lib/developer-transcripts"; import { canUseInfra } from "@/lib/infra-access"; +import { scheduleAutoTopUp } from "@/lib/infra-auto-top-up"; import { assertPublicMediaUrl, PublicMediaFetchError, @@ -348,9 +349,9 @@ async function startGate(appId: string): Promise { if (!(await appOwnerCanUseInfra(appId))) return NOT_AVAILABLE; if (!isSpeechConfigured()) return SPEECH_UNAVAILABLE; const refusal = await transcriptStartRefusal(appId); - return refusal - ? fail(402, refusal.code, refusal.message, ADD_CREDITS_HINT) - : null; + if (!refusal) return null; + scheduleAutoTopUp({ appId }); + return fail(402, refusal.code, refusal.message, ADD_CREDITS_HINT); } export async function createTranscriptFromUrl( diff --git a/apps/web/lib/infra-video-upload.ts b/apps/web/lib/infra-video-upload.ts index bac24628743..3a5f54c6ef7 100644 --- a/apps/web/lib/infra-video-upload.ts +++ b/apps/web/lib/infra-video-upload.ts @@ -24,6 +24,7 @@ import { startDeveloperVideoProcessing, } from "@/lib/developer-video-start"; import { canUseInfra } from "@/lib/infra-access"; +import { scheduleAutoTopUp } from "@/lib/infra-auto-top-up"; import { lockInfraAccount } from "@/lib/infra-balance"; import { getInfraUploadAdmission, @@ -342,6 +343,7 @@ export async function createInfraVideo({ > { const admission = await getInfraUploadAdmission(appId); if (!admission.allowed) { + scheduleAutoTopUp({ appId }); return { status: 402, body: uploadAdmissionError(admission) }; } @@ -624,6 +626,7 @@ export async function completeInfraUpload({ if (await videoIsRecorded(db(), videoId)) return ok({ success: true }); const admission = await admitDeveloperVideoUpload(upload); if (!admission.allowed) { + scheduleAutoTopUp({ appId }); return { status: 402, body: uploadAdmissionError(admission) }; } } @@ -689,6 +692,7 @@ export async function completeInfraUpload({ await removeStoredFile(s3Key); await releaseDeveloperVideoUpload(videoId, claim); claim = null; + scheduleAutoTopUp({ appId }); const refusal = uploadAdmissionError(admission); return { status: 402, From 4e7267525a5fd5886727e393513aab65c7692c18 Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:33:11 -0500 Subject: [PATCH 03/26] feat: point to auto top-up after a credit purchase After a purchase, while auto top-up is off, the Billing page says it can add credit as soon as the balance falls below a threshold. The card notes no longer say the card is charged at most once a day. --- .../__tests__/unit/infra-top-up-nudge.test.ts | 42 +++++++++++++++++++ .../infra/[appId]/billing/auto-top-up.tsx | 12 +++--- .../infra/[appId]/billing/billing-view.tsx | 36 +++++++++------- .../infra/[appId]/billing/top-up.tsx | 25 ++++++++++- 4 files changed, 93 insertions(+), 22 deletions(-) create mode 100644 apps/web/__tests__/unit/infra-top-up-nudge.test.ts diff --git a/apps/web/__tests__/unit/infra-top-up-nudge.test.ts b/apps/web/__tests__/unit/infra-top-up-nudge.test.ts new file mode 100644 index 00000000000..504edf10cea --- /dev/null +++ b/apps/web/__tests__/unit/infra-top-up-nudge.test.ts @@ -0,0 +1,42 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const navigation = vi.hoisted(() => ({ query: "" })); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn(), refresh: vi.fn() }), + useSearchParams: () => new URLSearchParams(navigation.query), +})); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); +vi.mock("@tanstack/react-query", () => ({ + useMutation: () => ({ mutate: vi.fn(), isPending: false }), +})); + +import { TopUp } from "@/app/(infra)/dashboard/infra/[appId]/billing/top-up"; + +const NUDGE = + "to add credit as soon as the balance falls below a threshold you set."; + +const render = (autoTopUpHref: string | null) => + renderToStaticMarkup(createElement(TopUp, { appId: "app-1", autoTopUpHref })); + +describe("the nudge after a purchase", () => { + beforeEach(() => { + navigation.query = ""; + }); + + it("points to auto top-up after a purchase while it is off", () => { + navigation.query = "purchase=success"; + const html = render("#auto-top-up"); + + expect(html).toContain(NUDGE); + expect(html).toContain('href="#auto-top-up"'); + }); + + it("stays out of the way before a purchase and when auto top-up is on", () => { + expect(render("#auto-top-up")).not.toContain(NUDGE); + navigation.query = "purchase=success"; + expect(render(null)).not.toContain(NUDGE); + }); +}); diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx index 3ad9dc0b54b..a444c5a09b6 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx @@ -143,11 +143,11 @@ export function AutoTopUp({

Saving a card charges nothing and leaves auto top-up off. Once you - switch it on, Cap adds ${initialAmount} from that card whenever this + switch it on, Cap adds ${initialAmount} from that card as soon as this app's balance falls below ${initialThreshold}. You can change both - amounts, the card is charged at most once a day, and auto top-up - switches itself off if a charge is declined. Remove the card at any - time. + amounts, the card is charged at most ten times in any 24 hours, and + auto top-up switches itself off if a charge is declined. Remove the + card at any time.

); @@ -263,8 +263,8 @@ export function AutoTopUp({

- A card is saved for this app. It is charged at most once a day, and auto - top-up switches itself off if a charge is declined. + A card is saved for this app. It is charged at most ten times in any 24 + hours, and auto top-up switches itself off if a charge is declined.

); diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx index d72659cb07b..42e8bb372f5 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx @@ -49,6 +49,7 @@ const RATE_PRICE = clsx( ); const LEDGER_ANCHOR = "transactions"; +const AUTO_TOP_UP_ANCHOR = "auto-top-up"; export type BillingAccount = { balanceMicroCredits: number; @@ -112,7 +113,12 @@ export function BillingView({
{account ? ( <> - +
@@ -187,19 +193,21 @@ export function BillingView({
{account && ( - - - +
+ + + +
)}
diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx index 6a0c40eb2ad..39fb4da76b8 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx @@ -13,9 +13,15 @@ import { TOP_UP_PRESETS_CENTS, topUpButtonLabel, } from "@/lib/infra-top-up"; -import { Button, FOCUS, Input, MONO } from "../../_components/ui"; +import { Button, FOCUS, Input, MONO, TEXT_LINK } from "../../_components/ui"; -export function TopUp({ appId }: { appId: string }) { +export function TopUp({ + appId, + autoTopUpHref, +}: { + appId: string; + autoTopUpHref: string | null; +}) { const router = useRouter(); const searchParams = useSearchParams(); const amountId = useId(); @@ -26,6 +32,9 @@ export function TopUp({ appId }: { appId: string }) { const codeNoteId = useId(); const [codeOpen, setCodeOpen] = useState(false); const [code, setCode] = useState(""); + const [purchased] = useState( + () => searchParams.get("purchase") === "success", + ); useEffect(() => { if (searchParams.get("purchase") === "success") { @@ -173,6 +182,18 @@ export function TopUp({ appId }: { appId: string }) { )}
+ {purchased && autoTopUpHref && ( +

+ Switch on{" "} + + auto top-up + {" "} + to add credit as soon as the balance falls below a threshold you set. +

+ )} ); } From cb18f694a7757b91d732b8d3335ec2ffc980ff39 Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:33:11 -0500 Subject: [PATCH 04/26] docs: say auto top-up runs as soon as the balance falls below the threshold --- apps/web/__tests__/unit/developer-billing-docs.test.ts | 7 +++++++ apps/web/content/docs/api/billing.mdx | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/web/__tests__/unit/developer-billing-docs.test.ts b/apps/web/__tests__/unit/developer-billing-docs.test.ts index 525d0799846..c4290f90fc9 100644 --- a/apps/web/__tests__/unit/developer-billing-docs.test.ts +++ b/apps/web/__tests__/unit/developer-billing-docs.test.ts @@ -43,6 +43,7 @@ import { } from "@/lib/developer-transcript-hold"; import { TRANSCRIPT_MIN_START_BALANCE_MICRO_CREDITS } from "@/lib/developer-transcripts"; import { + AUTO_TOP_UP_DAILY_LIMIT, AUTO_TOP_UP_MAX_AMOUNT_CENTS, AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS, AUTO_TOP_UP_MIN_AMOUNT_CENTS, @@ -276,6 +277,12 @@ describe("the Billing page", () => { expect(billing).toContain( `${INFRA_GRACE_DAYS} days after the app first fell short`, ); + expect(billing).toContain( + "- **When.** As soon as the available balance falls below your threshold.", + ); + expect(billing).toContain( + `up to ${AUTO_TOP_UP_DAILY_LIMIT} top-ups in any 24 hours`, + ); expect(billing).toContain(`${LEDGER_PAGE_SIZE} to a page`); expect(billing).toContain( `reads ${formatCredits(2_999_901)} and never $30.00`, diff --git a/apps/web/content/docs/api/billing.mdx b/apps/web/content/docs/api/billing.mdx index a18c88b1462..b4c7e19daab 100644 --- a/apps/web/content/docs/api/billing.mdx +++ b/apps/web/content/docs/api/billing.mdx @@ -119,9 +119,9 @@ Cap Pro includes $5 of credit every month. It is for any account with Cap Pro, w Auto top-up adds credit from a saved card when an app's balance runs low. It is off until you switch it on, and saving a card charges nothing. -- **When.** Cap checks at 00:41, 06:41, 12:41 and 18:41 UTC. If the available balance is under your threshold, the card is charged. +- **When.** As soon as the available balance falls below your threshold. Cap also checks at 00:41, 06:41, 12:41 and 18:41 UTC. - **How much.** The amount you set, from $5 to $1,000. The threshold is from $1 to $1,000. The console suggests $25 when the balance is under $5. -- **How often.** At most one attempt in any 24 hours. +- **How often.** Each time the balance falls below the threshold, up to 10 top-ups in any 24 hours. After a declined charge, the next attempt waits 24 hours. - **How to stop.** Switch it off or remove the card on the Billing page. If a charge is declined, auto top-up switches itself off and no credit is added. ## When credit runs out From 78c665f362b7b8410f1923e0eddc16eb5b2606c5 Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:37:03 -0500 Subject: [PATCH 05/26] fix: give up on an auto top-up payment request after 20 seconds A hung Stripe call could use up the storage run's time. A timeout is logged like any other failure to reach Stripe, writes no ledger row, and the next attempt sends the same idempotency key. --- .../__tests__/unit/infra-auto-top-up.test.ts | 29 +++++++++++++++++++ apps/web/lib/infra-auto-top-up.ts | 2 ++ 2 files changed, 31 insertions(+) diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index 7fa45975451..5519b215854 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -50,6 +50,7 @@ vi.mock("@/lib/developer-credits-product", () => ({ import type Stripe from "stripe"; import { AUTO_TOP_UP_DAILY_LIMIT, + AUTO_TOP_UP_STRIPE_TIMEOUT_MS, autoTopUpFailureText, autoTopUpIdempotencyKey, autoTopUpSettingsError, @@ -297,6 +298,7 @@ describe("auto top-up charge", () => { }); expect(options).toEqual({ idempotencyKey: autoTopUpIdempotencyKey("account-1", 0, params), + timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, }); expect(ledger.account.balanceMicroCredits).toBe(530_000); expect(topUps()).toEqual([ @@ -370,6 +372,33 @@ describe("auto top-up charge", () => { expect(topUps()).toHaveLength(1); }); + it("gives up on a payment request that times out, writes nothing and retries with the same key", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign( + new Error("Request aborted due to timeout being reached (20000ms)"), + { type: "StripeConnectionError" }, + ), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(errors).toHaveBeenCalledWith( + "[infra-billing] Auto top-up could not be attempted", + expect.objectContaining({ accountId: "account-1" }), + ); + expect(ledger.rows).toHaveLength(0); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + const [timedOut, retry] = mocks.createPaymentIntent.mock.calls; + expect(timedOut?.[1].timeout).toBe(20_000); + expect(retry?.[1].idempotencyKey).toBe(timedOut?.[1].idempotencyKey); + errors.mockRestore(); + }); + it("treats a payment already being made under the same key as in progress", async () => { mocks.createPaymentIntent.mockRejectedValue( Object.assign(new Error("in use"), { diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 031f786adfa..dae39f1ad66 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -41,6 +41,7 @@ export const AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS = 1_000 * MICRO_CREDITS_PER_DOLLAR; export const AUTO_TOP_UP_DAILY_LIMIT = 10; +export const AUTO_TOP_UP_STRIPE_TIMEOUT_MS = 20_000; const DAY_MS = 24 * 60 * 60 * 1000; export type AutoTopUpResult = @@ -505,6 +506,7 @@ export async function runAutoTopUp( snapshot.sequence, request, ), + timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, }); } catch (error) { if (isKeyInUse(error)) return { state: "skipped", reason: "in_progress" }; From c406ad52bf73f3f27ebd22bc8f12f01c15845fa0 Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:45:42 -0500 Subject: [PATCH 06/26] fix: one Cap Infra auto top-up per attempt, whatever the settings The Stripe key is now the account and its count of settled top-ups only, so a trigger that reads the same attempt after the amount or card changed gets Stripe's idempotency error instead of a second payment, and that error counts as an attempt in progress. A payment still processing blocks the next one however long it takes. Outside a request, scheduleAutoTopUp now finishes the attempt before it returns instead of leaving a detached promise. --- .../infra-auto-top-up-database.test.ts | 33 ++++++- .../__tests__/unit/infra-auto-top-up.test.ts | 89 +++++++++++++----- .../__tests__/unit/infra-billing-fake-db.ts | 20 +++++ apps/web/lib/infra-auto-top-up.ts | 90 ++++++++++++------- 4 files changed, 173 insertions(+), 59 deletions(-) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index ed762e2a53a..1481d7ff975 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -16,7 +16,8 @@ const fixture = vi.hoisted(() => ({ database: undefined as unknown, keys: new Map< string, - { state: "pending" } | { state: "done"; intent: FakeIntent } + | { state: "pending"; params: string } + | { state: "done"; params: string; intent: FakeIntent } >(), created: [] as FakeIntent[], createdKeys: [] as string[], @@ -47,6 +48,13 @@ vi.mock("@cap/utils", () => ({ ) => { const key = options.idempotencyKey; const entry = fixture.keys.get(key); + const sent = JSON.stringify(params); + if (entry && entry.params !== sent) { + throw Object.assign(new Error("Keys must match parameters"), { + type: "StripeIdempotencyError", + statusCode: 400, + }); + } if (entry?.state === "pending") { throw Object.assign(new Error("Key in use"), { type: "StripeInvalidRequestError", @@ -55,7 +63,7 @@ vi.mock("@cap/utils", () => ({ }); } if (entry?.state === "done") return entry.intent; - fixture.keys.set(key, { state: "pending" }); + fixture.keys.set(key, { state: "pending", params: sent }); await new Promise((resolve) => setTimeout(resolve, 40)); const intent: FakeIntent = { id: `pi_${fixture.created.length + 1}`, @@ -67,7 +75,7 @@ vi.mock("@cap/utils", () => ({ }; fixture.created.push(intent); fixture.createdKeys.push(key); - fixture.keys.set(key, { state: "done", intent }); + fixture.keys.set(key, { state: "done", params: sent, intent }); return intent; }, }, @@ -170,6 +178,25 @@ describe.runIf(Boolean(databaseUrl))( } }); + it("creates one charge when the amount changes while triggers race", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + const first = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + await new Promise((resolve) => setTimeout(resolve, 10)); + await pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = 2500 WHERE id = ?", + [accountId], + ); + const changed = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + await Promise.all([...first, ...changed]); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + }); + it("tops up again the same day with a new key once the balance is low again", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const { accountId } = await makeAccount(30_000); diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index 5519b215854..d28b53ab961 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -187,16 +187,13 @@ describe("auto top-up settings", () => { ).toBe(null); }); - it("keys each charge by account, attempt and the exact request", () => { - const request = { amount: 500, customer: "cus_1" }; - const key = autoTopUpIdempotencyKey("account-1", 3, request); - - expect(key).toMatch(/^infra-auto-top-up:account-1:3:[0-9a-f]{24}$/); - expect(autoTopUpIdempotencyKey("account-1", 3, { ...request })).toBe(key); - expect(autoTopUpIdempotencyKey("account-1", 4, request)).not.toBe(key); - expect( - autoTopUpIdempotencyKey("account-1", 3, { ...request, amount: 600 }), - ).not.toBe(key); + it("keys each charge by account and attempt, whatever the settings", () => { + expect(autoTopUpIdempotencyKey("account-1", 3)).toBe( + "infra-auto-top-up:account-1:3", + ); + expect(autoTopUpIdempotencyKey("account-1", 4)).not.toBe( + autoTopUpIdempotencyKey("account-1", 3), + ); }); it("explains a decline in plain words", () => { @@ -297,7 +294,7 @@ describe("auto top-up charge", () => { }, }); expect(options).toEqual({ - idempotencyKey: autoTopUpIdempotencyKey("account-1", 0, params), + idempotencyKey: autoTopUpIdempotencyKey("account-1", 0), timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, }); expect(ledger.account.balanceMicroCredits).toBe(530_000); @@ -417,6 +414,36 @@ describe("auto top-up charge", () => { expect(mocks.sendEmail).not.toHaveBeenCalled(); }); + it("cannot start a second payment when the settings change during an unresolved attempt", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("Request aborted due to timeout"), { + type: "StripeConnectionError", + }), + ); + vi.spyOn(console, "error").mockImplementationOnce(() => {}); + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + + ledger.account.autoTopUpAmountCents = 2_500; + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign( + new Error( + "Keys for idempotent requests can only be used with the same parameters they were first used with.", + ), + { type: "StripeIdempotencyError", statusCode: 400 }, + ), + ); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "in_progress", + }); + + const [first, changed] = mocks.createPaymentIntent.mock.calls; + expect(changed?.[0].amount).toBe(2_500); + expect(changed?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); + expect(ledger.rows).toHaveLength(0); + expect(ledger.account.autoTopUpEnabled).toBe(true); + }); + it(`stops at ${AUTO_TOP_UP_DAILY_LIMIT} automatic top-ups in any 24 hours, then waits`, async () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); for (let i = 1; i <= AUTO_TOP_UP_DAILY_LIMIT; i++) { @@ -480,7 +507,7 @@ describe("auto top-up charge", () => { expect(ledger.account.balanceMicroCredits).toBe(530_000); }); - it("does not start a second payment after a processing one, even on the next UTC day", async () => { + it("does not start a second payment while one is still processing, however long it takes", async () => { ledger.now = new Date("2026-11-11T23:50:00Z"); mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); await runAutoTopUp("account-1", now()); @@ -496,6 +523,10 @@ describe("auto top-up charge", () => { expect(await runAutoTopUp("account-1", now())).toMatchObject({ reason: "awaiting_payment", }); + later(48); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); expect(ledger.rows).toHaveLength(1); @@ -697,7 +728,7 @@ describe("auto top-up after a charge", () => { tasks.push(task); }); - scheduleAutoTopUp({ accountId: "account-1" }); + await scheduleAutoTopUp({ accountId: "account-1" }); await settle(); expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); @@ -712,7 +743,7 @@ describe("auto top-up after a charge", () => { tasks.push(task); }); - scheduleAutoTopUp({ appId: "app-1" }); + await scheduleAutoTopUp({ appId: "app-1" }); await tasks[0]?.(); expect(mocks.createPaymentIntent.mock.calls[0]?.[0].metadata).toMatchObject( @@ -720,11 +751,23 @@ describe("auto top-up after a charge", () => { ); }); - it("still tops up outside a request", async () => { + it("outside a request, finishes the top-up before it returns", async () => { mocks.after.mockImplementation(outsideRequest); - scheduleAutoTopUp({ accountId: "account-1" }); - await vi.waitFor(() => expect(topUps()).toHaveLength(1)); + await scheduleAutoTopUp({ accountId: "account-1" }); + + expect(mocks.after).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(topUps()).toHaveLength(1); + }); + + it("inside a request, returns at once and leaves the top-up to after()", async () => { + mocks.after.mockImplementation(() => {}); + + await scheduleAutoTopUp({ accountId: "account-1" }); + + expect(mocks.after).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); }); it("never throws into the caller and logs what went wrong", async () => { @@ -738,12 +781,12 @@ describe("auto top-up after a charge", () => { }; mocks.db = broken; - expect(() => scheduleAutoTopUp({ accountId: "account-1" })).not.toThrow(); - await vi.waitFor(() => - expect(errors).toHaveBeenCalledWith( - "[infra-billing] Auto top-up after a charge failed", - expect.objectContaining({ target: { accountId: "account-1" } }), - ), + await expect( + scheduleAutoTopUp({ accountId: "account-1" }), + ).resolves.toBeUndefined(); + expect(errors).toHaveBeenCalledWith( + "[infra-billing] Auto top-up after a charge failed", + expect.objectContaining({ target: { accountId: "account-1" } }), ); expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); mocks.db = fakeDb; diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index ce7bfc21ff9..1482ecf8e68 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -175,6 +175,26 @@ function runSelect(query: Query): Record[] { types.includes(row.referenceType) && row.createdAt.getTime() >= since, ); } + if (keys === "pendingId") { + return ledger.rows + .filter( + (row) => + row.type === "topup_pending" && + row.referenceType === "auto_top_up_pending", + ) + .map((row) => ({ pendingId: row.referenceId })); + } + if (keys === "settledId") { + const ids = rendered.params.slice(3); + return ledger.rows + .filter( + (row) => + (row.referenceType === "auto_top_up" || + row.referenceType === "auto_top_up_failed") && + ids.includes(row.referenceId), + ) + .map((row) => ({ settledId: row.referenceId })); + } if (keys === "settledAttempts") { const types = rendered.params.slice(1); return [ diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index dae39f1ad66..8fb8d055eb3 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { db } from "@cap/database"; import { nanoId } from "@cap/database/helpers"; import { @@ -95,16 +94,8 @@ export function autoTopUpSettingsError({ return null; } -export function autoTopUpIdempotencyKey( - accountId: string, - sequence: number, - request: unknown, -) { - const digest = createHash("sha256") - .update(JSON.stringify(request)) - .digest("hex") - .slice(0, 24); - return `infra-auto-top-up:${accountId}:${sequence}:${digest}`; +export function autoTopUpIdempotencyKey(accountId: string, sequence: number) { + return `infra-auto-top-up:${accountId}:${sequence}`; } export function autoTopUpFailureText(reason: string | null | undefined) { @@ -120,10 +111,18 @@ export function autoTopUpFailureText(reason: string | null | undefined) { } } -function isKeyInUse(error: unknown) { +function isUnresolvedAttempt(error: unknown) { if (typeof error !== "object" || error === null) return false; - const fields = error as { code?: unknown; statusCode?: unknown }; - return fields.code === "idempotency_key_in_use" || fields.statusCode === 409; + const fields = error as { + type?: unknown; + code?: unknown; + statusCode?: unknown; + }; + return ( + fields.code === "idempotency_key_in_use" || + fields.statusCode === 409 || + fields.type === "StripeIdempotencyError" + ); } function declineFrom(error: unknown) { @@ -201,7 +200,7 @@ async function attemptHistory( eq(developerCreditTransactions.accountId, accountId), inArray( developerCreditTransactions.referenceType, - ATTEMPT_REFERENCE_TYPES, + SETTLED_REFERENCE_TYPES, ), gte( developerCreditTransactions.createdAt, @@ -221,12 +220,39 @@ async function attemptHistory( ), ), ); - - const settledIds = new Set( - recent - .filter((row) => row.type !== "topup_pending") - .map((row) => row.referenceId), + const pending = await executor + .select({ pendingId: developerCreditTransactions.referenceId }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "topup_pending"), + eq( + developerCreditTransactions.referenceType, + AUTO_TOP_UP_PENDING_REFERENCE_TYPE, + ), + ), + ); + const pendingIds = pending.flatMap((row) => + row.pendingId ? [row.pendingId] : [], ); + const resolved = pendingIds.length + ? await executor + .select({ settledId: developerCreditTransactions.referenceId }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + inArray( + developerCreditTransactions.referenceType, + SETTLED_REFERENCE_TYPES, + ), + inArray(developerCreditTransactions.referenceId, pendingIds), + ), + ) + : []; + const resolvedIds = new Set(resolved.map((row) => row.settledId)); + const latest = (type: string) => Math.max( 0, @@ -237,9 +263,7 @@ async function attemptHistory( const lastDecline = latest("topup_failed"); return { sequence: Number(settled?.settledAttempts ?? 0), - awaitingPayment: recent.some( - (row) => row.type === "topup_pending" && !settledIds.has(row.referenceId), - ), + awaitingPayment: pendingIds.some((id) => !resolvedIds.has(id)), declinedRecently: lastDecline > 0 && lastDecline >= latest("topup"), recentTopUps: recent.filter((row) => row.type === "topup").length, }; @@ -501,15 +525,13 @@ export async function runAutoTopUp( let paymentIntent: Stripe.PaymentIntent; try { paymentIntent = await stripe().paymentIntents.create(request, { - idempotencyKey: autoTopUpIdempotencyKey( - accountId, - snapshot.sequence, - request, - ), + idempotencyKey: autoTopUpIdempotencyKey(accountId, snapshot.sequence), timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, }); } catch (error) { - if (isKeyInUse(error)) return { state: "skipped", reason: "in_progress" }; + if (isUnresolvedAttempt(error)) { + return { state: "skipped", reason: "in_progress" }; + } const decline = declineFrom(error); if (!decline) { console.error("[infra-billing] Auto top-up could not be attempted", { @@ -602,8 +624,10 @@ async function creditAccountOf(target: AutoTopUpTarget) { export type AutoTopUpTarget = { accountId: string } | { appId: string }; -export function scheduleAutoTopUp(target: AutoTopUpTarget) { - const task = async () => { +export async function scheduleAutoTopUp( + target: AutoTopUpTarget, +): Promise { + const attempt = async () => { try { const accountId = await creditAccountOf(target); if (!accountId) return; @@ -622,9 +646,9 @@ export function scheduleAutoTopUp(target: AutoTopUpTarget) { } }; try { - after(task); + after(attempt); } catch { - void task(); + await attempt(); } } From ffebcbad1927aa521dc24c1ea0f00990ebfe446d Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:45:42 -0500 Subject: [PATCH 07/26] fix: wait for the auto top-up where no request can keep it alive --- apps/web/lib/developer-transcript-billing.ts | 6 +++--- apps/web/lib/developer-transcript-store.ts | 2 +- apps/web/lib/infra-video-upload.ts | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/web/lib/developer-transcript-billing.ts b/apps/web/lib/developer-transcript-billing.ts index d45065c8043..ec7683d85a7 100644 --- a/apps/web/lib/developer-transcript-billing.ts +++ b/apps/web/lib/developer-transcript-billing.ts @@ -187,7 +187,7 @@ export async function holdTranscriptCredits(input: { return { state: "held", reservedMicroCredits: needed }; }); - if (hold.state !== "skipped") scheduleAutoTopUp({ appId }); + if (hold.state !== "skipped") await scheduleAutoTopUp({ appId }); return hold; } @@ -249,7 +249,7 @@ export async function reserveTranscriptCredits(input: { return { state: "reserved", reservedMicroCredits: cost }; }); - if (outcome.state !== "skipped") scheduleAutoTopUp({ appId }); + if (outcome.state !== "skipped") await scheduleAutoTopUp({ appId }); return outcome; } @@ -362,7 +362,7 @@ export async function settleTranscript(input: { settled.state === "insufficient_credits" || (settled.state === "complete" && settled.chargedMicroCredits > 0) ) { - scheduleAutoTopUp({ appId }); + await scheduleAutoTopUp({ appId }); } return settled; } diff --git a/apps/web/lib/developer-transcript-store.ts b/apps/web/lib/developer-transcript-store.ts index ba7a5f0df2f..0c11b856f02 100644 --- a/apps/web/lib/developer-transcript-store.ts +++ b/apps/web/lib/developer-transcript-store.ts @@ -350,7 +350,7 @@ async function startGate(appId: string): Promise { if (!isSpeechConfigured()) return SPEECH_UNAVAILABLE; const refusal = await transcriptStartRefusal(appId); if (!refusal) return null; - scheduleAutoTopUp({ appId }); + await scheduleAutoTopUp({ appId }); return fail(402, refusal.code, refusal.message, ADD_CREDITS_HINT); } diff --git a/apps/web/lib/infra-video-upload.ts b/apps/web/lib/infra-video-upload.ts index 3a5f54c6ef7..7a72019b412 100644 --- a/apps/web/lib/infra-video-upload.ts +++ b/apps/web/lib/infra-video-upload.ts @@ -343,7 +343,7 @@ export async function createInfraVideo({ > { const admission = await getInfraUploadAdmission(appId); if (!admission.allowed) { - scheduleAutoTopUp({ appId }); + await scheduleAutoTopUp({ appId }); return { status: 402, body: uploadAdmissionError(admission) }; } @@ -626,7 +626,7 @@ export async function completeInfraUpload({ if (await videoIsRecorded(db(), videoId)) return ok({ success: true }); const admission = await admitDeveloperVideoUpload(upload); if (!admission.allowed) { - scheduleAutoTopUp({ appId }); + await scheduleAutoTopUp({ appId }); return { status: 402, body: uploadAdmissionError(admission) }; } } @@ -692,7 +692,7 @@ export async function completeInfraUpload({ await removeStoredFile(s3Key); await releaseDeveloperVideoUpload(videoId, claim); claim = null; - scheduleAutoTopUp({ appId }); + await scheduleAutoTopUp({ appId }); const refusal = uploadAdmissionError(admission); return { status: 402, From 239ed19d2a9b5b833507fc81fbb77742b74ff0df Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:49:45 -0500 Subject: [PATCH 08/26] test: an amount change after a lost auto top-up response makes no second payment --- .../infra-auto-top-up-database.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index 1481d7ff975..4c065fa762a 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -21,6 +21,7 @@ const fixture = vi.hoisted(() => ({ >(), created: [] as FakeIntent[], createdKeys: [] as string[], + loseNextResponse: false, })); vi.mock("@cap/database", () => ({ @@ -76,6 +77,12 @@ vi.mock("@cap/utils", () => ({ fixture.created.push(intent); fixture.createdKeys.push(key); fixture.keys.set(key, { state: "done", params: sent, intent }); + if (fixture.loseNextResponse) { + fixture.loseNextResponse = false; + throw Object.assign(new Error("Request aborted due to timeout"), { + type: "StripeConnectionError", + }); + } return intent; }, }, @@ -197,6 +204,44 @@ describe.runIf(Boolean(databaseUrl))( expect(await balanceOf(accountId)).toBe(530_000); }); + it("makes no second payment when the amount changes after a lost response", async () => { + const { runAutoTopUp, settleAutoTopUpPaymentIntent } = await import( + "@/lib/infra-auto-top-up" + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + expect(await topUpsOf(accountId)).toHaveLength(0); + + await pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = 2500 WHERE id = ?", + [accountId], + ); + expect(await runAutoTopUp(accountId)).toEqual({ + state: "skipped", + reason: "in_progress", + }); + expect(fixture.created.length - before).toBe(1); + + const [lost] = fixture.created.slice(before); + if (!lost) throw new Error("No payment was created"); + await settleAutoTopUpPaymentIntent( + lost as unknown as Parameters[0], + "succeeded", + ); + await setBalance(accountId, 10); + expect(await runAutoTopUp(accountId)).toMatchObject({ + state: "succeeded", + }); + expect( + fixture.created.slice(before).map((intent) => intent.amount), + ).toEqual([500, 2500]); + errors.mockRestore(); + }); + it("tops up again the same day with a new key once the balance is low again", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const { accountId } = await makeAccount(30_000); From 49060ff3ec92bd6ef8d45d30ee2e6c2be1d8e01d Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:52:35 -0500 Subject: [PATCH 09/26] fix: settle an auto top-up still pending after 72 hours from Stripe A pending payment blocks the next auto top-up until it settles. So an account is never stuck behind a payment whose webhook never arrives, a run that finds one older than 72 hours reads it from Stripe once and records the outcome, or leaves it if Stripe still reports it as processing. --- .../__tests__/unit/infra-auto-top-up.test.ts | 109 +++++++++++++++++- .../__tests__/unit/infra-billing-fake-db.ts | 7 +- apps/web/lib/infra-auto-top-up.ts | 56 ++++++++- 3 files changed, 164 insertions(+), 8 deletions(-) diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index d28b53ab961..d103d3f0505 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -5,6 +5,7 @@ const mocks = vi.hoisted(() => ({ ids: 0, emailsEnabled: false, createPaymentIntent: vi.fn(), + retrievePaymentIntent: vi.fn(), retrieveSetupIntent: vi.fn(), retrievePaymentMethod: vi.fn(), detachPaymentMethod: vi.fn(), @@ -28,7 +29,10 @@ vi.mock("@cap/env", () => ({ })); vi.mock("@cap/utils", () => ({ stripe: () => ({ - paymentIntents: { create: mocks.createPaymentIntent }, + paymentIntents: { + create: mocks.createPaymentIntent, + retrieve: mocks.retrievePaymentIntent, + }, setupIntents: { retrieve: mocks.retrieveSetupIntent }, paymentMethods: { detach: mocks.detachPaymentMethod, @@ -125,6 +129,7 @@ beforeEach(() => { mocks.emailsEnabled = false; mocks.createPaymentIntent.mockReset(); mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "succeeded")); + mocks.retrievePaymentIntent.mockReset(); mocks.sendEmail.mockReset(); mocks.sendEmail.mockResolvedValue({ data: { id: "email-1" }, error: null }); mocks.retrieveSetupIntent.mockReset(); @@ -505,6 +510,101 @@ describe("auto top-up charge", () => { expect(topUps()).toHaveLength(1); expect(ledger.account.balanceMicroCredits).toBe(530_000); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + }); + + describe("a payment still pending after 72 hours", () => { + const leavePending = async () => { + mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); + await runAutoTopUp("account-1", now()); + mocks.createPaymentIntent.mockReset(); + }; + + it("asks Stripe once and records a payment that succeeded", async () => { + await leavePending(); + later(71.9); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + + later(0.2); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded"), + ); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + expect(mocks.retrievePaymentIntent.mock.calls[0]?.[0]).toBe("pi_1"); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + }); + + it("records a payment that failed and switches auto top-up off", async () => { + await leavePending(); + later(72.1); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "requires_payment_method", { + last_payment_error: { + decline_code: "insufficient_funds", + } as Stripe.PaymentIntent.LastPaymentError, + }), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(failures()).toEqual([ + expect.objectContaining({ + referenceId: "pi_1", + metadata: expect.objectContaining({ reason: "insufficient_funds" }), + }), + ]); + expect(topUps()).toHaveLength(0); + expect(ledger.account.autoTopUpEnabled).toBe(false); + expect(ledger.account.balanceMicroCredits).toBe(30_000); + }); + + it("leaves a payment Stripe still reports as processing", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await leavePending(); + later(80); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "processing"), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(topUps()).toHaveLength(0); + expect(failures()).toHaveLength(0); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "reconciled", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + warn.mockRestore(); + }); }); it("does not start a second payment while one is still processing, however long it takes", async () => { @@ -523,10 +623,15 @@ describe("auto top-up charge", () => { expect(await runAutoTopUp("account-1", now())).toMatchObject({ reason: "awaiting_payment", }); - later(48); + later(1.5); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); + later(46.5); expect(await runAutoTopUp("account-1", now())).toMatchObject({ reason: "awaiting_payment", }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); expect(ledger.rows).toHaveLength(1); diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index 1482ecf8e68..f64d91e8f59 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -175,14 +175,17 @@ function runSelect(query: Query): Record[] { types.includes(row.referenceType) && row.createdAt.getTime() >= since, ); } - if (keys === "pendingId") { + if (keys === "pendingId,pendingSince") { return ledger.rows .filter( (row) => row.type === "topup_pending" && row.referenceType === "auto_top_up_pending", ) - .map((row) => ({ pendingId: row.referenceId })); + .map((row) => ({ + pendingId: row.referenceId, + pendingSince: row.createdAt, + })); } if (keys === "settledId") { const ids = rendered.params.slice(3); diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 8fb8d055eb3..62951ef13ba 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -41,6 +41,7 @@ export const AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS = export const AUTO_TOP_UP_DAILY_LIMIT = 10; export const AUTO_TOP_UP_STRIPE_TIMEOUT_MS = 20_000; +export const AUTO_TOP_UP_PENDING_RECONCILE_MS = 72 * 60 * 60 * 1000; const DAY_MS = 24 * 60 * 60 * 1000; export type AutoTopUpResult = @@ -55,6 +56,7 @@ export type AutoTopUpResult = | "above_threshold" | "disputed" | "awaiting_payment" + | "reconciled" | "recent_decline" | "daily_limit" | "in_progress"; @@ -221,7 +223,10 @@ async function attemptHistory( ), ); const pending = await executor - .select({ pendingId: developerCreditTransactions.referenceId }) + .select({ + pendingId: developerCreditTransactions.referenceId, + pendingSince: developerCreditTransactions.createdAt, + }) .from(developerCreditTransactions) .where( and( @@ -263,7 +268,16 @@ async function attemptHistory( const lastDecline = latest("topup_failed"); return { sequence: Number(settled?.settledAttempts ?? 0), - awaitingPayment: pendingIds.some((id) => !resolvedIds.has(id)), + unresolved: pending.flatMap((row) => + row.pendingId && !resolvedIds.has(row.pendingId) + ? [ + { + paymentIntentId: row.pendingId, + since: new Date(row.pendingSince), + }, + ] + : [], + ), declinedRecently: lastDecline > 0 && lastDecline >= latest("topup"), recentTopUps: recent.filter((row) => row.type === "topup").length, }; @@ -418,6 +432,29 @@ function grantAutoTopUp({ }); } +async function reconcilePendingAutoTopUp( + accountId: string, + paymentIntentId: string, +) { + const paymentIntent = await stripe().paymentIntents.retrieve( + paymentIntentId, + {}, + { timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS }, + ); + if (paymentIntent.metadata?.accountId !== accountId) return; + if (paymentIntent.status === "processing") { + console.warn("[infra-billing] Auto top-up payment still processing", { + accountId, + paymentIntentId, + }); + return; + } + await settleAutoTopUpPaymentIntent( + paymentIntent, + paymentIntent.status === "succeeded" ? "succeeded" : "failed", + ); +} + export async function runAutoTopUp( accountId: string, now: Date = new Date(), @@ -487,8 +524,19 @@ export async function runAutoTopUp( return { state: "skipped", reason: "above_threshold" }; } if (snapshot.disputed) return { state: "skipped", reason: "disputed" }; - if (snapshot.awaitingPayment) { - return { state: "skipped", reason: "awaiting_payment" }; + if (snapshot.unresolved.length > 0) { + const stale = snapshot.unresolved.filter( + (payment) => + now.getTime() - payment.since.getTime() >= + AUTO_TOP_UP_PENDING_RECONCILE_MS, + ); + if (stale.length === 0) { + return { state: "skipped", reason: "awaiting_payment" }; + } + for (const payment of stale) { + await reconcilePendingAutoTopUp(accountId, payment.paymentIntentId); + } + return { state: "skipped", reason: "reconciled" }; } if (snapshot.declinedRecently) { return { state: "skipped", reason: "recent_decline" }; From 27077a92444895d0420fa8d8717097e979c5cab1 Mon Sep 17 00:00:00 2001 From: x Date: Thu, 12 Nov 2026 23:54:53 -0500 Subject: [PATCH 10/26] fix: key auto top-ups by recorded top-ups and declines The key was already counting declines alongside top-ups. It now names both counts, so a retry after a recorded decline visibly sends a new key while a settings change during an attempt keeps the old one. --- .../__tests__/unit/infra-auto-top-up.test.ts | 35 +++++++++++++------ .../__tests__/unit/infra-billing-fake-db.ts | 10 +++--- apps/web/lib/infra-auto-top-up.ts | 27 ++++++++++---- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index d103d3f0505..9d3d59af93e 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -192,12 +192,14 @@ describe("auto top-up settings", () => { ).toBe(null); }); - it("keys each charge by account and attempt, whatever the settings", () => { - expect(autoTopUpIdempotencyKey("account-1", 3)).toBe( - "infra-auto-top-up:account-1:3", - ); - expect(autoTopUpIdempotencyKey("account-1", 4)).not.toBe( - autoTopUpIdempotencyKey("account-1", 3), + it("keys each charge by the account's recorded top-ups and declines, whatever the settings", () => { + expect( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 1 }), + ).toBe("infra-auto-top-up:account-1:3:1"); + expect( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 2 }), + ).not.toBe( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 1 }), ); }); @@ -295,11 +297,14 @@ describe("auto top-up charge", () => { metadata: { type: "developer_auto_top_up", accountId: "account-1", - sequence: "0", + attempt: "0:0", }, }); expect(options).toEqual({ - idempotencyKey: autoTopUpIdempotencyKey("account-1", 0), + idempotencyKey: autoTopUpIdempotencyKey("account-1", { + topUps: 0, + declines: 0, + }), timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, }); expect(ledger.account.balanceMicroCredits).toBe(530_000); @@ -354,7 +359,7 @@ describe("auto top-up charge", () => { }); const [first, second] = mocks.createPaymentIntent.mock.calls; - expect(second?.[0].metadata.sequence).toBe("1"); + expect(second?.[0].metadata.attempt).toBe("1:0"); expect(second?.[1].idempotencyKey).not.toBe(first?.[1].idempotencyKey); expect(topUps()).toHaveLength(2); expect(now().toISOString().slice(0, 10)).toBe("2026-11-11"); @@ -550,6 +555,13 @@ describe("auto top-up charge", () => { expect(ledger.account.balanceMicroCredits).toBe(530_000); expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + await settleAutoTopUpPaymentIntent( + intent("pi_1", "succeeded"), + "succeeded", + ); + expect(topUps()).toHaveLength(1); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + ledger.account.balanceMicroCredits = 10; mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); expect(await runAutoTopUp("account-1", now())).toMatchObject({ @@ -757,7 +769,10 @@ describe("auto top-up charge", () => { paymentIntentId: "pi_2", }); const [declined, next] = mocks.createPaymentIntent.mock.calls; - expect(next?.[1].idempotencyKey).not.toBe(declined?.[1].idempotencyKey); + expect(declined?.[1].idempotencyKey).toBe( + "infra-auto-top-up:account-1:0:0", + ); + expect(next?.[1].idempotencyKey).toBe("infra-auto-top-up:account-1:0:1"); }); it("treats a charge that needs the customer as a failure", async () => { diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index f64d91e8f59..75107905c25 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -198,13 +198,13 @@ function runSelect(query: Query): Record[] { ) .map((row) => ({ settledId: row.referenceId })); } - if (keys === "settledAttempts") { - const types = rendered.params.slice(1); + if (keys === "declines,topUps") { + const count = (type: string) => + ledger.rows.filter((row) => row.referenceType === type).length; return [ { - settledAttempts: ledger.rows.filter((row) => - types.includes(row.referenceType), - ).length, + topUps: count("auto_top_up"), + declines: count("auto_top_up_failed"), }, ]; } diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 62951ef13ba..c8af88441a2 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -96,8 +96,13 @@ export function autoTopUpSettingsError({ return null; } -export function autoTopUpIdempotencyKey(accountId: string, sequence: number) { - return `infra-auto-top-up:${accountId}:${sequence}`; +export type AutoTopUpAttempt = { topUps: number; declines: number }; + +export function autoTopUpIdempotencyKey( + accountId: string, + { topUps, declines }: AutoTopUpAttempt, +) { + return `infra-auto-top-up:${accountId}:${topUps}:${declines}`; } export function autoTopUpFailureText(reason: string | null | undefined) { @@ -211,7 +216,14 @@ async function attemptHistory( ), ); const [settled] = await executor - .select({ settledAttempts: sql`COUNT(*)` }) + .select({ + topUps: sql< + number | string | null + >`SUM(${developerCreditTransactions.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE})`, + declines: sql< + number | string | null + >`SUM(${developerCreditTransactions.referenceType} = ${AUTO_TOP_UP_FAILED_REFERENCE_TYPE})`, + }) .from(developerCreditTransactions) .where( and( @@ -267,7 +279,10 @@ async function attemptHistory( ); const lastDecline = latest("topup_failed"); return { - sequence: Number(settled?.settledAttempts ?? 0), + attempt: { + topUps: Number(settled?.topUps ?? 0), + declines: Number(settled?.declines ?? 0), + }, unresolved: pending.flatMap((row) => row.pendingId && !resolvedIds.has(row.pendingId) ? [ @@ -566,14 +581,14 @@ export async function runAutoTopUp( accountId, appId: account.appId, amountCents: String(amountCents), - sequence: String(snapshot.sequence), + attempt: `${snapshot.attempt.topUps}:${snapshot.attempt.declines}`, ...(productId ? { productId } : {}), }, } satisfies Stripe.PaymentIntentCreateParams; let paymentIntent: Stripe.PaymentIntent; try { paymentIntent = await stripe().paymentIntents.create(request, { - idempotencyKey: autoTopUpIdempotencyKey(accountId, snapshot.sequence), + idempotencyKey: autoTopUpIdempotencyKey(accountId, snapshot.attempt), timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, }); } catch (error) { From de01b1a137e943d983b3b701d65ea397387367e1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:17:45 -0500 Subject: [PATCH 11/26] test: the sweep and the locked check read the same auto top-up waits On a real database, a decline makes the locked check wait, a 25 hour pending payment keeps the account out of the sweep, and one past 72 hours is picked up by the sweep with no other trigger and reconciled from Stripe. --- .../infra-auto-top-up-database.test.ts | 131 ++++++++++++++++++ .../__tests__/unit/infra-auto-top-up.test.ts | 23 +++ 2 files changed, 154 insertions(+) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index 4c065fa762a..da34312f1c0 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -22,6 +22,8 @@ const fixture = vi.hoisted(() => ({ created: [] as FakeIntent[], createdKeys: [] as string[], loseNextResponse: false, + stored: new Map(), + retrieved: [] as string[], })); vi.mock("@cap/database", () => ({ @@ -43,6 +45,12 @@ vi.mock("@/lib/infra-billing-emails", () => ({ vi.mock("@cap/utils", () => ({ stripe: () => ({ paymentIntents: { + retrieve: async (id: string) => { + fixture.retrieved.push(id); + const intent = fixture.stored.get(id); + if (!intent) throw new Error(`No such payment_intent: ${id}`); + return intent; + }, create: async ( params: { amount: number; metadata: Record }, options: { idempotencyKey: string }, @@ -242,6 +250,129 @@ describe.runIf(Boolean(databaseUrl))( errors.mockRestore(); }); + const addRow = ( + accountId: string, + type: string, + referenceType: string, + referenceId: string, + hoursAgo: number, + ) => + pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, ?, 0, 30000, ?, ?, '{}', UTC_TIMESTAMP() - INTERVAL ? HOUR)`, + [id(), accountId, type, referenceId, referenceType, hoursAgo], + ); + + it("makes the locked check wait on the same rules the sweep reads", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const declined = await makeAccount(30_000); + await addRow( + declined.accountId, + "topup_failed", + "auto_top_up_failed", + `declined-${declined.accountId}`, + 2, + ); + const young = await makeAccount(30_000); + await addRow( + young.accountId, + "topup_pending", + "auto_top_up_pending", + `pi_young_${young.accountId}`, + 25, + ); + const old = await makeAccount(30_000); + const oldPayment = `pi_old_${old.accountId}`; + await addRow( + old.accountId, + "topup_pending", + "auto_top_up_pending", + oldPayment, + 73, + ); + fixture.stored.set(oldPayment, { + id: oldPayment, + status: "succeeded", + amount: 500, + amount_received: 500, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: old.accountId }, + }); + const before = fixture.created.length; + + expect(await runAutoTopUp(declined.accountId)).toEqual({ + state: "skipped", + reason: "recent_decline", + }); + expect(await runAutoTopUp(young.accountId)).toEqual({ + state: "skipped", + reason: "awaiting_payment", + }); + expect(await runAutoTopUp(old.accountId)).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(await topUpsOf(old.accountId)).toEqual([oldPayment]); + expect(fixture.created.length).toBe(before); + }); + + it("lets the sweep reconcile a payment pending for over 72 hours, and only that", async () => { + const { runDueAutoTopUps } = await import("@/lib/infra-auto-top-up"); + + const stale = await makeAccount(30_000); + const stalePayment = `pi_stale_${stale.accountId}`; + await addRow( + stale.accountId, + "topup_pending", + "auto_top_up_pending", + stalePayment, + 73, + ); + fixture.stored.set(stalePayment, { + id: stalePayment, + status: "succeeded", + amount: 500, + amount_received: 500, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: stale.accountId }, + }); + const young = await makeAccount(30_000); + await addRow( + young.accountId, + "topup_pending", + "auto_top_up_pending", + `pi_young_${young.accountId}`, + 25, + ); + const declined = await makeAccount(30_000); + await addRow( + declined.accountId, + "topup_failed", + "auto_top_up_failed", + `declined-${declined.accountId}`, + 2, + ); + const free = await makeAccount(30_000); + const before = fixture.created.length; + + await runDueAutoTopUps(new Date()); + + expect(fixture.retrieved).not.toContain(`pi_young_${young.accountId}`); + expect(fixture.retrieved.at(-1)).toBe(stalePayment); + expect(await topUpsOf(stale.accountId)).toEqual([stalePayment]); + expect(await topUpsOf(young.accountId)).toEqual([]); + expect(await topUpsOf(declined.accountId)).toEqual([]); + expect(await topUpsOf(free.accountId)).toHaveLength(1); + const charged = fixture.created + .slice(before) + .map((intent) => intent.metadata.accountId); + expect(charged).toContain(free.accountId); + expect(charged).not.toContain(stale.accountId); + expect(charged).not.toContain(young.accountId); + expect(charged).not.toContain(declined.accountId); + }); + it("tops up again the same day with a new key once the balance is low again", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const { accountId } = await makeAccount(30_000); diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index d3cbd447344..b7e811d5074 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -571,6 +571,29 @@ describe("auto top-up charge", () => { expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); }); + it("is picked up by the storage sweep once it passes 72 hours, with no other trigger", async () => { + await leavePending(); + later(71); + expect(await runDueAutoTopUps(now())).toEqual({ + attempted: 0, + succeeded: 0, + failed: 0, + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + + later(1.1); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded"), + ); + await runDueAutoTopUps(now()); + + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + }); + it("records a payment that failed and switches auto top-up off", async () => { await leavePending(); later(72.1); From 5a23e88dc5cdc72dfd0e673be7f5c42adbccf2d8 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:43:27 -0500 Subject: [PATCH 12/26] fix: cap an owner's automatic top-ups at $1,000 a day across their apps A fourth wait in the shared fragment sums the owner's automatic top-ups in the last 24 hours, including payments still processing, plus this attempt's amount. Past $1,000 the attempt waits and logs owner_daily_limit; auto top-up stays on and no email is sent. Credit bought by hand does not count. An attempt now locks every credit account of the owner in id order and keeps the lock through the Stripe create and the grant, so two apps of one owner cannot race past the ceiling. The Stripe request timeout is 15 s with no SDK retry, to keep that lock inside the database's 20 s transaction limit. --- .../infra-auto-top-up-database.test.ts | 108 ++++++- .../__tests__/unit/infra-auto-top-up.test.ts | 64 +++- .../__tests__/unit/infra-billing-fake-db.ts | 26 +- apps/web/lib/developer-credits.ts | 8 +- apps/web/lib/infra-auto-top-up.ts | 273 +++++++++++------- 5 files changed, 358 insertions(+), 121 deletions(-) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index da34312f1c0..53a5641df70 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -105,16 +105,29 @@ describe.runIf(Boolean(databaseUrl))( () => { let pool: Pool; const accounts: string[] = []; + const owners: string[] = []; - const makeAccount = async (balanceMicroCredits: number) => { + const makeOwner = async () => { const userId = id(); - const appId = id(); - const accountId = id(); - accounts.push(accountId); + owners.push(userId); await pool.query( "INSERT INTO users (id, email, activeOrganizationId) VALUES (?, ?, '')", [userId, `${userId}@example.com`], ); + return userId; + }; + + const makeAccount = async ( + balanceMicroCredits: number, + { + ownerId, + amountCents = 500, + }: { ownerId?: string; amountCents?: number } = {}, + ) => { + const userId = ownerId ?? (await makeOwner()); + const appId = id(); + const accountId = id(); + accounts.push(accountId); await pool.query( "INSERT INTO developer_apps (id, ownerId, name, environment) VALUES (?, ?, 'Top-up test', 'production')", [appId, userId], @@ -123,10 +136,10 @@ describe.runIf(Boolean(databaseUrl))( `INSERT INTO developer_credit_accounts (id, appId, ownerId, balanceMicroCredits, stripeCustomerId, stripePaymentMethodId, autoTopUpEnabled, autoTopUpThresholdMicroCredits, autoTopUpAmountCents) - VALUES (?, ?, ?, ?, 'cus_test', 'pm_test', 1, 500000, 500)`, - [accountId, appId, userId, balanceMicroCredits], + VALUES (?, ?, ?, ?, 'cus_test', 'pm_test', 1, 500000, ?)`, + [accountId, appId, userId, balanceMicroCredits, amountCents], ); - return { accountId, appId }; + return { accountId, appId, ownerId: userId }; }; const balanceOf = async (accountId: string) => { @@ -165,10 +178,13 @@ describe.runIf(Boolean(databaseUrl))( [accountId], ); await pool.query( - "DELETE a, c, u FROM developer_credit_accounts c JOIN developer_apps a ON a.id = c.appId JOIN users u ON u.id = c.ownerId WHERE c.id = ?", + "DELETE a, c FROM developer_credit_accounts c JOIN developer_apps a ON a.id = c.appId WHERE c.id = ?", [accountId], ); } + for (const ownerId of owners) { + await pool.query("DELETE FROM users WHERE id = ?", [ownerId]); + } await pool.end(); }); @@ -373,6 +389,82 @@ describe.runIf(Boolean(databaseUrl))( expect(charged).not.toContain(declined.accountId); }); + it("stops an owner's automatic top-ups at $1,000 a day across all their apps", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + expect((await runAutoTopUp(first.accountId)).state).toBe("succeeded"); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + expect(fixture.created.length - before).toBe(1); + const [[enabled]] = (await pool.query( + "SELECT autoTopUpEnabled AS on_ FROM developer_credit_accounts WHERE id = ?", + [second.accountId], + )) as unknown as [[{ on_: number }]]; + expect(enabled.on_).toBe(1); + expect(warn).toHaveBeenCalledWith( + "[infra-billing] Auto top-up waits for its daily limit", + { accountId: second.accountId, reason: "owner_daily_limit" }, + ); + warn.mockRestore(); + }); + + it("lets two apps of one owner racing near the ceiling make one charge", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + const results = await Promise.all([ + runAutoTopUp(first.accountId), + runAutoTopUp(second.accountId), + ]); + + expect(fixture.created.length - before).toBe(1); + expect(results.map((result) => result.state).sort()).toEqual([ + "skipped", + "succeeded", + ]); + expect(results).toContainEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + warn.mockRestore(); + }); + + it("leaves credit bought by hand out of the owner's ceiling", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const ownerId = await makeOwner(); + const bought = await makeAccount(30_000, { ownerId }); + const automatic = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 90000000, 90030000, ?, 'stripe_payment_intent', '{}')`, + [id(), bought.accountId, `pi_manual_${bought.accountId}`], + ); + + expect((await runAutoTopUp(automatic.accountId)).state).toBe("succeeded"); + }); + it("tops up again the same day with a new key once the balance is low again", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const { accountId } = await makeAccount(30_000); diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index b7e811d5074..3ecf657c8c6 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -306,6 +306,7 @@ describe("auto top-up charge", () => { declines: 0, }), timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + maxNetworkRetries: 0, }); expect(ledger.account.balanceMicroCredits).toBe(530_000); expect(topUps()).toEqual([ @@ -383,7 +384,7 @@ describe("auto top-up charge", () => { const errors = vi.spyOn(console, "error").mockImplementation(() => {}); mocks.createPaymentIntent.mockRejectedValueOnce( Object.assign( - new Error("Request aborted due to timeout being reached (20000ms)"), + new Error("Request aborted due to timeout being reached (15000ms)"), { type: "StripeConnectionError" }, ), ); @@ -401,7 +402,7 @@ describe("auto top-up charge", () => { state: "succeeded", }); const [timedOut, retry] = mocks.createPaymentIntent.mock.calls; - expect(timedOut?.[1].timeout).toBe(20_000); + expect(timedOut?.[1].timeout).toBe(15_000); expect(retry?.[1].idempotencyKey).toBe(timedOut?.[1].idempotencyKey); errors.mockRestore(); }); @@ -524,6 +525,65 @@ describe("auto top-up charge", () => { }); }); + describe("the owner's daily ceiling", () => { + it("waits once automatic top-ups would pass $1,000 in 24 hours, without switching anything off", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + ledger.account.autoTopUpAmountCents = 60_000; + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded", { amount_received: 60_000 }), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + + ledger.account.balanceMicroCredits = 10; + later(1); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(failures()).toHaveLength(0); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + "[infra-billing] Auto top-up waits for its daily limit", + { accountId: "account-1", reason: "owner_daily_limit" }, + ); + + later(23.1); + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_2", "succeeded", { amount_received: 60_000 }), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + warn.mockRestore(); + }); + + it("does not count credit bought by hand", async () => { + ledger.rows.push({ + id: "manual", + accountId: "account-1", + type: "topup", + amountMicroCredits: 90_000_000, + balanceAfterMicroCredits: 90_030_000, + referenceId: "pi_manual", + referenceType: "stripe_payment_intent", + createdAt: new Date(ledger.now), + }); + ledger.account.autoTopUpAmountCents = 60_000; + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded", { amount_received: 60_000 }), + ); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + }); + }); + describe("a payment still pending after 72 hours", () => { const leavePending = async () => { mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index 9e40e7bea02..ba5fb2eeb48 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -153,6 +153,26 @@ function autoTopUpWaits() { limit: topUps.filter((row) => row.createdAt.getTime() > now - 24 * HOUR_MS) .length >= 10, + owner: + ledger.rows + .filter( + (row) => + row.createdAt.getTime() > now - 24 * HOUR_MS && + ((row.type === "topup" && row.referenceType === "auto_top_up") || + (row.type === "topup_pending" && + row.referenceType === "auto_top_up_pending" && + !settledIds.has(row.referenceId))), + ) + .reduce( + (total, row) => + total + + (row.type === "topup" + ? row.amountMicroCredits + : Number(row.metadata?.amountCents ?? 0) * 1000), + 0, + ) + + ledger.account.autoTopUpAmountCents * 1000 > + 100_000_000, }; } @@ -165,7 +185,7 @@ function runSelect(query: Query): Record[] { if (query.table === developerCreditAccounts) { const { exists, ...account } = ledger.account; if (!exists) return []; - if (keys === "declineWait,id,limitWait,pendingWait") { + if (keys === "declineWait,id,limitWait,ownerWait,pendingWait") { if (rendered.params.at(-1) !== account.id) return []; const waits = autoTopUpWaits(); return [ @@ -174,6 +194,7 @@ function runSelect(query: Query): Record[] { pendingWait: Number(waits.pending), declineWait: Number(waits.decline), limitWait: Number(waits.limit), + ownerWait: Number(waits.owner), }, ]; } @@ -194,7 +215,8 @@ function runSelect(query: Query): Record[] { account.autoTopUpThresholdMicroCredits + ledger.heldMicroCredits && !waits.pending && !waits.decline && - !waits.limit; + !waits.limit && + !waits.owner; return due ? [ { diff --git a/apps/web/lib/developer-credits.ts b/apps/web/lib/developer-credits.ts index de32295489d..523929b02f2 100644 --- a/apps/web/lib/developer-credits.ts +++ b/apps/web/lib/developer-credits.ts @@ -16,18 +16,20 @@ export async function addCreditsToAccount({ referenceId, referenceType, metadata, + executor, }: { accountId: string; amountCents: number; referenceId?: string; referenceType?: DeveloperCreditReferenceType; metadata?: Record; + executor?: DeveloperCreditTransaction; }): Promise { const microCreditsToAdd = Math.floor( (amountCents / 100) * MICRO_CREDITS_PER_DOLLAR, ); - const newBalance = await db().transaction(async (tx) => { + const grant = async (tx: DeveloperCreditTransaction) => { if (referenceId && referenceType) { const [account] = await tx .select({ @@ -86,9 +88,9 @@ export async function addCreditsToAccount({ }); return updated.balanceMicroCredits; - }); + }; - return newBalance; + return executor ? grant(executor) : db().transaction(grant); } type DeveloperCreditTransaction = Parameters< diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 19594b93198..3d3891540ab 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -41,11 +41,26 @@ export const AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS = 1_000 * MICRO_CREDITS_PER_DOLLAR; export const AUTO_TOP_UP_DAILY_LIMIT = 10; -export const AUTO_TOP_UP_STRIPE_TIMEOUT_MS = 20_000; +export const AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS = 100_000; +const MICRO_CREDITS_PER_CENT = MICRO_CREDITS_PER_DOLLAR / 100; +export const AUTO_TOP_UP_STRIPE_TIMEOUT_MS = 15_000; export const AUTO_TOP_UP_PENDING_RECONCILE_MS = 72 * 60 * 60 * 1000; const DAY_MS = 24 * 60 * 60 * 1000; const DUE_PAGE_SIZE = 200; +type AutoTopUpTransaction = Parameters< + Parameters["transaction"]>[0] +>[0]; + +type SkipReason = Extract["reason"]; + +type LockedAttempt = + | { kind: "skipped"; reason: SkipReason } + | { kind: "reconcile"; paymentIntentIds: string[] } + | { kind: "done"; result: AutoTopUpResult } + | { kind: "declined"; paymentIntent: Stripe.PaymentIntent } + | { kind: "unpaid"; error: unknown }; + export type AutoTopUpResult = | { state: "skipped"; @@ -61,6 +76,7 @@ export type AutoTopUpResult = | "reconciled" | "recent_decline" | "daily_limit" + | "owner_daily_limit" | "in_progress"; } | { state: "succeeded"; paymentIntentId: string; balanceMicroCredits: number } @@ -191,6 +207,8 @@ function ledgerAlias(name: string) { referenceId: column(developerCreditTransactions.referenceId), referenceType: column(developerCreditTransactions.referenceType), createdAt: column(developerCreditTransactions.createdAt), + amountMicroCredits: column(developerCreditTransactions.amountMicroCredits), + metadata: column(developerCreditTransactions.metadata), }; } @@ -201,7 +219,14 @@ const inList = (values: readonly string[]) => ); export function autoTopUpWaits(now: Date) { - const accountRow = sql`${sql.identifier(getTableName(developerCreditAccounts))}.${sql.identifier(developerCreditAccounts.id.name)}`; + const accounts = sql.identifier(getTableName(developerCreditAccounts)); + const accountColumn = (field: { name: string }) => + sql`${accounts}.${sql.identifier(field.name)}`; + const accountRow = accountColumn(developerCreditAccounts.id); + const ownerAccount = (field: { name: string }) => + sql`${sql.identifier("owner_account")}.${sql.identifier(field.name)}`; + const ownerEntry = ledgerAlias("owner_entry"); + const ownerSettled = ledgerAlias("owner_settled"); const pendingAttempt = ledgerAlias("pending_attempt"); const settledAttempt = ledgerAlias("settled_attempt"); const declinedAttempt = ledgerAlias("declined_attempt"); @@ -214,13 +239,14 @@ export function autoTopUpWaits(now: Date) { return { pending: sql`exists (select 1 from ${pendingAttempt.from} where ${pendingAttempt.accountId} = ${accountRow} and ${pendingAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${pendingAttempt.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${pendingAttempt.createdAt} > ${reconcileFrom} and not exists (select 1 from ${settledAttempt.from} where ${settledAttempt.accountId} = ${pendingAttempt.accountId} and ${settledAttempt.referenceId} = ${pendingAttempt.referenceId} and ${settledAttempt.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`, decline: sql`exists (select 1 from ${declinedAttempt.from} where ${declinedAttempt.accountId} = ${accountRow} and ${declinedAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_FAILED_REFERENCE_TYPE]} and ${declinedAttempt.referenceType} = ${AUTO_TOP_UP_FAILED_REFERENCE_TYPE} and ${declinedAttempt.createdAt} > ${dayAgo} and not exists (select 1 from ${laterTopUp.from} where ${laterTopUp.accountId} = ${declinedAttempt.accountId} and ${laterTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${laterTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${laterTopUp.createdAt} > ${declinedAttempt.createdAt}))`, + owner: sql`(select coalesce(sum(case when ${ownerEntry.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} then ${ownerEntry.amountMicroCredits} else cast(json_unquote(json_extract(${ownerEntry.metadata}, '$.amountCents')) as unsigned) * ${MICRO_CREDITS_PER_CENT} end), 0) from ${accounts} ${sql.identifier("owner_account")} join ${ownerEntry.from} on ${ownerEntry.accountId} = ${ownerAccount(developerCreditAccounts.id)} where ${ownerAccount(developerCreditAccounts.ownerId)} = ${accountColumn(developerCreditAccounts.ownerId)} and ${ownerEntry.type} in (${inList([ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE], ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]])}) and ${ownerEntry.createdAt} > ${dayAgo} and (${ownerEntry.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} or (${ownerEntry.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and not exists (select 1 from ${ownerSettled.from} where ${ownerSettled.accountId} = ${ownerEntry.accountId} and ${ownerSettled.referenceId} = ${ownerEntry.referenceId} and ${ownerSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)}))))) + ${accountColumn(developerCreditAccounts.autoTopUpAmountCents)} * ${MICRO_CREDITS_PER_CENT} > ${AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS * MICRO_CREDITS_PER_CENT}`, limit: sql`(select count(*) from ${recentTopUp.from} where ${recentTopUp.accountId} = ${accountRow} and ${recentTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${recentTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${recentTopUp.createdAt} > ${dayAgo}) >= ${AUTO_TOP_UP_DAILY_LIMIT}`, }; } export function autoTopUpReleased(now: Date) { const waits = autoTopUpWaits(now); - return sql`not (${waits.pending} or ${waits.decline} or ${waits.limit})`; + return sql`not (${waits.pending} or ${waits.decline} or ${waits.limit} or ${waits.owner})`; } export function autoTopUpLastAttemptAt() { @@ -403,13 +429,15 @@ async function recordPendingAutoTopUp({ paymentIntentId, amountCents, period, + executor, }: { accountId: string; paymentIntentId: string; amountCents: number; period: string; + executor?: AutoTopUpTransaction; }) { - await db().transaction(async (tx) => { + const record = async (tx: AutoTopUpTransaction) => { const [account] = await tx .select({ balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, @@ -446,7 +474,8 @@ async function recordPendingAutoTopUp({ referenceType: AUTO_TOP_UP_PENDING_REFERENCE_TYPE, metadata: { amountCents, period }, }); - }); + }; + await (executor ? record(executor) : db().transaction(record)); } function grantAutoTopUp({ @@ -454,11 +483,13 @@ function grantAutoTopUp({ paymentIntentId, amountCents, period, + executor, }: { accountId: string; paymentIntentId: string; amountCents: number; period: string | null; + executor?: AutoTopUpTransaction; }) { return addCreditsToAccount({ accountId, @@ -466,6 +497,7 @@ function grantAutoTopUp({ referenceId: paymentIntentId, referenceType: AUTO_TOP_UP_REFERENCE_TYPE, metadata: { amountCents, autoTopUp: true, period }, + executor, }); } @@ -506,6 +538,7 @@ export async function runAutoTopUp( autoTopUpThresholdMicroCredits: developerCreditAccounts.autoTopUpThresholdMicroCredits, autoTopUpAmountCents: developerCreditAccounts.autoTopUpAmountCents, + ownerId: developerCreditAccounts.ownerId, email: users.email, }) .from(developerCreditAccounts) @@ -528,7 +561,9 @@ export async function runAutoTopUp( if (!canUseInfra(account.email)) { return { state: "skipped", reason: "not_allowed" }; } - if (!account.stripeCustomerId || !account.stripePaymentMethodId) { + const customerId = account.stripeCustomerId; + const paymentMethodId = account.stripePaymentMethodId; + if (!customerId || !paymentMethodId) { return { state: "skipped", reason: "no_card" }; } const amountCents = account.autoTopUpAmountCents; @@ -542,7 +577,14 @@ export async function runAutoTopUp( ) { return { state: "skipped", reason: "invalid_settings" }; } - const snapshot = await db().transaction(async (tx) => { + const period = utcDateKey(now); + const outcome = await db().transaction(async (tx): Promise => { + await tx + .select({ id: developerCreditAccounts.id }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.ownerId, account.ownerId)) + .orderBy(asc(developerCreditAccounts.id)) + .for("update"); const waits = autoTopUpWaits(now); const [locked] = await tx .select({ @@ -550,123 +592,142 @@ export async function runAutoTopUp( pendingWait: waits.pending, declineWait: waits.decline, limitWait: waits.limit, + ownerWait: waits.owner, }) .from(developerCreditAccounts) .where(eq(developerCreditAccounts.id, accountId)) - .limit(1) - .for("update"); - if (!locked) return null; - return { - available: (await availableBalance(account.appId, tx)) ?? 0, - disputed: Boolean(await disputedSince(tx, accountId)), - waits: { - pending: Boolean(Number(locked.pendingWait)), - decline: Boolean(Number(locked.declineWait)), - limit: Boolean(Number(locked.limitWait)), + .limit(1); + if (!locked) return { kind: "skipped", reason: "not_found" }; + const available = (await availableBalance(account.appId, tx)) ?? 0; + if (available >= account.autoTopUpThresholdMicroCredits) { + return { kind: "skipped", reason: "above_threshold" }; + } + if (await disputedSince(tx, accountId)) { + return { kind: "skipped", reason: "disputed" }; + } + if (Number(locked.pendingWait)) { + return { kind: "skipped", reason: "awaiting_payment" }; + } + if (Number(locked.declineWait)) { + return { kind: "skipped", reason: "recent_decline" }; + } + if (Number(locked.limitWait)) { + return { kind: "skipped", reason: "daily_limit" }; + } + if (Number(locked.ownerWait)) { + return { kind: "skipped", reason: "owner_daily_limit" }; + } + const history = await attemptHistory(tx, accountId); + if (history.unresolved.length > 0) { + return { kind: "reconcile", paymentIntentIds: history.unresolved }; + } + + const productId = developerCreditsProductId(); + const request = { + amount: amountCents, + currency: "usd", + customer: customerId, + payment_method: paymentMethodId, + payment_method_types: ["card"], + off_session: true, + confirm: true, + description: "Cap Infra credits, auto top-up", + receipt_email: account.email, + metadata: { + type: AUTO_TOP_UP_METADATA_TYPE, + accountId, + appId: account.appId, + amountCents: String(amountCents), + attempt: `${history.attempt.topUps}:${history.attempt.declines}`, + ...(productId ? { productId } : {}), }, - ...(await attemptHistory(tx, accountId)), - }; - }); - if (!snapshot) return { state: "skipped", reason: "not_found" }; - if (snapshot.available >= account.autoTopUpThresholdMicroCredits) { - return { state: "skipped", reason: "above_threshold" }; - } - if (snapshot.disputed) return { state: "skipped", reason: "disputed" }; - if (snapshot.waits.pending) { - return { state: "skipped", reason: "awaiting_payment" }; - } - if (snapshot.waits.decline) { - return { state: "skipped", reason: "recent_decline" }; - } - if (snapshot.waits.limit) { - console.warn( - "[infra-billing] Auto top-up waits: the daily limit of automatic top-ups is reached", - { accountId, limit: AUTO_TOP_UP_DAILY_LIMIT }, - ); - return { state: "skipped", reason: "daily_limit" }; - } - if (snapshot.unresolved.length > 0) { - for (const payment of snapshot.unresolved) { - await reconcilePendingAutoTopUp(accountId, payment); + } satisfies Stripe.PaymentIntentCreateParams; + let paymentIntent: Stripe.PaymentIntent; + try { + paymentIntent = await stripe().paymentIntents.create(request, { + idempotencyKey: autoTopUpIdempotencyKey(accountId, history.attempt), + timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + maxNetworkRetries: 0, + }); + } catch (error) { + return { kind: "unpaid", error }; } - return { state: "skipped", reason: "reconciled" }; - } - const period = utcDateKey(now); - const productId = developerCreditsProductId(); - const request = { - amount: amountCents, - currency: "usd", - customer: account.stripeCustomerId, - payment_method: account.stripePaymentMethodId, - payment_method_types: ["card"], - off_session: true, - confirm: true, - description: "Cap Infra credits, auto top-up", - receipt_email: account.email, - metadata: { - type: AUTO_TOP_UP_METADATA_TYPE, - accountId, - appId: account.appId, - amountCents: String(amountCents), - attempt: `${snapshot.attempt.topUps}:${snapshot.attempt.declines}`, - ...(productId ? { productId } : {}), - }, - } satisfies Stripe.PaymentIntentCreateParams; - let paymentIntent: Stripe.PaymentIntent; - try { - paymentIntent = await stripe().paymentIntents.create(request, { - idempotencyKey: autoTopUpIdempotencyKey(accountId, snapshot.attempt), - timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, - }); - } catch (error) { - if (isUnresolvedAttempt(error)) { - return { state: "skipped", reason: "in_progress" }; + if (paymentIntent.status === "succeeded") { + const balanceMicroCredits = await grantAutoTopUp({ + accountId, + paymentIntentId: paymentIntent.id, + amountCents: paymentIntent.amount_received, + period, + executor: tx, + }); + return { + kind: "done", + result: { + state: "succeeded", + paymentIntentId: paymentIntent.id, + balanceMicroCredits, + }, + }; } - const decline = declineFrom(error); - if (!decline) { - console.error("[infra-billing] Auto top-up could not be attempted", { + if (paymentIntent.status === "processing") { + await recordPendingAutoTopUp({ accountId, - error, + paymentIntentId: paymentIntent.id, + amountCents, + period, + executor: tx, }); - return { state: "error" }; + return { + kind: "done", + result: { state: "pending", paymentIntentId: paymentIntent.id }, + }; } - await failAutoTopUp({ accountId, amountCents, period, ...decline }); - return { state: "failed", reason: decline.reason }; - } + return { kind: "declined", paymentIntent }; + }); - if (paymentIntent.status === "succeeded") { - const balanceMicroCredits = await grantAutoTopUp({ + if (outcome.kind === "skipped") { + if ( + outcome.reason === "daily_limit" || + outcome.reason === "owner_daily_limit" + ) { + console.warn("[infra-billing] Auto top-up waits for its daily limit", { + accountId, + reason: outcome.reason, + }); + } + return { state: "skipped", reason: outcome.reason }; + } + if (outcome.kind === "reconcile") { + for (const paymentIntentId of outcome.paymentIntentIds) { + await reconcilePendingAutoTopUp(accountId, paymentIntentId); + } + return { state: "skipped", reason: "reconciled" }; + } + if (outcome.kind === "done") return outcome.result; + if (outcome.kind === "declined") { + await failAutoTopUp({ accountId, - paymentIntentId: paymentIntent.id, - amountCents: paymentIntent.amount_received, + paymentIntentId: outcome.paymentIntent.id, + reason: outcome.paymentIntent.status, + amountCents, period, }); - return { - state: "succeeded", - paymentIntentId: paymentIntent.id, - balanceMicroCredits, - }; + return { state: "failed", reason: outcome.paymentIntent.status }; } - - if (paymentIntent.status === "processing") { - await recordPendingAutoTopUp({ + if (isUnresolvedAttempt(outcome.error)) { + return { state: "skipped", reason: "in_progress" }; + } + const decline = declineFrom(outcome.error); + if (!decline) { + console.error("[infra-billing] Auto top-up could not be attempted", { accountId, - paymentIntentId: paymentIntent.id, - amountCents, - period, + error: outcome.error, }); - return { state: "pending", paymentIntentId: paymentIntent.id }; + return { state: "error" }; } - - await failAutoTopUp({ - accountId, - paymentIntentId: paymentIntent.id, - reason: paymentIntent.status, - amountCents, - period, - }); - return { state: "failed", reason: paymentIntent.status }; + await failAutoTopUp({ accountId, amountCents, period, ...decline }); + return { state: "failed", reason: decline.reason }; } type DueCursor = { attemptKey: string; id: string }; From 3037099dcfbf471a26dea407ab43152f599024bd Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:43:27 -0500 Subject: [PATCH 13/26] docs: say automatic top-ups stop at $1,000 a day across all your apps --- apps/web/__tests__/unit/developer-billing-docs.test.ts | 3 ++- .../dashboard/infra/[appId]/billing/auto-top-up.tsx | 9 +++++---- apps/web/content/docs/api/billing.mdx | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/web/__tests__/unit/developer-billing-docs.test.ts b/apps/web/__tests__/unit/developer-billing-docs.test.ts index c4290f90fc9..46707ccfbc4 100644 --- a/apps/web/__tests__/unit/developer-billing-docs.test.ts +++ b/apps/web/__tests__/unit/developer-billing-docs.test.ts @@ -48,6 +48,7 @@ import { AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS, AUTO_TOP_UP_MIN_AMOUNT_CENTS, AUTO_TOP_UP_MIN_THRESHOLD_MICRO_CREDITS, + AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS, } from "@/lib/infra-auto-top-up"; import { LEDGER_PAGE_SIZE } from "@/lib/infra-ledger"; import { @@ -281,7 +282,7 @@ describe("the Billing page", () => { "- **When.** As soon as the available balance falls below your threshold.", ); expect(billing).toContain( - `up to ${AUTO_TOP_UP_DAILY_LIMIT} top-ups in any 24 hours`, + `up to ${AUTO_TOP_UP_DAILY_LIMIT} top-ups in any 24 hours, and at most ${wholeDollars(AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS)} a day across all your apps.`, ); expect(billing).toContain(`${LEDGER_PAGE_SIZE} to a page`); expect(billing).toContain( diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx index a444c5a09b6..d16fa649792 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx @@ -145,9 +145,9 @@ export function AutoTopUp({ Saving a card charges nothing and leaves auto top-up off. Once you switch it on, Cap adds ${initialAmount} from that card as soon as this app's balance falls below ${initialThreshold}. You can change both - amounts, the card is charged at most ten times in any 24 hours, and - auto top-up switches itself off if a charge is declined. Remove the - card at any time. + amounts. The card is charged at most ten times in any 24 hours, and at + most $1,000 a day across all your apps. Auto top-up switches itself + off if a charge is declined. Remove the card at any time.

); @@ -264,7 +264,8 @@ export function AutoTopUp({

A card is saved for this app. It is charged at most ten times in any 24 - hours, and auto top-up switches itself off if a charge is declined. + hours, and at most $1,000 a day across all your apps. Auto top-up + switches itself off if a charge is declined.

); diff --git a/apps/web/content/docs/api/billing.mdx b/apps/web/content/docs/api/billing.mdx index b4c7e19daab..5a136b4b995 100644 --- a/apps/web/content/docs/api/billing.mdx +++ b/apps/web/content/docs/api/billing.mdx @@ -121,7 +121,7 @@ Auto top-up adds credit from a saved card when an app's balance runs low. It is - **When.** As soon as the available balance falls below your threshold. Cap also checks at 00:41, 06:41, 12:41 and 18:41 UTC. - **How much.** The amount you set, from $5 to $1,000. The threshold is from $1 to $1,000. The console suggests $25 when the balance is under $5. -- **How often.** Each time the balance falls below the threshold, up to 10 top-ups in any 24 hours. After a declined charge, the next attempt waits 24 hours. +- **How often.** Each time the balance falls below the threshold, up to 10 top-ups in any 24 hours, and at most $1,000 a day across all your apps. Credit you buy yourself does not count toward that. After a declined charge, the next attempt waits 24 hours. - **How to stop.** Switch it off or remove the card on the Billing page. If a charge is declined, auto top-up switches itself off and no credit is added. ## When credit runs out From 8481b9dc79810fcc661b6dd53cc9a3898fd0e1e7 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:08:35 -0500 Subject: [PATCH 14/26] fix: reserve an auto top-up under the owner lock and call Stripe outside it The owner lock now covers only the four waits and inserting a reservation, a topup_pending row keyed by the attempt's idempotency key with the planned amount in its metadata, and commits at once. Stripe is called outside any transaction. A success grants and removes the reservation, a decline records itself and removes it, a payment still processing relinks it to the PaymentIntent, and a timeout or in-progress answer leaves it for the next run to replay with the same key. A request that never reached Stripe voids it. Reservations count toward the owner's ceiling but do not hold the account's own next run. Settling locks the account row before the ledger row, like every other writer. --- .../unit/infra-auto-top-up-sweep.test.ts | 6 +- .../__tests__/unit/infra-auto-top-up.test.ts | 81 ++++- .../__tests__/unit/infra-billing-fake-db.ts | 37 +- apps/web/lib/infra-auto-top-up.ts | 326 ++++++++++-------- 4 files changed, 302 insertions(+), 148 deletions(-) diff --git a/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts index 61a75a8280e..2afa48ab0d6 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts @@ -157,7 +157,7 @@ describe("auto top-up sweep selection", () => { "(select max(`developer_credit_transactions`.`createdAt`) from `developer_credit_transactions` where `developer_credit_transactions`.`accountId` = `developer_credit_accounts`.`id` and `developer_credit_transactions`.`type` in (?, ?, ?) and `developer_credit_transactions`.`referenceType` in (?, ?, ?))", ); expect(sql).toContain( - "not (exists (select 1 from `developer_credit_transactions` `pending_attempt` where `pending_attempt`.`accountId` = `developer_credit_accounts`.`id` and `pending_attempt`.`type` = ? and `pending_attempt`.`referenceType` = ? and `pending_attempt`.`createdAt` > ? and not exists (select 1 from `developer_credit_transactions` `settled_attempt` where `settled_attempt`.`accountId` = `pending_attempt`.`accountId` and `settled_attempt`.`referenceId` = `pending_attempt`.`referenceId`", + "not (exists (select 1 from `developer_credit_transactions` `pending_attempt` where `pending_attempt`.`accountId` = `developer_credit_accounts`.`id` and `pending_attempt`.`type` = ? and `pending_attempt`.`referenceType` = ? and `pending_attempt`.`createdAt` > ? and `pending_attempt`.`referenceId` not like ? and not exists (select 1 from `developer_credit_transactions` `settled_attempt` where `settled_attempt`.`accountId` = `pending_attempt`.`accountId` and `settled_attempt`.`referenceId` = `pending_attempt`.`referenceId`", ); expect(sql).toContain( "exists (select 1 from `developer_credit_transactions` `declined_attempt` where `declined_attempt`.`accountId` = `developer_credit_accounts`.`id` and `declined_attempt`.`type` = ? and `declined_attempt`.`referenceType` = ? and `declined_attempt`.`createdAt` > ? and not exists (select 1 from `developer_credit_transactions` `later_top_up` where `later_top_up`.`accountId` = `declined_attempt`.`accountId` and `later_top_up`.`type` = ?", @@ -171,6 +171,10 @@ describe("auto top-up sweep selection", () => { expect(params).toContain(sqlTime(new Date(NOW.getTime() - 72 * HOUR_MS))); expect(params).toContain(sqlTime(new Date(NOW.getTime() - 24 * HOUR_MS))); expect(params).toContain(AUTO_TOP_UP_DAILY_LIMIT); + expect(params).toContain("infra-auto-top-up:%"); + expect(sql).toContain( + "from `developer_credit_accounts` `owner_account` join `developer_credit_transactions` `owner_entry` on `owner_entry`.`accountId` = `owner_account`.`id` where `owner_account`.`ownerId` = `developer_credit_accounts`.`ownerId`", + ); }); it("tries the least recently tried account first", async () => { diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index 3ecf657c8c6..624c06e31ef 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -94,6 +94,7 @@ const intent = ( type: "developer_auto_top_up", accountId: "account-1", period: "2026-11-11", + attempt: "0:0", }, last_payment_error: null, ...extra, @@ -120,6 +121,12 @@ const arm = () => { const topUps = () => ledger.rows.filter((row) => row.referenceType === "auto_top_up"); +const reservations = () => + ledger.rows.filter( + (row) => + row.referenceType === "auto_top_up_pending" && + String(row.referenceId).startsWith("infra-auto-top-up:"), + ); const failures = () => ledger.rows.filter((row) => row.referenceType === "auto_top_up_failed"); @@ -366,6 +373,57 @@ describe("auto top-up charge", () => { expect(now().toISOString().slice(0, 10)).toBe("2026-11-11"); }); + it("reserves the attempt before calling Stripe and removes the reservation with the grant", async () => { + mocks.createPaymentIntent.mockImplementation(async () => { + expect(reservations()).toEqual([ + expect.objectContaining({ + type: "topup_pending", + amountMicroCredits: 0, + referenceId: "infra-auto-top-up:account-1:0:0", + }), + ]); + return intent("pi_1", "succeeded"); + }); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + expect(reservations()).toHaveLength(0); + expect(ledger.rows.map((row) => row.type)).toEqual(["topup"]); + }); + + it("voids the reservation when the request never reached Stripe", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + new Error("STRIPE_SECRET_KEY is not set"), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(ledger.rows).toHaveLength(0); + errors.mockRestore(); + }); + + it("lets the webhook settle a reservation whose response was lost", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("timeout"), { type: "StripeConnectionError" }), + ); + await runAutoTopUp("account-1", now()); + expect(reservations()).toHaveLength(1); + + await settleAutoTopUpPaymentIntent( + intent("pi_1", "succeeded"), + "succeeded", + ); + + expect(reservations()).toHaveLength(0); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + errors.mockRestore(); + }); + it("repeats the same key when retried before the attempt settles", async () => { mocks.createPaymentIntent.mockRejectedValueOnce( Object.assign(new Error("network"), { type: "StripeConnectionError" }), @@ -394,7 +452,14 @@ describe("auto top-up charge", () => { "[infra-billing] Auto top-up could not be attempted", expect.objectContaining({ accountId: "account-1" }), ); - expect(ledger.rows).toHaveLength(0); + expect(ledger.rows).toEqual([ + expect.objectContaining({ + type: "topup_pending", + amountMicroCredits: 0, + referenceId: "infra-auto-top-up:account-1:0:0", + metadata: { amountCents: 500, period: "2026-11-11" }, + }), + ]); expect(ledger.account.autoTopUpEnabled).toBe(true); expect(mocks.sendEmail).not.toHaveBeenCalled(); @@ -421,7 +486,8 @@ describe("auto top-up charge", () => { reason: "in_progress", }); expect(ledger.account.autoTopUpEnabled).toBe(true); - expect(ledger.rows).toHaveLength(0); + expect(reservations()).toHaveLength(1); + expect(ledger.rows).toHaveLength(1); expect(mocks.sendEmail).not.toHaveBeenCalled(); }); @@ -451,7 +517,13 @@ describe("auto top-up charge", () => { const [first, changed] = mocks.createPaymentIntent.mock.calls; expect(changed?.[0].amount).toBe(2_500); expect(changed?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); - expect(ledger.rows).toHaveLength(0); + expect(reservations()).toEqual([ + expect.objectContaining({ + referenceId: "infra-auto-top-up:account-1:0:0", + metadata: { amountCents: 500, period: "2026-11-11" }, + }), + ]); + expect(ledger.rows).toHaveLength(1); expect(ledger.account.autoTopUpEnabled).toBe(true); }); @@ -878,7 +950,8 @@ describe("auto top-up charge", () => { expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); expect(ledger.account.autoTopUpEnabled).toBe(true); - expect(ledger.rows).toHaveLength(0); + expect(reservations()).toHaveLength(1); + expect(ledger.rows).toHaveLength(1); }); it("ignores payments that are not auto top-ups or did not settle in dollars", async () => { diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index ba5fb2eeb48..a1e04bb9484 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -143,6 +143,7 @@ function autoTopUpWaits() { pending: of("topup_pending", "auto_top_up_pending").some( (row) => row.createdAt.getTime() > now - 72 * HOUR_MS && + !String(row.referenceId).startsWith("infra-auto-top-up:") && !settledIds.has(row.referenceId), ), decline: of("topup_failed", "auto_top_up_failed").some( @@ -413,7 +414,19 @@ function update(table: unknown) { return { set(values: Record) { return { - async where() { + async where(condition?: SQL) { + if (table === developerCreditTransactions && condition) { + const [accountId, referenceId, referenceType] = + dialect.sqlToQuery(condition).params; + const rows = ledger.rows.filter( + (row) => + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === referenceType, + ); + for (const row of rows) Object.assign(row, values); + return [{ affectedRows: rows.length }]; + } if (table !== developerCreditAccounts) return [{ affectedRows: 0 }]; const { balanceMicroCredits, ...rest } = values; if (balanceMicroCredits !== undefined) { @@ -464,10 +477,31 @@ function insert(table: unknown) { }; } +function remove(table: unknown) { + return { + async where(condition: SQL) { + if (table !== developerCreditTransactions) return [{ affectedRows: 0 }]; + const [accountId, referenceId, referenceType] = + dialect.sqlToQuery(condition).params; + const before = ledger.rows.length; + ledger.rows = ledger.rows.filter( + (row) => + !( + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === referenceType + ), + ); + return [{ affectedRows: before - ledger.rows.length }]; + }, + }; +} + type FakeDb = { select: typeof select; update: typeof update; insert: typeof insert; + delete: typeof remove; transaction: (run: (tx: FakeDb) => Promise) => Promise; }; @@ -475,5 +509,6 @@ export const fakeDb: FakeDb = { select, update, insert, + delete: remove, transaction: (run) => run(fakeDb), }; diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 3d3891540ab..094b10fad21 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -57,9 +57,7 @@ type SkipReason = Extract["reason"]; type LockedAttempt = | { kind: "skipped"; reason: SkipReason } | { kind: "reconcile"; paymentIntentIds: string[] } - | { kind: "done"; result: AutoTopUpResult } - | { kind: "declined"; paymentIntent: Stripe.PaymentIntent } - | { kind: "unpaid"; error: unknown }; + | { kind: "reserved"; key: string; attempt: AutoTopUpAttempt }; export type AutoTopUpResult = | { @@ -116,11 +114,57 @@ export function autoTopUpSettingsError({ export type AutoTopUpAttempt = { topUps: number; declines: number }; +const RESERVATION_PREFIX = "infra-auto-top-up:"; + export function autoTopUpIdempotencyKey( accountId: string, { topUps, declines }: AutoTopUpAttempt, ) { - return `infra-auto-top-up:${accountId}:${topUps}:${declines}`; + return `${RESERVATION_PREFIX}${accountId}:${topUps}:${declines}`; +} + +function reservationKeyOf(paymentIntent: Stripe.PaymentIntent) { + const { accountId, attempt } = paymentIntent.metadata ?? {}; + const [topUps, declines] = (attempt ?? "").split(":").map(Number); + if (!accountId || !Number.isInteger(topUps) || !Number.isInteger(declines)) { + return null; + } + return autoTopUpIdempotencyKey(accountId, { + topUps: topUps ?? 0, + declines: declines ?? 0, + }); +} + +const reservationOf = (accountId: string, key: string) => + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.referenceId, key), + eq( + developerCreditTransactions.referenceType, + AUTO_TOP_UP_PENDING_REFERENCE_TYPE, + ), + ); + +async function dropReservation( + executor: AutoTopUpTransaction | ReturnType, + accountId: string, + key: string | null, +) { + if (!key) return; + await executor + .delete(developerCreditTransactions) + .where(reservationOf(accountId, key)); +} + +async function linkReservation( + accountId: string, + key: string, + paymentIntentId: string, +) { + await db() + .update(developerCreditTransactions) + .set({ referenceId: paymentIntentId }) + .where(reservationOf(accountId, key)); } export function autoTopUpFailureText(reason: string | null | undefined) { @@ -150,6 +194,24 @@ function isUnresolvedAttempt(error: unknown) { ); } +const UNSENT_STRIPE_ERRORS = new Set([ + "StripeRateLimitError", + "StripeAuthenticationError", + "StripePermissionError", +]); + +function neverReachedStripe(error: unknown) { + const type = + typeof error === "object" && error !== null + ? (error as { type?: unknown }).type + : undefined; + return ( + typeof type !== "string" || + !type.startsWith("Stripe") || + UNSENT_STRIPE_ERRORS.has(type) + ); +} + function declineFrom(error: unknown) { if (typeof error !== "object" || error === null) return null; const fields = error as { @@ -237,7 +299,7 @@ export function autoTopUpWaits(now: Date) { new Date(now.getTime() - AUTO_TOP_UP_PENDING_RECONCILE_MS), ); return { - pending: sql`exists (select 1 from ${pendingAttempt.from} where ${pendingAttempt.accountId} = ${accountRow} and ${pendingAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${pendingAttempt.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${pendingAttempt.createdAt} > ${reconcileFrom} and not exists (select 1 from ${settledAttempt.from} where ${settledAttempt.accountId} = ${pendingAttempt.accountId} and ${settledAttempt.referenceId} = ${pendingAttempt.referenceId} and ${settledAttempt.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`, + pending: sql`exists (select 1 from ${pendingAttempt.from} where ${pendingAttempt.accountId} = ${accountRow} and ${pendingAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${pendingAttempt.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${pendingAttempt.createdAt} > ${reconcileFrom} and ${pendingAttempt.referenceId} not like ${`${RESERVATION_PREFIX}%`} and not exists (select 1 from ${settledAttempt.from} where ${settledAttempt.accountId} = ${pendingAttempt.accountId} and ${settledAttempt.referenceId} = ${pendingAttempt.referenceId} and ${settledAttempt.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`, decline: sql`exists (select 1 from ${declinedAttempt.from} where ${declinedAttempt.accountId} = ${accountRow} and ${declinedAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_FAILED_REFERENCE_TYPE]} and ${declinedAttempt.referenceType} = ${AUTO_TOP_UP_FAILED_REFERENCE_TYPE} and ${declinedAttempt.createdAt} > ${dayAgo} and not exists (select 1 from ${laterTopUp.from} where ${laterTopUp.accountId} = ${declinedAttempt.accountId} and ${laterTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${laterTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${laterTopUp.createdAt} > ${declinedAttempt.createdAt}))`, owner: sql`(select coalesce(sum(case when ${ownerEntry.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} then ${ownerEntry.amountMicroCredits} else cast(json_unquote(json_extract(${ownerEntry.metadata}, '$.amountCents')) as unsigned) * ${MICRO_CREDITS_PER_CENT} end), 0) from ${accounts} ${sql.identifier("owner_account")} join ${ownerEntry.from} on ${ownerEntry.accountId} = ${ownerAccount(developerCreditAccounts.id)} where ${ownerAccount(developerCreditAccounts.ownerId)} = ${accountColumn(developerCreditAccounts.ownerId)} and ${ownerEntry.type} in (${inList([ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE], ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]])}) and ${ownerEntry.createdAt} > ${dayAgo} and (${ownerEntry.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} or (${ownerEntry.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and not exists (select 1 from ${ownerSettled.from} where ${ownerSettled.accountId} = ${ownerEntry.accountId} and ${ownerSettled.referenceId} = ${ownerEntry.referenceId} and ${ownerSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)}))))) + ${accountColumn(developerCreditAccounts.autoTopUpAmountCents)} * ${MICRO_CREDITS_PER_CENT} > ${AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS * MICRO_CREDITS_PER_CENT}`, limit: sql`(select count(*) from ${recentTopUp.from} where ${recentTopUp.accountId} = ${accountRow} and ${recentTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${recentTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${recentTopUp.createdAt} > ${dayAgo}) >= ${AUTO_TOP_UP_DAILY_LIMIT}`, @@ -342,7 +404,9 @@ async function attemptHistory(executor: InfraDbExecutor, accountId: string) { topUps: Number(settled?.topUps ?? 0), declines: Number(settled?.declines ?? 0), }, - unresolved: pendingIds.filter((id) => !resolvedIds.has(id)), + unresolved: pendingIds.filter( + (id) => !resolvedIds.has(id) && !id.startsWith(RESERVATION_PREFIX), + ), }; } @@ -365,12 +429,14 @@ async function failAutoTopUp({ reason, amountCents, period, + reservationKey = null, }: { accountId: string; paymentIntentId: string | null; reason: string; amountCents: number; period: string; + reservationKey?: string | null; }) { const referenceId = paymentIntentId ?? `declined-${period}`; const recorded = await db().transaction(async (tx) => { @@ -383,6 +449,7 @@ async function failAutoTopUp({ .limit(1) .for("update"); if (!account) return false; + await dropReservation(tx, accountId, reservationKey); const [existing] = await tx .select({ id: developerCreditTransactions.id }) @@ -424,60 +491,6 @@ async function failAutoTopUp({ return recorded; } -async function recordPendingAutoTopUp({ - accountId, - paymentIntentId, - amountCents, - period, - executor, -}: { - accountId: string; - paymentIntentId: string; - amountCents: number; - period: string; - executor?: AutoTopUpTransaction; -}) { - const record = async (tx: AutoTopUpTransaction) => { - const [account] = await tx - .select({ - balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, - }) - .from(developerCreditAccounts) - .where(eq(developerCreditAccounts.id, accountId)) - .limit(1) - .for("update"); - if (!account) return; - - const [existing] = await tx - .select({ id: developerCreditTransactions.id }) - .from(developerCreditTransactions) - .where( - and( - eq(developerCreditTransactions.accountId, accountId), - eq(developerCreditTransactions.referenceId, paymentIntentId), - inArray( - developerCreditTransactions.referenceType, - ATTEMPT_REFERENCE_TYPES, - ), - ), - ) - .limit(1); - if (existing) return; - - await tx.insert(developerCreditTransactions).values({ - id: nanoId(), - accountId, - type: "topup_pending", - amountMicroCredits: 0, - balanceAfterMicroCredits: account.balanceMicroCredits, - referenceId: paymentIntentId, - referenceType: AUTO_TOP_UP_PENDING_REFERENCE_TYPE, - metadata: { amountCents, period }, - }); - }; - await (executor ? record(executor) : db().transaction(record)); -} - function grantAutoTopUp({ accountId, paymentIntentId, @@ -621,69 +634,32 @@ export async function runAutoTopUp( if (history.unresolved.length > 0) { return { kind: "reconcile", paymentIntentIds: history.unresolved }; } - - const productId = developerCreditsProductId(); - const request = { - amount: amountCents, - currency: "usd", - customer: customerId, - payment_method: paymentMethodId, - payment_method_types: ["card"], - off_session: true, - confirm: true, - description: "Cap Infra credits, auto top-up", - receipt_email: account.email, - metadata: { - type: AUTO_TOP_UP_METADATA_TYPE, - accountId, - appId: account.appId, - amountCents: String(amountCents), - attempt: `${history.attempt.topUps}:${history.attempt.declines}`, - ...(productId ? { productId } : {}), - }, - } satisfies Stripe.PaymentIntentCreateParams; - let paymentIntent: Stripe.PaymentIntent; - try { - paymentIntent = await stripe().paymentIntents.create(request, { - idempotencyKey: autoTopUpIdempotencyKey(accountId, history.attempt), - timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, - maxNetworkRetries: 0, - }); - } catch (error) { - return { kind: "unpaid", error }; - } - - if (paymentIntent.status === "succeeded") { - const balanceMicroCredits = await grantAutoTopUp({ - accountId, - paymentIntentId: paymentIntent.id, - amountCents: paymentIntent.amount_received, - period, - executor: tx, - }); - return { - kind: "done", - result: { - state: "succeeded", - paymentIntentId: paymentIntent.id, - balanceMicroCredits, - }, - }; - } - if (paymentIntent.status === "processing") { - await recordPendingAutoTopUp({ + const key = autoTopUpIdempotencyKey(accountId, history.attempt); + const [reserved] = await tx + .select({ id: developerCreditTransactions.id }) + .from(developerCreditTransactions) + .where(reservationOf(accountId, key)) + .limit(1); + if (!reserved) { + const [balance] = await tx + .select({ + balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, + }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.id, accountId)) + .limit(1); + await tx.insert(developerCreditTransactions).values({ + id: nanoId(), accountId, - paymentIntentId: paymentIntent.id, - amountCents, - period, - executor: tx, + type: "topup_pending", + amountMicroCredits: 0, + balanceAfterMicroCredits: balance?.balanceMicroCredits ?? 0, + referenceId: key, + referenceType: AUTO_TOP_UP_PENDING_REFERENCE_TYPE, + metadata: { amountCents, period }, }); - return { - kind: "done", - result: { state: "pending", paymentIntentId: paymentIntent.id }, - }; } - return { kind: "declined", paymentIntent }; + return { kind: "reserved", key, attempt: history.attempt }; }); if (outcome.kind === "skipped") { @@ -704,30 +680,91 @@ export async function runAutoTopUp( } return { state: "skipped", reason: "reconciled" }; } - if (outcome.kind === "done") return outcome.result; - if (outcome.kind === "declined") { - await failAutoTopUp({ + + const { key, attempt } = outcome; + const productId = developerCreditsProductId(); + const request = { + amount: amountCents, + currency: "usd", + customer: customerId, + payment_method: paymentMethodId, + payment_method_types: ["card"], + off_session: true, + confirm: true, + description: "Cap Infra credits, auto top-up", + receipt_email: account.email, + metadata: { + type: AUTO_TOP_UP_METADATA_TYPE, accountId, - paymentIntentId: outcome.paymentIntent.id, - reason: outcome.paymentIntent.status, - amountCents, - period, + appId: account.appId, + amountCents: String(amountCents), + attempt: `${attempt.topUps}:${attempt.declines}`, + ...(productId ? { productId } : {}), + }, + } satisfies Stripe.PaymentIntentCreateParams; + let paymentIntent: Stripe.PaymentIntent; + try { + paymentIntent = await stripe().paymentIntents.create(request, { + idempotencyKey: key, + timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + maxNetworkRetries: 0, }); - return { state: "failed", reason: outcome.paymentIntent.status }; - } - if (isUnresolvedAttempt(outcome.error)) { - return { state: "skipped", reason: "in_progress" }; - } - const decline = declineFrom(outcome.error); - if (!decline) { + } catch (error) { + if (isUnresolvedAttempt(error)) { + return { state: "skipped", reason: "in_progress" }; + } + const decline = declineFrom(error); + if (decline) { + await failAutoTopUp({ + accountId, + amountCents, + period, + reservationKey: key, + ...decline, + }); + return { state: "failed", reason: decline.reason }; + } + if (neverReachedStripe(error)) { + await dropReservation(db(), accountId, key); + } console.error("[infra-billing] Auto top-up could not be attempted", { accountId, - error: outcome.error, + error, }); return { state: "error" }; } - await failAutoTopUp({ accountId, amountCents, period, ...decline }); - return { state: "failed", reason: decline.reason }; + + if (paymentIntent.status === "succeeded") { + const balanceMicroCredits = await db().transaction(async (tx) => { + const balance = await grantAutoTopUp({ + accountId, + paymentIntentId: paymentIntent.id, + amountCents: paymentIntent.amount_received, + period, + executor: tx, + }); + await dropReservation(tx, accountId, key); + return balance; + }); + return { + state: "succeeded", + paymentIntentId: paymentIntent.id, + balanceMicroCredits, + }; + } + if (paymentIntent.status === "processing") { + await linkReservation(accountId, key, paymentIntent.id); + return { state: "pending", paymentIntentId: paymentIntent.id }; + } + await failAutoTopUp({ + accountId, + paymentIntentId: paymentIntent.id, + reason: paymentIntent.status, + amountCents, + period, + reservationKey: key, + }); + return { state: "failed", reason: paymentIntent.status }; } type DueCursor = { attemptKey: string; id: string }; @@ -847,16 +884,21 @@ export async function settleAutoTopUpPaymentIntent( ) { return false; } - await grantAutoTopUp({ - accountId, - paymentIntentId: paymentIntent.id, - amountCents: paymentIntent.amount_received, - period: period ?? null, + await db().transaction(async (tx) => { + await grantAutoTopUp({ + accountId, + paymentIntentId: paymentIntent.id, + amountCents: paymentIntent.amount_received, + period: period ?? null, + executor: tx, + }); + await dropReservation(tx, accountId, reservationKeyOf(paymentIntent)); }); return true; } await failAutoTopUp({ + reservationKey: reservationKeyOf(paymentIntent), accountId, paymentIntentId: paymentIntent.id, reason: From 714052843581c2ff6b9acf173412b77032a6c368 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:08:35 -0500 Subject: [PATCH 15/26] test: a slow Stripe call no longer holds the owner's accounts, and a crash recovers with one charge --- .../infra-auto-top-up-database.test.ts | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index 53a5641df70..d4d8aeb9643 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -22,6 +22,8 @@ const fixture = vi.hoisted(() => ({ created: [] as FakeIntent[], createdKeys: [] as string[], loseNextResponse: false, + delayMs: 40, + calledAt: [] as number[], stored: new Map(), retrieved: [] as string[], })); @@ -73,7 +75,8 @@ vi.mock("@cap/utils", () => ({ } if (entry?.state === "done") return entry.intent; fixture.keys.set(key, { state: "pending", params: sent }); - await new Promise((resolve) => setTimeout(resolve, 40)); + fixture.calledAt.push(Date.now()); + await new Promise((resolve) => setTimeout(resolve, fixture.delayMs)); const intent: FakeIntent = { id: `pi_${fixture.created.length + 1}`, status: "succeeded", @@ -465,6 +468,86 @@ describe.runIf(Boolean(databaseUrl))( expect((await runAutoTopUp(automatic.accountId)).state).toBe("succeeded"); }); + it("holds the owner lock only while reserving, not through a slow Stripe call", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { lockInfraAccount } = await import("@/lib/infra-balance"); + const ownerId = await makeOwner(); + const slow = await makeAccount(30_000, { ownerId }); + const other = await makeAccount(30_000, { ownerId }); + fixture.delayMs = 2_000; + try { + const startedAt = Date.now(); + const before = fixture.calledAt.length; + const topUp = runAutoTopUp(slow.accountId); + while (fixture.calledAt.length === before) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + const lockHeldMs = (fixture.calledAt.at(-1) ?? 0) - startedAt; + + const waitStarted = Date.now(); + await (fixture.database as MySql2Database).transaction(async (tx) => { + await lockInfraAccount(tx, other.appId); + }); + const otherAppWaitedMs = Date.now() - waitStarted; + + expect((await topUp).state).toBe("succeeded"); + const stripeCallMs = Date.now() - (fixture.calledAt.at(-1) ?? 0); + console.log( + JSON.stringify({ lockHeldMs, otherAppWaitedMs, stripeCallMs }), + ); + expect(stripeCallMs).toBeGreaterThanOrEqual(1_900); + expect(otherAppWaitedMs).toBeLessThan(200); + } finally { + fixture.delayMs = 40; + } + }); + + it("recovers a reservation left by a crash before Stripe was called, with one charge", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const { accountId } = await makeAccount(30_000); + const key = autoTopUpIdempotencyKey(accountId, { + topUps: 0, + declines: 0, + }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":500}')`, + [id(), accountId, key], + ); + const before = fixture.createdKeys.length; + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect((await runAutoTopUp(accountId)).state).toBe("skipped"); + + expect(fixture.createdKeys.slice(before)).toEqual([key]); + expect(await topUpsOf(accountId)).toHaveLength(1); + const [rows] = await pool.query( + "SELECT COUNT(*) AS n FROM developer_credit_transactions WHERE accountId = ? AND type = 'topup_pending'", + [accountId], + ); + expect(Number((rows as { n: number }[])[0]?.n)).toBe(0); + }); + + it("recovers a reservation whose payment Stripe made before a crash, with one charge", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + expect(await topUpsOf(accountId)).toHaveLength(0); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + errors.mockRestore(); + }); + it("tops up again the same day with a new key once the balance is low again", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const { accountId } = await makeAccount(30_000); From e2cb49f7e98107bb901748dc8f20fefc07093df1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:26:48 -0500 Subject: [PATCH 16/26] fix: reserve the exact charge under the owner lock and recover before any limit The owner lock now re-reads the account's switch, threshold, amount and card, checks the four waits against that amount, and stores the amount, customer, card, receipt email and description in the reservation. Stripe gets exactly those, and a retry replays them under the same key, so the amount the owner ceiling counted is the amount charged. The ceiling counts the account's own reservation once, as the attempt's amount. Payments and reservations pending for 72 hours are reconciled before the switch, the balance, the decline wait or any limit, and the sweep selects those accounts whatever else applies. A reservation older than a day is no longer replayed; after 72 hours it is settled from Stripe's search by its attempt, or voided if Stripe has no payment. The charge description names the app. --- .../unit/infra-auto-top-up-sweep.test.ts | 5 +- .../__tests__/unit/infra-auto-top-up.test.ts | 142 +++++++- .../__tests__/unit/infra-billing-fake-db.ts | 42 ++- apps/web/lib/infra-auto-top-up.ts | 303 +++++++++++++----- 4 files changed, 388 insertions(+), 104 deletions(-) diff --git a/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts index 2afa48ab0d6..551c6d50d96 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts @@ -65,7 +65,7 @@ vi.mock("@cap/database", async () => { .map((account) => [account.id]), }; } - if (sql.includes("`autoTopUpAmountCents`")) { + if (sql.includes("`developer_apps`.`name`")) { state.reads.push(String(params[0])); vi.setSystemTime(Date.now() + state.msPerAccount); return { rows: [] }; @@ -172,6 +172,9 @@ describe("auto top-up sweep selection", () => { expect(params).toContain(sqlTime(new Date(NOW.getTime() - 24 * HOUR_MS))); expect(params).toContain(AUTO_TOP_UP_DAILY_LIMIT); expect(params).toContain("infra-auto-top-up:%"); + expect(sql).toContain( + "or exists (select 1 from `developer_credit_transactions` `stale_payment` where `stale_payment`.`accountId` = `developer_credit_accounts`.`id` and `stale_payment`.`type` = ? and `stale_payment`.`referenceType` = ? and `stale_payment`.`createdAt` <= ?", + ); expect(sql).toContain( "from `developer_credit_accounts` `owner_account` join `developer_credit_transactions` `owner_entry` on `owner_entry`.`accountId` = `owner_account`.`id` where `owner_account`.`ownerId` = `developer_credit_accounts`.`ownerId`", ); diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index 624c06e31ef..8eea6434fb3 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -6,6 +6,7 @@ const mocks = vi.hoisted(() => ({ emailsEnabled: false, createPaymentIntent: vi.fn(), retrievePaymentIntent: vi.fn(), + searchPaymentIntents: vi.fn(), retrieveSetupIntent: vi.fn(), retrievePaymentMethod: vi.fn(), detachPaymentMethod: vi.fn(), @@ -32,6 +33,7 @@ vi.mock("@cap/utils", () => ({ paymentIntents: { create: mocks.createPaymentIntent, retrieve: mocks.retrievePaymentIntent, + search: mocks.searchPaymentIntents, }, setupIntents: { retrieve: mocks.retrieveSetupIntent }, paymentMethods: { @@ -55,6 +57,7 @@ import type Stripe from "stripe"; import { AUTO_TOP_UP_DAILY_LIMIT, AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + autoTopUpDescription, autoTopUpFailureText, autoTopUpIdempotencyKey, autoTopUpSettingsError, @@ -137,6 +140,8 @@ beforeEach(() => { mocks.createPaymentIntent.mockReset(); mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "succeeded")); mocks.retrievePaymentIntent.mockReset(); + mocks.searchPaymentIntents.mockReset(); + mocks.searchPaymentIntents.mockResolvedValue({ data: [] }); mocks.sendEmail.mockReset(); mocks.sendEmail.mockResolvedValue({ data: { id: "email-1" }, error: null }); mocks.retrieveSetupIntent.mockReset(); @@ -210,6 +215,17 @@ describe("auto top-up settings", () => { ); }); + it("names the app in the charge description, within Stripe's limit", () => { + expect(autoTopUpDescription(" Acme Video ")).toBe( + "Cap Infra credits for Acme Video, auto top-up", + ); + expect(autoTopUpDescription("")).toBe("Cap Infra credits, auto top-up"); + expect(autoTopUpDescription(null)).toBe("Cap Infra credits, auto top-up"); + const long = autoTopUpDescription("x".repeat(5_000)); + expect(long.length).toBeLessThanOrEqual(1_000); + expect(long.endsWith(", auto top-up")).toBe(true); + }); + it("explains a decline in plain words", () => { expect(autoTopUpFailureText("authentication_required")).toContain("bank"); expect(autoTopUpFailureText("insufficient_funds")).toContain( @@ -457,7 +473,12 @@ describe("auto top-up charge", () => { type: "topup_pending", amountMicroCredits: 0, referenceId: "infra-auto-top-up:account-1:0:0", - metadata: { amountCents: 500, period: "2026-11-11" }, + metadata: expect.objectContaining({ + amountCents: 500, + customerId: "cus_1", + paymentMethodId: "pm_1", + period: "2026-11-11", + }), }), ]); expect(ledger.account.autoTopUpEnabled).toBe(true); @@ -491,7 +512,7 @@ describe("auto top-up charge", () => { expect(mocks.sendEmail).not.toHaveBeenCalled(); }); - it("cannot start a second payment when the settings change during an unresolved attempt", async () => { + it("retries an unresolved attempt with the reserved amount and card, not changed settings", async () => { mocks.createPaymentIntent.mockRejectedValueOnce( Object.assign(new Error("Request aborted due to timeout"), { type: "StripeConnectionError", @@ -501,6 +522,36 @@ describe("auto top-up charge", () => { expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); ledger.account.autoTopUpAmountCents = 2_500; + ledger.account.stripePaymentMethodId = "pm_new"; + mocks.createPaymentIntent.mockResolvedValueOnce( + intent("pi_1", "succeeded"), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_1", + }); + + const [first, retry] = mocks.createPaymentIntent.mock.calls; + expect(retry?.[0]).toEqual(first?.[0]); + expect(retry?.[0]).toMatchObject({ amount: 500, payment_method: "pm_1" }); + expect(retry?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); + expect(reservations()).toHaveLength(0); + expect(topUps()).toHaveLength(1); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValueOnce( + intent("pi_2", "succeeded", { amount_received: 2_500 }), + ); + await runAutoTopUp("account-1", now()); + const next = mocks.createPaymentIntent.mock.calls[2]; + expect(next?.[0]).toMatchObject({ + amount: 2_500, + payment_method: "pm_new", + }); + expect(next?.[1].idempotencyKey).not.toBe(first?.[1].idempotencyKey); + }); + + it("treats Stripe refusing a changed request under the same key as in progress", async () => { mocks.createPaymentIntent.mockRejectedValueOnce( Object.assign( new Error( @@ -509,22 +560,14 @@ describe("auto top-up charge", () => { { type: "StripeIdempotencyError", statusCode: 400 }, ), ); + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "skipped", reason: "in_progress", }); - - const [first, changed] = mocks.createPaymentIntent.mock.calls; - expect(changed?.[0].amount).toBe(2_500); - expect(changed?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); - expect(reservations()).toEqual([ - expect.objectContaining({ - referenceId: "infra-auto-top-up:account-1:0:0", - metadata: { amountCents: 500, period: "2026-11-11" }, - }), - ]); - expect(ledger.rows).toHaveLength(1); + expect(reservations()).toHaveLength(1); expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(mocks.sendEmail).not.toHaveBeenCalled(); }); it(`stops at ${AUTO_TOP_UP_DAILY_LIMIT} automatic top-ups in any 24 hours, then waits`, async () => { @@ -656,6 +699,79 @@ describe("auto top-up charge", () => { }); }); + describe("recovering an attempt whose answer was lost", () => { + const loseAnswer = async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("timeout"), { type: "StripeConnectionError" }), + ); + await runAutoTopUp("account-1", now()); + errors.mockRestore(); + mocks.createPaymentIntent.mockClear(); + }; + + it("waits instead of replaying once the reservation is a day old", async () => { + await loseAnswer(); + later(25); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "awaiting_payment", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(mocks.searchPaymentIntents).not.toHaveBeenCalled(); + }); + + it("after 72 hours, finds the payment in Stripe and grants it once", async () => { + await loseAnswer(); + later(72.1); + mocks.searchPaymentIntents.mockResolvedValue({ + data: [intent("pi_found", "succeeded")], + }); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.searchPaymentIntents.mock.calls[0]?.[0]).toEqual({ + query: + "metadata['accountId']:'account-1' AND metadata['attempt']:'0:0'", + limit: 1, + }); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_found" }), + ]); + expect(reservations()).toHaveLength(0); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + }); + + it("after 72 hours, voids a reservation Stripe never charged", async () => { + await loseAnswer(); + later(72.1); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(ledger.rows).toHaveLength(0); + expect(topUps()).toHaveLength(0); + }); + + it("recovers before the switch, the decline wait or any limit can stop it", async () => { + await loseAnswer(); + ledger.account.autoTopUpEnabled = false; + ledger.account.balanceMicroCredits = 99_000_000; + later(72.1); + mocks.searchPaymentIntents.mockResolvedValue({ + data: [intent("pi_found", "succeeded")], + }); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "reconciled", + }); + expect(topUps()).toHaveLength(1); + }); + }); + describe("a payment still pending after 72 hours", () => { const leavePending = async () => { mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index a1e04bb9484..25ef9122328 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -159,6 +159,7 @@ function autoTopUpWaits() { .filter( (row) => row.createdAt.getTime() > now - 24 * HOUR_MS && + !String(row.referenceId).startsWith("infra-auto-top-up:") && ((row.type === "topup" && row.referenceType === "auto_top_up") || (row.type === "topup_pending" && row.referenceType === "auto_top_up_pending" && @@ -172,7 +173,14 @@ function autoTopUpWaits() { : Number(row.metadata?.amountCents ?? 0) * 1000), 0, ) + - ledger.account.autoTopUpAmountCents * 1000 > + Number( + ledger.rows.find( + (row) => + row.type === "topup_pending" && + String(row.referenceId).startsWith("infra-auto-top-up:"), + )?.metadata?.amountCents ?? ledger.account.autoTopUpAmountCents, + ) * + 1000 > 100_000_000, }; } @@ -186,12 +194,21 @@ function runSelect(query: Query): Record[] { if (query.table === developerCreditAccounts) { const { exists, ...account } = ledger.account; if (!exists) return []; - if (keys === "declineWait,id,limitWait,ownerWait,pendingWait") { + if ( + keys === + "amountCents,balanceMicroCredits,customerId,declineWait,enabled,id,limitWait,ownerWait,paymentMethodId,pendingWait,thresholdMicroCredits" + ) { if (rendered.params.at(-1) !== account.id) return []; const waits = autoTopUpWaits(); return [ { id: account.id, + enabled: account.autoTopUpEnabled, + thresholdMicroCredits: account.autoTopUpThresholdMicroCredits, + amountCents: account.autoTopUpAmountCents, + customerId: account.stripeCustomerId, + paymentMethodId: account.stripePaymentMethodId, + balanceMicroCredits: account.balanceMicroCredits, pendingWait: Number(waits.pending), declineWait: Number(waits.decline), limitWait: Number(waits.limit), @@ -296,14 +313,31 @@ function runSelect(query: Query): Record[] { ), ).slice(0, 1); } - if (keys === "pendingId") { + if (keys === "pendingId,pendingSince") { return ledger.rows .filter( (row) => row.type === "topup_pending" && row.referenceType === "auto_top_up_pending", ) - .map((row) => ({ pendingId: row.referenceId })); + .map((row) => ({ + pendingId: row.referenceId, + pendingSince: row.createdAt, + })); + } + if (keys === "reservation,reservedAt") { + const [accountId, referenceId] = rendered.params; + return ledger.rows + .filter( + (row) => + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === "auto_top_up_pending", + ) + .map((row) => ({ + reservation: row.metadata, + reservedAt: row.createdAt, + })); } if (keys === "settledId") { const ids = rendered.params.slice(3); diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 094b10fad21..43667284c96 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -16,6 +16,8 @@ import { inArray, isNotNull, isNull, + or, + type SQL, sql, } from "drizzle-orm"; import { after } from "next/server"; @@ -56,8 +58,12 @@ type SkipReason = Extract["reason"]; type LockedAttempt = | { kind: "skipped"; reason: SkipReason } - | { kind: "reconcile"; paymentIntentIds: string[] } - | { kind: "reserved"; key: string; attempt: AutoTopUpAttempt }; + | { + kind: "reserved"; + key: string; + attempt: AutoTopUpAttempt; + request: ReservedRequest; + }; export type AutoTopUpResult = | { @@ -289,6 +295,10 @@ export function autoTopUpWaits(now: Date) { sql`${sql.identifier("owner_account")}.${sql.identifier(field.name)}`; const ownerEntry = ledgerAlias("owner_entry"); const ownerSettled = ledgerAlias("owner_settled"); + const ownReservation = ledgerAlias("own_reservation"); + const reservedLike = `${RESERVATION_PREFIX}%`; + const jsonCents = (column: SQL) => + sql`cast(json_unquote(json_extract(${column}, '$.amountCents')) as unsigned)`; const pendingAttempt = ledgerAlias("pending_attempt"); const settledAttempt = ledgerAlias("settled_attempt"); const declinedAttempt = ledgerAlias("declined_attempt"); @@ -301,11 +311,21 @@ export function autoTopUpWaits(now: Date) { return { pending: sql`exists (select 1 from ${pendingAttempt.from} where ${pendingAttempt.accountId} = ${accountRow} and ${pendingAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${pendingAttempt.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${pendingAttempt.createdAt} > ${reconcileFrom} and ${pendingAttempt.referenceId} not like ${`${RESERVATION_PREFIX}%`} and not exists (select 1 from ${settledAttempt.from} where ${settledAttempt.accountId} = ${pendingAttempt.accountId} and ${settledAttempt.referenceId} = ${pendingAttempt.referenceId} and ${settledAttempt.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`, decline: sql`exists (select 1 from ${declinedAttempt.from} where ${declinedAttempt.accountId} = ${accountRow} and ${declinedAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_FAILED_REFERENCE_TYPE]} and ${declinedAttempt.referenceType} = ${AUTO_TOP_UP_FAILED_REFERENCE_TYPE} and ${declinedAttempt.createdAt} > ${dayAgo} and not exists (select 1 from ${laterTopUp.from} where ${laterTopUp.accountId} = ${declinedAttempt.accountId} and ${laterTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${laterTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${laterTopUp.createdAt} > ${declinedAttempt.createdAt}))`, - owner: sql`(select coalesce(sum(case when ${ownerEntry.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} then ${ownerEntry.amountMicroCredits} else cast(json_unquote(json_extract(${ownerEntry.metadata}, '$.amountCents')) as unsigned) * ${MICRO_CREDITS_PER_CENT} end), 0) from ${accounts} ${sql.identifier("owner_account")} join ${ownerEntry.from} on ${ownerEntry.accountId} = ${ownerAccount(developerCreditAccounts.id)} where ${ownerAccount(developerCreditAccounts.ownerId)} = ${accountColumn(developerCreditAccounts.ownerId)} and ${ownerEntry.type} in (${inList([ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE], ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]])}) and ${ownerEntry.createdAt} > ${dayAgo} and (${ownerEntry.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} or (${ownerEntry.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and not exists (select 1 from ${ownerSettled.from} where ${ownerSettled.accountId} = ${ownerEntry.accountId} and ${ownerSettled.referenceId} = ${ownerEntry.referenceId} and ${ownerSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)}))))) + ${accountColumn(developerCreditAccounts.autoTopUpAmountCents)} * ${MICRO_CREDITS_PER_CENT} > ${AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS * MICRO_CREDITS_PER_CENT}`, + owner: sql`(select coalesce(sum(case when ${ownerEntry.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} then ${ownerEntry.amountMicroCredits} else ${jsonCents(ownerEntry.metadata)} * ${MICRO_CREDITS_PER_CENT} end), 0) from ${accounts} ${sql.identifier("owner_account")} join ${ownerEntry.from} on ${ownerEntry.accountId} = ${ownerAccount(developerCreditAccounts.id)} where ${ownerAccount(developerCreditAccounts.ownerId)} = ${accountColumn(developerCreditAccounts.ownerId)} and ${ownerEntry.type} in (${inList([ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE], ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]])}) and ${ownerEntry.createdAt} > ${dayAgo} and not (${ownerEntry.accountId} = ${accountRow} and ${ownerEntry.referenceId} like ${reservedLike}) and (${ownerEntry.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} or (${ownerEntry.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and not exists (select 1 from ${ownerSettled.from} where ${ownerSettled.accountId} = ${ownerEntry.accountId} and ${ownerSettled.referenceId} = ${ownerEntry.referenceId} and ${ownerSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)}))))) + coalesce((select ${jsonCents(ownReservation.metadata)} from ${ownReservation.from} where ${ownReservation.accountId} = ${accountRow} and ${ownReservation.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${ownReservation.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${ownReservation.referenceId} like ${reservedLike} limit 1), ${accountColumn(developerCreditAccounts.autoTopUpAmountCents)}) * ${MICRO_CREDITS_PER_CENT} > ${AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS * MICRO_CREDITS_PER_CENT}`, limit: sql`(select count(*) from ${recentTopUp.from} where ${recentTopUp.accountId} = ${accountRow} and ${recentTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${recentTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${recentTopUp.createdAt} > ${dayAgo}) >= ${AUTO_TOP_UP_DAILY_LIMIT}`, }; } +export function autoTopUpNeedsReconcile(now: Date) { + const stale = ledgerAlias("stale_payment"); + const staleSettled = ledgerAlias("stale_settled"); + const accountRow = sql`${sql.identifier(getTableName(developerCreditAccounts))}.${sql.identifier(developerCreditAccounts.id.name)}`; + const reconcileFrom = sqlTime( + new Date(now.getTime() - AUTO_TOP_UP_PENDING_RECONCILE_MS), + ); + return sql`exists (select 1 from ${stale.from} where ${stale.accountId} = ${accountRow} and ${stale.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${stale.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${stale.createdAt} <= ${reconcileFrom} and not exists (select 1 from ${staleSettled.from} where ${staleSettled.accountId} = ${stale.accountId} and ${staleSettled.referenceId} = ${stale.referenceId} and ${staleSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`; +} + export function autoTopUpReleased(now: Date) { const waits = autoTopUpWaits(now); return sql`not (${waits.pending} or ${waits.decline} or ${waits.limit} or ${waits.owner})`; @@ -367,8 +387,18 @@ async function attemptHistory(executor: InfraDbExecutor, accountId: string) { ), ), ); - const pending = await executor - .select({ pendingId: developerCreditTransactions.referenceId }) + return { + topUps: Number(settled?.topUps ?? 0), + declines: Number(settled?.declines ?? 0), + }; +} + +async function unresolvedPayments(accountId: string) { + const pending = await db() + .select({ + pendingId: developerCreditTransactions.referenceId, + pendingSince: developerCreditTransactions.createdAt, + }) .from(developerCreditTransactions) .where( and( @@ -384,7 +414,7 @@ async function attemptHistory(executor: InfraDbExecutor, accountId: string) { row.pendingId ? [row.pendingId] : [], ); const resolved = pendingIds.length - ? await executor + ? await db() .select({ settledId: developerCreditTransactions.referenceId }) .from(developerCreditTransactions) .where( @@ -399,15 +429,51 @@ async function attemptHistory(executor: InfraDbExecutor, accountId: string) { ) : []; const resolvedIds = new Set(resolved.map((row) => row.settledId)); - return { - attempt: { - topUps: Number(settled?.topUps ?? 0), - declines: Number(settled?.declines ?? 0), + return pending.flatMap((row) => + row.pendingId && !resolvedIds.has(row.pendingId) + ? [{ referenceId: row.pendingId, since: new Date(row.pendingSince) }] + : [], + ); +} + +async function reconcileReservation(accountId: string, key: string) { + const attempt = key.slice(`${RESERVATION_PREFIX}${accountId}:`.length); + const found = await stripe().paymentIntents.search( + { + query: `metadata['accountId']:'${accountId}' AND metadata['attempt']:'${attempt}'`, + limit: 1, }, - unresolved: pendingIds.filter( - (id) => !resolvedIds.has(id) && !id.startsWith(RESERVATION_PREFIX), - ), - }; + { timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS }, + ); + const [paymentIntent] = found.data; + if (!paymentIntent) { + await dropReservation(db(), accountId, key); + return; + } + if (paymentIntent.status === "processing") { + await linkReservation(accountId, key, paymentIntent.id); + return; + } + await settleAutoTopUpPaymentIntent( + paymentIntent, + paymentIntent.status === "succeeded" ? "succeeded" : "failed", + ); +} + +async function reconcileStalePayments(accountId: string, now: Date) { + const stale = (await unresolvedPayments(accountId)).filter( + (payment) => + now.getTime() - payment.since.getTime() >= + AUTO_TOP_UP_PENDING_RECONCILE_MS, + ); + for (const payment of stale) { + if (payment.referenceId.startsWith(RESERVATION_PREFIX)) { + await reconcileReservation(accountId, payment.referenceId); + } else { + await reconcilePendingAutoTopUp(accountId, payment.referenceId); + } + } + return stale.length > 0; } export async function getAutoTopUpFailure(accountId: string) { @@ -537,6 +603,26 @@ async function reconcilePendingAutoTopUp( ); } +const RESERVATION_REPLAY_MS = DAY_MS; +const STRIPE_DESCRIPTION_LIMIT = 1_000; + +export function autoTopUpDescription(appName: string | null | undefined) { + const suffix = ", auto top-up"; + const prefix = "Cap Infra credits for "; + const name = (appName ?? "").replace(/\s+/g, " ").trim(); + if (!name) return `Cap Infra credits${suffix}`; + const room = STRIPE_DESCRIPTION_LIMIT - prefix.length - suffix.length; + return `${prefix}${name.slice(0, room).trim()}${suffix}`; +} + +type ReservedRequest = { + amountCents: number; + customerId: string; + paymentMethodId: string; + receiptEmail: string; + description: string; +}; + export async function runAutoTopUp( accountId: string, now: Date = new Date(), @@ -544,13 +630,7 @@ export async function runAutoTopUp( const [account] = await db() .select({ appId: developerCreditAccounts.appId, - balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, - stripeCustomerId: developerCreditAccounts.stripeCustomerId, - stripePaymentMethodId: developerCreditAccounts.stripePaymentMethodId, - autoTopUpEnabled: developerCreditAccounts.autoTopUpEnabled, - autoTopUpThresholdMicroCredits: - developerCreditAccounts.autoTopUpThresholdMicroCredits, - autoTopUpAmountCents: developerCreditAccounts.autoTopUpAmountCents, + appName: developerApps.name, ownerId: developerCreditAccounts.ownerId, email: users.email, }) @@ -569,27 +649,12 @@ export async function runAutoTopUp( .limit(1); if (!account) return { state: "skipped", reason: "not_found" }; - if (!account.autoTopUpEnabled) - return { state: "skipped", reason: "disabled" }; + if (await reconcileStalePayments(accountId, now)) { + return { state: "skipped", reason: "reconciled" }; + } if (!canUseInfra(account.email)) { return { state: "skipped", reason: "not_allowed" }; } - const customerId = account.stripeCustomerId; - const paymentMethodId = account.stripePaymentMethodId; - if (!customerId || !paymentMethodId) { - return { state: "skipped", reason: "no_card" }; - } - const amountCents = account.autoTopUpAmountCents; - if ( - autoTopUpSettingsError({ - enabled: true, - thresholdMicroCredits: account.autoTopUpThresholdMicroCredits, - amountCents, - hasCard: true, - }) - ) { - return { state: "skipped", reason: "invalid_settings" }; - } const period = utcDateKey(now); const outcome = await db().transaction(async (tx): Promise => { await tx @@ -599,9 +664,16 @@ export async function runAutoTopUp( .orderBy(asc(developerCreditAccounts.id)) .for("update"); const waits = autoTopUpWaits(now); - const [locked] = await tx + const [current] = await tx .select({ id: developerCreditAccounts.id, + enabled: developerCreditAccounts.autoTopUpEnabled, + thresholdMicroCredits: + developerCreditAccounts.autoTopUpThresholdMicroCredits, + amountCents: developerCreditAccounts.autoTopUpAmountCents, + customerId: developerCreditAccounts.stripeCustomerId, + paymentMethodId: developerCreditAccounts.stripePaymentMethodId, + balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, pendingWait: waits.pending, declineWait: waits.decline, limitWait: waits.limit, @@ -610,56 +682,79 @@ export async function runAutoTopUp( .from(developerCreditAccounts) .where(eq(developerCreditAccounts.id, accountId)) .limit(1); - if (!locked) return { kind: "skipped", reason: "not_found" }; + if (!current) return { kind: "skipped", reason: "not_found" }; + if (!current.enabled) return { kind: "skipped", reason: "disabled" }; + if (!current.customerId || !current.paymentMethodId) { + return { kind: "skipped", reason: "no_card" }; + } + if ( + autoTopUpSettingsError({ + enabled: true, + thresholdMicroCredits: current.thresholdMicroCredits, + amountCents: current.amountCents, + hasCard: true, + }) + ) { + return { kind: "skipped", reason: "invalid_settings" }; + } const available = (await availableBalance(account.appId, tx)) ?? 0; - if (available >= account.autoTopUpThresholdMicroCredits) { + if (available >= current.thresholdMicroCredits) { return { kind: "skipped", reason: "above_threshold" }; } if (await disputedSince(tx, accountId)) { return { kind: "skipped", reason: "disputed" }; } - if (Number(locked.pendingWait)) { + if (Number(current.pendingWait)) { return { kind: "skipped", reason: "awaiting_payment" }; } - if (Number(locked.declineWait)) { + if (Number(current.declineWait)) { return { kind: "skipped", reason: "recent_decline" }; } - if (Number(locked.limitWait)) { + if (Number(current.limitWait)) { return { kind: "skipped", reason: "daily_limit" }; } - if (Number(locked.ownerWait)) { + if (Number(current.ownerWait)) { return { kind: "skipped", reason: "owner_daily_limit" }; } - const history = await attemptHistory(tx, accountId); - if (history.unresolved.length > 0) { - return { kind: "reconcile", paymentIntentIds: history.unresolved }; - } - const key = autoTopUpIdempotencyKey(accountId, history.attempt); + const attempt = await attemptHistory(tx, accountId); + const key = autoTopUpIdempotencyKey(accountId, attempt); const [reserved] = await tx - .select({ id: developerCreditTransactions.id }) + .select({ + reservedAt: developerCreditTransactions.createdAt, + reservation: developerCreditTransactions.metadata, + }) .from(developerCreditTransactions) .where(reservationOf(accountId, key)) .limit(1); - if (!reserved) { - const [balance] = await tx - .select({ - balanceMicroCredits: developerCreditAccounts.balanceMicroCredits, - }) - .from(developerCreditAccounts) - .where(eq(developerCreditAccounts.id, accountId)) - .limit(1); - await tx.insert(developerCreditTransactions).values({ - id: nanoId(), - accountId, - type: "topup_pending", - amountMicroCredits: 0, - balanceAfterMicroCredits: balance?.balanceMicroCredits ?? 0, - referenceId: key, - referenceType: AUTO_TOP_UP_PENDING_REFERENCE_TYPE, - metadata: { amountCents, period }, - }); + if (reserved) { + if ( + now.getTime() - new Date(reserved.reservedAt).getTime() >= + RESERVATION_REPLAY_MS + ) { + return { kind: "skipped", reason: "awaiting_payment" }; + } + const request = reservedRequestOf(reserved.reservation); + if (!request) return { kind: "skipped", reason: "in_progress" }; + return { kind: "reserved", key, attempt, request }; } - return { kind: "reserved", key, attempt: history.attempt }; + const request: ReservedRequest = { + amountCents: current.amountCents, + customerId: current.customerId, + paymentMethodId: current.paymentMethodId, + receiptEmail: account.email, + description: autoTopUpDescription(account.appName), + }; + await tx.insert(developerCreditTransactions).values({ + id: nanoId(), + accountId, + type: "topup_pending", + amountMicroCredits: 0, + balanceAfterMicroCredits: current.balanceMicroCredits, + referenceId: key, + referenceType: AUTO_TOP_UP_PENDING_REFERENCE_TYPE, + metadata: { ...request, period }, + }); + return { kind: "reserved", key, attempt, request }; }); if (outcome.kind === "skipped") { @@ -674,25 +769,20 @@ export async function runAutoTopUp( } return { state: "skipped", reason: outcome.reason }; } - if (outcome.kind === "reconcile") { - for (const paymentIntentId of outcome.paymentIntentIds) { - await reconcilePendingAutoTopUp(accountId, paymentIntentId); - } - return { state: "skipped", reason: "reconciled" }; - } - const { key, attempt } = outcome; + const { key, attempt, request: reserved } = outcome; + const amountCents = reserved.amountCents; const productId = developerCreditsProductId(); const request = { amount: amountCents, currency: "usd", - customer: customerId, - payment_method: paymentMethodId, + customer: reserved.customerId, + payment_method: reserved.paymentMethodId, payment_method_types: ["card"], off_session: true, confirm: true, - description: "Cap Infra credits, auto top-up", - receipt_email: account.email, + description: reserved.description, + receipt_email: reserved.receiptEmail, metadata: { type: AUTO_TOP_UP_METADATA_TYPE, accountId, @@ -767,6 +857,42 @@ export async function runAutoTopUp( return { state: "failed", reason: paymentIntent.status }; } +function reservedRequestOf(value: unknown): ReservedRequest | null { + let metadata: unknown = value; + if (typeof metadata === "string") { + try { + metadata = JSON.parse(metadata); + } catch { + return null; + } + } + if (typeof metadata !== "object" || metadata === null) return null; + const fields = metadata as Record; + const { + amountCents, + customerId, + paymentMethodId, + receiptEmail, + description, + } = fields; + if ( + typeof amountCents !== "number" || + typeof customerId !== "string" || + typeof paymentMethodId !== "string" || + typeof receiptEmail !== "string" || + typeof description !== "string" + ) { + return null; + } + return { + amountCents, + customerId, + paymentMethodId, + receiptEmail, + description, + }; +} + type DueCursor = { attemptKey: string; id: string }; function dueAutoTopUpsAfter(now: Date, cursor: DueCursor | null) { @@ -785,11 +911,16 @@ function dueAutoTopUpsAfter(now: Date, cursor: DueCursor | null) { .leftJoin(held, eq(held.appId, developerCreditAccounts.appId)) .where( and( - eq(developerCreditAccounts.autoTopUpEnabled, true), - isNotNull(developerCreditAccounts.stripePaymentMethodId), isNull(developerApps.deletedAt), - sql`${developerCreditAccounts.balanceMicroCredits} < ${developerCreditAccounts.autoTopUpThresholdMicroCredits} + coalesce(${held.heldMicroCredits}, 0)`, - autoTopUpReleased(now), + or( + and( + eq(developerCreditAccounts.autoTopUpEnabled, true), + isNotNull(developerCreditAccounts.stripePaymentMethodId), + sql`${developerCreditAccounts.balanceMicroCredits} < ${developerCreditAccounts.autoTopUpThresholdMicroCredits} + coalesce(${held.heldMicroCredits}, 0)`, + autoTopUpReleased(now), + ), + autoTopUpNeedsReconcile(now), + ), cursor ? sql`(${attemptKey} > ${cursor.attemptKey} or (${attemptKey} = ${cursor.attemptKey} and ${developerCreditAccounts.id} > ${cursor.id}))` : undefined, From 6c9ae945e0b40dfbf0fbce32606c61e8d30976c8 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:26:48 -0500 Subject: [PATCH 17/26] fix: keep pending top-up rows out of the customer ledger Reservations and payments still processing are zero-amount rows that only the billing code needs. The console, the CSV export and the transactions API no longer list them, and the API reference no longer lists the pending row type. --- apps/web/content/docs/api/rest-api.mdx | 1 - apps/web/lib/developer-openapi.ts | 2 +- apps/web/lib/infra-ledger-store.ts | 8 ++++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/web/content/docs/api/rest-api.mdx b/apps/web/content/docs/api/rest-api.mdx index f85bd5603e4..698a066abc7 100644 --- a/apps/web/content/docs/api/rest-api.mdx +++ b/apps/web/content/docs/api/rest-api.mdx @@ -1031,7 +1031,6 @@ curl "https://cap.so/api/developer/v1/transactions?limit=2" \ | `topup` | Code redeemed | `stripe_checkout_session` | The checkout a code covered in full | | `topup` | Auto top-up | `auto_top_up` | The payment | | `topup_failed` | Auto top-up declined | `auto_top_up_failed` | The payment, or `declined-` and the UTC day when there was none | -| `topup_pending` | Auto top-up pending | `auto_top_up_pending` | The payment | | `pro_credit` | Cap Pro credit | `pro_monthly_credit` | The month, as `YYYY-MM` | | `storage_daily` | Storage | `storage_day` | The UTC day, as `YYYY-MM-DD` | | `transcription` | Transcription | `developer_transcript` | The transcript ID | diff --git a/apps/web/lib/developer-openapi.ts b/apps/web/lib/developer-openapi.ts index a7dd4f86516..caada242d1c 100644 --- a/apps/web/lib/developer-openapi.ts +++ b/apps/web/lib/developer-openapi.ts @@ -1125,7 +1125,7 @@ export const developerOpenApiSpec = (baseUrl: string) => ({ type: { type: "string", description: - "The kind of row: topup, topup_failed, topup_pending, pro_credit, storage_daily, transcription or adjustment. Treat a value you do not know as a row to show as it is.", + "The kind of row: topup, topup_failed, pro_credit, storage_daily, transcription or adjustment. Treat a value you do not know as a row to show as it is.", }, description: { type: "string", diff --git a/apps/web/lib/infra-ledger-store.ts b/apps/web/lib/infra-ledger-store.ts index 7a972f93e9b..b99578571c8 100644 --- a/apps/web/lib/infra-ledger-store.ts +++ b/apps/web/lib/infra-ledger-store.ts @@ -3,7 +3,7 @@ import { developerCreditAccounts, developerCreditTransactions, } from "@cap/database/schema"; -import { and, asc, desc, eq, gt, lt, or } from "drizzle-orm"; +import { and, asc, desc, eq, gt, lt, or, sql } from "drizzle-orm"; import { type LedgerRow, orderLedgerRows } from "@/lib/infra-ledger"; const EXPORT_BATCH_SIZE = 1_000; @@ -20,7 +20,11 @@ const columns = { createdAt: developerCreditTransactions.createdAt, }; -const ofApp = (appId: string) => eq(developerCreditAccounts.appId, appId); +const ofApp = (appId: string) => + and( + eq(developerCreditAccounts.appId, appId), + sql`${developerCreditTransactions.type} <> 'topup_pending'`, + ); const appLedger = () => db() From 2593a95d281e9bd29de4927ab2e996a2bb1790e4 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:26:48 -0500 Subject: [PATCH 18/26] test: one amount for the ceiling and the charge, lost answers count, recovery is never gated --- .../infra-auto-top-up-database.test.ts | 182 ++++++++++++++++-- 1 file changed, 168 insertions(+), 14 deletions(-) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index d4d8aeb9643..5041935b608 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -21,6 +21,8 @@ const fixture = vi.hoisted(() => ({ >(), created: [] as FakeIntent[], createdKeys: [] as string[], + sent: [] as Record[], + duringCreate: null as null | (() => Promise), loseNextResponse: false, delayMs: 40, calledAt: [] as number[], @@ -75,6 +77,10 @@ vi.mock("@cap/utils", () => ({ } if (entry?.state === "done") return entry.intent; fixture.keys.set(key, { state: "pending", params: sent }); + fixture.sent.push(params as unknown as Record); + const during = fixture.duringCreate; + fixture.duringCreate = null; + await during?.(); fixture.calledAt.push(Date.now()); await new Promise((resolve) => setTimeout(resolve, fixture.delayMs)); const intent: FakeIntent = { @@ -232,9 +238,7 @@ describe.runIf(Boolean(databaseUrl))( }); it("makes no second payment when the amount changes after a lost response", async () => { - const { runAutoTopUp, settleAutoTopUpPaymentIntent } = await import( - "@/lib/infra-auto-top-up" - ); + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const errors = vi.spyOn(console, "error").mockImplementation(() => {}); const { accountId } = await makeAccount(30_000); const before = fixture.created.length; @@ -247,18 +251,12 @@ describe.runIf(Boolean(databaseUrl))( "UPDATE developer_credit_accounts SET autoTopUpAmountCents = 2500 WHERE id = ?", [accountId], ); - expect(await runAutoTopUp(accountId)).toEqual({ - state: "skipped", - reason: "in_progress", + expect(await runAutoTopUp(accountId)).toMatchObject({ + state: "succeeded", }); expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); - const [lost] = fixture.created.slice(before); - if (!lost) throw new Error("No payment was created"); - await settleAutoTopUpPaymentIntent( - lost as unknown as Parameters[0], - "succeeded", - ); await setBalance(accountId, 10); expect(await runAutoTopUp(accountId)).toMatchObject({ state: "succeeded", @@ -511,11 +509,27 @@ describe.runIf(Boolean(databaseUrl))( topUps: 0, declines: 0, }); + const [[owner]] = (await pool.query( + "SELECT u.email FROM users u JOIN developer_credit_accounts c ON c.ownerId = u.id WHERE c.id = ?", + [accountId], + )) as unknown as [[{ email: string }]]; await pool.query( `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, balanceAfterMicroCredits, referenceId, referenceType, metadata) - VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":500}')`, - [id(), accountId, key], + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', ?)`, + [ + id(), + accountId, + key, + JSON.stringify({ + amountCents: 500, + customerId: "cus_test", + paymentMethodId: "pm_test", + receiptEmail: owner.email, + description: "Cap Infra credits for Top-up test, auto top-up", + period: "2026-11-13", + }), + ], ); const before = fixture.createdKeys.length; @@ -548,6 +562,146 @@ describe.runIf(Boolean(databaseUrl))( errors.mockRestore(); }); + it("checks and charges one amount, read under the lock, when the setting changes mid-attempt", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const app = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const other = await makeAccount(30_000, { ownerId }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 50000000, 50030000, ?, 'auto_top_up', '{}')`, + [id(), other.accountId, `pi_today_${other.accountId}`], + ); + const setAmount = (cents: number) => + pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = ? WHERE id = ?", + [cents, app.accountId], + ); + + expect(await runAutoTopUp(app.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + await setAmount(500); + fixture.duringCreate = () => setAmount(60_000).then(() => undefined); + expect((await runAutoTopUp(app.accountId)).state).toBe("succeeded"); + + expect(fixture.sent.at(-1)?.amount).toBe(500); + const [[total]] = (await pool.query( + `SELECT SUM(t.amountMicroCredits) AS micro FROM developer_credit_transactions t + JOIN developer_credit_accounts c ON c.id = t.accountId + WHERE c.ownerId = ? AND t.type = 'topup' AND t.referenceType = 'auto_top_up'`, + [ownerId], + )) as unknown as [[{ micro: number }]]; + expect(Number(total.micro)).toBe(50_500_000); + warn.mockRestore(); + }); + + it("keeps counting a charge whose answer was lost against the owner's ceiling", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(first.accountId)).toEqual({ state: "error" }); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(second.accountId)).toEqual([]); + errors.mockRestore(); + warn.mockRestore(); + }); + + it("lets the sweep reconcile a 72 hour old payment even when every limit applies", async () => { + const { runDueAutoTopUps } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const stale = await makeAccount(30_000, { + ownerId, + amountCents: 100_000, + }); + const other = await makeAccount(30_000, { ownerId }); + const stalePayment = `pi_stale1000_${stale.accountId}`; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":100000}', UTC_TIMESTAMP() - INTERVAL 73 HOUR)`, + [id(), stale.accountId, stalePayment], + ); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 500000, 530000, ?, 'auto_top_up', '{}')`, + [id(), other.accountId, `pi_today_${other.accountId}`], + ); + fixture.stored.set(stalePayment, { + id: stalePayment, + status: "succeeded", + amount: 100_000, + amount_received: 100_000, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: stale.accountId }, + }); + await pool.query( + "UPDATE developer_credit_accounts SET balanceMicroCredits = 99000000, autoTopUpEnabled = 0 WHERE id = ?", + [stale.accountId], + ); + + await runDueAutoTopUps(new Date()); + + expect(await topUpsOf(stale.accountId)).toEqual([stalePayment]); + expect(await balanceOf(stale.accountId)).toBe(99_000_000 + 100_000_000); + warn.mockRestore(); + }); + + it("never shows a pending row in the customer's ledger", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { listAppLedgerPage, readAppLedger } = await import( + "@/lib/infra-ledger-store" + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId, appId } = await makeAccount(30_000); + fixture.loseNextResponse = true; + await runAutoTopUp(accountId); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_pending', 0, 30000, 'pi_processing', 'auto_top_up_pending', '{}'), + (?, ?, 'storage_daily', -100, 29900, '2026-11-12', 'storage_day', '{}')`, + [id(), accountId, id(), accountId], + ); + + const page = await listAppLedgerPage(appId, { limit: 50 }); + const exported = []; + for await (const rows of readAppLedger(appId)) exported.push(...rows); + + expect(page.rows.map((row) => row.type)).toEqual(["storage_daily"]); + expect(exported.map((row) => row.type)).toEqual(["storage_daily"]); + errors.mockRestore(); + }); + + it("names the app in the charge description", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect(fixture.sent.at(-1)?.description).toBe( + "Cap Infra credits for Top-up test, auto top-up", + ); + }); + it("tops up again the same day with a new key once the balance is low again", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const { accountId } = await makeAccount(30_000); From 9c259719eb0b1d9d3a08e166627fcfacd5380368 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:32:18 -0500 Subject: [PATCH 19/26] test: a $600 retry is not held back by its own reservation, in the guard or the sweep --- .../infra-auto-top-up-database.test.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index 5041935b608..68975410233 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -24,6 +24,7 @@ const fixture = vi.hoisted(() => ({ sent: [] as Record[], duringCreate: null as null | (() => Promise), loseNextResponse: false, + failNextCall: false, delayMs: 40, calledAt: [] as number[], stored: new Map(), @@ -60,6 +61,12 @@ vi.mock("@cap/utils", () => ({ options: { idempotencyKey: string }, ) => { const key = options.idempotencyKey; + if (fixture.failNextCall) { + fixture.failNextCall = false; + throw Object.assign(new Error("socket hang up"), { + type: "StripeConnectionError", + }); + } const entry = fixture.keys.get(key); const sent = JSON.stringify(params); if (entry && entry.params !== sent) { @@ -600,6 +607,60 @@ describe.runIf(Boolean(databaseUrl))( warn.mockRestore(); }); + it("retries a $600 attempt whose Stripe call failed, once, while it still holds back the owner's other app", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + fixture.failNextCall = true; + expect(await runAutoTopUp(first.accountId)).toEqual({ state: "error" }); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + expect(await runAutoTopUp(first.accountId)).toMatchObject({ + state: "succeeded", + }); + expect(fixture.created.length - before).toBe(1); + expect(fixture.created.at(-1)?.amount).toBe(60_000); + expect(await topUpsOf(first.accountId)).toHaveLength(1); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + errors.mockRestore(); + warn.mockRestore(); + }); + + it("lets the sweep retry a $600 reservation whose only blocker is itself", async () => { + const { runAutoTopUp, runDueAutoTopUps } = await import( + "@/lib/infra-auto-top-up" + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000, { amountCents: 60_000 }); + const before = fixture.created.length; + + fixture.failNextCall = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + await runDueAutoTopUps(new Date()); + + expect(await topUpsOf(accountId)).toHaveLength(1); + expect( + fixture.created + .slice(before) + .filter((intent) => intent.metadata.accountId === accountId), + ).toHaveLength(1); + errors.mockRestore(); + }); + it("keeps counting a charge whose answer was lost against the owner's ceiling", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const errors = vi.spyOn(console, "error").mockImplementation(() => {}); From 01c87a98f09d4df729b0fbce58d4ed86a093b6c6 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:34:48 -0500 Subject: [PATCH 20/26] fix: recovery voids or relinks only the stale reservation row it read Voiding by key could delete a newer reservation made under the same key after another recovery run voided the stale one. Recovery now targets the exact row id it found. --- .../__tests__/unit/infra-billing-fake-db.ts | 13 ++++---- apps/web/lib/infra-auto-top-up.ts | 30 ++++++++++++++----- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index 25ef9122328..96cfc787c0b 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -313,7 +313,7 @@ function runSelect(query: Query): Record[] { ), ).slice(0, 1); } - if (keys === "pendingId,pendingSince") { + if (keys === "pendingId,pendingRow,pendingSince") { return ledger.rows .filter( (row) => @@ -321,6 +321,7 @@ function runSelect(query: Query): Record[] { row.referenceType === "auto_top_up_pending", ) .map((row) => ({ + pendingRow: row.id, pendingId: row.referenceId, pendingSince: row.createdAt, })); @@ -450,13 +451,14 @@ function update(table: unknown) { return { async where(condition?: SQL) { if (table === developerCreditTransactions && condition) { - const [accountId, referenceId, referenceType] = + const [accountId, referenceId, referenceType, rowId] = dialect.sqlToQuery(condition).params; const rows = ledger.rows.filter( (row) => row.accountId === accountId && row.referenceId === referenceId && - row.referenceType === referenceType, + row.referenceType === referenceType && + (rowId === undefined || row.id === rowId), ); for (const row of rows) Object.assign(row, values); return [{ affectedRows: rows.length }]; @@ -515,7 +517,7 @@ function remove(table: unknown) { return { async where(condition: SQL) { if (table !== developerCreditTransactions) return [{ affectedRows: 0 }]; - const [accountId, referenceId, referenceType] = + const [accountId, referenceId, referenceType, rowId] = dialect.sqlToQuery(condition).params; const before = ledger.rows.length; ledger.rows = ledger.rows.filter( @@ -523,7 +525,8 @@ function remove(table: unknown) { !( row.accountId === accountId && row.referenceId === referenceId && - row.referenceType === referenceType + row.referenceType === referenceType && + (rowId === undefined || row.id === rowId) ), ); return [{ affectedRows: before - ledger.rows.length }]; diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 43667284c96..ac06f895d59 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -141,7 +141,7 @@ function reservationKeyOf(paymentIntent: Stripe.PaymentIntent) { }); } -const reservationOf = (accountId: string, key: string) => +const reservationOf = (accountId: string, key: string, rowId?: string) => and( eq(developerCreditTransactions.accountId, accountId), eq(developerCreditTransactions.referenceId, key), @@ -149,28 +149,31 @@ const reservationOf = (accountId: string, key: string) => developerCreditTransactions.referenceType, AUTO_TOP_UP_PENDING_REFERENCE_TYPE, ), + rowId ? eq(developerCreditTransactions.id, rowId) : undefined, ); async function dropReservation( executor: AutoTopUpTransaction | ReturnType, accountId: string, key: string | null, + rowId?: string, ) { if (!key) return; await executor .delete(developerCreditTransactions) - .where(reservationOf(accountId, key)); + .where(reservationOf(accountId, key, rowId)); } async function linkReservation( accountId: string, key: string, paymentIntentId: string, + rowId?: string, ) { await db() .update(developerCreditTransactions) .set({ referenceId: paymentIntentId }) - .where(reservationOf(accountId, key)); + .where(reservationOf(accountId, key, rowId)); } export function autoTopUpFailureText(reason: string | null | undefined) { @@ -396,6 +399,7 @@ async function attemptHistory(executor: InfraDbExecutor, accountId: string) { async function unresolvedPayments(accountId: string) { const pending = await db() .select({ + pendingRow: developerCreditTransactions.id, pendingId: developerCreditTransactions.referenceId, pendingSince: developerCreditTransactions.createdAt, }) @@ -431,12 +435,22 @@ async function unresolvedPayments(accountId: string) { const resolvedIds = new Set(resolved.map((row) => row.settledId)); return pending.flatMap((row) => row.pendingId && !resolvedIds.has(row.pendingId) - ? [{ referenceId: row.pendingId, since: new Date(row.pendingSince) }] + ? [ + { + rowId: row.pendingRow, + referenceId: row.pendingId, + since: new Date(row.pendingSince), + }, + ] : [], ); } -async function reconcileReservation(accountId: string, key: string) { +async function reconcileReservation( + accountId: string, + key: string, + rowId: string, +) { const attempt = key.slice(`${RESERVATION_PREFIX}${accountId}:`.length); const found = await stripe().paymentIntents.search( { @@ -447,11 +461,11 @@ async function reconcileReservation(accountId: string, key: string) { ); const [paymentIntent] = found.data; if (!paymentIntent) { - await dropReservation(db(), accountId, key); + await dropReservation(db(), accountId, key, rowId); return; } if (paymentIntent.status === "processing") { - await linkReservation(accountId, key, paymentIntent.id); + await linkReservation(accountId, key, paymentIntent.id, rowId); return; } await settleAutoTopUpPaymentIntent( @@ -468,7 +482,7 @@ async function reconcileStalePayments(accountId: string, now: Date) { ); for (const payment of stale) { if (payment.referenceId.startsWith(RESERVATION_PREFIX)) { - await reconcileReservation(accountId, payment.referenceId); + await reconcileReservation(accountId, payment.referenceId, payment.rowId); } else { await reconcilePendingAutoTopUp(accountId, payment.referenceId); } From fcb95c6e224a27fce34ca2ae3301efc4f8dc3eb7 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:34:48 -0500 Subject: [PATCH 21/26] test: recovery leaves a new reservation under the same key alone --- .../infra-auto-top-up-database.test.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index 68975410233..ccacd134184 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -25,6 +25,7 @@ const fixture = vi.hoisted(() => ({ duringCreate: null as null | (() => Promise), loseNextResponse: false, failNextCall: false, + duringSearch: null as null | (() => Promise), delayMs: 40, calledAt: [] as number[], stored: new Map(), @@ -50,6 +51,12 @@ vi.mock("@/lib/infra-billing-emails", () => ({ vi.mock("@cap/utils", () => ({ stripe: () => ({ paymentIntents: { + search: async () => { + const during = fixture.duringSearch; + fixture.duringSearch = null; + await during?.(); + return { data: [] }; + }, retrieve: async (id: string) => { fixture.retrieved.push(id); const intent = fixture.stored.get(id); @@ -661,6 +668,39 @@ describe.runIf(Boolean(databaseUrl))( errors.mockRestore(); }); + it("voids only the stale reservation, never a new one made under the same key", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const { accountId } = await makeAccount(30_000); + const key = autoTopUpIdempotencyKey(accountId, { + topUps: 0, + declines: 0, + }); + const reserve = (rowId: string, hoursAgo: number) => + pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":500}', UTC_TIMESTAMP() - INTERVAL ? HOUR)`, + [rowId, accountId, key, hoursAgo], + ); + const stale = id(); + const fresh = id(); + await reserve(stale, 73); + fixture.duringSearch = () => reserve(fresh, 0).then(() => undefined); + + expect(await runAutoTopUp(accountId)).toEqual({ + state: "skipped", + reason: "reconciled", + }); + + const [rows] = await pool.query( + "SELECT id FROM developer_credit_transactions WHERE accountId = ? AND referenceId = ?", + [accountId, key], + ); + expect((rows as { id: string }[]).map((row) => row.id)).toEqual([fresh]); + }); + it("keeps counting a charge whose answer was lost against the owner's ceiling", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const errors = vi.spyOn(console, "error").mockImplementation(() => {}); From 40dab1fd9e6e8136a4891b84d9ceb4c68a1c744d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:05:38 -0500 Subject: [PATCH 22/26] fix: count unresolved top-ups of any age against the owner's ceiling Settled automatic top-ups count for 24 hours. A reservation or a payment still processing counts until it settles or is reconciled, because it may already have been charged. --- apps/web/__tests__/unit/infra-billing-fake-db.ts | 5 +++-- apps/web/lib/infra-auto-top-up.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index 2a515b93246..c09a10ebe7f 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -158,9 +158,10 @@ function autoTopUpWaits() { ledger.rows .filter( (row) => - row.createdAt.getTime() > now - 24 * HOUR_MS && !String(row.referenceId).startsWith("infra-auto-top-up:") && - ((row.type === "topup" && row.referenceType === "auto_top_up") || + ((row.type === "topup" && + row.referenceType === "auto_top_up" && + row.createdAt.getTime() > now - 24 * HOUR_MS) || (row.type === "topup_pending" && row.referenceType === "auto_top_up_pending" && !settledIds.has(row.referenceId))), diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index ac06f895d59..24fb580b282 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -314,7 +314,7 @@ export function autoTopUpWaits(now: Date) { return { pending: sql`exists (select 1 from ${pendingAttempt.from} where ${pendingAttempt.accountId} = ${accountRow} and ${pendingAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${pendingAttempt.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${pendingAttempt.createdAt} > ${reconcileFrom} and ${pendingAttempt.referenceId} not like ${`${RESERVATION_PREFIX}%`} and not exists (select 1 from ${settledAttempt.from} where ${settledAttempt.accountId} = ${pendingAttempt.accountId} and ${settledAttempt.referenceId} = ${pendingAttempt.referenceId} and ${settledAttempt.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)})))`, decline: sql`exists (select 1 from ${declinedAttempt.from} where ${declinedAttempt.accountId} = ${accountRow} and ${declinedAttempt.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_FAILED_REFERENCE_TYPE]} and ${declinedAttempt.referenceType} = ${AUTO_TOP_UP_FAILED_REFERENCE_TYPE} and ${declinedAttempt.createdAt} > ${dayAgo} and not exists (select 1 from ${laterTopUp.from} where ${laterTopUp.accountId} = ${declinedAttempt.accountId} and ${laterTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${laterTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${laterTopUp.createdAt} > ${declinedAttempt.createdAt}))`, - owner: sql`(select coalesce(sum(case when ${ownerEntry.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} then ${ownerEntry.amountMicroCredits} else ${jsonCents(ownerEntry.metadata)} * ${MICRO_CREDITS_PER_CENT} end), 0) from ${accounts} ${sql.identifier("owner_account")} join ${ownerEntry.from} on ${ownerEntry.accountId} = ${ownerAccount(developerCreditAccounts.id)} where ${ownerAccount(developerCreditAccounts.ownerId)} = ${accountColumn(developerCreditAccounts.ownerId)} and ${ownerEntry.type} in (${inList([ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE], ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]])}) and ${ownerEntry.createdAt} > ${dayAgo} and not (${ownerEntry.accountId} = ${accountRow} and ${ownerEntry.referenceId} like ${reservedLike}) and (${ownerEntry.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} or (${ownerEntry.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and not exists (select 1 from ${ownerSettled.from} where ${ownerSettled.accountId} = ${ownerEntry.accountId} and ${ownerSettled.referenceId} = ${ownerEntry.referenceId} and ${ownerSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)}))))) + coalesce((select ${jsonCents(ownReservation.metadata)} from ${ownReservation.from} where ${ownReservation.accountId} = ${accountRow} and ${ownReservation.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${ownReservation.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${ownReservation.referenceId} like ${reservedLike} limit 1), ${accountColumn(developerCreditAccounts.autoTopUpAmountCents)}) * ${MICRO_CREDITS_PER_CENT} > ${AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS * MICRO_CREDITS_PER_CENT}`, + owner: sql`(select coalesce(sum(case when ${ownerEntry.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} then ${ownerEntry.amountMicroCredits} else ${jsonCents(ownerEntry.metadata)} * ${MICRO_CREDITS_PER_CENT} end), 0) from ${accounts} ${sql.identifier("owner_account")} join ${ownerEntry.from} on ${ownerEntry.accountId} = ${ownerAccount(developerCreditAccounts.id)} where ${ownerAccount(developerCreditAccounts.ownerId)} = ${accountColumn(developerCreditAccounts.ownerId)} and ${ownerEntry.type} in (${inList([ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE], ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]])}) and not (${ownerEntry.accountId} = ${accountRow} and ${ownerEntry.referenceId} like ${reservedLike}) and ((${ownerEntry.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${ownerEntry.createdAt} > ${dayAgo}) or (${ownerEntry.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and not exists (select 1 from ${ownerSettled.from} where ${ownerSettled.accountId} = ${ownerEntry.accountId} and ${ownerSettled.referenceId} = ${ownerEntry.referenceId} and ${ownerSettled.referenceType} in (${inList(SETTLED_REFERENCE_TYPES)}))))) + coalesce((select ${jsonCents(ownReservation.metadata)} from ${ownReservation.from} where ${ownReservation.accountId} = ${accountRow} and ${ownReservation.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_PENDING_REFERENCE_TYPE]} and ${ownReservation.referenceType} = ${AUTO_TOP_UP_PENDING_REFERENCE_TYPE} and ${ownReservation.referenceId} like ${reservedLike} limit 1), ${accountColumn(developerCreditAccounts.autoTopUpAmountCents)}) * ${MICRO_CREDITS_PER_CENT} > ${AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS * MICRO_CREDITS_PER_CENT}`, limit: sql`(select count(*) from ${recentTopUp.from} where ${recentTopUp.accountId} = ${accountRow} and ${recentTopUp.type} = ${ATTEMPT_LEDGER_TYPES[AUTO_TOP_UP_REFERENCE_TYPE]} and ${recentTopUp.referenceType} = ${AUTO_TOP_UP_REFERENCE_TYPE} and ${recentTopUp.createdAt} > ${dayAgo}) >= ${AUTO_TOP_UP_DAILY_LIMIT}`, }; } From 5ad39a3d71b12e676316d0406291c05ac6b7ce1b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:05:38 -0500 Subject: [PATCH 23/26] test: a 25 hour old unresolved payment holds the owner's other app back until it resolves --- .../infra-auto-top-up-database.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index ccacd134184..f87d47ee485 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -701,6 +701,41 @@ describe.runIf(Boolean(databaseUrl))( expect((rows as { id: string }[]).map((row) => row.id)).toEqual([fresh]); }); + it("counts an unresolved payment of any age against the owner's ceiling until it resolves", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const processing = `pi_processing_${first.accountId}`; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":60000}', UTC_TIMESTAMP() - INTERVAL 25 HOUR)`, + [id(), first.accountId, processing], + ); + const before = fixture.created.length; + + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(fixture.created.length).toBe(before); + + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_failed', 0, 30000, ?, 'auto_top_up_failed', '{}')`, + [id(), first.accountId, processing], + ); + expect((await runAutoTopUp(second.accountId)).state).toBe("succeeded"); + expect(fixture.created.length - before).toBe(1); + warn.mockRestore(); + }); + it("keeps counting a charge whose answer was lost against the owner's ceiling", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const errors = vi.spyOn(console, "error").mockImplementation(() => {}); From 5dc5fee2bd8ee243dae7e48d1950431f7710dabc Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:15:27 -0500 Subject: [PATCH 24/26] fix: recover a deleted app's stale payments and never charge a deleted app --- apps/web/lib/infra-auto-top-up.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/apps/web/lib/infra-auto-top-up.ts b/apps/web/lib/infra-auto-top-up.ts index 24fb580b282..7408b71da56 100644 --- a/apps/web/lib/infra-auto-top-up.ts +++ b/apps/web/lib/infra-auto-top-up.ts @@ -78,6 +78,7 @@ export type AutoTopUpResult = | "disputed" | "awaiting_payment" | "reconciled" + | "app_deleted" | "recent_decline" | "daily_limit" | "owner_daily_limit" @@ -645,6 +646,7 @@ export async function runAutoTopUp( .select({ appId: developerCreditAccounts.appId, appName: developerApps.name, + appDeletedAt: developerApps.deletedAt, ownerId: developerCreditAccounts.ownerId, email: users.email, }) @@ -654,18 +656,14 @@ export async function runAutoTopUp( eq(developerCreditAccounts.appId, developerApps.id), ) .innerJoin(users, eq(developerCreditAccounts.ownerId, users.id)) - .where( - and( - eq(developerCreditAccounts.id, accountId), - isNull(developerApps.deletedAt), - ), - ) + .where(eq(developerCreditAccounts.id, accountId)) .limit(1); if (!account) return { state: "skipped", reason: "not_found" }; if (await reconcileStalePayments(accountId, now)) { return { state: "skipped", reason: "reconciled" }; } + if (account.appDeletedAt) return { state: "skipped", reason: "app_deleted" }; if (!canUseInfra(account.email)) { return { state: "skipped", reason: "not_allowed" }; } @@ -677,6 +675,14 @@ export async function runAutoTopUp( .where(eq(developerCreditAccounts.ownerId, account.ownerId)) .orderBy(asc(developerCreditAccounts.id)) .for("update"); + const [app] = await tx + .select({ appDeletedAt: developerApps.deletedAt }) + .from(developerApps) + .where(eq(developerApps.id, account.appId)) + .limit(1); + if (!app || app.appDeletedAt) { + return { kind: "skipped", reason: "app_deleted" }; + } const waits = autoTopUpWaits(now); const [current] = await tx .select({ @@ -925,9 +931,9 @@ function dueAutoTopUpsAfter(now: Date, cursor: DueCursor | null) { .leftJoin(held, eq(held.appId, developerCreditAccounts.appId)) .where( and( - isNull(developerApps.deletedAt), or( and( + isNull(developerApps.deletedAt), eq(developerCreditAccounts.autoTopUpEnabled, true), isNotNull(developerCreditAccounts.stripePaymentMethodId), sql`${developerCreditAccounts.balanceMicroCredits} < ${developerCreditAccounts.autoTopUpThresholdMicroCredits} + coalesce(${held.heldMicroCredits}, 0)`, From 7c1a25e18697c7a93137a923dd7131cc6e86d68a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:15:27 -0500 Subject: [PATCH 25/26] test: a deleted app is reconciled after 72 hours and never charged, even when deleted just before the lock --- .../__tests__/unit/infra-auto-top-up.test.ts | 39 +++++++++++++++++++ .../__tests__/unit/infra-billing-fake-db.ts | 11 ++++++ 2 files changed, 50 insertions(+) diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index 8eea6434fb3..73352c57394 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -281,6 +281,25 @@ describe("auto top-up charge", () => { expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); }); + it("never charges the card of a deleted app", async () => { + arm(); + ledger.account.appDeletedAt = new Date("2026-11-10T00:00:00Z"); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + + ledger.account.appDeletedAt = null; + ledger.appDeletedBeforeLock = true; + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(ledger.rows).toHaveLength(0); + }); + it("does not charge the card of an app whose payment was disputed", async () => { arm(); ledger.rows.push({ @@ -744,6 +763,26 @@ describe("auto top-up charge", () => { expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); }); + it("recovers a deleted app's lost attempt, then never charges it again", async () => { + await loseAnswer(); + later(72.1); + ledger.account.appDeletedAt = now(); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.searchPaymentIntents).toHaveBeenCalledTimes(1); + expect(ledger.rows).toHaveLength(0); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(ledger.rows).toHaveLength(0); + }); + it("after 72 hours, voids a reservation Stripe never charged", async () => { await loseAnswer(); later(72.1); diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index c09a10ebe7f..495b7a66990 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -1,4 +1,5 @@ import { + developerApps, developerCreditAccounts, developerCreditTransactions, developerDailyStorageSnapshots, @@ -52,6 +53,7 @@ const freshAccount = () => ({ autoTopUpThresholdMicroCredits: 0, autoTopUpAmountCents: 0, email: "richie@cap.so", + appDeletedAt: null as Date | null, }); export const ledger = { @@ -61,6 +63,7 @@ export const ledger = { snapshots: [] as SnapshotRow[], videos: [] as VideoGroup[], heldMicroCredits: 0, + appDeletedBeforeLock: false, }; export function resetLedger() { @@ -68,6 +71,7 @@ export function resetLedger() { ledger.account = freshAccount(); ledger.rows = []; ledger.snapshots = []; + ledger.appDeletedBeforeLock = false; ledger.videos = []; ledger.heldMicroCredits = 0; } @@ -424,6 +428,13 @@ function runSelect(query: Query): Record[] { } } + if (query.table === developerApps && keys === "appDeletedAt") { + const deletedAt = ledger.appDeletedBeforeLock + ? ledger.now + : ledger.account.appDeletedAt; + return [{ appDeletedAt: deletedAt }]; + } + if (query.table === developerTranscripts) { return [{ total: ledger.heldMicroCredits }]; } From 7043911a0233a2ace82aac8a46c548d7132987d5 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:15:27 -0500 Subject: [PATCH 26/26] test: the sweep voids a deleted app's lost $600 attempt so the owner's other app can top up --- .../infra-auto-top-up-database.test.ts | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts index f87d47ee485..2a2b75aa112 100644 --- a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -701,6 +701,73 @@ describe.runIf(Boolean(databaseUrl))( expect((rows as { id: string }[]).map((row) => row.id)).toEqual([fresh]); }); + it("lets the sweep void a deleted app's lost $600 attempt, then tops up the owner's other app and never charges the deleted one", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp, runDueAutoTopUps } = + await import("@/lib/infra-auto-top-up"); + const ownerId = await makeOwner(); + const deleted = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const other = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const key = autoTopUpIdempotencyKey(deleted.accountId, { + topUps: 0, + declines: 0, + }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":60000}', UTC_TIMESTAMP() - INTERVAL 73 HOUR)`, + [id(), deleted.accountId, key], + ); + await pool.query( + "UPDATE developer_apps SET deletedAt = UTC_TIMESTAMP() WHERE id = ?", + [deleted.appId], + ); + const before = fixture.created.length; + + expect(await runAutoTopUp(other.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + let searched = 0; + fixture.duringSearch = async () => { + searched += 1; + }; + await runDueAutoTopUps(new Date()); + + expect(searched).toBe(1); + const [rows] = await pool.query( + "SELECT id FROM developer_credit_transactions WHERE accountId = ?", + [deleted.accountId], + ); + expect(rows).toEqual([]); + + await runDueAutoTopUps(new Date()); + + const charged = fixture.created.slice(before); + expect(charged.map((intent) => intent.metadata.accountId)).toEqual([ + other.accountId, + ]); + expect(charged[0]?.amount).toBe(60_000); + expect(await topUpsOf(other.accountId)).toHaveLength(1); + + await setBalance(deleted.accountId, 30_000); + expect(await runAutoTopUp(deleted.accountId)).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + expect( + fixture.created.filter( + (intent) => intent.metadata.accountId === deleted.accountId, + ), + ).toEqual([]); + }); + it("counts an unresolved payment of any age against the owner's ceiling until it resolves", async () => { const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); const warn = vi.spyOn(console, "warn").mockImplementation(() => {});