fix: Cap Infra auto top-up as soon as the balance falls below the threshold

apps/web/__tests__/unit/developer-billing-docs.test.ts

@@ -43,10 +43,12 @@ import {
 } from "@/lib/developer-transcript-hold";
 import { TRANSCRIPT_MIN_START_BALANCE_MICRO_CREDITS } from "@/lib/developer-transcripts";
 import {
+	AUTO_TOP_UP_DAILY_LIMIT,
 	AUTO_TOP_UP_MAX_AMOUNT_CENTS,
 	AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS,
 	AUTO_TOP_UP_MIN_AMOUNT_CENTS,
 	AUTO_TOP_UP_MIN_THRESHOLD_MICRO_CREDITS,
+	AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS,
 } from "@/lib/infra-auto-top-up";
 import { LEDGER_PAGE_SIZE } from "@/lib/infra-ledger";
 import {
@@ -278,6 +280,12 @@ describe("the Billing page", () => {
 		expect(billing).toContain(
 			`${INFRA_GRACE_DAYS} days after the app first fell short`,
 		);
+		expect(billing).toContain(
+			"- **When.** As soon as the available balance falls below your threshold.",
+		);
+		expect(billing).toContain(
+			`up to ${AUTO_TOP_UP_DAILY_LIMIT} top-ups in any 24 hours, and at most ${wholeDollars(AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS)} a day across all your apps.`,
+		);
 		expect(billing).toContain(`${LEDGER_PAGE_SIZE} to a page`);
 		expect(billing).toContain(
 			`reads ${formatCredits(2_999_901)} and never $30.00`,

apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts

@@ -31,6 +31,11 @@ const mocks = vi.hoisted(() => ({
 	claim: vi.fn(),
 	release: vi.fn(),
 	startProcessing: vi.fn(),
+	scheduleAutoTopUp: vi.fn(),
+}));
+
+vi.mock("@/lib/infra-auto-top-up", () => ({
+	scheduleAutoTopUp: mocks.scheduleAutoTopUp,
 }));
 
 vi.mock("@/lib/developer-video-start", () => ({
@@ -286,6 +291,7 @@ const refused = {
 
 describe("developer SDK multipart completion", () => {
 	beforeEach(() => {
+		mocks.scheduleAutoTopUp.mockReset();
 		mocks.balance = 10_000_000;
 		mocks.accountExists = true;
 		mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 1_000 }];
@@ -339,6 +345,13 @@ describe("developer SDK multipart completion", () => {
 		mocks.unrecorded = [];
 	});
 
+	it("asks for no top-up when an upload is admitted", async () => {
+		mocks.admission.mockResolvedValue(admitted);
+
+		expect((await complete({ durationInSecs: 60 })).status).toBe(200);
+		expect(mocks.scheduleAutoTopUp).not.toHaveBeenCalled();
+	});
+
 	it("charges nothing when an upload completes", async () => {
 		mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 10_000_000 }];
 
@@ -443,6 +456,7 @@ describe("developer SDK multipart completion", () => {
 		expect(mocks.startProcessing).not.toHaveBeenCalled();
 		expect(mocks.videoUpdates).toEqual([]);
 		expect(mocks.storedDuration).toBe(null);
+		expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" });
 	});
 
 	it("keeps the file when the upload is refused before storage assembles it", async () => {
@@ -454,6 +468,7 @@ describe("developer SDK multipart completion", () => {
 		await expect(response.json()).resolves.not.toHaveProperty("fileRemoved");
 		expect(mocks.complete).not.toHaveBeenCalled();
 		expect(mocks.deleteObject).not.toHaveBeenCalled();
+		expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" });
 	});
 
 	it("still answers 402 when the refused file cannot be removed", async () => {

apps/web/__tests__/unit/developer-transcript-settle.test.ts

@@ -12,6 +12,11 @@ const context = vi.hoisted(() => ({
 		reset: () => void;
 	},
 	ledgerIds: 0,
+	scheduleAutoTopUp: vi.fn(),
+}));
+
+vi.mock("@/lib/infra-auto-top-up", () => ({
+	scheduleAutoTopUp: context.scheduleAutoTopUp,
 }));
 
 vi.mock("@cap/database", async () => {
@@ -99,6 +104,7 @@ const settle = (durationSeconds = ONE_HOUR, transcriptId = "tr-1") =>
 beforeEach(() => {
 	fake().reset();
 	context.ledgerIds = 0;
+	context.scheduleAutoTopUp.mockReset();
 	fake().seed(developerCreditAccounts, {
 		id: "account-1",
 		appId: "app-1",
@@ -403,6 +409,48 @@ describe("transcript settlement", () => {
 	});
 });
 
+describe("auto top-up after a transcript charge or hold", () => {
+	const scheduled = () => context.scheduleAutoTopUp.mock.calls;
+
+	it("asks for a top-up once after a settle that charged, not on a retry", async () => {
+		await settle();
+		await settle();
+
+		expect(scheduled()).toEqual([[{ appId: "app-1" }]]);
+	});
+
+	it("asks for a top-up when the balance could not cover the transcript", async () => {
+		const current = account();
+		if (current) current.balanceMicroCredits = 9_999;
+
+		await settle();
+
+		expect(scheduled()).toEqual([[{ appId: "app-1" }]]);
+	});
+
+	it("asks for a top-up after a hold or a reservation, and not for skipped work", async () => {
+		addTranscript("tr-2");
+		await holdTranscriptCredits({ transcriptId: "tr-1", microCredits: 1_000 });
+		await reserveTranscriptCredits({
+			transcriptId: "tr-2",
+			durationSeconds: ONE_HOUR,
+		});
+		expect(scheduled()).toEqual([[{ appId: "app-1" }], [{ appId: "app-1" }]]);
+
+		addTranscript("tr-failed", { status: "error" });
+		await holdTranscriptCredits({
+			transcriptId: "tr-failed",
+			microCredits: 1_000,
+		});
+		await reserveTranscriptCredits({
+			transcriptId: "tr-failed",
+			durationSeconds: ONE_HOUR,
+		});
+		await settle(ONE_HOUR, "tr-failed");
+		expect(scheduled()).toHaveLength(2);
+	});
+});
+
 describe("transcript holds", () => {
 	const reserve = (transcriptId: string, durationSeconds: number) =>
 		reserveTranscriptCredits({ transcriptId, durationSeconds });

apps/web/__tests__/unit/developer-transcript-store.test.ts

@@ -19,6 +19,11 @@ const context = vi.hoisted(() => ({
 	presigned: [] as { key: string; contentLength: number }[],
 	failSigning: false,
 	onDelete: null as null | (() => Promise<void>),
+	scheduleAutoTopUp: vi.fn(),
+}));
+
+vi.mock("@/lib/infra-auto-top-up", () => ({
+	scheduleAutoTopUp: context.scheduleAutoTopUp,
 }));
 
 vi.mock("@cap/database", async () => {
@@ -163,6 +168,7 @@ const fromUrl = (url = URL_A, idempotencyKey?: string, appId = "app-1") =>
 beforeEach(() => {
 	fake().reset();
 	context.ids = 0;
+	context.scheduleAutoTopUp.mockReset();
 	context.speechConfigured = true;
 	context.hosted = true;
 	context.start.mockReset();
@@ -305,6 +311,9 @@ describe("creating a transcript from a URL", () => {
 		});
 		expect(transcripts()).toHaveLength(0);
 		expect(context.start).not.toHaveBeenCalled();
+		expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({
+			appId: "app-1",
+		});
 	});
 
 	it("says how much to add below the minimum balance", async () => {
@@ -953,12 +962,16 @@ describe("starting an uploaded transcript", () => {
 		const id = await created();
 		context.objectSize = 1024;
 		setBalance(0);
+		context.scheduleAutoTopUp.mockReset();
 
 		expect(await startTranscriptUpload("app-1", id)).toMatchObject({
 			ok: false,
 			status: 402,
 		});
 		expect(context.start).not.toHaveBeenCalled();
+		expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({
+			appId: "app-1",
+		});
 	});
 
 	it("can be retried after the job could not be started", async () => {

apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts

@@ -1,9 +1,13 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
-type Account = { id: string; lastAttemptAt: Date | null };
+type Account = { id: string; lastAttemptAt: Date | null; waiting?: boolean };
 
 const state = vi.hoisted(() => ({
-	accounts: [] as { id: string; lastAttemptAt: Date | null }[],
+	accounts: [] as {
+		id: string;
+		lastAttemptAt: Date | null;
+		waiting?: boolean;
+	}[],
 	pages: [] as {
 		sql: string;
 		params: unknown[];
@@ -25,21 +29,14 @@ vi.mock("@cap/database", async () => {
 	const client = drizzle(async (sql, params) => {
 		if (sql.startsWith("select `developer_credit_accounts`.`id`, cast(")) {
 			const limit = Number(params.at(-1));
-			const keyset = sql.includes(" or (");
+			const keyset = sql.includes("and `developer_credit_accounts`.`id` > ?))");
 			const after = keyset
 				? { key: String(params.at(-4)), id: String(params.at(-2)) }
 				: null;
-			const cutoff = String(
-				params.find(
-					(param) =>
-						typeof param === "string" &&
-						/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/.test(param),
-				),
-			);
 			state.pages.push({ sql, params, after, limit });
 			const rows = state.accounts
+				.filter((account) => !account.waiting)
 				.map((account) => ({ id: account.id, key: keyOf(account) }))
-				.filter((row) => row.key <= cutoff)
 				.filter(
 					(row) =>
 						!after ||
@@ -68,7 +65,7 @@ vi.mock("@cap/database", async () => {
 					.map((account) => [account.id]),
 			};
 		}
-		if (sql.includes("`autoTopUpAmountCents`")) {
+		if (sql.includes("`developer_apps`.`name`")) {
 			state.reads.push(String(params[0]));
 			vi.setSystemTime(Date.now() + state.msPerAccount);
 			return { rows: [] };
@@ -92,7 +89,7 @@ vi.mock("@/lib/infra-billing-emails", () => ({
 }));
 
 import {
-	autoTopUpRetryCutoff,
+	AUTO_TOP_UP_DAILY_LIMIT,
 	getAutoTopUpFailure,
 	runDueAutoTopUps,
 } from "@/lib/infra-auto-top-up";
@@ -104,12 +101,17 @@ const accounts = (
 	count: number,
 	from = 0,
 	lastAttemptAt: Date | null = null,
+	waiting = false,
 ): Account[] =>
 	Array.from({ length: count }, (_, index) => ({
 		id: `account-${String(from + index).padStart(4, "0")}`,
 		lastAttemptAt,
+		waiting,
 	}));
 
+const sqlTime = (date: Date) =>
+	date.toISOString().slice(0, 23).replace("T", " ");
+
 beforeEach(() => {
 	vi.useFakeTimers();
 	vi.setSystemTime(NOW);
@@ -136,7 +138,7 @@ describe("auto top-up sweep selection", () => {
 		expect(state.reads).toHaveLength(450);
 	});
 
-	it("asks the database for enabled accounts under their threshold after held credit, not tried since the retry cutoff", async () => {
+	it("asks the database for enabled accounts under their threshold after held credit that nothing makes wait", async () => {
 		state.accounts = [];
 
 		await runDueAutoTopUps(NOW);
@@ -154,14 +156,27 @@ describe("auto top-up sweep selection", () => {
 		expect(sql).toContain(
 			"(select max(`developer_credit_transactions`.`createdAt`) from `developer_credit_transactions` where `developer_credit_transactions`.`accountId` = `developer_credit_accounts`.`id` and `developer_credit_transactions`.`type` in (?, ?, ?) and `developer_credit_transactions`.`referenceType` in (?, ?, ?))",
 		);
+		expect(sql).toContain(
+			"not (exists (select 1 from `developer_credit_transactions` `pending_attempt` where `pending_attempt`.`accountId` = `developer_credit_accounts`.`id` and `pending_attempt`.`type` = ? and `pending_attempt`.`referenceType` = ? and `pending_attempt`.`createdAt` > ? and `pending_attempt`.`referenceId` not like ? and not exists (select 1 from `developer_credit_transactions` `settled_attempt` where `settled_attempt`.`accountId` = `pending_attempt`.`accountId` and `settled_attempt`.`referenceId` = `pending_attempt`.`referenceId`",
+		);
+		expect(sql).toContain(
+			"exists (select 1 from `developer_credit_transactions` `declined_attempt` where `declined_attempt`.`accountId` = `developer_credit_accounts`.`id` and `declined_attempt`.`type` = ? and `declined_attempt`.`referenceType` = ? and `declined_attempt`.`createdAt` > ? and not exists (select 1 from `developer_credit_transactions` `later_top_up` where `later_top_up`.`accountId` = `declined_attempt`.`accountId` and `later_top_up`.`type` = ?",
+		);
+		expect(sql).toContain(
+			"(select count(*) from `developer_credit_transactions` `recent_top_up` where `recent_top_up`.`accountId` = `developer_credit_accounts`.`id` and `recent_top_up`.`type` = ? and `recent_top_up`.`referenceType` = ? and `recent_top_up`.`createdAt` > ?) >= ?",
+		);
 		expect(sql).toMatch(
 			/order by attemptKey, `developer_credit_accounts`\.`id` asc limit \?$/,
 		);
-		expect(params).toContain(
-			autoTopUpRetryCutoff(NOW).toISOString().slice(0, 23).replace("T", " "),
+		expect(params).toContain(sqlTime(new Date(NOW.getTime() - 72 * HOUR_MS)));
+		expect(params).toContain(sqlTime(new Date(NOW.getTime() - 24 * HOUR_MS)));
+		expect(params).toContain(AUTO_TOP_UP_DAILY_LIMIT);
+		expect(params).toContain("infra-auto-top-up:%");
+		expect(sql).toContain(
+			"or exists (select 1 from `developer_credit_transactions` `stale_payment` where `stale_payment`.`accountId` = `developer_credit_accounts`.`id` and `stale_payment`.`type` = ? and `stale_payment`.`referenceType` = ? and `stale_payment`.`createdAt` <= ?",
 		);
-		expect(autoTopUpRetryCutoff(NOW)).toEqual(
-			new Date(NOW.getTime() - 24 * HOUR_MS),
+		expect(sql).toContain(
+			"from `developer_credit_accounts` `owner_account` join `developer_credit_transactions` `owner_entry` on `owner_entry`.`accountId` = `owner_account`.`id` where `owner_account`.`ownerId` = `developer_credit_accounts`.`ownerId`",
 		);
 	});
 
@@ -190,11 +205,12 @@ describe("auto top-up sweep selection", () => {
 			"account-0005",
 			"account-0003",
 			"account-0001",
+			"account-0004",
 		]);
 	});
 
-	it("reaches 50 due accounts in one run behind 300 accounts tried inside the retry window", async () => {
-		const tried = accounts(300, 0, new Date(NOW.getTime() - HOUR_MS));
+	it("reaches 50 due accounts in one run behind 300 accounts that must wait", async () => {
+		const tried = accounts(300, 0, new Date(NOW.getTime() - HOUR_MS), true);
 		const due = accounts(50, 300);
 		state.accounts = [...tried, ...due];
 		state.msPerAccount = 250;