From 8b5926c9f99232cb0f4845702cb4f29d63233e1c Mon Sep 17 00:00:00 2001 From: wargloom Date: Tue, 8 Sep 2026 23:51:55 +0300 Subject: [PATCH 01/22] refactor(doctor): share account repair workflow --- lib/tools/codex-doctor.ts | 94 +++----------------------------------- lib/tools/doctor-repair.ts | 89 ++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 88 deletions(-) create mode 100644 lib/tools/doctor-repair.ts diff --git a/lib/tools/codex-doctor.ts b/lib/tools/codex-doctor.ts index 570f1943..082d4c9e 100644 --- a/lib/tools/codex-doctor.ts +++ b/lib/tools/codex-doctor.ts @@ -13,7 +13,6 @@ import { import { AccountManager } from "../accounts.js"; import { MODEL_FAMILIES } from "../prompts/codex.js"; import { - clearRefreshedAccountsStaleState, findDisabledAccountsWithFreshCredential, findDisabledTokenSourceDuplicates, findConflictingBusinessMemberCredentials, @@ -39,11 +38,10 @@ import { type RoutingVisibilitySnapshot, } from "../runtime.js"; import { - buildRefreshInputs, findAccountIndexByIdentity, - refreshAndPersistAccount, type RefreshAccountIdentity, } from "./refresh-account.js"; +import { repairDoctorAccounts } from "./doctor-repair.js"; import type { ToolContext } from "./index.js"; interface DoctorDiagnostics { @@ -192,36 +190,10 @@ export function createCodexDoctorTool(ctx: ToolContext): ToolDefinition { let diagnostics = await loadDiagnostics(); if (fix && diagnostics.storage && diagnostics.storage.accounts.length > 0) { - const refreshResults: Array<{ - index: number; - identity: RefreshAccountIdentity; - }> = []; - const reloginNeeded: number[] = []; - const verificationFailureIdentities: RefreshAccountIdentity[] = []; - const inputs = buildRefreshInputs(diagnostics.storage.accounts); - - for (const input of inputs) { - if (!input) continue; - const outcome = await refreshAndPersistAccount(input); - if (outcome.status === "refreshed") { - refreshResults.push({ - index: outcome.index, - identity: outcome.result.identity, - }); - } else if (outcome.status === "skipped") { - // Skip intentionally-disabled accounts: refreshing them is wrong - // (e.g. the disabled token-source duplicate would get a spurious - // "re-login" directive when its dead token fails, when the correct - // remedy is `codex-remove`), and stale-state must never be cleared - // on an entry the user disabled on purpose. - } else { - verificationFailureIdentities.push(outcome.identity); - reloginNeeded.push(outcome.index + 1); - fixErrors.push( - `Account ${outcome.index + 1}: ${outcome.error} — run \`opencode auth login\` to re-authenticate.`, - ); - } - } + const repair = await repairDoctorAccounts(diagnostics.storage.accounts); + const { reloginNeeded, verificationFailureIdentities } = repair; + appliedFixes.push(...repair.appliedFixes); + fixErrors.push(...repair.fixErrors); if (verificationFailureIdentities.length > 0) { extraFindings.push({ @@ -232,61 +204,7 @@ export function createCodexDoctorTool(ctx: ToolContext): ToolDefinition { }); } - if (refreshResults.length > 0) { - appliedFixes.push( - `Refreshed and persisted ${refreshResults.length} account token(s).`, - ); - - // A successful refresh proves the credential is alive, so clear any - // stale cooldown / rate-limit state that would otherwise keep the - // recovered account out of rotation (issue #171). Apply this to a - // fresh storage snapshot so non-credential state written by other - // processes is preserved. - try { - const staleSummary = await withAccountStorageTransaction( - async (current, persist) => { - if (!current) { - throw new Error("Account storage is unavailable"); - } - const refreshedRecords = []; - for (const refreshed of refreshResults) { - const idx = findAccountIndexByIdentity( - current.accounts, - refreshed.identity, - ); - const record = idx >= 0 ? current.accounts[idx] : undefined; - if (record && record.enabled !== false) { - refreshedRecords.push(record); - } - } - const summary = clearRefreshedAccountsStaleState(refreshedRecords); - if ( - summary.cooldownsCleared > 0 || - summary.rateLimitKeysCleared > 0 - ) { - await persist(current); - } - return summary; - }, - ); - if (staleSummary.cooldownsCleared > 0) { - appliedFixes.push( - `Cleared cooldown on ${staleSummary.cooldownsCleared} recovered account(s).`, - ); - } - if (staleSummary.rateLimitKeysCleared > 0) { - appliedFixes.push( - `Cleared ${staleSummary.rateLimitKeysCleared} stale rate-limit marker(s).`, - ); - } - } catch (error) { - fixErrors.push( - `Failed to persist stale-state repairs: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } - + if (repair.refreshedCount > 0) { // Stale TUI quota cache can reference an account index/count that no // longer matches the pool, making diagnostics misleading (#171). try { diff --git a/lib/tools/doctor-repair.ts b/lib/tools/doctor-repair.ts new file mode 100644 index 00000000..40238ef1 --- /dev/null +++ b/lib/tools/doctor-repair.ts @@ -0,0 +1,89 @@ +import { withAccountStorageTransaction, type AccountMetadataV3 } from "../storage.js"; +import { clearRefreshedAccountsStaleState } from "../accounts/stale-state.js"; +import { + buildRefreshInputs, + findAccountIndexByIdentity, + refreshAndPersistAccount, + type RefreshAccountIdentity, +} from "./refresh-account.js"; + +export async function repairDoctorAccounts(accounts: AccountMetadataV3[]) { + const refreshedAccounts: { + readonly identity: RefreshAccountIdentity; + readonly staleState: Pick; + }[] = []; + const verificationFailureIdentities: RefreshAccountIdentity[] = []; + const reloginNeeded: number[] = []; + const appliedFixes: string[] = []; + const fixErrors: string[] = []; + + for (const input of buildRefreshInputs(accounts)) { + const account = accounts[input.index]; + if (!account) continue; + const staleState = { + coolingDownUntil: account.coolingDownUntil, + cooldownReason: account.cooldownReason, + rateLimitResetTimes: { ...account.rateLimitResetTimes }, + }; + const outcome = await refreshAndPersistAccount(input); + switch (outcome.status) { + case "refreshed": + refreshedAccounts.push({ + identity: { ...outcome.result.identity, refreshToken: outcome.result.refreshToken }, + staleState, + }); + break; + case "skipped": + break; + case "failed": + verificationFailureIdentities.push(outcome.identity); + reloginNeeded.push(outcome.index + 1); + // Upstream errors can echo arbitrary credential material, even without token labels. + fixErrors.push(`Account ${outcome.index + 1}: refresh verification or credential persistence failed — run \`opencode auth login\` to re-authenticate.`); + break; + default: { + const exhaustive: never = outcome; + return exhaustive; + } + } + } + + if (refreshedAccounts.length > 0) { + appliedFixes.push(`Refreshed and persisted ${refreshedAccounts.length} account token(s).`); + try { + const staleSummary = await withAccountStorageTransaction(async (current, persist) => { + if (!current) throw new Error("Account storage is unavailable"); + const refreshedRecords: AccountMetadataV3[] = []; + for (const { identity, staleState } of refreshedAccounts) { + const index = findAccountIndexByIdentity(current.accounts, identity); + const record = current.accounts[index]; + if (!record || record.enabled === false) continue; + // A concurrent health update is newer evidence than this repair's snapshot. + const stateUnchanged = record.coolingDownUntil === staleState.coolingDownUntil && + record.cooldownReason === staleState.cooldownReason && + Object.keys({ ...record.rateLimitResetTimes, ...staleState.rateLimitResetTimes }).every( + (key) => record.rateLimitResetTimes?.[key] === staleState.rateLimitResetTimes?.[key], + ); + if (stateUnchanged) refreshedRecords.push(record); + } + const hasStaleState = refreshedRecords.some((record) => + record.coolingDownUntil !== undefined || record.cooldownReason !== undefined || + Object.keys(record.rateLimitResetTimes ?? {}).length > 0, + ); + const summary = clearRefreshedAccountsStaleState(refreshedRecords); + if (hasStaleState) await persist(current); + return summary; + }); + if (staleSummary.cooldownsCleared > 0) { + appliedFixes.push(`Cleared cooldown on ${staleSummary.cooldownsCleared} recovered account(s).`); + } + if (staleSummary.rateLimitKeysCleared > 0) { + appliedFixes.push(`Cleared ${staleSummary.rateLimitKeysCleared} stale rate-limit marker(s).`); + } + } catch { + fixErrors.push("Failed to persist stale-state repairs."); + } + } + + return { refreshedCount: refreshedAccounts.length, verificationFailureIdentities, reloginNeeded, appliedFixes, fixErrors }; +} From 2eafa8e081ecff9029ebb7063d88e0b32a2a947c Mon Sep 17 00:00:00 2001 From: wargloom Date: Tue, 8 Sep 2026 23:52:06 +0300 Subject: [PATCH 02/22] fix(cli): apply standalone doctor repairs --- scripts/install-oc-codex-multi-auth-core.js | 38 ++++++- test/standalone-cli.test.ts | 104 ++++++++++++++++++++ 2 files changed, 139 insertions(+), 3 deletions(-) diff --git a/scripts/install-oc-codex-multi-auth-core.js b/scripts/install-oc-codex-multi-auth-core.js index 050aabc5..f7e3c1a1 100644 --- a/scripts/install-oc-codex-multi-auth-core.js +++ b/scripts/install-oc-codex-multi-auth-core.js @@ -440,6 +440,8 @@ function printStandaloneResult(command, payload, json) { } } if (payload.error) console.log(`Error: ${payload.error}`); + for (const fix of payload.appliedFixes ?? []) console.log(`Fixed: ${fix}`); + for (const error of payload.fixErrors ?? []) console.log(`Repair failed: ${error}`); if (payload.nextAction) console.log(`Next: ${payload.nextAction}`); } @@ -771,7 +773,33 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { } const { env = process.env } = options; const storagePath = getStandaloneStoragePath(parsed, env); - const { storage, error } = await readStandaloneStorage(storagePath); + let { storage, error } = await readStandaloneStorage(storagePath); + const appliedFixes = []; + const fixErrors = []; + if (command === "doctor" && parsed.fix && storage && !error) { + const previousKeychain = process.env.CODEX_KEYCHAIN; + try { + const loadDoctorRuntime = options.loadDoctorRuntime ?? (() => loadDistModules( + ["storage.js", "tools/doctor-repair.js", "shutdown.js"], "doctor", + )); + const [storageMod, repairMod, shutdownMod] = await loadDoctorRuntime(); + // A CLI file selection must not read or replace the global keychain pool. + process.env.CODEX_KEYCHAIN = "0"; + storageMod.setStoragePathDirect(storagePath); + shutdownMod.setShutdownOwnsProcess(true); + const current = await storageMod.loadAccounts(); + if (!current) throw new Error("Account storage is unavailable"); + const repair = await repairMod.repairDoctorAccounts(current.accounts); + appliedFixes.push(...repair.appliedFixes); + fixErrors.push(...repair.fixErrors); + } catch { + fixErrors.push("Doctor repair could not complete. Check the selected storage file and installed runtime."); + } finally { + if (previousKeychain === undefined) delete process.env.CODEX_KEYCHAIN; + else process.env.CODEX_KEYCHAIN = previousKeychain; + } + ({ storage, error } = await readStandaloneStorage(storagePath)); + } const accounts = summarizeStandaloneAccounts(storage, parsed.includeSensitive, parsed.tag); const totalAccounts = Array.isArray(storage?.accounts) ? storage.accounts.length : 0; const payload = { @@ -790,7 +818,11 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { } else if (command === "doctor") { payload.message = error ? "Storage could not be parsed." : totalAccounts > 0 ? "Local diagnostics completed." : "No accounts configured."; payload.deep = parsed.deep; - payload.fixApplied = parsed.fix ? false : undefined; + payload.fixApplied = parsed.fix ? appliedFixes.length > 0 : undefined; + if (parsed.fix) { + payload.appliedFixes = appliedFixes; + payload.fixErrors = fixErrors; + } payload.nextAction = totalAccounts > 0 ? "Run oc-codex-multi-auth health --json for scriptable checks." : "Run opencode auth login."; } else if (command === "health") { payload.healthyCount = accounts.filter((account) => account.enabled && account.hasRefreshToken).length; @@ -799,7 +831,7 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { payload.message = totalAccounts > 0 ? "Account storage loaded." : "No accounts configured."; } printStandaloneResult(command, payload, parsed.json); - return { exitCode: error ? 1 : 0, action: command, storagePath }; + return { exitCode: error || fixErrors.length > 0 ? 1 : 0, action: command, storagePath }; } // Top-level keys inside `provider.openai` that the installer owns absolutely. diff --git a/test/standalone-cli.test.ts b/test/standalone-cli.test.ts index 94eb8508..c5fddfb2 100644 --- a/test/standalone-cli.test.ts +++ b/test/standalone-cli.test.ts @@ -1,8 +1,18 @@ +/// import { afterEach, describe, expect, it, vi } from "vitest"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +// Exercise the shipped import boundary with source implementations, not stale dist. +async function loadSourceDoctorRuntime() { + return Promise.all([ + import("../lib/storage.js"), + import("../lib/tools/doctor-repair.js"), + import("../lib/shutdown.js"), + ]); +} + async function createTempHome() { return mkdtemp(join(tmpdir(), "oc-codex-standalone-")); } @@ -31,6 +41,7 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { afterEach(async () => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); if (tempHome) { await rm(tempHome, { recursive: true, force: true }); tempHome = null; @@ -311,6 +322,99 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { ...over, }); + it.each(["acct_warm", undefined])("doctor: repairs stale state and persists rotated credentials only in --config-path (%s)", async (accountId) => { + // Given a selected pool distinct from both the home pool and runtime default. + vi.resetModules(); + vi.stubEnv("CODEX_KEYCHAIN", "1"); + tempHome = await createTempHome(); + await seedPool(tempHome, [freshAccount({ refreshToken: "home-secret" })]); + const homePath = join(tempHome, ".opencode", "oc-codex-multi-auth-accounts.json"); + const homeBefore = await readFile(homePath, "utf-8"); + const poolPath = join(tempHome, "selected-pool.json"); + const resetAt = Date.now() + 86_400_000; + await writeFile(poolPath, JSON.stringify({ version: 3, activeIndex: 0, accounts: [ + freshAccount({ accountId, coolingDownUntil: resetAt, cooldownReason: "auth-failure", rateLimitResetTimes: { codex: resetAt } }), + ] })); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(JSON.stringify({ + access_token: "rotated-access-secret", refresh_token: "rotated-refresh-secret", expires_in: 3600, + }), { status: 200 })); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + + // When explicitly repairing this pool, even an unexpired token is verified. + const result = await runInstaller(["doctor", "--fix", "--json", "--config-path", poolPath], { + env: { ...process.env, HOME: tempHome, USERPROFILE: tempHome }, + loadDoctorRuntime: loadSourceDoctorRuntime, + }); + + // Then the repair is durable, reported, and confined to the selected file. + const stored = JSON.parse(await readFile(poolPath, "utf-8")); + expect.soft(stored.accounts[0].rateLimitResetTimes).toEqual({}); + expect.soft(JSON.parse(String(logSpy.mock.calls.at(-1)?.[0])).fixApplied).toBe(true); + expect(stored.accounts[0]).toMatchObject({ accessToken: "rotated-access-secret", refreshToken: "rotated-refresh-secret" }); + expect(stored.accounts[0].coolingDownUntil).toBeUndefined(); + expect(stored.accounts[0].cooldownReason).toBeUndefined(); + expect(result).toMatchObject({ action: "doctor", exitCode: 0, storagePath: poolPath }); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(String(fetchSpy.mock.calls[0]?.[1]?.body)).toContain("rt-warm"); + expect(await readFile(homePath, "utf-8")).toBe(homeBefore); + expect(JSON.stringify(logSpy.mock.calls)).not.toMatch(/rotated-access-secret|rotated-refresh-secret|rt-warm|home-secret/); + }); + + it("doctor: preserves failed and disabled accounts while reporting partial repair failure", async () => { + // Given one recoverable, one failing, and one intentionally disabled account. + vi.resetModules(); + tempHome = await createTempHome(); + const poolPath = join(tempHome, "selected-pool.json"); + const stale = { coolingDownUntil: Date.now() + 86_400_000, cooldownReason: "auth-failure", rateLimitResetTimes: { codex: Date.now() + 86_400_000 } }; + const failed = freshAccount({ ...stale, accountId: "acct_failed", refreshToken: "failed-refresh-secret" }); + const disabled = freshAccount({ ...stale, accountId: "acct_disabled", refreshToken: "disabled-refresh-secret", enabled: false }); + await writeFile(poolPath, JSON.stringify({ version: 3, activeIndex: 0, accounts: [freshAccount(stale), failed, disabled] })); + const fetchSpy = vi.spyOn(globalThis, "fetch") + .mockResolvedValueOnce(new Response(JSON.stringify({ access_token: "rotated-access-secret", refresh_token: "rotated-refresh-secret", expires_in: 3600 }))) + .mockRejectedValueOnce(new Error("failed-refresh-secret at-warm access_token=upstream-access-secret")); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + + // When repair encounters a refresh failure, it continues but exits nonzero. + const result = await runInstaller(["doctor", "--fix", "--json", "--config-path", poolPath], { + env: { ...process.env, HOME: tempHome, USERPROFILE: tempHome }, + loadDoctorRuntime: loadSourceDoctorRuntime, + }); + + // Then only the verified account loses stale state and no secret is reported. + expect(result).toMatchObject({ exitCode: 1 }); + const stored = JSON.parse(await readFile(poolPath, "utf-8")); + expect(stored.accounts[0].rateLimitResetTimes).toEqual({}); + expect(stored.accounts[1]).toMatchObject(failed); + expect(stored.accounts[2]).toMatchObject(disabled); + expect(fetchSpy).toHaveBeenCalledTimes(2); + const output = JSON.parse(String(logSpy.mock.calls.at(-1)?.[0])); + expect(output.fixApplied).toBe(true); + expect(output.fixErrors).toEqual([expect.stringContaining("Account 2")]); + expect(JSON.stringify(output)).not.toMatch(/failed-refresh-secret|at-warm|upstream-access-secret|rotated-access-secret|rotated-refresh-secret|disabled-refresh-secret/); + }); + + it("doctor: remains read-only without --fix", async () => { + // Given a stale pool that would require verification to repair. + vi.resetModules(); + tempHome = await createTempHome(); + await seedPool(tempHome, [freshAccount({ rateLimitResetTimes: { codex: Date.now() + 86_400_000 } })]); + const poolPath = join(tempHome, ".opencode", "oc-codex-multi-auth-accounts.json"); + const before = await readFile(poolPath, "utf-8"); + const fetchSpy = vi.spyOn(globalThis, "fetch"); + vi.spyOn(console, "log").mockImplementation(() => {}); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + + // When only diagnostics are requested. + await runInstaller(["doctor", "--json", "--config-path", poolPath]); + + // Then neither credentials nor storage are touched. + expect(fetchSpy).not.toHaveBeenCalled(); + expect(await readFile(poolPath, "utf-8")).toBe(before); + }); + it("warm: empty pool reports 0/0/0 and exits 0 (no network)", async () => { vi.resetModules(); tempHome = await createTempHome(); From ab6e40118770cc5e4100cc9784ebed39c11bad7d Mon Sep 17 00:00:00 2001 From: wargloom Date: Wed, 9 Sep 2026 00:15:26 +0300 Subject: [PATCH 03/22] fix(cli): preserve default keychain routing --- scripts/install-oc-codex-multi-auth-core.js | 8 +++-- test/standalone-cli.test.ts | 33 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/scripts/install-oc-codex-multi-auth-core.js b/scripts/install-oc-codex-multi-auth-core.js index f7e3c1a1..96ebbe97 100644 --- a/scripts/install-oc-codex-multi-auth-core.js +++ b/scripts/install-oc-codex-multi-auth-core.js @@ -784,7 +784,7 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { )); const [storageMod, repairMod, shutdownMod] = await loadDoctorRuntime(); // A CLI file selection must not read or replace the global keychain pool. - process.env.CODEX_KEYCHAIN = "0"; + if (parsed.configPath) process.env.CODEX_KEYCHAIN = "0"; storageMod.setStoragePathDirect(storagePath); shutdownMod.setShutdownOwnsProcess(true); const current = await storageMod.loadAccounts(); @@ -795,8 +795,10 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { } catch { fixErrors.push("Doctor repair could not complete. Check the selected storage file and installed runtime."); } finally { - if (previousKeychain === undefined) delete process.env.CODEX_KEYCHAIN; - else process.env.CODEX_KEYCHAIN = previousKeychain; + if (parsed.configPath) { + if (previousKeychain === undefined) delete process.env.CODEX_KEYCHAIN; + else process.env.CODEX_KEYCHAIN = previousKeychain; + } } ({ storage, error } = await readStandaloneStorage(storagePath)); } diff --git a/test/standalone-cli.test.ts b/test/standalone-cli.test.ts index c5fddfb2..d10d552e 100644 --- a/test/standalone-cli.test.ts +++ b/test/standalone-cli.test.ts @@ -361,6 +361,39 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { expect(JSON.stringify(logSpy.mock.calls)).not.toMatch(/rotated-access-secret|rotated-refresh-secret|rt-warm|home-secret/); }); + it("doctor: preserves enabled keychain routing when repairing the default path", async () => { + // Given enabled keychain routing and an isolated default pool. + vi.resetModules(); + vi.stubEnv("CODEX_KEYCHAIN", "1"); + tempHome = await createTempHome(); + await seedPool(tempHome, [freshAccount()]); + vi.spyOn(console, "log").mockImplementation(() => {}); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + const keychainRouting: (string | undefined)[] = []; + const accounts = [freshAccount()]; + + // When repair uses injected runtime seams, never the real keychain. + const result = await runInstaller(["doctor", "--fix", "--json"], { + env: { ...process.env, HOME: tempHome, USERPROFILE: tempHome }, + loadDoctorRuntime: async () => [ + { setStoragePathDirect: vi.fn(), loadAccounts: async () => { + keychainRouting.push(process.env.CODEX_KEYCHAIN); + return { accounts }; + } }, + { repairDoctorAccounts: async () => { + keychainRouting.push(process.env.CODEX_KEYCHAIN); + return { appliedFixes: [], fixErrors: [] }; + } }, + { setShutdownOwnsProcess: vi.fn() }, + ], + }); + + // Then both loading and repair retain enabled keychain routing. + expect(keychainRouting).toEqual(["1", "1"]); + expect(process.env.CODEX_KEYCHAIN).toBe("1"); + expect(result).toMatchObject({ action: "doctor", exitCode: 0 }); + }); + it("doctor: preserves failed and disabled accounts while reporting partial repair failure", async () => { // Given one recoverable, one failing, and one intentionally disabled account. vi.resetModules(); From a3c9ea4382c1ad2ed21c19e6f9b775bf111dc5ce Mon Sep 17 00:00:00 2001 From: wargloom Date: Wed, 9 Sep 2026 00:15:39 +0300 Subject: [PATCH 04/22] docs(cli): document standalone doctor repairs --- docs/tools-and-cli.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/tools-and-cli.md b/docs/tools-and-cli.md index 494921e5..c5b07c64 100644 --- a/docs/tools-and-cli.md +++ b/docs/tools-and-cli.md @@ -180,7 +180,7 @@ Choose only one of `--plugin-only`, `--modern`, `--full`, or `--legacy`. Use `up | `--json` | Machine-readable JSON output | | `--include-sensitive` | Include sensitive identity fields in JSON where applicable | | `--deep` | Deeper diagnostics (used with `doctor`; implied by `diag`) | -| `--fix` | Request fix application where supported (may be a no-op for some safe CLI paths) | +| `--fix` | With `doctor`, refresh enabled accounts and clear stale cooldown and rate-limit markers only after successful verification. Exit nonzero if any repair fails. | | `--tag ` | Filter accounts by tag when listing | | `--config-path ` | Point at a specific accounts storage path | | `--help` / `-h` | Print usage | @@ -192,9 +192,12 @@ oc-codex-multi-auth status --json oc-codex-multi-auth list --tag work oc-codex-multi-auth warm --json oc-codex-multi-auth doctor --deep +oc-codex-multi-auth doctor --fix --config-path ./accounts.json npx -y oc-codex-multi-auth@latest warm ``` +For `doctor --fix`, an explicit `--config-path` repairs only the selected JSON pool and bypasses keychain routing. Without `--config-path`, repair preserves enabled keychain routing. + --- ## Related runtime concepts From 420dfcc7968936ccdc39d7c5e34a34fb900f0337 Mon Sep 17 00:00:00 2001 From: wargloom Date: Wed, 9 Sep 2026 00:35:21 +0300 Subject: [PATCH 05/22] fix(cli): load default doctor storage backend --- scripts/install-oc-codex-multi-auth-core.js | 14 +++-- test/standalone-cli.test.ts | 65 +++++++++++++++++---- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/scripts/install-oc-codex-multi-auth-core.js b/scripts/install-oc-codex-multi-auth-core.js index 96ebbe97..049f022d 100644 --- a/scripts/install-oc-codex-multi-auth-core.js +++ b/scripts/install-oc-codex-multi-auth-core.js @@ -773,10 +773,11 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { } const { env = process.env } = options; const storagePath = getStandaloneStoragePath(parsed, env); - let { storage, error } = await readStandaloneStorage(storagePath); + let storage = null; + let error = null; const appliedFixes = []; const fixErrors = []; - if (command === "doctor" && parsed.fix && storage && !error) { + if (command === "doctor" && parsed.fix) { const previousKeychain = process.env.CODEX_KEYCHAIN; try { const loadDoctorRuntime = options.loadDoctorRuntime ?? (() => loadDistModules( @@ -787,11 +788,13 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { if (parsed.configPath) process.env.CODEX_KEYCHAIN = "0"; storageMod.setStoragePathDirect(storagePath); shutdownMod.setShutdownOwnsProcess(true); - const current = await storageMod.loadAccounts(); - if (!current) throw new Error("Account storage is unavailable"); - const repair = await repairMod.repairDoctorAccounts(current.accounts); + storage = await storageMod.loadAccounts(); + if (!storage) throw new Error("Account storage is unavailable"); + const repair = await repairMod.repairDoctorAccounts(storage.accounts); appliedFixes.push(...repair.appliedFixes); fixErrors.push(...repair.fixErrors); + storage = await storageMod.loadAccounts(); + if (!storage) throw new Error("Account storage is unavailable"); } catch { fixErrors.push("Doctor repair could not complete. Check the selected storage file and installed runtime."); } finally { @@ -800,6 +803,7 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { else process.env.CODEX_KEYCHAIN = previousKeychain; } } + } else { ({ storage, error } = await readStandaloneStorage(storagePath)); } const accounts = summarizeStandaloneAccounts(storage, parsed.includeSensitive, parsed.tag); diff --git a/test/standalone-cli.test.ts b/test/standalone-cli.test.ts index d10d552e..694d7364 100644 --- a/test/standalone-cli.test.ts +++ b/test/standalone-cli.test.ts @@ -361,16 +361,21 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { expect(JSON.stringify(logSpy.mock.calls)).not.toMatch(/rotated-access-secret|rotated-refresh-secret|rt-warm|home-secret/); }); - it("doctor: preserves enabled keychain routing when repairing the default path", async () => { - // Given enabled keychain routing and an isolated default pool. + it("doctor: repairs and summarizes the default keychain pool when no JSON file exists", async () => { + // Given enabled keychain routing with accounts only in the injected backend. vi.resetModules(); vi.stubEnv("CODEX_KEYCHAIN", "1"); tempHome = await createTempHome(); - await seedPool(tempHome, [freshAccount()]); - vi.spyOn(console, "log").mockImplementation(() => {}); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); const keychainRouting: (string | undefined)[] = []; - const accounts = [freshAccount()]; + const accounts = [freshAccount({ accountLabel: "Keychain account", rateLimitResetTimes: { codex: 123 } })]; + let snapshot = { accounts }; + const repairDoctorAccounts = vi.fn(async () => { + keychainRouting.push(process.env.CODEX_KEYCHAIN); + snapshot = { accounts: [freshAccount({ accountLabel: "Keychain account", rateLimitResetTimes: {} })] }; + return { appliedFixes: ["Cleared stale rate-limit markers."], fixErrors: [] }; + }); // When repair uses injected runtime seams, never the real keychain. const result = await runInstaller(["doctor", "--fix", "--json"], { @@ -378,22 +383,58 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { loadDoctorRuntime: async () => [ { setStoragePathDirect: vi.fn(), loadAccounts: async () => { keychainRouting.push(process.env.CODEX_KEYCHAIN); - return { accounts }; - } }, - { repairDoctorAccounts: async () => { - keychainRouting.push(process.env.CODEX_KEYCHAIN); - return { appliedFixes: [], fixErrors: [] }; + return snapshot; } }, + { repairDoctorAccounts }, { setShutdownOwnsProcess: vi.fn() }, ], }); - // Then both loading and repair retain enabled keychain routing. - expect(keychainRouting).toEqual(["1", "1"]); + // Then repair runs and the summary uses the post-repair backend snapshot. + expect(repairDoctorAccounts).toHaveBeenCalledWith(accounts); + const output = JSON.parse(String(logSpy.mock.calls.at(-1)?.[0])); + expect(output).toMatchObject({ + totalAccounts: 1, + fixApplied: true, + accounts: [{ label: "Keychain account" }], + }); + expect(output.accounts[0].rateLimitResetTimes).toEqual({}); + expect(keychainRouting).toEqual(["1", "1", "1"]); expect(process.env.CODEX_KEYCHAIN).toBe("1"); expect(result).toMatchObject({ action: "doctor", exitCode: 0 }); }); + it.each(["discovery", "repair", "snapshot"])("doctor: redacts runtime %s failures without a JSON pool", async (stage) => { + // Given an injected backend that fails at one repair boundary. + vi.resetModules(); + vi.stubEnv("CODEX_KEYCHAIN", "1"); + tempHome = await createTempHome(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + const failure = new Error("upstream-private-token-text"); + const loadAccounts = vi.fn().mockResolvedValue({ accounts: [freshAccount()] }); + if (stage === "discovery") loadAccounts.mockRejectedValue(failure); + if (stage === "snapshot") loadAccounts.mockResolvedValueOnce({ accounts: [freshAccount()] }).mockRejectedValue(failure); + const repairDoctorAccounts = vi.fn().mockResolvedValue({ appliedFixes: [], fixErrors: [] }); + if (stage === "repair") repairDoctorAccounts.mockRejectedValue(failure); + + // When default-path repair runs without reading any real credentials. + const result = await runInstaller(["doctor", "--fix", "--json"], { + env: { ...process.env, HOME: tempHome, USERPROFILE: tempHome }, + loadDoctorRuntime: async () => [ + { setStoragePathDirect: vi.fn(), loadAccounts }, + { repairDoctorAccounts }, + { setShutdownOwnsProcess: vi.fn() }, + ], + }); + + // Then failure is nonzero and redacted, with keychain routing preserved. + expect(result.exitCode).toBe(1); + expect(JSON.parse(String(logSpy.mock.calls.at(-1)?.[0])).fixErrors).toHaveLength(1); + expect(JSON.stringify(logSpy.mock.calls)).not.toContain(failure.message); + expect(process.env.CODEX_KEYCHAIN).toBe("1"); + }); + it("doctor: preserves failed and disabled accounts while reporting partial repair failure", async () => { // Given one recoverable, one failing, and one intentionally disabled account. vi.resetModules(); From 10544c28907ecac0e9b93f25a4b206a66d01f02b Mon Sep 17 00:00:00 2001 From: wargloom Date: Wed, 9 Sep 2026 00:46:21 +0300 Subject: [PATCH 06/22] fix(cli): allow empty doctor account pools --- scripts/install-oc-codex-multi-auth-core.js | 14 ++--- test/standalone-cli.test.ts | 63 +++++++++++++++++++++ 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/scripts/install-oc-codex-multi-auth-core.js b/scripts/install-oc-codex-multi-auth-core.js index 049f022d..28be7ffa 100644 --- a/scripts/install-oc-codex-multi-auth-core.js +++ b/scripts/install-oc-codex-multi-auth-core.js @@ -773,11 +773,15 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { } const { env = process.env } = options; const storagePath = getStandaloneStoragePath(parsed, env); + const repairRequested = command === "doctor" && parsed.fix; let storage = null; let error = null; + if (parsed.configPath || !repairRequested) { + ({ storage, error } = await readStandaloneStorage(storagePath)); + } const appliedFixes = []; const fixErrors = []; - if (command === "doctor" && parsed.fix) { + if (repairRequested && !error) { const previousKeychain = process.env.CODEX_KEYCHAIN; try { const loadDoctorRuntime = options.loadDoctorRuntime ?? (() => loadDistModules( @@ -789,12 +793,10 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { storageMod.setStoragePathDirect(storagePath); shutdownMod.setShutdownOwnsProcess(true); storage = await storageMod.loadAccounts(); - if (!storage) throw new Error("Account storage is unavailable"); - const repair = await repairMod.repairDoctorAccounts(storage.accounts); + const repair = await repairMod.repairDoctorAccounts(storage?.accounts ?? []); appliedFixes.push(...repair.appliedFixes); fixErrors.push(...repair.fixErrors); - storage = await storageMod.loadAccounts(); - if (!storage) throw new Error("Account storage is unavailable"); + storage = (await storageMod.loadAccounts()) ?? storage; } catch { fixErrors.push("Doctor repair could not complete. Check the selected storage file and installed runtime."); } finally { @@ -803,8 +805,6 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { else process.env.CODEX_KEYCHAIN = previousKeychain; } } - } else { - ({ storage, error } = await readStandaloneStorage(storagePath)); } const accounts = summarizeStandaloneAccounts(storage, parsed.includeSensitive, parsed.tag); const totalAccounts = Array.isArray(storage?.accounts) ? storage.accounts.length : 0; diff --git a/test/standalone-cli.test.ts b/test/standalone-cli.test.ts index 694d7364..bfca3ff5 100644 --- a/test/standalone-cli.test.ts +++ b/test/standalone-cli.test.ts @@ -404,6 +404,69 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { expect(result).toMatchObject({ action: "doctor", exitCode: 0 }); }); + it("doctor: recommends login without errors when the default runtime pool is empty", async () => { + // Given a fresh installation with no JSON file or runtime accounts. + vi.resetModules(); + vi.stubEnv("CODEX_KEYCHAIN", "1"); + tempHome = await createTempHome(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("Unexpected OAuth request")); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + const repairMod = await import("../lib/tools/doctor-repair.js"); + const repairDoctorAccounts = vi.fn(repairMod.repairDoctorAccounts); + const loadAccounts = vi.fn().mockResolvedValue(null); + + // When the real repair helper receives accounts from the injected empty backend. + const result = await runInstaller(["doctor", "--fix", "--json"], { + env: { ...process.env, HOME: tempHome, USERPROFILE: tempHome }, + loadDoctorRuntime: async () => [ + { setStoragePathDirect: vi.fn(), loadAccounts }, + { repairDoctorAccounts }, + { setShutdownOwnsProcess: vi.fn() }, + ], + }); + + // Then null discovery and snapshot are successful, without OAuth requests. + expect(result.exitCode).toBe(0); + expect(repairDoctorAccounts).toHaveBeenCalledWith([]); + expect(loadAccounts).toHaveBeenCalledTimes(2); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(JSON.parse(String(logSpy.mock.calls.at(-1)?.[0]))).toMatchObject({ + totalAccounts: 0, accounts: [], fixApplied: false, fixErrors: [], error: null, + nextAction: "Run opencode auth login.", + }); + expect(process.env.CODEX_KEYCHAIN).toBe("1"); + }); + + it("doctor: reports malformed explicit JSON without attempting repair", async () => { + // Given an explicitly selected file that cannot be parsed as JSON. + vi.resetModules(); + tempHome = await createTempHome(); + const poolPath = join(tempHome, "malformed-pool.json"); + await writeFile(poolPath, "{", "utf-8"); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + const repairDoctorAccounts = vi.fn().mockResolvedValue({ appliedFixes: [], fixErrors: [] }); + const loadDoctorRuntime = vi.fn(async () => [ + { setStoragePathDirect: vi.fn(), loadAccounts: async () => null }, + { repairDoctorAccounts }, + { setShutdownOwnsProcess: vi.fn() }, + ]); + + // When repair is requested for the malformed file. + const result = await runInstaller(["doctor", "--fix", "--json", "--config-path", poolPath], { + loadDoctorRuntime, + }); + + // Then parsing fails before runtime discovery or repair can run. + expect(result.exitCode).toBe(1); + expect(loadDoctorRuntime).not.toHaveBeenCalled(); + expect(repairDoctorAccounts).not.toHaveBeenCalled(); + expect(JSON.parse(String(logSpy.mock.calls.at(-1)?.[0]))).toMatchObject({ + error: expect.any(String), message: "Storage could not be parsed.", fixApplied: false, fixErrors: [], + }); + }); + it.each(["discovery", "repair", "snapshot"])("doctor: redacts runtime %s failures without a JSON pool", async (stage) => { // Given an injected backend that fails at one repair boundary. vi.resetModules(); From 73e5fe0d5a10f98c2b6c1ee9b716f86e48aec46a Mon Sep 17 00:00:00 2001 From: wargloom Date: Thu, 10 Sep 2026 23:49:09 +0300 Subject: [PATCH 07/22] feat(accounts): add account-wide quota-exhaustion state --- lib/accounts/rate-limits.ts | 37 +++++++++++++++++++++++++++++++++++++ lib/schemas.ts | 1 + lib/storage/migrations.ts | 6 ++++++ 3 files changed, 44 insertions(+) diff --git a/lib/accounts/rate-limits.ts b/lib/accounts/rate-limits.ts index 9cad2145..39547383 100644 --- a/lib/accounts/rate-limits.ts +++ b/lib/accounts/rate-limits.ts @@ -53,6 +53,43 @@ export function clearExpiredRateLimits(entity: RateLimitedEntity): void { } } +export interface QuotaExhaustibleEntity { + /** Ms epoch until which this account's shared subscription quota is spent. */ + quotaExhaustedUntil?: number; +} + +/** + * Whether an account's shared subscription quota is currently spent. + * + * This is an ACCOUNT-WIDE block sourced from the `/wham/usage` + * primary/secondary window, deliberately distinct from the per-family / + * per-model transient blocks tracked in {@link RateLimitState}. Read sites + * report it separately so a 30-second 429 is never conflated with a week-long + * subscription-quota exhaustion. + */ +export function isQuotaExhausted( + entity: QuotaExhaustibleEntity, + now: number = nowMs(), +): boolean { + const until = entity.quotaExhaustedUntil; + return typeof until === "number" && Number.isFinite(until) && now < until; +} + +/** + * Drop an elapsed (or non-finite) quota-exhaustion stamp so it does not leak + * into snapshots or persistence, mirroring {@link clearExpiredRateLimits} for + * the per-family map. + */ +export function clearExpiredQuotaExhaustion( + entity: QuotaExhaustibleEntity, + now: number = nowMs(), +): void { + const until = entity.quotaExhaustedUntil; + if (until !== undefined && (!Number.isFinite(until) || now >= until)) { + delete entity.quotaExhaustedUntil; + } +} + export function isRateLimitedForQuotaKey(entity: RateLimitedEntity, key: QuotaKey): boolean { const resetTime = entity.rateLimitResetTimes[key]; return resetTime !== undefined && nowMs() < resetTime; diff --git a/lib/schemas.ts b/lib/schemas.ts index d2b40fbd..ecff3db9 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -146,6 +146,7 @@ export const AccountMetadataV3Schema = z.object({ lastSwitchReason: SwitchReasonSchema.optional(), rateLimitResetTimes: RateLimitStateV3Schema.optional(), coolingDownUntil: z.number().optional(), + quotaExhaustedUntil: z.number().optional(), cooldownReason: CooldownReasonSchema.optional(), }); diff --git a/lib/storage/migrations.ts b/lib/storage/migrations.ts index 1a25868b..e0eac4be 100644 --- a/lib/storage/migrations.ts +++ b/lib/storage/migrations.ts @@ -134,6 +134,12 @@ export interface AccountMetadataV3 { lastSwitchReason?: "rate-limit" | "initial" | "rotation"; rateLimitResetTimes?: RateLimitStateV3; coolingDownUntil?: number; + /** + * Ms epoch until which this account's shared subscription quota (the + * `/wham/usage` primary/secondary window) is spent. Account-wide, distinct + * from the per-family/per-model blocks in `rateLimitResetTimes`. + */ + quotaExhaustedUntil?: number; cooldownReason?: CooldownReason; } From fbbc5c4660121cc44813c7a84d01fec7dda18c28 Mon Sep 17 00:00:00 2001 From: wargloom Date: Thu, 10 Sep 2026 23:49:11 +0300 Subject: [PATCH 08/22] fix(usage): record spent subscription quota once, not per family --- lib/codex-usage.ts | 51 ++++++++++++++++++++++------------------ test/codex-usage.test.ts | 13 +++++----- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/lib/codex-usage.ts b/lib/codex-usage.ts index d967c97b..fb200b8e 100644 --- a/lib/codex-usage.ts +++ b/lib/codex-usage.ts @@ -10,7 +10,6 @@ import { createUsageRequestTimeoutError, } from "./error-sentinels.js"; import { logWarn } from "./logger.js"; -import { MODEL_FAMILIES } from "./prompts/codex.js"; import { isQuotaWindowExhausted, MAX_QUOTA_RESET_HORIZON_MS, @@ -363,11 +362,19 @@ export function getUsageQuotaExhaustedResetAtMs( } /** - * Persist a base block for every model family on stored entries sharing the - * queried usage quota. Rotation tracks each family independently, whereas the - * `/wham/usage` primary/secondary subscription quota is shared by all models. - * Writing every base key ensures the next round-robin selection cannot use a - * different model family to spend an already-exhausted account's Credits. + * Persist the account-wide subscription-quota exhaustion stamp on stored + * entries sharing the queried usage quota. Rotation tracks each model family + * independently in `rateLimitResetTimes`, whereas the `/wham/usage` + * primary/secondary subscription quota is shared by all models — so it is + * recorded ONCE, on the dedicated `quotaExhaustedUntil` field, rather than + * forged into a per-family rate-limit block for every model. Read sites treat + * an active stamp as a blocking condition reported separately from a transient + * 429. + * + * The stored value is kept at its monotonic maximum, with the same validity + * guards as {@link AccountRotation.markQuotaExhausted}: finite, strictly in the + * future, and within {@link MAX_QUOTA_RESET_HORIZON_MS} so an absurd stamp + * cannot strand the account. * * This uses a storage transaction rather than saving the caller's usage * snapshot: usage inspection can refresh a single-use token, while another @@ -380,29 +387,27 @@ export async function persistUsageQuotaExhaustion( const usageKey = getUsageAccountDedupeKey(account); if (!usageKey) return false; + if (!Number.isFinite(resetAtMs)) return false; + const resetAt = Math.floor(resetAtMs); + const now = Date.now(); + if (resetAt <= now) return false; + if (resetAt - now > MAX_QUOTA_RESET_HORIZON_MS) return false; + return withAccountStorageTransaction(async (current, persist) => { if (!current) return false; let changed = false; for (const storedAccount of current.accounts) { if (getUsageAccountDedupeKey(storedAccount) !== usageKey) continue; - const rateLimitResetTimes = { ...(storedAccount.rateLimitResetTimes ?? {}) }; - let accountChanged = false; - for (const family of MODEL_FAMILIES) { - const existingResetAtMs = rateLimitResetTimes[family]; - if ( - typeof existingResetAtMs === "number" && - Number.isFinite(existingResetAtMs) && - existingResetAtMs >= resetAtMs - ) { - continue; - } - rateLimitResetTimes[family] = resetAtMs; - accountChanged = true; - } - if (accountChanged) { - storedAccount.rateLimitResetTimes = rateLimitResetTimes; - changed = true; + const existing = storedAccount.quotaExhaustedUntil; + if ( + typeof existing === "number" && + Number.isFinite(existing) && + existing >= resetAt + ) { + continue; } + storedAccount.quotaExhaustedUntil = resetAt; + changed = true; } if (changed) await persist(current); return changed; diff --git a/test/codex-usage.test.ts b/test/codex-usage.test.ts index db512f75..a6daab18 100644 --- a/test/codex-usage.test.ts +++ b/test/codex-usage.test.ts @@ -18,7 +18,6 @@ import { type UsagePayload, } from "../lib/codex-usage.js"; import { loadAccounts, saveAccounts, type AccountStorageV3 } from "../lib/storage.js"; -import { MODEL_FAMILIES } from "../lib/prompts/codex.js"; import { setStoragePathDirect } from "../lib/storage/state.js"; describe("codex usage helpers", () => { @@ -139,7 +138,7 @@ describe("codex usage helpers", () => { ).toBeUndefined(); }); - it("persists a quota block for every model family without shortening a longer block", async () => { + it("persists an account-wide quota-exhaustion stamp without stamping per-family rate limits", async () => { const directory = await mkdtemp(join(tmpdir(), "usage-quota-persist-")); try { setStoragePathDirect(join(directory, "accounts.json")); @@ -156,11 +155,11 @@ describe("codex usage helpers", () => { expect(await persistUsageQuotaExhaustion(account, resetAtMs - 60_000)).toBe(false); const persisted = await loadAccounts(); - expect(persisted?.accounts[0]?.rateLimitResetTimes).toEqual( - expect.objectContaining( - Object.fromEntries(MODEL_FAMILIES.map((family) => [family, resetAtMs])), - ), - ); + // The account-wide subscription-quota fact lands on its own field, kept + // at the monotonic maximum reset stamp. + expect(persisted?.accounts[0]?.quotaExhaustedUntil).toBe(resetAtMs); + // It no longer forges a per-family rate-limit block for every model. + expect(persisted?.accounts[0]?.rateLimitResetTimes ?? {}).toEqual({}); } finally { setStoragePathDirect(null); await rm(directory, { recursive: true, force: true }); From 079436fdbc9c8cfae6412d82a6aee14f0b72c399 Mon Sep 17 00:00:00 2001 From: wargloom Date: Thu, 10 Sep 2026 23:49:19 +0300 Subject: [PATCH 09/22] fix(rotation): report quota exhaustion apart from rate limits --- lib/accounts/rotation.ts | 14 ++ lib/accounts/state.ts | 15 +++ lib/parallel-probe.ts | 6 +- test/accounts-quota-exhaustion.test.ts | 171 +++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 test/accounts-quota-exhaustion.test.ts diff --git a/lib/accounts/rotation.ts b/lib/accounts/rotation.ts index e4159add..a3fd0553 100644 --- a/lib/accounts/rotation.ts +++ b/lib/accounts/rotation.ts @@ -20,8 +20,10 @@ import { MAX_QUOTA_RESET_HORIZON_MS } from "../quota-windows.js"; import type { CooldownReason } from "../storage.js"; import { nowMs } from "../utils.js"; import { + clearExpiredQuotaExhaustion, clearExpiredRateLimits, getQuotaKey, + isQuotaExhausted, isRateLimitedForFamily, type RateLimitReason, } from "./rate-limits.js"; @@ -49,6 +51,8 @@ export class AccountRotation { ): boolean { if (account.enabled === false) return false; clearExpiredRateLimits(account); + clearExpiredQuotaExhaustion(account); + if (isQuotaExhausted(account)) return false; if (isRateLimitedForFamily(account, family, model)) return false; if (this.state.isAccountCoolingDown(account)) return false; const quotaKey = model ? `${family}:${model}` : family; @@ -559,6 +563,16 @@ export class AccountRotation { waitTimes.push(Math.max(0, account.coolingDownUntil - now)); } + // An account whose shared subscription quota is spent is blocked + // account-wide until the stamp resets; surface that wait so a + // quota-exhausted-only pool waits instead of returning 0 (503). + if ( + typeof account.quotaExhaustedUntil === "number" && + account.quotaExhaustedUntil > now + ) { + waitTimes.push(account.quotaExhaustedUntil - now); + } + // An account blocked only by a depleted local token bucket becomes // available again after refill; include that wait so a fully // token-depleted pool waits for refill instead of returning 0 (503). diff --git a/lib/accounts/state.ts b/lib/accounts/state.ts index b05a4283..6eea1528 100644 --- a/lib/accounts/state.ts +++ b/lib/accounts/state.ts @@ -12,8 +12,10 @@ import type { AccountIdSource, OAuthAuthDetails } from "../types.js"; import { nowMs } from "../utils.js"; import { clampNonNegativeInt, + clearExpiredQuotaExhaustion, clearExpiredRateLimits, getQuotaKey, + isQuotaExhausted, isRateLimitedForFamily, type RateLimitReason, } from "./rate-limits.js"; @@ -54,6 +56,7 @@ export interface ManagedAccount { lastSwitchReason?: "rate-limit" | "initial" | "rotation"; lastRateLimitReason?: RateLimitReason; rateLimitResetTimes: RateLimitStateV3; + quotaExhaustedUntil?: number; coolingDownUntil?: number; cooldownReason?: CooldownReason; } @@ -67,6 +70,7 @@ export interface AccountSelectionExplainability { healthScore: number; tokensAvailable: number; rateLimitedUntil?: number; + quotaExhaustedUntil?: number; coolingDownUntil?: number; cooldownReason?: CooldownReason; lastUsed: number; @@ -385,6 +389,7 @@ export class AccountState { lastUsed: clampNonNegativeInt(account.lastUsed, 0), lastSwitchReason: account.lastSwitchReason, rateLimitResetTimes: account.rateLimitResetTimes ?? {}, + quotaExhaustedUntil: account.quotaExhaustedUntil, coolingDownUntil: account.coolingDownUntil, cooldownReason: account.cooldownReason, }; @@ -527,6 +532,7 @@ export class AccountState { return this.accounts.map((account) => { clearExpiredRateLimits(account); + clearExpiredQuotaExhaustion(account); const enabled = account.enabled !== false; const reasons: string[] = []; let rateLimitedUntil: number | undefined; @@ -550,8 +556,13 @@ export class AccountState { ? account.coolingDownUntil : undefined; + const quotaExhaustedUntil = isQuotaExhausted(account, now) + ? account.quotaExhaustedUntil + : undefined; + if (!enabled) reasons.push("disabled"); if (rateLimitedUntil !== undefined) reasons.push("rate-limited"); + if (quotaExhaustedUntil !== undefined) reasons.push("quota-exhausted"); if (coolingDownUntil !== undefined) { reasons.push( account.cooldownReason ? `cooldown:${account.cooldownReason}` : "cooldown", @@ -564,6 +575,7 @@ export class AccountState { const eligible = enabled && rateLimitedUntil === undefined && + quotaExhaustedUntil === undefined && coolingDownUntil === undefined && tokensAvailable >= 1; if (reasons.length === 0) reasons.push("eligible"); @@ -577,6 +589,7 @@ export class AccountState { healthScore: healthTracker.getScore(account.index, quotaKey), tokensAvailable, rateLimitedUntil, + quotaExhaustedUntil, coolingDownUntil, cooldownReason: coolingDownUntil !== undefined ? account.cooldownReason : undefined, lastUsed: account.lastUsed, @@ -834,6 +847,8 @@ export class AccountState { ): boolean { if (account.enabled === false) return false; clearExpiredRateLimits(account); + clearExpiredQuotaExhaustion(account); + if (isQuotaExhausted(account)) return false; if (isRateLimitedForFamily(account, family, model)) return false; if (this.isAccountCoolingDown(account)) return false; return true; diff --git a/lib/parallel-probe.ts b/lib/parallel-probe.ts index a939ec31..ebbf7abf 100644 --- a/lib/parallel-probe.ts +++ b/lib/parallel-probe.ts @@ -6,7 +6,7 @@ import { getTokenTracker, type AccountWithMetrics, } from "./rotation.js"; -import { clearExpiredRateLimits, isRateLimitedForFamily } from "./accounts/rate-limits.js"; +import { clearExpiredQuotaExhaustion, clearExpiredRateLimits, isQuotaExhausted, isRateLimitedForFamily } from "./accounts/rate-limits.js"; const log = createLogger("parallel-probe"); @@ -48,9 +48,11 @@ export function getTopCandidates( for (const account of accounts) { clearExpiredRateLimits(account); + clearExpiredQuotaExhaustion(account); const isRateLimited = isRateLimitedForFamily(account, modelFamily, model); const isCoolingDown = account.coolingDownUntil !== undefined && account.coolingDownUntil > Date.now(); - const isAvailable = !isRateLimited && !isCoolingDown; + const isQuotaBlocked = isQuotaExhausted(account); + const isAvailable = !isRateLimited && !isCoolingDown && !isQuotaBlocked; accountsWithMetrics.push({ index: account.index, diff --git a/test/accounts-quota-exhaustion.test.ts b/test/accounts-quota-exhaustion.test.ts new file mode 100644 index 00000000..58935f43 --- /dev/null +++ b/test/accounts-quota-exhaustion.test.ts @@ -0,0 +1,171 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { AccountManager } from "../lib/accounts.js"; +import { resetTrackers } from "../lib/rotation.js"; +import { MODEL_FAMILIES } from "../lib/prompts/codex.js"; + +// Storage writes are irrelevant to these in-memory rotation/eligibility checks; +// stub the persistence surface so no real accounts file is touched. +vi.mock("../lib/storage.js", async (importOriginal) => { + const actual = await importOriginal(); + const saveAccounts = vi.fn().mockResolvedValue(undefined); + return { + ...actual, + saveAccounts, + loadAccounts: vi.fn().mockResolvedValue(null), + withAccountStorageTransaction: vi.fn( + async ( + handler: ( + current: null, + persist: (storage: unknown) => Promise, + ) => Promise, + ) => handler(null, saveAccounts as (storage: unknown) => Promise), + ), + }; +}); + +/** + * Regressions for the account-wide `quotaExhaustedUntil` state, which must be + * kept distinct from the per-family/per-model transient `rateLimitResetTimes` + * map. See the root-cause note on `persistUsageQuotaExhaustion`. + */ +describe("account-wide quota exhaustion state", () => { + beforeEach(() => { + resetTrackers(); + }); + + afterEach(() => { + vi.useRealTimers(); + resetTrackers(); + }); + + // (b) + it("blocks selection for every family and reports a quota-exhausted reason distinct from rate-limited", () => { + const now = Date.now(); + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { + refreshToken: "token-1", + accountId: "acct-1", + addedAt: now, + lastUsed: now, + quotaExhaustedUntil: now + 7 * 24 * 60 * 60 * 1000, + }, + ], + }); + + // Not selectable for any model family. + for (const family of MODEL_FAMILIES) { + expect(manager.getCurrentOrNextForFamily(family), family).toBeNull(); + } + + const explain = manager.getSelectionExplainability("codex", null, now); + expect(explain[0]?.eligible).toBe(false); + expect(explain[0]?.reasons).toContain("quota-exhausted"); + expect(explain[0]?.reasons).not.toContain("rate-limited"); + }); + + // (c) + it("does not set quotaExhaustedUntil on a transient 429 and leaves other families usable", () => { + const now = Date.now(); + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { refreshToken: "token-1", accountId: "acct-1", addedAt: now, lastUsed: now }, + ], + }); + const account = manager.getCurrentAccount()!; + + manager.markRateLimitedWithReason(account, 60_000, "codex", "quota"); + + // A transient rate limit is per-family, never the account-wide field. + expect(account.quotaExhaustedUntil).toBeUndefined(); + expect(account.rateLimitResetTimes["codex"]).toBeDefined(); + + // A different family is unaffected by the codex 429. + expect(manager.getCurrentOrNextForFamily("gpt-5.1")?.accountId).toBe("acct-1"); + }); + + // (d) + it("getMinWaitTimeForFamily returns the quota-exhaustion wait when it is the only block", () => { + const now = Date.now(); + const wait = 6 * 60 * 60 * 1000; + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { + refreshToken: "token-1", + accountId: "acct-1", + addedAt: now, + lastUsed: now, + quotaExhaustedUntil: now + wait, + }, + ], + }); + + // Whole pool is blocked, so selection fails. + expect(manager.getCurrentOrNextForFamily("codex")).toBeNull(); + + // The wait must reflect the quota reset, not 0 (which upstream turns into + // a 503 instead of a retryable 429 with a hint). + const minWait = manager.getMinWaitTimeForFamily("codex"); + expect(minWait).toBeGreaterThan(0); + expect(minWait).toBeLessThanOrEqual(wait); + }); + + // (e) + it("ignores and clears an expired quotaExhaustedUntil", () => { + const now = Date.now(); + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { + refreshToken: "token-1", + accountId: "acct-1", + addedAt: now, + lastUsed: now, + quotaExhaustedUntil: now - 60_000, + }, + ], + }); + const account = manager.getCurrentAccount()!; + + // An elapsed quota block does not keep the account out of rotation. + expect(manager.getCurrentOrNextForFamily("codex")?.accountId).toBe("acct-1"); + expect(account.quotaExhaustedUntil).toBeUndefined(); + + const explain = manager.getSelectionExplainability("codex", null, now); + expect(explain[0]?.reasons).not.toContain("quota-exhausted"); + }); + + // (h) + it("keeps blocking a legacy record with every family stamped and no quotaExhaustedUntil", () => { + const now = Date.now(); + const resetAt = now + 6 * 24 * 60 * 60 * 1000; + const rateLimitResetTimes: Record = {}; + for (const family of MODEL_FAMILIES) rateLimitResetTimes[family] = resetAt; + + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { + refreshToken: "token-1", + accountId: "acct-1", + addedAt: now, + lastUsed: now, + rateLimitResetTimes, + }, + ], + }); + + // Legacy blanket stamp still blocks via the per-family map, exactly as before. + expect(manager.getCurrentOrNextForFamily("codex")).toBeNull(); + const explain = manager.getSelectionExplainability("codex", null, now); + expect(explain[0]?.reasons).toContain("rate-limited"); + }); +}); From d2657d3f821db253afd84eb4b4752a515061f96b Mon Sep 17 00:00:00 2001 From: wargloom Date: Thu, 10 Sep 2026 23:49:31 +0300 Subject: [PATCH 10/22] fix(storage): merge quota exhaustion monotonically across processes --- lib/accounts/persistence.ts | 26 ++++++++++++++++++++++++++ lib/auth/login-runner.ts | 16 ++++++++++++++++ lib/storage/flagged.ts | 2 ++ test/credential-clobber.test.ts | 19 +++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/lib/accounts/persistence.ts b/lib/accounts/persistence.ts index 08c26d09..ce947a62 100644 --- a/lib/accounts/persistence.ts +++ b/lib/accounts/persistence.ts @@ -78,6 +78,7 @@ export class AccountPersistence { Object.keys(account.rateLimitResetTimes).length > 0 ? { ...account.rateLimitResetTimes } : undefined, + quotaExhaustedUntil: account.quotaExhaustedUntil, coolingDownUntil: account.coolingDownUntil, cooldownReason: account.cooldownReason, })), @@ -167,6 +168,16 @@ export class AccountPersistence { merged.cooldownReason = mine.cooldownReason; } + // Account-wide quota exhaustion is monotonic like the per-family + // blocks: keep whichever side's stamp runs longer. + if ( + typeof mine.quotaExhaustedUntil === "number" && + mine.quotaExhaustedUntil > now && + mine.quotaExhaustedUntil > (record.quotaExhaustedUntil ?? 0) + ) { + merged.quotaExhaustedUntil = mine.quotaExhaustedUntil; + } + if (typeof mine.lastUsed === "number" && mine.lastUsed > (record.lastUsed ?? 0)) { merged.lastUsed = mine.lastUsed; merged.lastSwitchReason = mine.lastSwitchReason ?? record.lastSwitchReason; @@ -210,6 +221,21 @@ export class AccountPersistence { for (const mine of outgoing.accounts) { if (!mine) continue; const theirs = diskByIdentity.get(getWorkspaceIdentityKey(mine)); + + // Adopt a longer on-disk quota-exhaustion stamp for the same reason the + // per-family blocks below are adopted (#218): a second process may have + // recorded a week-long block after this one loaded. Handled before the + // `theirResets` guard so a disk record carrying only a quota stamp is + // not skipped. + const theirQuota = theirs?.quotaExhaustedUntil; + if ( + typeof theirQuota === "number" && + Number.isFinite(theirQuota) && + theirQuota > now && + theirQuota > (mine.quotaExhaustedUntil ?? 0) + ) { + mine.quotaExhaustedUntil = theirQuota; + } const theirResets = theirs?.rateLimitResetTimes; if (!theirResets) continue; diff --git a/lib/auth/login-runner.ts b/lib/auth/login-runner.ts index a2f5e663..73712861 100644 --- a/lib/auth/login-runner.ts +++ b/lib/auth/login-runner.ts @@ -43,6 +43,7 @@ type MergeableAccountRecord = { lastSwitchReason?: string; rateLimitResetTimes?: Record; coolingDownUntil?: number; + quotaExhaustedUntil?: number; cooldownReason?: string; tokenRotatedAt?: number; }; @@ -107,6 +108,20 @@ export function mergeStoredAccountPair( ); const mergedCoolingDownUntil = mergedCoolingDownUntilValue > 0 ? mergedCoolingDownUntilValue : undefined; + const targetQuotaExhaustedUntil = + typeof target.quotaExhaustedUntil === "number" && Number.isFinite(target.quotaExhaustedUntil) + ? target.quotaExhaustedUntil + : 0; + const sourceQuotaExhaustedUntil = + typeof source.quotaExhaustedUntil === "number" && Number.isFinite(source.quotaExhaustedUntil) + ? source.quotaExhaustedUntil + : 0; + const mergedQuotaExhaustedUntilValue = Math.max( + targetQuotaExhaustedUntil, + sourceQuotaExhaustedUntil, + ); + const mergedQuotaExhaustedUntil = + mergedQuotaExhaustedUntilValue > 0 ? mergedQuotaExhaustedUntilValue : undefined; const mergedCooldownReason = (() => { if (mergedCoolingDownUntilValue <= 0) { return target.cooldownReason ?? source.cooldownReason; @@ -152,6 +167,7 @@ export function mergeStoredAccountPair( lastSwitchReason: target.lastSwitchReason ?? source.lastSwitchReason, rateLimitResetTimes: mergedRateLimitResetTimes, coolingDownUntil: mergedCoolingDownUntil, + quotaExhaustedUntil: mergedQuotaExhaustedUntil, cooldownReason: mergedCooldownReason, }; } diff --git a/lib/storage/flagged.ts b/lib/storage/flagged.ts index 67ff604a..a595c00c 100644 --- a/lib/storage/flagged.ts +++ b/lib/storage/flagged.ts @@ -142,6 +142,8 @@ function normalizeFlaggedStorage(data: unknown): FlaggedAccountStorageV1 { rateLimitResetTimes, coolingDownUntil: typeof rawAccount.coolingDownUntil === "number" ? rawAccount.coolingDownUntil : undefined, + quotaExhaustedUntil: + typeof rawAccount.quotaExhaustedUntil === "number" ? rawAccount.quotaExhaustedUntil : undefined, cooldownReason, flaggedAt, flaggedReason: typeof rawAccount.flaggedReason === "string" ? rawAccount.flaggedReason : undefined, diff --git a/test/credential-clobber.test.ts b/test/credential-clobber.test.ts index d0ed5319..be0a3787 100644 --- a/test/credential-clobber.test.ts +++ b/test/credential-clobber.test.ts @@ -186,6 +186,25 @@ describe("AccountPersistence rate-limit merge (multi-process clobber guard)", () ).toBeUndefined(); }); + it("keeps a longer on-disk quota-exhaustion stamp instead of clobbering it", async () => { + // A shorter in-memory quota block must not pull a longer on-disk one + // forward, mirroring the per-family rate-limit monotonic merge (#218). + const state = makeState([ + makeStoredAccount({ quotaExhaustedUntil: Date.now() + 30_000 }), + ]); + const persistence = new AccountPersistence(state); + + diskStateRef.current = { + version: 3, + accounts: [makeStoredAccount({ quotaExhaustedUntil: WEEKLY_RESET })], + activeIndex: 0, + } satisfies AccountStorageV3; + + await persistence.saveToDisk(); + + expect(persistedStorage()?.accounts[0]?.quotaExhaustedUntil).toBe(WEEKLY_RESET); + }); + // Documents a known limitation rather than desired behavior. A record with // neither organizationId nor accountId is identified by its refresh token // (lib/storage/identity.ts), so once another process rotates that token From a0152f14d20a501659cd3d43c70034546c6eab08 Mon Sep 17 00:00:00 2001 From: wargloom Date: Thu, 10 Sep 2026 23:49:33 +0300 Subject: [PATCH 11/22] fix(doctor): clear stale quota exhaustion on verified accounts --- lib/accounts/stale-state.ts | 22 ++++++++++++++++++++-- lib/tools/doctor-repair.ts | 8 +++++++- test/stale-state.test.ts | 18 +++++++++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/lib/accounts/stale-state.ts b/lib/accounts/stale-state.ts index 1de84769..09be0cf9 100644 --- a/lib/accounts/stale-state.ts +++ b/lib/accounts/stale-state.ts @@ -33,6 +33,7 @@ export interface StaleStateAccount { coolingDownUntil?: number; cooldownReason?: string; rateLimitResetTimes?: Record; + quotaExhaustedUntil?: number; } export interface ClearedStaleState { @@ -40,6 +41,8 @@ export interface ClearedStaleState { clearedCooldown: boolean; /** Number of rate-limit reset entries removed. */ clearedRateLimitKeys: number; + /** True when an active account-wide quota-exhaustion stamp was cleared. */ + clearedQuotaExhaustion: boolean; } /** @@ -67,6 +70,12 @@ export function clearRefreshedAccountStaleState( delete account.cooldownReason; } + const hadActiveQuotaExhaustion = + typeof account.quotaExhaustedUntil === "number" && account.quotaExhaustedUntil > now; + if (account.quotaExhaustedUntil !== undefined) { + delete account.quotaExhaustedUntil; + } + let clearedRateLimitKeys = 0; if (account.rateLimitResetTimes) { clearedRateLimitKeys = Object.keys(account.rateLimitResetTimes).length; @@ -78,6 +87,7 @@ export function clearRefreshedAccountStaleState( return { clearedCooldown: hadActiveCooldown, clearedRateLimitKeys, + clearedQuotaExhaustion: hadActiveQuotaExhaustion, }; } @@ -86,6 +96,8 @@ export interface StaleStateRepairSummary { cooldownsCleared: number; /** Total rate-limit reset entries removed across all accounts. */ rateLimitKeysCleared: number; + /** Accounts that had an active quota-exhaustion stamp cleared. */ + quotaExhaustionsCleared: number; } /** @@ -98,12 +110,14 @@ export function clearRefreshedAccountsStaleState( ): StaleStateRepairSummary { let cooldownsCleared = 0; let rateLimitKeysCleared = 0; + let quotaExhaustionsCleared = 0; for (const account of accounts) { const cleared = clearRefreshedAccountStaleState(account); if (cleared.clearedCooldown) cooldownsCleared += 1; rateLimitKeysCleared += cleared.clearedRateLimitKeys; + if (cleared.clearedQuotaExhaustion) quotaExhaustionsCleared += 1; } - return { cooldownsCleared, rateLimitKeysCleared }; + return { cooldownsCleared, rateLimitKeysCleared, quotaExhaustionsCleared }; } /** @@ -224,6 +238,7 @@ export interface StaleStateScanAccount { coolingDownUntil?: number; cooldownReason?: string; rateLimitResetTimes?: Record; + quotaExhaustedUntil?: number; } /** @@ -252,6 +267,9 @@ export function findStaleRecoverableAccounts( const hasFutureCooldown = typeof account.coolingDownUntil === "number" && account.coolingDownUntil > now; + const hasFutureQuotaExhaustion = + typeof account.quotaExhaustedUntil === "number" && account.quotaExhaustedUntil > now; + let hasFutureRateLimit = false; if (account.rateLimitResetTimes) { for (const reset of Object.values(account.rateLimitResetTimes)) { @@ -262,7 +280,7 @@ export function findStaleRecoverableAccounts( } } - if (hasFutureCooldown || hasFutureRateLimit) { + if (hasFutureCooldown || hasFutureRateLimit || hasFutureQuotaExhaustion) { blocked.push(i); } } diff --git a/lib/tools/doctor-repair.ts b/lib/tools/doctor-repair.ts index 40238ef1..87d1ce1b 100644 --- a/lib/tools/doctor-repair.ts +++ b/lib/tools/doctor-repair.ts @@ -10,7 +10,7 @@ import { export async function repairDoctorAccounts(accounts: AccountMetadataV3[]) { const refreshedAccounts: { readonly identity: RefreshAccountIdentity; - readonly staleState: Pick; + readonly staleState: Pick; }[] = []; const verificationFailureIdentities: RefreshAccountIdentity[] = []; const reloginNeeded: number[] = []; @@ -24,6 +24,7 @@ export async function repairDoctorAccounts(accounts: AccountMetadataV3[]) { coolingDownUntil: account.coolingDownUntil, cooldownReason: account.cooldownReason, rateLimitResetTimes: { ...account.rateLimitResetTimes }, + quotaExhaustedUntil: account.quotaExhaustedUntil, }; const outcome = await refreshAndPersistAccount(input); switch (outcome.status) { @@ -61,6 +62,7 @@ export async function repairDoctorAccounts(accounts: AccountMetadataV3[]) { // A concurrent health update is newer evidence than this repair's snapshot. const stateUnchanged = record.coolingDownUntil === staleState.coolingDownUntil && record.cooldownReason === staleState.cooldownReason && + record.quotaExhaustedUntil === staleState.quotaExhaustedUntil && Object.keys({ ...record.rateLimitResetTimes, ...staleState.rateLimitResetTimes }).every( (key) => record.rateLimitResetTimes?.[key] === staleState.rateLimitResetTimes?.[key], ); @@ -68,6 +70,7 @@ export async function repairDoctorAccounts(accounts: AccountMetadataV3[]) { } const hasStaleState = refreshedRecords.some((record) => record.coolingDownUntil !== undefined || record.cooldownReason !== undefined || + record.quotaExhaustedUntil !== undefined || Object.keys(record.rateLimitResetTimes ?? {}).length > 0, ); const summary = clearRefreshedAccountsStaleState(refreshedRecords); @@ -80,6 +83,9 @@ export async function repairDoctorAccounts(accounts: AccountMetadataV3[]) { if (staleSummary.rateLimitKeysCleared > 0) { appliedFixes.push(`Cleared ${staleSummary.rateLimitKeysCleared} stale rate-limit marker(s).`); } + if (staleSummary.quotaExhaustionsCleared > 0) { + appliedFixes.push(`Cleared quota-exhaustion state on ${staleSummary.quotaExhaustionsCleared} recovered account(s).`); + } } catch { fixErrors.push("Failed to persist stale-state repairs."); } diff --git a/test/stale-state.test.ts b/test/stale-state.test.ts index f9460fe7..a02afee2 100644 --- a/test/stale-state.test.ts +++ b/test/stale-state.test.ts @@ -59,10 +59,21 @@ describe("clearRefreshedAccountStaleState", () => { it("is a no-op for a clean account", () => { const account: StaleStateAccount = {}; const result = clearRefreshedAccountStaleState(account); - expect(result).toEqual({ clearedCooldown: false, clearedRateLimitKeys: 0 }); + expect(result).toEqual({ clearedCooldown: false, clearedRateLimitKeys: 0, clearedQuotaExhaustion: false }); expect(account).toEqual({}); }); + it("clears an active account-wide quota-exhaustion stamp and counts it", () => { + const account: StaleStateAccount = { + quotaExhaustedUntil: Date.now() + 7 * 24 * 60 * 60 * 1000, + }; + + const result = clearRefreshedAccountStaleState(account); + + expect(result.clearedQuotaExhaustion).toBe(true); + expect(account.quotaExhaustedUntil).toBeUndefined(); + }); + it("aggregates across multiple accounts", () => { const accounts: StaleStateAccount[] = [ { coolingDownUntil: Date.now() + 600_000, cooldownReason: "auth-failure" }, @@ -194,6 +205,11 @@ describe("findStaleRecoverableAccounts", () => { expect(findStaleRecoverableAccounts(accounts, NOW)).toEqual([0]); }); + it("flags an account blocked only by a future quota-exhaustion stamp", () => { + const accounts = [{ enabled: true, quotaExhaustedUntil: FUTURE }]; + expect(findStaleRecoverableAccounts(accounts, NOW)).toEqual([0]); + }); + it("ignores expired cooldown/rate-limit (the request path clears those)", () => { const accounts = [ { enabled: true, coolingDownUntil: PAST, cooldownReason: "auth-failure" }, From da0d31bc95c129fa2dbb91e73180b85ccb602963 Mon Sep 17 00:00:00 2001 From: wargloom Date: Thu, 10 Sep 2026 23:49:36 +0300 Subject: [PATCH 12/22] fix(tui): label quota exhaustion instead of calling it a rate limit --- index.ts | 26 +++++++++++++++++++++ lib/tools/codex-list.ts | 15 ++++++++++++ lib/tools/codex-status.ts | 9 +++++++ lib/tools/index.ts | 8 +++++++ scripts/install-oc-codex-multi-auth-core.js | 2 ++ test/index.test.ts | 10 ++++---- test/standalone-cli.test.ts | 8 +++---- test/tools-codex-list.test.ts | 2 ++ 8 files changed, 72 insertions(+), 8 deletions(-) diff --git a/index.ts b/index.ts index 51e8d71e..16477a09 100644 --- a/index.ts +++ b/index.ts @@ -1189,6 +1189,30 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { return `resets in ${formatWaitTime(remaining)}`; }; + // Account-wide subscription-quota exhaustion is a DIFFERENT block from the + // per-family rate limits above: it lives on its own field and is reported + // with its own label so a spent weekly quota is never shown as a transient + // 429 ("rate limit"). + const getQuotaExhaustedUntil = ( + account: { quotaExhaustedUntil?: number }, + now: number, + ): number | null => { + const until = account.quotaExhaustedUntil; + if (typeof until !== "number" || !Number.isFinite(until) || until <= now) { + return null; + } + return until; + }; + + const formatQuotaExhaustionEntry = ( + account: { quotaExhaustedUntil?: number }, + now: number, + ): string | null => { + const until = getQuotaExhaustedUntil(account, now); + if (until === null) return null; + return `quota exhausted, resets in ${formatWaitTime(until - now)}`; + }; + const applyUiRuntimeFromConfig = ( pluginConfig: ReturnType, ): UiRuntimeOptions => { @@ -1811,6 +1835,8 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { resolveActiveIndex, getRateLimitResetTimeForFamily, formatRateLimitEntry, + getQuotaExhaustedUntil, + formatQuotaExhaustionEntry, buildJsonAccountIdentity, buildRoutingVisibilitySnapshot, appendRoutingVisibilityText, diff --git a/lib/tools/codex-list.ts b/lib/tools/codex-list.ts index ee559b91..a7c67168 100644 --- a/lib/tools/codex-list.ts +++ b/lib/tools/codex-list.ts @@ -26,6 +26,7 @@ export function createCodexListTool(ctx: ToolContext): ToolDefinition { formatCommandAccountLabel, resolveMaskEmail, formatRateLimitEntry, + formatQuotaExhaustionEntry, buildJsonAccountIdentity, } = ctx; return tool({ @@ -165,11 +166,13 @@ export function createCodexListTool(ctx: ToolContext): ToolDefinition { storagePath: storePath, accounts: filteredEntries.map(({ account, index }) => { const rateLimit = formatRateLimitEntry(account, now); + const quotaExhausted = formatQuotaExhaustionEntry(account, now); const cooldown = formatCooldown(account, now); const statuses: string[] = []; if (index === activeIndex) statuses.push("active"); if (account.enabled === false) statuses.push("disabled"); if (rateLimit) statuses.push("rate-limited"); + if (quotaExhausted) statuses.push("quota-exhausted"); if (cooldown) statuses.push("cooldown"); if (statuses.length === 0) statuses.push("ok"); return { @@ -182,6 +185,7 @@ export function createCodexListTool(ctx: ToolContext): ToolDefinition { planType: account.planType ?? null, plan: formatPlanType(account.planType) ?? null, rateLimit: rateLimit ?? null, + quotaExhausted: quotaExhausted ?? null, cooldown: cooldown ?? null, tags: Array.isArray(account.accountTags) ? [...account.accountTags] @@ -213,8 +217,12 @@ export function createCodexListTool(ctx: ToolContext): ToolDefinition { if (account.enabled === false) badges.push(formatUiBadge(ui, "disabled", "danger")); const rateLimit = formatRateLimitEntry(account, now); + const quotaExhausted = formatQuotaExhaustionEntry(account, now); if (rateLimit) badges.push(formatUiBadge(ui, "rate-limited", "warning")); + if (quotaExhausted) + badges.push(formatUiBadge(ui, "quota-exhausted", "warning")); + badges.push(formatUiBadge(ui, "rate-limited", "warning")); if ( typeof account.coolingDownUntil === "number" && account.coolingDownUntil > now @@ -235,6 +243,11 @@ export function createCodexListTool(ctx: ToolContext): ToolDefinition { ` ${paintUiText(ui, `rate limit: ${rateLimit}`, "muted")}`, ); } + if (quotaExhausted) { + lines.push( + ` ${paintUiText(ui, `quota: ${quotaExhausted}`, "muted")}`, + ); + } }); lines.push(""); @@ -293,8 +306,10 @@ export function createCodexListTool(ctx: ToolContext): ToolDefinition { const label = formatCommandAccountLabel(account, index, { maskEmail }); const statuses: string[] = []; const rateLimit = formatRateLimitEntry(account, now); + const quotaExhausted = formatQuotaExhaustionEntry(account, now); if (index === activeIndex) statuses.push("active"); if (rateLimit) statuses.push("rate-limited"); + if (quotaExhausted) statuses.push("quota-exhausted"); if ( typeof account.coolingDownUntil === "number" && account.coolingDownUntil > now diff --git a/lib/tools/codex-status.ts b/lib/tools/codex-status.ts index 4dd61c60..fb0adae0 100644 --- a/lib/tools/codex-status.ts +++ b/lib/tools/codex-status.ts @@ -31,6 +31,7 @@ export function createCodexStatusTool(ctx: ToolContext): ToolDefinition { formatCommandAccountLabel, resolveMaskEmail, formatRateLimitEntry, + formatQuotaExhaustionEntry, getRateLimitResetTimeForFamily, buildJsonAccountIdentity, buildRoutingVisibilitySnapshot, @@ -146,6 +147,7 @@ export function createCodexStatusTool(ctx: ToolContext): ToolDefinition { planType: account.planType ?? null, plan: formatPlanType(account.planType) ?? null, rateLimit: formatRateLimitEntry(account, now) ?? null, + quotaExhausted: formatQuotaExhaustionEntry(account, now) ?? null, cooldown: formatCooldown(account, now) ?? null, lastUsedAgeMs: typeof account.lastUsed === "number" && account.lastUsed > 0 @@ -212,9 +214,13 @@ export function createCodexStatusTool(ctx: ToolContext): ToolDefinition { if (account.enabled === false) badges.push(formatUiBadge(ui, "disabled", "danger")); const rateLimit = formatRateLimitEntry(account, now) ?? "none"; + const quotaExhausted = formatQuotaExhaustionEntry(account, now) ?? "none"; const cooldown = formatCooldown(account, now) ?? "none"; if (rateLimit !== "none") badges.push(formatUiBadge(ui, "rate-limited", "warning")); + if (quotaExhausted !== "none") + badges.push(formatUiBadge(ui, "quota-exhausted", "warning")); + badges.push(formatUiBadge(ui, "rate-limited", "warning")); if (cooldown !== "none") badges.push(formatUiBadge(ui, "cooldown", "warning")); if (badges.length === 0) @@ -228,6 +234,9 @@ export function createCodexStatusTool(ctx: ToolContext): ToolDefinition { lines.push( ` ${formatUiKeyValue(ui, "rate limit", rateLimit, rateLimit === "none" ? "muted" : "warning")}`, ); + lines.push( + ` ${formatUiKeyValue(ui, "quota", quotaExhausted, quotaExhausted === "none" ? "muted" : "warning")}`, + ); lines.push( ` ${formatUiKeyValue(ui, "cooldown", cooldown, cooldown === "none" ? "muted" : "warning")}`, ); diff --git a/lib/tools/index.ts b/lib/tools/index.ts index 3c9066b4..99418e95 100644 --- a/lib/tools/index.ts +++ b/lib/tools/index.ts @@ -143,6 +143,14 @@ export interface ToolContext { now: number, family?: ModelFamily, ) => string | null; + getQuotaExhaustedUntil: ( + account: { quotaExhaustedUntil?: number }, + now: number, + ) => number | null; + formatQuotaExhaustionEntry: ( + account: { quotaExhaustedUntil?: number }, + now: number, + ) => string | null; buildJsonAccountIdentity: ( index: number, options?: { diff --git a/scripts/install-oc-codex-multi-auth-core.js b/scripts/install-oc-codex-multi-auth-core.js index 28be7ffa..2bfdf7e9 100644 --- a/scripts/install-oc-codex-multi-auth-core.js +++ b/scripts/install-oc-codex-multi-auth-core.js @@ -417,6 +417,7 @@ function summarizeStandaloneAccounts(storage, includeSensitive, tag) { tags: Array.isArray(account?.accountTags) ? account.accountTags : [], note: account?.accountNote, rateLimitResetTimes: account?.rateLimitResetTimes ?? {}, + quotaExhaustedUntil: account?.quotaExhaustedUntil, }; }); } @@ -660,6 +661,7 @@ export async function runLimitsCommand(parsed, options = {}) { label: account.accountLabel ?? `Account ${index + 1}`, email: maskValue(account.email, parsed.includeSensitive), rateLimitResetTimes: account.rateLimitResetTimes ?? {}, + quotaExhaustedUntil: account.quotaExhaustedUntil, }; try { const { accessToken } = await usageMod.ensureCodexUsageAccessToken({ storage, account }); diff --git a/test/index.test.ts b/test/index.test.ts index 9805e96f..15ee0e79 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -314,6 +314,7 @@ const mockStorage = { coolingDownUntil?: number; cooldownReason?: string; rateLimitResetTimes?: Record; + quotaExhaustedUntil?: number; lastSwitchReason?: string; }>, activeIndex: 0, @@ -1783,10 +1784,11 @@ describe("OpenAIOAuthPlugin", () => { await plugin.tool["codex-limits"].execute(); - expect(mockStorage.accounts[0]?.rateLimitResetTimes).toMatchObject({ - codex: weeklyResetAt * 1000, - "gpt-5.1": weeklyResetAt * 1000, - }); + // The account-wide subscription-quota fact now lands on its own field + // instead of being forged into a per-family rate-limit block for every + // model, so a spent weekly quota is not mislabeled as a transient 429. + expect(mockStorage.accounts[0]?.quotaExhaustedUntil).toBe(weeklyResetAt * 1000); + expect(mockStorage.accounts[0]?.rateLimitResetTimes ?? {}).toEqual({}); }); it("returns json output for usage windows", async () => { diff --git a/test/standalone-cli.test.ts b/test/standalone-cli.test.ts index bfca3ff5..2643aa1d 100644 --- a/test/standalone-cli.test.ts +++ b/test/standalone-cli.test.ts @@ -760,10 +760,10 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { "utf-8", ), ); - expect(stored.accounts[0]?.rateLimitResetTimes).toMatchObject({ - codex: weeklyResetAt * 1000, - "gpt-5.6-terra": weeklyResetAt * 1000, - }); + // The shared subscription quota is ONE account-wide fact, so it is stored + // once and must not be forged into a per-family rate-limit block. + expect(stored.accounts[0]?.quotaExhaustedUntil).toBe(weeklyResetAt * 1000); + expect(stored.accounts[0]?.rateLimitResetTimes ?? {}).toEqual({}); }); it("limits: renders the windows in text output rather than a bare account list (#209)", async () => { diff --git a/test/tools-codex-list.test.ts b/test/tools-codex-list.test.ts index 2605950c..ece3cb63 100644 --- a/test/tools-codex-list.test.ts +++ b/test/tools-codex-list.test.ts @@ -44,6 +44,8 @@ function buildCtx(options: { v2Enabled?: boolean } = {}): ToolContext { resolveActiveIndex: () => 0, formatCommandAccountLabel, formatRateLimitEntry: () => null, + getQuotaExhaustedUntil: () => null, + formatQuotaExhaustionEntry: () => null, buildJsonAccountIdentity: ( index: number, opts: { includeSensitive?: boolean; account?: { email?: string } } = {}, From b3dbd2235c1748eab1f030a24bb5649cd26380f6 Mon Sep 17 00:00:00 2001 From: wargloom Date: Fri, 11 Sep 2026 03:23:11 +0300 Subject: [PATCH 13/22] refactor(request): share one fallback chain-walk policy --- lib/request/fetch-helpers.ts | 108 +++++++++++++++++++++++++++-------- test/fetch-helpers.test.ts | 70 +++++++++++++++++++++++ 2 files changed, 155 insertions(+), 23 deletions(-) diff --git a/lib/request/fetch-helpers.ts b/lib/request/fetch-helpers.ts index a5175bf5..8ad8ecbd 100644 --- a/lib/request/fetch-helpers.ts +++ b/lib/request/fetch-helpers.ts @@ -406,6 +406,85 @@ export function getUnsupportedCodexModelInfo( }; } +/** + * Whether the default auto-fallback (the one that does NOT require + * `unsupportedCodexPolicy: "fallback"`) currently applies to `currentModel`. + * + * Exported so a caller degrading for a reason OTHER than an entitlement 400 — + * notably a fully quota-blocked pool — gates on exactly the same entry models + * and opt-out env vars instead of inventing a second policy. + */ +export function isDefaultAutoFallbackModel( + currentModel: string, + attemptedModels?: Iterable, +): boolean { + const attempted = new Set(); + for (const model of attemptedModels ?? []) { + const normalized = canonicalizeModelName(model); + if (normalized) attempted.add(normalized); + } + const entryModel = resolveAutoFallbackEntryModel( + canonicalizeModelName(currentModel) ?? currentModel, + attempted, + ); + const optOutEnv = entryModel + ? DEFAULT_AUTO_FALLBACK_ENTRY_OPT_OUT_ENV[entryModel] + : undefined; + return !!optOutEnv && process.env[optOutEnv] !== "1"; +} + +export interface PickFallbackChainTargetOptions { + currentModel: string; + attemptedModels?: Iterable; + customChain?: Record; + fallbackToGpt52OnUnsupportedGpt53?: boolean; +} + +/** + * Walk the fallback chain and return the next model worth trying. + * + * This is the single chain-walking policy: both the entitlement fallback and + * the quota/rate-limit fallback go through it, so the two can never drift. + * It decides only what comes NEXT in the chain — whether degrading is allowed + * at all is the caller's gate. + */ +export function pickFallbackChainTarget( + options: PickFallbackChainTargetOptions, +): string | undefined { + const currentModel = canonicalizeModelName(options.currentModel); + if (!currentModel) return undefined; + + const attempted = new Set(); + for (const model of options.attemptedModels ?? []) { + const normalized = canonicalizeModelName(model); + if (normalized) attempted.add(normalized); + } + + const chain = normalizeFallbackChain(options.customChain); + const targets = chain[currentModel] ?? []; + // `Array.isArray`, not just a length check. `currentModel` comes from the + // caller's `body.model`, so it can be any `Object.prototype` member name. On + // a plain object `chain["constructor"]` returns the Object constructor: a + // truthy non-array whose `.length` is 1, so an emptiness check passes it + // through and the `for...of` below throws `targets is not iterable` inside + // the request path. The chain is null-prototype now as well; this guard also + // covers a `customChain` value that is not an array. + if (!Array.isArray(targets) || targets.length === 0) return undefined; + + for (const target of targets) { + if (!options.fallbackToGpt52OnUnsupportedGpt53 && + currentModel === "gpt-5.3-codex" && + target === "gpt-5.2-codex") { + continue; + } + if (target === currentModel) continue; + if (attempted.has(target)) continue; + return target; + } + + return undefined; +} + export function resolveUnsupportedCodexFallbackModel( options: ResolveUnsupportedCodexFallbackOptions, ): string | undefined { @@ -444,29 +523,12 @@ export function resolveUnsupportedCodexFallbackModel( return undefined; } - const chain = normalizeFallbackChain(options.customChain); - const targets = chain[currentModel] ?? []; - // `Array.isArray`, not just a length check. `currentModel` comes from the - // caller's `body.model`, so it can be any `Object.prototype` member name. On - // a plain object `chain["constructor"]` returns the Object constructor: a - // truthy non-array whose `.length` is 1, so an emptiness check passes it - // through and the `for...of` below throws `targets is not iterable` inside - // the request path. The chain is null-prototype now as well; this guard also - // covers a `customChain` value that is not an array. - if (!Array.isArray(targets) || targets.length === 0) return undefined; - - for (const target of targets) { - if (!options.fallbackToGpt52OnUnsupportedGpt53 && - currentModel === "gpt-5.3-codex" && - target === "gpt-5.2-codex") { - continue; - } - if (target === currentModel) continue; - if (attempted.has(target)) continue; - return target; - } - - return undefined; + return pickFallbackChainTarget({ + currentModel, + attemptedModels: attempted, + customChain: options.customChain, + fallbackToGpt52OnUnsupportedGpt53: options.fallbackToGpt52OnUnsupportedGpt53, + }); } /** diff --git a/test/fetch-helpers.test.ts b/test/fetch-helpers.test.ts index 4fa81fb5..6e79b6f6 100644 --- a/test/fetch-helpers.test.ts +++ b/test/fetch-helpers.test.ts @@ -15,6 +15,8 @@ import { createEntitlementErrorResponse, getUnsupportedCodexModelInfo, resolveUnsupportedCodexFallbackModel, + isDefaultAutoFallbackModel, + pickFallbackChainTarget, extractUnsupportedCodexModelFromText, shouldFallbackToGpt52OnUnsupportedGpt53, } from '../lib/request/fetch-helpers.js'; @@ -2048,3 +2050,71 @@ describe("quota header authority (issues #16/#17 vs #218)", () => { expect(quotaHeadersAuthoritative).toBeFalsy(); }); }); + + +describe("fallback chain reuse for non-entitlement degradation", () => { + afterEach(() => { + delete process.env.CODEX_AUTH_DISABLE_GPT56_AUTO_FALLBACK; + }); + + it("walks the chain without needing an entitlement error body", () => { + // Given a default selector that owns a chain row. + // When the caller degrades for a reason other than a 400. + const target = pickFallbackChainTarget({ currentModel: "gpt-5.6-sol" }); + // Then it still returns the next chain model. + expect(target).toBeTruthy(); + expect(target).not.toBe("gpt-5.6-sol"); + }); + + it("never returns a model already attempted, so a caller cannot cycle", () => { + const first = pickFallbackChainTarget({ currentModel: "gpt-5.6-sol" }); + const second = pickFallbackChainTarget({ + currentModel: "gpt-5.6-sol", + attemptedModels: ["gpt-5.6-sol", String(first)], + }); + expect(second).not.toBe(first); + expect(second).not.toBe("gpt-5.6-sol"); + }); + + it("exhausts to undefined once every chain target was attempted", () => { + const attempted = new Set(["gpt-5.6-sol"]); + // Bounded walk: each pass must consume one target or stop. + for (let i = 0; i < 20; i += 1) { + const next = pickFallbackChainTarget({ + currentModel: "gpt-5.6-sol", + attemptedModels: attempted, + }); + if (!next) break; + expect(attempted.has(next)).toBe(false); + attempted.add(next); + } + expect( + pickFallbackChainTarget({ + currentModel: "gpt-5.6-sol", + attemptedModels: attempted, + }), + ).toBeUndefined(); + }); + + it("treats a chainless model as non-degradable", () => { + expect( + pickFallbackChainTarget({ currentModel: "definitely-not-a-model" }), + ).toBeUndefined(); + }); + + it("survives a model id naming an Object.prototype member", () => { + expect(pickFallbackChainTarget({ currentModel: "constructor" })).toBeUndefined(); + expect(pickFallbackChainTarget({ currentModel: "__proto__" })).toBeUndefined(); + }); + + it("gates degradation on the default selector entry models", () => { + expect(isDefaultAutoFallbackModel("gpt-5.6-sol")).toBe(true); + // A directly chosen, non-entry model must never be swapped silently. + expect(isDefaultAutoFallbackModel("gpt-5.1")).toBe(false); + }); + + it("honours the same opt-out env var as the entitlement auto-fallback", () => { + process.env.CODEX_AUTH_DISABLE_GPT56_AUTO_FALLBACK = "1"; + expect(isDefaultAutoFallbackModel("gpt-5.6-sol")).toBe(false); + }); +}); \ No newline at end of file From 4ec0eaad0b9a396978375d572df22b6e397cb54e Mon Sep 17 00:00:00 2001 From: wargloom Date: Fri, 11 Sep 2026 03:23:15 +0300 Subject: [PATCH 14/22] feat(rotation): fall back model when every account is blocked --- index.ts | 255 ++++++++++++++++++++++++++------------- test/index-retry.test.ts | 2 + test/index.test.ts | 78 ++++++++++++ 3 files changed, 251 insertions(+), 84 deletions(-) diff --git a/index.ts b/index.ts index 16477a09..54ac5b98 100644 --- a/index.ts +++ b/index.ts @@ -165,6 +165,8 @@ import { createAbortError, getUnsupportedCodexModelInfo, resolveUnsupportedCodexFallbackModel, + isDefaultAutoFallbackModel, + pickFallbackChainTarget, refreshAndUpdateToken, rewriteUrlForCodex, shouldRefreshToken, @@ -2318,6 +2320,103 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { attemptedUnsupportedFallbackModels.add(model); } + // Degrading the model mid-request touches several coupled pieces: + // the attempted-model set, the routing snapshot, the model's own + // instructions, the reasoning clamp (only the 5.6 tiers accept + // `max`, so an un-clamped sol -> gpt-5.5 hop turns a graceful + // fallback into a hard 400) and the per-model body shape. Both the + // entitlement fallback and the quota fallback go through here so + // the two can never drift apart on any of them. + const applyModelFallback = async ( + previousModel: string, + target: string, + reason: string, + ): Promise => { + attemptedUnsupportedFallbackModels.add(previousModel); + attemptedUnsupportedFallbackModels.add(target); + + model = target; + modelFamily = getModelFamily(model); + quotaKey = `${modelFamily}:${model}`; + fallbackApplied = true; + fallbackFrom = previousModel; + fallbackTo = model; + fallbackReason = reason; + const fallbackInstructions = await getCodexInstructions(model); + + if (transformedBody && typeof transformedBody === "object") { + transformedBody = { + ...transformedBody, + model, + instructions: fallbackInstructions, + input: upsertBackendModelIdentityMessage( + transformedBody.input, + model, + ), + }; + } else { + let fallbackBody: Record = { + model, + instructions: fallbackInstructions, + }; + if (requestInit?.body && typeof requestInit.body === "string") { + try { + const parsed = JSON.parse(requestInit.body) as Record; + fallbackBody = { + ...parsed, + model, + instructions: fallbackInstructions, + }; + if (Array.isArray(fallbackBody.input)) { + fallbackBody.input = upsertBackendModelIdentityMessage( + fallbackBody.input, + model, + ); + } + } catch { + // Keep minimal fallback body if parsing fails. + } + } + transformedBody = fallbackBody as RequestBody; + } + + const clampedReasoning = clampReasoningForModel( + transformedBody.reasoning, + model, + ); + if (clampedReasoning !== transformedBody.reasoning) { + transformedBody = { + ...transformedBody, + reasoning: clampedReasoning, + }; + } + + requestInit = { + ...(requestInit ?? {}), + body: JSON.stringify(shapeBodyForModel(transformedBody)), + }; + if (runtimeMetrics.lastSelectionSnapshot) { + runtimeMetrics.lastSelectionSnapshot = { + ...runtimeMetrics.lastSelectionSnapshot, + family: modelFamily, + model: model ?? null, + requestedModel, + effectiveModel: model ?? null, + quotaKey, + fallbackApplied, + fallbackFrom, + fallbackTo, + fallbackReason, + }; + } + }; + + // A degraded model must not degrade again without bound, even if a + // custom chain is cyclic. The attempted set already prevents + // revisiting a model; this caps the total hops per request. + const MAX_QUOTA_FALLBACK_SWITCHES = 3; + let quotaFallbackSwitches = 0; + while (true) { let accountCount = accountManager.getAccountCount(); const attempted = new Set(); @@ -2921,92 +3020,12 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { if (fallbackModel) { const previousModel = model ?? "gpt-5-codex"; const previousModelFamily = modelFamily; - attemptedUnsupportedFallbackModels.add(previousModel); - attemptedUnsupportedFallbackModels.add(fallbackModel); accountManager.refundToken(account, previousModelFamily, previousModel); - - model = fallbackModel; - modelFamily = getModelFamily(model); - quotaKey = `${modelFamily}:${model}`; - fallbackApplied = true; - fallbackFrom = previousModel; - fallbackTo = model; - fallbackReason = "fallback-unsupported-model-entitlement"; - const fallbackInstructions = await getCodexInstructions(model); - - if (transformedBody && typeof transformedBody === "object") { - transformedBody = { - ...transformedBody, - model, - instructions: fallbackInstructions, - input: upsertBackendModelIdentityMessage( - transformedBody.input, - model, - ), - }; - } else { - let fallbackBody: Record = { - model, - instructions: fallbackInstructions, - }; - if (requestInit?.body && typeof requestInit.body === "string") { - try { - const parsed = JSON.parse(requestInit.body) as Record; - fallbackBody = { - ...parsed, - model, - instructions: fallbackInstructions, - }; - if (Array.isArray(fallbackBody.input)) { - fallbackBody.input = upsertBackendModelIdentityMessage( - fallbackBody.input, - model, - ); - } - } catch { - // Keep minimal fallback body if parsing fails. - } - } - transformedBody = fallbackBody as RequestBody; - } - - // The carried-over reasoning effort was clamped for the ORIGINAL - // model; the fallback target may not accept it (`max` exists only - // on the 5.6 tiers, so a sol -> gpt-5.5 hop must degrade it or the - // graceful fallback turns into a hard 400). - const clampedReasoning = clampReasoningForModel( - transformedBody.reasoning, - model, + await applyModelFallback( + previousModel, + fallbackModel, + "fallback-unsupported-model-entitlement", ); - if (clampedReasoning !== transformedBody.reasoning) { - transformedBody = { - ...transformedBody, - reasoning: clampedReasoning, - }; - } - - // Shape for whichever model this attempt targets. A 5.6 -> 5.5 fallback - // must go out in the classic shape, and a 5.6 -> 5.6 hop must re-fold - // the new model's instructions into `input` rather than leaving them - // at the top level. - requestInit = { - ...(requestInit ?? {}), - body: JSON.stringify(shapeBodyForModel(transformedBody)), - }; - if (runtimeMetrics.lastSelectionSnapshot) { - runtimeMetrics.lastSelectionSnapshot = { - ...runtimeMetrics.lastSelectionSnapshot, - family: modelFamily, - model: model ?? null, - requestedModel, - effectiveModel: model ?? null, - quotaKey, - fallbackApplied, - fallbackFrom, - fallbackTo, - fallbackReason, - }; - } runtimeMetrics.lastError = `Model fallback: ${previousModel} -> ${model}`; runtimeMetrics.lastErrorCategory = "model-fallback"; logWarn( @@ -3474,6 +3493,74 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { !fetchedAccountKeys.has(getAccountDiagnosticsKey(account)), ).length; + // Every account is blocked for this model. Before waiting out a + // block that can run for days (`retryAllAccountsMaxRetries` + // defaults to Infinity), degrade to the next chain model that is + // actually usable right now. Gated exactly like the entitlement + // auto-fallback -- same default-selector entry models, same + // opt-out env vars -- so a directly chosen model is never + // silently swapped. An account-wide quota block fails this test + // on every candidate, so it correctly falls through to the wait. + if ( + waitMs > 0 && + count > 0 && + model && + quotaFallbackSwitches < MAX_QUOTA_FALLBACK_SWITCHES && + isDefaultAutoFallbackModel( + model, + attemptedUnsupportedFallbackModels, + ) + ) { + const rejected = new Set(); + let usableFallback: string | undefined; + while (true) { + const candidate = pickFallbackChainTarget({ + currentModel: model, + attemptedModels: new Set([ + ...attemptedUnsupportedFallbackModels, + ...rejected, + ]), + customChain: unsupportedCodexFallbackChain, + fallbackToGpt52OnUnsupportedGpt53, + }); + if (!candidate) break; + // Only degrade to a model some account can serve NOW, + // otherwise the hop just moves the same block sideways. + if ( + accountManager.getMinWaitTimeForFamily( + getModelFamily(candidate), + candidate, + ) === 0 + ) { + usableFallback = candidate; + break; + } + rejected.add(candidate); + } + + if (usableFallback) { + const previousModel = model; + quotaFallbackSwitches++; + await applyModelFallback( + previousModel, + usableFallback, + "fallback-quota-exhausted", + ); + runtimeMetrics.lastError = `Model fallback: ${previousModel} -> ${model}`; + runtimeMetrics.lastErrorCategory = "model-fallback"; + logWarn( + `All ${count} account(s) are rate-limited or out of quota for ${previousModel}. Falling back to ${model}.`, + { + requestedModel: previousModel, + effectiveModel: model, + fallbackApplied: true, + fallbackReason: "fallback-quota-exhausted", + }, + ); + continue; + } + } + if ( retryAllAccountsRateLimited && count > 0 && diff --git a/test/index-retry.test.ts b/test/index-retry.test.ts index 8ff2c535..3e44ea95 100644 --- a/test/index-retry.test.ts +++ b/test/index-retry.test.ts @@ -58,6 +58,8 @@ vi.mock("../lib/request/fetch-helpers.js", () => ({ }, isInvalidatedAuthTokenError: (_errorBody: unknown, status?: number) => status === 401, resolveUnsupportedCodexFallbackModel: () => undefined, + isDefaultAutoFallbackModel: () => false, + pickFallbackChainTarget: () => undefined, getUnsupportedCodexModelInfo: () => ({ isUnsupported: false, unsupportedModel: undefined, diff --git a/test/index.test.ts b/test/index.test.ts index 15ee0e79..f584147e 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -293,6 +293,8 @@ vi.mock("../lib/request/rate-limit-backoff.js", () => ({ isInvalidatedAuthTokenError: vi.fn((_errorBody: unknown, status?: number) => status === 401), getUnsupportedCodexModelInfo: vi.fn(() => ({ isUnsupported: false })), resolveUnsupportedCodexFallbackModel: vi.fn(() => undefined), + isDefaultAutoFallbackModel: vi.fn(() => false), + pickFallbackChainTarget: vi.fn(() => undefined), shouldFallbackToGpt52OnUnsupportedGpt53: vi.fn(() => false), handleSuccessResponse: vi.fn(async (response: Response) => response), })); @@ -4874,6 +4876,82 @@ describe("OpenAIOAuthPlugin fetch handler", () => { } }); + it("degrades to the next chain model when every account is blocked for the requested one", async () => { + // Given a pool that is fully blocked for the requested default selector, + // while the next chain model is servable right now. + const fetchHelpers = await import("../lib/request/fetch-helpers.js"); + const { AccountManager } = await import("../lib/accounts.js"); + + const account = { + index: 0, + accountId: "acc-1", + email: "user@example.com", + refreshToken: "refresh-1", + }; + let askedModel = ""; + const selectable = () => (askedModel === "gpt-5.5" ? account : null); + const customManager = { + getAccountCount: () => 1, + getSelectionExplainability: (_family: string, model?: string | null) => { + askedModel = String(model ?? ""); + return []; + }, + getCurrentOrNextForFamilyHybrid: selectable, + getAccountForStrategy: selectable, + // Only the fallback model is servable; the requested one is blocked. + getMinWaitTimeForFamily: vi.fn((_family: string, model?: string | null) => + model === "gpt-5.5" ? 0 : 600_000, + ), + toAuthDetails: () => ({ + type: "oauth" as const, + access: "access-1", + refresh: account.refreshToken, + expires: Date.now() + 60_000, + }), + hasRefreshToken: () => true, + saveToDiskDebounced: vi.fn(), + updateFromAuth: vi.fn(), + clearAuthFailures: vi.fn(), + incrementAuthFailures: vi.fn(() => 1), + markAccountCoolingDown: vi.fn(), + markRateLimitedWithReason: vi.fn(), + recordRateLimit: vi.fn(), + consumeToken: vi.fn(() => true), + refundToken: vi.fn(), + markSwitched: vi.fn(), + removeAccount: vi.fn(() => false), + removeAccountsWithSameRefreshToken: vi.fn(() => 0), + recordFailure: vi.fn(), + recordSuccess: vi.fn(), + shouldShowAccountToast: vi.fn(() => false), + markToastShown: vi.fn(), + setActiveIndex: vi.fn(() => account), + getAccountsSnapshot: vi.fn(() => [account]), + }; + vi.spyOn(AccountManager, "loadFromDisk").mockResolvedValue(customManager as never); + vi.mocked(fetchHelpers.isDefaultAutoFallbackModel).mockReturnValue(true); + vi.mocked(fetchHelpers.pickFallbackChainTarget).mockReturnValue("gpt-5.5"); + vi.mocked(fetchHelpers.createCodexHeaders).mockImplementation(() => new Headers()); + globalThis.fetch = vi + .fn() + .mockResolvedValue(new Response(JSON.stringify({ content: "ok" }), { status: 200 })); + + // When the request asks for the blocked model. + const { sdk } = await setupPlugin(); + const response = await sdk.fetch!("https://api.openai.com/v1/chat", { + method: "POST", + body: JSON.stringify({ model: "gpt-5.6-sol" }), + }); + + // Then it is served on the fallback model instead of waiting out the block. + expect(response.status).toBe(200); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + const sent = JSON.parse( + String(vi.mocked(globalThis.fetch).mock.calls[0]?.[1]?.body), + ); + expect(sent.model).toBe("gpt-5.5"); + }); + it("cools down the account when grouped auth removal removes zero entries", async () => { const fetchHelpers = await import("../lib/request/fetch-helpers.js"); const { AccountManager } = await import("../lib/accounts.js"); From db596e2aade99050d8f10e27e0c482c34fc2b488 Mon Sep 17 00:00:00 2001 From: wargloom Date: Fri, 11 Sep 2026 03:24:11 +0300 Subject: [PATCH 15/22] docs(config): note auto-fallback covers exhausted quota --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index c71a2250..78a7107c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -281,7 +281,7 @@ The sample above intentionally sets `"retryAllAccountsMaxRetries": 3` as a bound | `unsupportedCodexPolicy` | `strict` | unsupported-model behavior: `strict` (return entitlement error) or `fallback` (retry with configured fallback chain) | | `fallbackOnUnsupportedCodexModel` | `false` | legacy fallback toggle mapped to `unsupportedCodexPolicy` (prefer using `unsupportedCodexPolicy`) | | `fallbackToGpt52OnUnsupportedGpt53` | `true` | legacy compatibility toggle for the `gpt-5.3-codex -> gpt-5.2-codex` edge when generic fallback is enabled | -| `unsupportedCodexFallbackChain` | `{}` | optional per-model fallback-chain override (map of `model -> [fallback1, fallback2, ...]`; default includes `gpt-6-astra` and the 5.6 tiers down to `gpt-5.5`, and `gpt-5.5`/`gpt-5-codex` down to `gpt-5.2`). The 5.6 tier, `gpt-5.5`, and canonical Codex auto-fallbacks are on by default for common entitlement gates; set `CODEX_AUTH_DISABLE_GPT6_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT56_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT55_AUTO_FALLBACK=1`, or `CODEX_AUTH_DISABLE_CODEX_AUTO_FALLBACK=1` to opt out. GPT-5.5 Pro and GPT-6 Astra Pro are not mapped: neither is a Codex-routable id. The Daybreak cyber tiers are deliberately chainless, so an unentitled account fails loudly rather than being answered by a general model. | +| `unsupportedCodexFallbackChain` | `{}` | optional per-model fallback-chain override (map of `model -> [fallback1, fallback2, ...]`; default includes `gpt-6-astra` and the 5.6 tiers down to `gpt-5.5`, and `gpt-5.5`/`gpt-5-codex` down to `gpt-5.2`). The 5.6 tier, `gpt-5.5`, and canonical Codex auto-fallbacks are on by default, both for common entitlement gates and when every account is rate-limited or out of quota for the requested model; set `CODEX_AUTH_DISABLE_GPT6_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT56_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT55_AUTO_FALLBACK=1`, or `CODEX_AUTH_DISABLE_CODEX_AUTO_FALLBACK=1` to opt out. A model chosen directly rather than through a default selector is never swapped, and the chain is only followed to a model some account can serve immediately. GPT-5.5 Pro and GPT-6 Astra Pro are not mapped: neither is a Codex-routable id. The Daybreak cyber tiers are deliberately chainless, so an unentitled account fails loudly rather than being answered by a general model. | | `sessionRecovery` | `true` | auto-recover from common api errors | | `autoResume` | `true` | auto-resume after thinking block recovery | | `tokenRefreshSkewMs` | `60000` | refresh tokens this many ms before expiry | From 02ce44cc3a2abfa6d10dec5d7c5b41c60751d60f Mon Sep 17 00:00:00 2001 From: wargloom Date: Fri, 11 Sep 2026 20:01:09 +0300 Subject: [PATCH 16/22] fix(tui): stop badging every account as rate-limited --- lib/tools/codex-list.ts | 1 - lib/tools/codex-status.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/tools/codex-list.ts b/lib/tools/codex-list.ts index a7c67168..a8922be0 100644 --- a/lib/tools/codex-list.ts +++ b/lib/tools/codex-list.ts @@ -222,7 +222,6 @@ export function createCodexListTool(ctx: ToolContext): ToolDefinition { badges.push(formatUiBadge(ui, "rate-limited", "warning")); if (quotaExhausted) badges.push(formatUiBadge(ui, "quota-exhausted", "warning")); - badges.push(formatUiBadge(ui, "rate-limited", "warning")); if ( typeof account.coolingDownUntil === "number" && account.coolingDownUntil > now diff --git a/lib/tools/codex-status.ts b/lib/tools/codex-status.ts index fb0adae0..e2b680c7 100644 --- a/lib/tools/codex-status.ts +++ b/lib/tools/codex-status.ts @@ -220,7 +220,6 @@ export function createCodexStatusTool(ctx: ToolContext): ToolDefinition { badges.push(formatUiBadge(ui, "rate-limited", "warning")); if (quotaExhausted !== "none") badges.push(formatUiBadge(ui, "quota-exhausted", "warning")); - badges.push(formatUiBadge(ui, "rate-limited", "warning")); if (cooldown !== "none") badges.push(formatUiBadge(ui, "cooldown", "warning")); if (badges.length === 0) From a9a2bf7742e05d37091136ac3815f35c1a0a18d7 Mon Sep 17 00:00:00 2001 From: wargloom Date: Fri, 11 Sep 2026 20:01:14 +0300 Subject: [PATCH 17/22] fix(rotation): record header quota exhaustion account-wide and gate fallback on upstream blocks --- index.ts | 66 +++++++---- lib/accounts/rotation.ts | 40 +++---- test/accounts-quota-exhaustion.test.ts | 17 +++ test/index.test.ts | 149 ++++++++++++++++++++++++- test/quota-reset-horizon.test.ts | 4 +- test/quota-windows.test.ts | 24 ++-- 6 files changed, 241 insertions(+), 59 deletions(-) diff --git a/index.ts b/index.ts index 54ac5b98..a090c44e 100644 --- a/index.ts +++ b/index.ts @@ -843,8 +843,8 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { * served request and on a refused one alike, so the moment an account hits * 0% left we can record it instead of rediscovering it with a failed request * on every subsequent prompt. The block lands on the persisted - * `rateLimitResetTimes` map, so it is remembered across restarts and shared - * with other processes, and it clears itself once the window rolls over. + * account-wide `quotaExhaustedUntil` field, so it is remembered across + * restarts and clears itself once the window rolls over. * * Call this only for responses whose headers are authoritative: one the * backend served, or one it refused for a confirmed usage limit. Every other @@ -875,7 +875,7 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { account.lastSwitchReason = "rate-limit"; manager.saveToDiskDebounced(); logWarn( - `Account ${account.index + 1} (${account.email ?? "unknown"}) has no ${family} quota left; skipping it for ${formatWaitTime(resetAtMs - Date.now())}.`, + `Account ${account.index + 1} has no shared subscription quota left; skipping it for ${formatWaitTime(resetAtMs - Date.now())}.`, ); return true; } catch (error) { @@ -2469,6 +2469,11 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { break; } attempted.add(account.index); + // Hybrid's last-resort result is not necessarily eligible. Requests + // must honor active blocks rather than sending it upstream anyway. + if (selectionExplainability.some((entry) => entry.index === account.index && !entry.eligible)) { + continue; + } runtimeMetrics.lastSelectedAccountIndex = account.index; runtimeMetrics.lastQuotaKey = quotaKey; if (runtimeMetrics.lastSelectionSnapshot) { @@ -3179,13 +3184,18 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { continue; } - accountManager.markRateLimitedWithReason( - account, - delayMs, - modelFamily, - parseRateLimitReason(rateLimit.code), - model, - ); + // Authoritative subscription exhaustion already has its own block. + // Do not duplicate its reset as a model/family transient 429; retain + // any genuine transient state written by other in-flight requests. + if (!quotaExhausted) { + accountManager.markRateLimitedWithReason( + account, + delayMs, + modelFamily, + parseRateLimitReason(rateLimit.code), + model, + ); + } accountManager.recordRateLimit(account, modelFamily, model); account.lastSwitchReason = "rate-limit"; runtimeMetrics.accountRotations++; @@ -3493,15 +3503,24 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { !fetchedAccountKeys.has(getAccountDiagnosticsKey(account)), ).length; - // Every account is blocked for this model. Before waiting out a + const enabledSelection = count > 0 ? accountManager + .getSelectionExplainability(modelFamily, model) + .filter((entry) => entry.enabled) : []; + const upstreamBlocked = enabledSelection.length > 0 && enabledSelection.every( + (entry) => entry.rateLimitedUntil !== undefined || entry.quotaExhaustedUntil !== undefined, + ); + + // Every enabled account has an active upstream block. Before waiting out a // block that can run for days (`retryAllAccountsMaxRetries` // defaults to Infinity), degrade to the next chain model that is // actually usable right now. Gated exactly like the entitlement // auto-fallback -- same default-selector entry models, same - // opt-out env vars -- so a directly chosen model is never - // silently swapped. An account-wide quota block fails this test + // opt-out env vars -- even when an entry ID was selected directly. + // Local token depletion and auth cooldown alone never trigger it. + // An account-wide quota block fails the eligibility test // on every candidate, so it correctly falls through to the wait. if ( + upstreamBlocked && waitMs > 0 && count > 0 && model && @@ -3523,15 +3542,22 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { customChain: unsupportedCodexFallbackChain, fallbackToGpt52OnUnsupportedGpt53, }); - if (!candidate) break; + if (!candidate || rejected.has(candidate)) break; // Only degrade to a model some account can serve NOW, // otherwise the hop just moves the same block sideways. - if ( - accountManager.getMinWaitTimeForFamily( - getModelFamily(candidate), - candidate, - ) === 0 - ) { + const candidatePool = getModelAccountPool(pluginConfig, candidate); + const strictCandidatePool = candidatePool.length > 0 && + getModelAccountPoolMode(pluginConfig, candidate) === "strict"; + const candidateAccounts = accountManager.getAccountsSnapshot(); + const candidateEligible = accountManager.getSelectionExplainability( + getModelFamily(candidate), candidate, + ).some((entry) => entry.eligible && (!strictCandidatePool || + candidateAccounts.some((account) => account.index === entry.index && + candidatePool.some((key) => matchesModelPoolAccountKey(account, key))))); + // A preferred pool may spill into general accounts; a strict + // pool must contain an eligible member. This does not select + // an account or advance any rotation cursor. + if (candidateEligible) { usableFallback = candidate; break; } diff --git a/lib/accounts/rotation.ts b/lib/accounts/rotation.ts index a3fd0553..e9f5fb44 100644 --- a/lib/accounts/rotation.ts +++ b/lib/accounts/rotation.ts @@ -445,31 +445,21 @@ export class AccountRotation { * Block an account until a quota window the backend reported as fully spent * resets (issue #218). * - * Differs from {@link markRateLimitedWithReason} in taking an ABSOLUTE reset - * stamp, so a week-long weekly-quota block is never rebuilt from a capped or - * backed-off retry delay. Like every writer it goes through - * {@link extendRateLimitReset}, so it neither shortens an existing block nor - * can be shortened by a later one. - * - * The block rides on the same persisted `rateLimitResetTimes` map as server - * 429s, so it survives restarts and is shared with other processes through - * the accounts file, and it expires on its own via `clearExpiredRateLimits`. - * - * Because that write is monotonic and persisted, it is also unforgiving: a - * reset stamp further out than any real window would strand the account for - * as long as it claims, with nothing in the product able to walk it back. The - * upper bound below is the same guard `parseQuotaResetAtMs` applies to the - * headers, repeated here because this is the method that makes a block - * permanent — the lower bound on the next line has always been checked for - * the same reason. + * Primary/secondary subscription windows are account-wide, irrespective of + * the request's family/model. Keep their absolute reset monotonically in + * `quotaExhaustedUntil`, persisted separately from transient server 429s and + * expired by `clearExpiredQuotaExhaustion`. Legacy family/model arguments + * remain accepted, but cannot narrow the subscription block's scope. + * Reject implausible timestamps using the parser's horizon guard so a bad + * header cannot strand an account indefinitely. * * @returns true when a new (or longer) block was written. */ markQuotaExhausted( account: ManagedAccount, resetAtMs: number, - family: ModelFamily, - model?: string | null, + _family: ModelFamily, + _model?: string | null, ): boolean { if (!Number.isFinite(resetAtMs)) return false; const resetAt = Math.floor(resetAtMs); @@ -477,13 +467,13 @@ export class AccountRotation { if (resetAt <= now) return false; if (resetAt - now > MAX_QUOTA_RESET_HORIZON_MS) return false; - let changed = false; - for (const key of this.getBlockedQuotaKeys(family, model)) { - if (this.extendRateLimitReset(account, key, resetAt)) changed = true; + const existing = account.quotaExhaustedUntil; + if (typeof existing === "number" && Number.isFinite(existing) && existing >= resetAt) { + return false; } - - if (changed) account.lastRateLimitReason = "quota"; - return changed; + account.quotaExhaustedUntil = resetAt; + account.lastRateLimitReason = "quota"; + return true; } markAccountCoolingDown( diff --git a/test/accounts-quota-exhaustion.test.ts b/test/accounts-quota-exhaustion.test.ts index 58935f43..b5f927f0 100644 --- a/test/accounts-quota-exhaustion.test.ts +++ b/test/accounts-quota-exhaustion.test.ts @@ -117,6 +117,23 @@ describe("account-wide quota exhaustion state", () => { }); // (e) + it("probes eligibility by dropping only expired stamps and never moving the rotation cursor", () => { + vi.useFakeTimers(); + const now = Date.now(); + const manager = new AccountManager(undefined, { + version: 3, activeIndex: 0, + accounts: [{ refreshToken: "token-1", addedAt: 1, lastUsed: 1, + quotaExhaustedUntil: now - 1, rateLimitResetTimes: { codex: now - 1 } }], + }); + const { quotaExhaustedUntil: _q, rateLimitResetTimes: _r, ...before } = manager.getAccountsSnapshot()[0]!; + expect(manager.getSelectionExplainability("codex")[0]?.eligible).toBe(true); + const { quotaExhaustedUntil, rateLimitResetTimes, ...after } = manager.getAccountsSnapshot()[0]!; + expect(after).toEqual(before); + expect(quotaExhaustedUntil).toBeUndefined(); + expect(rateLimitResetTimes).toEqual({}); + expect(manager.getCurrentAccount()?.index).toBe(0); + }); + it("ignores and clears an expired quotaExhaustedUntil", () => { const now = Date.now(); const manager = new AccountManager(undefined, { diff --git a/test/index.test.ts b/test/index.test.ts index f584147e..7f8023b4 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -225,7 +225,8 @@ vi.mock("../lib/context-overflow.js", () => ({ handleContextOverflow: vi.fn(async () => ({ handled: false })), })); -vi.mock("../lib/rotation.js", () => ({ +vi.mock("../lib/rotation.js", async (importOriginal) => ({ + ...await importOriginal(), addJitter: (ms: number) => ms, })); @@ -236,14 +237,14 @@ vi.mock("../lib/ui/select.js", () => ({ select: vi.fn(async () => null), })); -vi.mock("../lib/prompts/codex.js", () => ({ +vi.mock("../lib/prompts/codex.js", async (importOriginal) => ({ + ...await importOriginal(), getModelFamily: (model: string) => { if (model.includes("codex-max")) return "codex-max"; if (model.includes("codex")) return "codex"; return "gpt-5.1"; }, getCodexInstructions: vi.fn(async () => "test instructions"), - MODEL_FAMILIES: ["codex-max", "codex", "gpt-5.1"] as const, prewarmCodexInstructions: vi.fn(), })); @@ -1617,6 +1618,27 @@ describe("OpenAIOAuthPlugin", () => { }); }); + describe.each(["codex-list", "codex-status"] as const)("%s quota badges", (toolName) => { + it.each([ + { state: "clean", rate: false, quota: false }, + { state: "transient-only", rate: true, quota: false }, + { state: "quota-only", rate: false, quota: true }, + ])("renders exactly the $state badges", async ({ rate, quota }) => { + const config = await import("../lib/config.js"); + vi.spyOn(config, "getCodexTuiV2").mockReturnValue(true); + mockStorage.accounts = [{ + accountId: "acc-1", refreshToken: "refresh-1", + rateLimitResetTimes: rate ? { codex: Date.now() + 60_000 } : {}, + quotaExhaustedUntil: quota ? Date.now() + 600_000 : undefined, + }]; + const output = await plugin.tool[toolName].execute(); + const accountLine = output.split("\n").find((line) => line.includes("Account 1") && line.includes(toolName === "codex-list" ? "current" : "active")); + expect(accountLine).toBeDefined(); + expect(accountLine?.match(/rate-limited/g) ?? []).toHaveLength(rate ? 1 : 0); + expect(accountLine?.match(/quota-exhausted/g) ?? []).toHaveLength(quota ? 1 : 0); + }); + }); + describe("codex-status tool", () => { it("returns error when no accounts", async () => { mockStorage.accounts = []; @@ -4876,6 +4898,125 @@ describe("OpenAIOAuthPlugin fetch handler", () => { } }); + describe("quota fallback safety with real account eligibility", () => { + const entryModel = "gpt-5.6-sol"; + const makeManager = async (accounts: import("../lib/storage.js").AccountMetadataV3[]) => { + const prompts = await import("../lib/prompts/codex.js"); + const realPrompts = await vi.importActual("../lib/prompts/codex.js"); + vi.spyOn(prompts, "getModelFamily").mockImplementation(realPrompts.getModelFamily); + const actual = await vi.importActual("../lib/accounts.js"); + const { AccountManager, resolveRequestAccountId } = await import("../lib/accounts.js"); + vi.mocked(resolveRequestAccountId).mockImplementation((storedId) => storedId); + const manager = new actual.AccountManager(undefined, { version: 3, activeIndex: 0, accounts }); + vi.spyOn(AccountManager, "loadFromDisk").mockResolvedValue(manager); + vi.spyOn(manager, "saveToDiskDebounced").mockImplementation(() => {}); + const fetchHelpers = await import("../lib/request/fetch-helpers.js"); + const realHelpers = await vi.importActual("../lib/request/fetch-helpers.js"); + vi.mocked(fetchHelpers.transformRequestForCodex).mockImplementation(async (init) => ({ + updatedInit: init, body: JSON.parse(String(init?.body)), + })); + vi.mocked(fetchHelpers.isDefaultAutoFallbackModel).mockImplementation(realHelpers.isDefaultAutoFallbackModel); + vi.mocked(fetchHelpers.pickFallbackChainTarget).mockImplementation(realHelpers.pickFallbackChainTarget); + vi.mocked(fetchHelpers.handleErrorResponse).mockImplementation(realHelpers.handleErrorResponse); + vi.mocked(fetchHelpers.createCodexHeaders).mockImplementation((_init, accountId) => new Headers({ "x-test-account": accountId })); + const quotaCache = await import("../lib/tui-quota-cache.js"); + vi.spyOn(quotaCache, "writeTuiQuotaSnapshot").mockResolvedValue(undefined); + globalThis.fetch = vi.fn().mockImplementation(async () => new Response(JSON.stringify({ content: "ok" }))); + return manager; + }; + const accountRecord = (accountId = "acc-1") => ({ + accountId, refreshToken: `refresh-${accountId}`, accessToken: "access-test", + expiresAt: Date.now() + 3_600_000, addedAt: 1, lastUsed: 1, + }); + const send = async (sdk: Awaited>["sdk"], model = entryModel) => { + if (!sdk.fetch) throw new Error("Missing plugin fetch"); + return sdk.fetch("https://api.openai.com/v1/chat", { + method: "POST", body: JSON.stringify({ model }), + }); + }; + + beforeEach(async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-09-11T00:00:00Z")); + const { resetTrackers } = await import("../lib/rotation.js"); + resetTrackers(); + }); + afterEach(async () => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + const { resetTrackers } = await import("../lib/rotation.js"); + resetTrackers(); + }); + + it.each([200, 429])("persists authoritative %i shared quota and never falls back into the spent account", async (status) => { + const manager = await makeManager([accountRecord()]); + const resetAt = Date.now() + 604_800_000; + vi.mocked(globalThis.fetch).mockImplementationOnce(async () => new Response( + JSON.stringify(status === 429 ? { error: { code: "usage_limit_reached" } } : { content: "ok" }), + { status, headers: { "x-codex-secondary-used-percent": "100", "x-codex-secondary-reset-at": String(resetAt) } }, + )); + const { sdk } = await setupPlugin(); + expect((await send(sdk)).status).toBe(status); + expect((await send(sdk, "gpt-5.6-terra")).status).toBe(429); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + expect(manager.getAccountsSnapshot()[0]?.quotaExhaustedUntil).toBe(resetAt); + expect(manager.getAccountsSnapshot()[0]?.rateLimitResetTimes).toEqual({}); + expect(manager.getSelectionExplainability(entryModel, entryModel)[0]?.reasons).toContain("quota-exhausted"); + expect(manager.getSelectionExplainability(entryModel, entryModel)[0]?.reasons).not.toContain("rate-limited"); + expect(manager.saveToDiskDebounced).toHaveBeenCalled(); + await manager.saveToDisk(); + expect(mockStorage.accounts[0]?.quotaExhaustedUntil).toBe(resetAt); + }); + + it.each(["token-bucket", "cooldown"])("does not downgrade for %s-only blocking", async (block) => { + const manager = await makeManager([accountRecord()]); + const account = manager.getCurrentAccount(); + if (!account) throw new Error("Missing test account"); + if (block === "cooldown") manager.markAccountCoolingDown(account, 600_000, "auth-failure"); + else { + const { getTokenTracker } = await import("../lib/rotation.js"); + getTokenTracker().drain(account.index, `${entryModel}:${entryModel}`, 100); + } + const config = await import("../lib/config.js"); + vi.spyOn(config, "getRetryAllAccountsRateLimited").mockReturnValue(false); + vi.spyOn(config, "getRotationStrategy").mockReturnValue("sticky"); + const { sdk } = await setupPlugin(); + expect((await send(sdk)).status).toBe(429); + expect(globalThis.fetch).not.toHaveBeenCalled(); + const helpers = await import("../lib/request/fetch-helpers.js"); + expect(helpers.pickFallbackChainTarget).not.toHaveBeenCalled(); + }); + + it.each(["blocked", "unresolved", "disabled"])("skips a %s strict target and serves a subsequent eligible strict candidate", async (state) => { + await makeManager([ + { ...accountRecord(), rateLimitResetTimes: { [`${entryModel}:${entryModel}`]: Date.now() + 600_000, "gpt-5.6-terra:gpt-5.6-terra": Date.now() + 600_000 } }, + { ...accountRecord("acc-2"), rateLimitResetTimes: { [`${entryModel}:${entryModel}`]: Date.now() + 600_000 } }, + { ...accountRecord("disabled"), enabled: false }, + ]); + const config = await import("../lib/config.js"); + vi.spyOn(config, "getRotationStrategy").mockReturnValue("sticky"); + vi.mocked(config.getModelAccountPool).mockImplementation((_config, model) => model === "gpt-5.6-terra" + ? [state === "blocked" ? "acc-1" : state === "unresolved" ? "missing" : "disabled"] + : model === "gpt-5.6-luna" ? ["acc-2"] : []); + vi.mocked(config.getModelAccountPoolMode).mockReturnValue("strict"); + const { sdk } = await setupPlugin(); + expect((await send(sdk)).status).toBe(200); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + const init = vi.mocked(globalThis.fetch).mock.calls[0]?.[1]; + expect(JSON.parse(String(init?.body)).model).toBe("gpt-5.6-luna"); + expect(new Headers(init?.headers).get("x-test-account")).toBe("acc-2"); + }); + + it.each([undefined, "CODEX_AUTH_DISABLE_GPT56_AUTO_FALLBACK"])("preserves genuine model fallback and opt-out %s", async (optOut) => { + await makeManager([{ ...accountRecord(), rateLimitResetTimes: { [`${entryModel}:${entryModel}`]: Date.now() + 600_000 } }]); + if (optOut) vi.stubEnv(optOut, "1"); + const { sdk } = await setupPlugin(); + expect((await send(sdk)).status).toBe(optOut ? 429 : 200); + expect(globalThis.fetch).toHaveBeenCalledTimes(optOut ? 0 : 1); + if (!optOut) expect(JSON.parse(String(vi.mocked(globalThis.fetch).mock.calls[0]?.[1]?.body)).model).toBe("gpt-5.6-terra"); + }); + }); + it("degrades to the next chain model when every account is blocked for the requested one", async () => { // Given a pool that is fully blocked for the requested default selector, // while the next chain model is servable right now. @@ -4894,7 +5035,7 @@ describe("OpenAIOAuthPlugin fetch handler", () => { getAccountCount: () => 1, getSelectionExplainability: (_family: string, model?: string | null) => { askedModel = String(model ?? ""); - return []; + return [{ index: 0, enabled: true, eligible: model === "gpt-5.5", reasons: model === "gpt-5.5" ? ["eligible"] : ["rate-limited"], rateLimitedUntil: model === "gpt-5.5" ? undefined : Date.now() + 600_000 }]; }, getCurrentOrNextForFamilyHybrid: selectable, getAccountForStrategy: selectable, diff --git a/test/quota-reset-horizon.test.ts b/test/quota-reset-horizon.test.ts index aa9f41d6..b03071c8 100644 --- a/test/quota-reset-horizon.test.ts +++ b/test/quota-reset-horizon.test.ts @@ -142,7 +142,8 @@ describe("markQuotaExhausted horizon guard", () => { const resetAt = Date.now() + 7 * 24 * HOUR_MS; expect(manager.markQuotaExhausted(account, resetAt, "codex")).toBe(true); - expect(account.rateLimitResetTimes["codex"]).toBe(Math.floor(resetAt)); + expect(account.quotaExhaustedUntil).toBe(Math.floor(resetAt)); + expect(account.rateLimitResetTimes).toEqual({}); }); it("refuses a reset past the horizon, so nothing permanent is persisted", async () => { @@ -151,6 +152,7 @@ describe("markQuotaExhausted horizon guard", () => { const bogus = Date.now() + 4_000_000_000 * 1000; // ~127 years expect(manager.markQuotaExhausted(account, bogus, "codex")).toBe(false); + expect(account.quotaExhaustedUntil).toBeUndefined(); expect(account.rateLimitResetTimes["codex"]).toBeUndefined(); }); }); diff --git a/test/quota-windows.test.ts b/test/quota-windows.test.ts index 28078f75..02295f6f 100644 --- a/test/quota-windows.test.ts +++ b/test/quota-windows.test.ts @@ -294,11 +294,13 @@ describe("AccountManager.markQuotaExhausted (issue #218)", () => { expect(manager.getCurrentOrNext()?.refreshToken).toBe("token-2"); }); - it("records the block as a quota rate limit", () => { + it("records shared quota separately from rate limits", () => { const manager = buildManager(); const account = manager.getCurrentOrNext()!; manager.markQuotaExhausted(account, Date.now() + SEVEN_DAYS_MS, "codex"); expect(account.lastRateLimitReason).toBe("quota"); + expect(account.quotaExhaustedUntil).toBeGreaterThan(Date.now()); + expect(account.rateLimitResetTimes).toEqual({}); }); it("never shortens an existing longer block", () => { @@ -308,7 +310,7 @@ describe("AccountManager.markQuotaExhausted (issue #218)", () => { expect(manager.markQuotaExhausted(account, weeklyReset, "codex")).toBe(true); expect(manager.markQuotaExhausted(account, Date.now() + 60_000, "codex")).toBe(false); - expect(account.rateLimitResetTimes.codex).toBe(weeklyReset); + expect(account.quotaExhaustedUntil).toBe(weeklyReset); }); it("is not shortened by a later ordinary rate limit", () => { @@ -327,8 +329,10 @@ describe("AccountManager.markQuotaExhausted (issue #218)", () => { "gpt-5-codex", ); - expect(account.rateLimitResetTimes.codex).toBe(weeklyReset); - expect(account.rateLimitResetTimes["codex:gpt-5-codex"]).toBe(weeklyReset); + expect(account.quotaExhaustedUntil).toBe(weeklyReset); + expect(account.rateLimitResetTimes.codex).toBeLessThan(weeklyReset); + expect(account.rateLimitResetTimes["codex:gpt-5-codex"]).toBe(account.rateLimitResetTimes.codex); + expect(account.lastRateLimitReason).toBe("tokens"); expect(manager.getCurrentOrNext()?.refreshToken).toBe("token-2"); }); @@ -354,22 +358,24 @@ describe("AccountManager.markQuotaExhausted (issue #218)", () => { expect(account.rateLimitResetTimes.codex).toBeGreaterThan(shortReset); }); - it("ignores a reset that is already in the past", () => { + it.each([NaN, Infinity, -Infinity, -1000, 0])("ignores an invalid or elapsed reset %s", (offset) => { const manager = buildManager(); const account = manager.getCurrentOrNext()!; - expect(manager.markQuotaExhausted(account, Date.now() - 1000, "codex")).toBe(false); + expect(manager.markQuotaExhausted(account, Date.now() + offset, "codex")).toBe(false); + expect(account.quotaExhaustedUntil).toBeUndefined(); expect(account.rateLimitResetTimes.codex).toBeUndefined(); }); - it("blocks the model-scoped quota key too", () => { + it("blocks other models without manufacturing model-scoped rate limits", () => { const manager = buildManager(); const account = manager.getCurrentOrNext()!; const weeklyReset = Date.now() + SEVEN_DAYS_MS; manager.markQuotaExhausted(account, weeklyReset, "codex", "gpt-5-codex"); - expect(account.rateLimitResetTimes.codex).toBe(weeklyReset); - expect(account.rateLimitResetTimes["codex:gpt-5-codex"]).toBe(weeklyReset); + expect(account.quotaExhaustedUntil).toBe(weeklyReset); + expect(account.rateLimitResetTimes).toEqual({}); + expect(manager.getCurrentOrNextForFamily("gpt-5.1", "gpt-5.1")?.refreshToken).toBe("token-2"); }); }); From d276daf371705c13f94f3aca0ab9b449e4d91c1b Mon Sep 17 00:00:00 2001 From: wargloom Date: Fri, 11 Sep 2026 20:01:20 +0300 Subject: [PATCH 18/22] docs(config): describe fallback precedence and pool rules --- docs/configuration.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 78a7107c..72b79234 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -281,7 +281,7 @@ The sample above intentionally sets `"retryAllAccountsMaxRetries": 3` as a bound | `unsupportedCodexPolicy` | `strict` | unsupported-model behavior: `strict` (return entitlement error) or `fallback` (retry with configured fallback chain) | | `fallbackOnUnsupportedCodexModel` | `false` | legacy fallback toggle mapped to `unsupportedCodexPolicy` (prefer using `unsupportedCodexPolicy`) | | `fallbackToGpt52OnUnsupportedGpt53` | `true` | legacy compatibility toggle for the `gpt-5.3-codex -> gpt-5.2-codex` edge when generic fallback is enabled | -| `unsupportedCodexFallbackChain` | `{}` | optional per-model fallback-chain override (map of `model -> [fallback1, fallback2, ...]`; default includes `gpt-6-astra` and the 5.6 tiers down to `gpt-5.5`, and `gpt-5.5`/`gpt-5-codex` down to `gpt-5.2`). The 5.6 tier, `gpt-5.5`, and canonical Codex auto-fallbacks are on by default, both for common entitlement gates and when every account is rate-limited or out of quota for the requested model; set `CODEX_AUTH_DISABLE_GPT6_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT56_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT55_AUTO_FALLBACK=1`, or `CODEX_AUTH_DISABLE_CODEX_AUTO_FALLBACK=1` to opt out. A model chosen directly rather than through a default selector is never swapped, and the chain is only followed to a model some account can serve immediately. GPT-5.5 Pro and GPT-6 Astra Pro are not mapped: neither is a Codex-routable id. The Daybreak cyber tiers are deliberately chainless, so an unentitled account fails loudly rather than being answered by a general model. | +| `unsupportedCodexFallbackChain` | `{}` | optional per-model fallback-chain override (map of `model -> [fallback1, fallback2, ...]`; default includes `gpt-6-astra` and the 5.6 tiers down to `gpt-5.5`, and `gpt-5.5`/`gpt-5-codex` down to `gpt-5.2`). These entry IDs auto-fallback by default, even when selected directly, both for common entitlement gates and when every enabled account has an active upstream rate/quota block for the requested model; set `CODEX_AUTH_DISABLE_GPT6_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT56_AUTO_FALLBACK=1`, `CODEX_AUTH_DISABLE_GPT55_AUTO_FALLBACK=1`, or `CODEX_AUTH_DISABLE_CODEX_AUTO_FALLBACK=1` to opt out. Directly selected non-entry IDs stay strict under this auto gate. GPT-5.5 Pro and GPT-6 Astra Pro are not mapped: neither is a Codex-routable id. The Daybreak cyber tiers are deliberately chainless, so an unentitled account fails loudly rather than being answered by a general model. | | `sessionRecovery` | `true` | auto-recover from common api errors | | `autoResume` | `true` | auto-resume after thinking block recovery | | `tokenRefreshSkewMs` | `60000` | refresh tokens this many ms before expiry | @@ -295,6 +295,15 @@ The sample above intentionally sets `"retryAllAccountsMaxRetries": 3` as a bound | `streamStallTimeoutMs` | `45000` | max time to wait for next SSE chunk before aborting | | `quotaNotifications` | disabled | optional macOS Notification Center alerts for aggregate 5-hour and weekly pool quotas. `autoProtectCredits` defaults to `true` and polls the same endpoint to exclude fully spent subscription quotas from rotation; `intervalMs` defaults to 30 minutes with a 30-second minimum, `notifyEveryCheck` defaults to `false`, and `thresholds` defaults to `[25, 10, 0]` | +For upstream rate/quota blocks, automatic model fallback runs **before** configured +waiting (`retryAllAccountsRateLimited` and its wait/retry limits). It only moves to +a model with an eligible account under that target's pool policy: unavailable +strict pools are skipped, while preferred pools may use general accounts. An +unavailable strict pool for the current model remains a strict-pool error. Local +token-bucket depletion or authentication cooldown alone does not trigger model +fallback. Shared subscription exhaustion blocks the account across all models; +changing models cannot bypass it. + The quota guard queries each distinct enabled account with bounded concurrency every `intervalMs` (30 minutes by default), even when notifications are off. When the backend reports a fully spent 5-hour or weekly subscription window, From 85395c1f04521628e091afdba67b7e2e0405fdb1 Mon Sep 17 00:00:00 2001 From: ndycode Date: Sun, 13 Sep 2026 23:06:02 +0800 Subject: [PATCH 19/22] fix(cli): fail doctor --fix on a corrupt default storage file Default-path repair skipped the storage pre-read, so a malformed accounts.json fell through loadAccounts() (which swallows parse errors) and reported an empty pool with exit 0. Probe the file when discovery yields nothing: a parse error now exits 1 and no repair is attempted, matching the --config-path route. ENOENT stays silent. --- docs/tools-and-cli.md | 4 +-- scripts/install-oc-codex-multi-auth-core.js | 20 ++++++++++--- test/standalone-cli.test.ts | 31 +++++++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/docs/tools-and-cli.md b/docs/tools-and-cli.md index c5b07c64..52005bf4 100644 --- a/docs/tools-and-cli.md +++ b/docs/tools-and-cli.md @@ -180,7 +180,7 @@ Choose only one of `--plugin-only`, `--modern`, `--full`, or `--legacy`. Use `up | `--json` | Machine-readable JSON output | | `--include-sensitive` | Include sensitive identity fields in JSON where applicable | | `--deep` | Deeper diagnostics (used with `doctor`; implied by `diag`) | -| `--fix` | With `doctor`, refresh enabled accounts and clear stale cooldown and rate-limit markers only after successful verification. Exit nonzero if any repair fails. | +| `--fix` | With `doctor`, refresh enabled accounts and clear stale cooldown, rate-limit, and quota-exhaustion markers only after successful verification. A cleared quota stamp re-establishes itself on the next quota 429 or usage poll. Exit nonzero if any repair fails, or if the storage file cannot be parsed. | | `--tag ` | Filter accounts by tag when listing | | `--config-path ` | Point at a specific accounts storage path | | `--help` / `-h` | Print usage | @@ -196,7 +196,7 @@ oc-codex-multi-auth doctor --fix --config-path ./accounts.json npx -y oc-codex-multi-auth@latest warm ``` -For `doctor --fix`, an explicit `--config-path` repairs only the selected JSON pool and bypasses keychain routing. Without `--config-path`, repair preserves enabled keychain routing. +For `doctor --fix`, an explicit `--config-path` repairs only the selected JSON pool and bypasses keychain routing. Without `--config-path`, repair preserves enabled keychain routing, and a corrupt default storage file fails with a parse error instead of reporting an empty pool. --- diff --git a/scripts/install-oc-codex-multi-auth-core.js b/scripts/install-oc-codex-multi-auth-core.js index 2bfdf7e9..d95a1c57 100644 --- a/scripts/install-oc-codex-multi-auth-core.js +++ b/scripts/install-oc-codex-multi-auth-core.js @@ -795,10 +795,22 @@ export async function runStandaloneCommand(command, argv = [], options = {}) { storageMod.setStoragePathDirect(storagePath); shutdownMod.setShutdownOwnsProcess(true); storage = await storageMod.loadAccounts(); - const repair = await repairMod.repairDoctorAccounts(storage?.accounts ?? []); - appliedFixes.push(...repair.appliedFixes); - fixErrors.push(...repair.fixErrors); - storage = (await storageMod.loadAccounts()) ?? storage; + if (!storage) { + // `loadAccounts` swallows JSON parse/IO errors and returns null. In + // default-path mode the pre-read above was skipped (keychain routing + // may own the pool), so probe the JSON file here: a corrupt file must + // surface as a parse error (exit 1) instead of "No accounts + // configured" (exit 0). ENOENT stays silent - a missing file with an + // empty keychain legitimately means no accounts yet. + const probe = await readStandaloneStorage(storagePath); + if (probe.error) error = probe.error; + } + if (!error) { + const repair = await repairMod.repairDoctorAccounts(storage?.accounts ?? []); + appliedFixes.push(...repair.appliedFixes); + fixErrors.push(...repair.fixErrors); + storage = (await storageMod.loadAccounts()) ?? storage; + } } catch { fixErrors.push("Doctor repair could not complete. Check the selected storage file and installed runtime."); } finally { diff --git a/test/standalone-cli.test.ts b/test/standalone-cli.test.ts index 2643aa1d..ac252e74 100644 --- a/test/standalone-cli.test.ts +++ b/test/standalone-cli.test.ts @@ -467,6 +467,37 @@ describe("standalone oc-codex-multi-auth CLI commands", () => { }); }); + it("doctor: reports malformed default-path JSON as an error during --fix, not as success", async () => { + // Given a corrupt default storage file while the runtime swallows the + // parse failure (loadAccounts returns null instead of throwing). + vi.resetModules(); + tempHome = await createTempHome(); + const accountsPath = join(tempHome, ".opencode", "oc-codex-multi-auth-accounts.json"); + await mkdir(join(tempHome, ".opencode"), { recursive: true }); + await writeFile(accountsPath, "{", "utf-8"); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const { runInstaller } = await import("../scripts/install-oc-codex-multi-auth-core.js"); + const repairDoctorAccounts = vi.fn().mockResolvedValue({ appliedFixes: [], fixErrors: [] }); + + // When default-path repair discovers nothing because the file is unparseable. + const result = await runInstaller(["doctor", "--fix", "--json"], { + env: { ...process.env, HOME: tempHome, USERPROFILE: tempHome }, + loadDoctorRuntime: async () => [ + { setStoragePathDirect: vi.fn(), loadAccounts: async () => null }, + { repairDoctorAccounts }, + { setShutdownOwnsProcess: vi.fn() }, + ], + }); + + // Then the parse error surfaces with a nonzero exit instead of + // "No accounts configured" (exit 0), and no repair is attempted. + expect(result.exitCode).toBe(1); + expect(repairDoctorAccounts).not.toHaveBeenCalled(); + expect(JSON.parse(String(logSpy.mock.calls.at(-1)?.[0]))).toMatchObject({ + error: expect.any(String), message: "Storage could not be parsed.", fixApplied: false, fixErrors: [], + }); + }); + it.each(["discovery", "repair", "snapshot"])("doctor: redacts runtime %s failures without a JSON pool", async (stage) => { // Given an injected backend that fails at one repair boundary. vi.resetModules(); From 5acf01b18605db6ff40760b82531a25492d63036 Mon Sep 17 00:00:00 2001 From: ndycode Date: Sun, 13 Sep 2026 23:06:09 +0800 Subject: [PATCH 20/22] fix(health): report quota exhaustion apart from stale state findStaleRecoverableAccounts now flags only the legacy #171 dark state (stale cooldown/rate-limit stamps). A future quotaExhaustedUntil is written only by the usage poller or a quota 429, so it is real, not stale; a dedicated findQuotaExhaustedAccounts surfaces those slots with wording that says the stamp re-establishes after --fix clears it. --fix clearing behavior itself is unchanged. --- lib/accounts/stale-state.ts | 46 +++++++++++++++++++++++++++++++++---- lib/tools/codex-health.ts | 9 ++++++++ test/index.test.ts | 24 +++++++++++++++++++ test/stale-state.test.ts | 32 +++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 5 deletions(-) diff --git a/lib/accounts/stale-state.ts b/lib/accounts/stale-state.ts index 09be0cf9..c11e9309 100644 --- a/lib/accounts/stale-state.ts +++ b/lib/accounts/stale-state.ts @@ -247,6 +247,14 @@ export interface StaleStateScanAccount { * issue #171 that `codex-doctor --fix` can recover (a successful token refresh * proves the credential is alive, so the block is stale). * + * A future-dated `quotaExhaustedUntil` is deliberately NOT flagged here: unlike + * the legacy blanket `rateLimitResetTimes` stamps, quota exhaustion is written + * only by authoritative sources (the usage poller and quota-429 response + * headers with horizon guards), so a future stamp is real, not stale #171 + * state. Quota-blocked accounts are surfaced separately through + * {@link findQuotaExhaustedAccounts} so diagnostics do not tell a user with a + * genuine week-long block that the block is stale. + * * This is read-only (it mutates nothing) so both `codex-health` and the * non-`--fix` `codex-doctor` path can surface the finding and point the user at * the repair. Expired cooldowns / rate-limits are ignored because the normal @@ -267,9 +275,6 @@ export function findStaleRecoverableAccounts( const hasFutureCooldown = typeof account.coolingDownUntil === "number" && account.coolingDownUntil > now; - const hasFutureQuotaExhaustion = - typeof account.quotaExhaustedUntil === "number" && account.quotaExhaustedUntil > now; - let hasFutureRateLimit = false; if (account.rateLimitResetTimes) { for (const reset of Object.values(account.rateLimitResetTimes)) { @@ -280,7 +285,40 @@ export function findStaleRecoverableAccounts( } } - if (hasFutureCooldown || hasFutureRateLimit || hasFutureQuotaExhaustion) { + if (hasFutureCooldown || hasFutureRateLimit) { + blocked.push(i); + } + } + return blocked; +} + +/** + * Identify enabled accounts carrying an active (future-dated) account-wide + * quota-exhaustion stamp. These blocks are authoritative, not stale #171 + * state — they are written only by the usage poller or a quota 429 response + * and re-establish themselves after being cleared — so diagnostics must label + * them as quota exhaustion rather than "recoverable stale state". + * + * Read-only: mutates nothing. `codex-doctor --fix` still clears these stamps + * after a successful token verification (an explicit user action), which is + * why they are worth surfacing, but the next quota 429 or usage poll simply + * re-stamps the account. + * + * @returns the 0-based indexes of enabled accounts blocked by quota exhaustion. + */ +export function findQuotaExhaustedAccounts( + accounts: StaleStateScanAccount[], + now: number = nowMs(), +): number[] { + const blocked: number[] = []; + for (let i = 0; i < accounts.length; i += 1) { + const account = accounts[i]; + if (!account) continue; + if (account.enabled === false) continue; + if ( + typeof account.quotaExhaustedUntil === "number" && + account.quotaExhaustedUntil > now + ) { blocked.push(i); } } diff --git a/lib/tools/codex-health.ts b/lib/tools/codex-health.ts index 495a9e5b..05820175 100644 --- a/lib/tools/codex-health.ts +++ b/lib/tools/codex-health.ts @@ -10,6 +10,7 @@ import { findDisabledTokenSourceDuplicates, findConflictingBusinessMemberCredentials, findStaleRecoverableAccounts, + findQuotaExhaustedAccounts, } from "../accounts/stale-state.js"; import { formatUiHeader, formatUiItem, paintUiText } from "../ui/format.js"; import { normalizeToolOutputFormat, renderJsonOutput } from "../runtime.js"; @@ -152,11 +153,13 @@ export function createCodexHealthTool(ctx: ToolContext): ToolDefinition { // (issue #171). Token verification is destructive to single-use refresh // tokens, but this tool persists rotations before reporting health. const staleRecoverable = findStaleRecoverableAccounts(storage.accounts); + const quotaExhausted = findQuotaExhaustedAccounts(storage.accounts); const duplicateSlots = findDisabledTokenSourceDuplicates(storage.accounts); const memberCredentialConflicts = findConflictingBusinessMemberCredentials( storage.accounts, ); const staleSlots = staleRecoverable.map((index) => index + 1); + const quotaSlots = quotaExhausted.map((index) => index + 1); const dupSlots = duplicateSlots.map((index) => index + 1); const memberConflictSlots = memberCredentialConflicts.map((indices) => indices.map((index) => index + 1), @@ -166,6 +169,11 @@ export function createCodexHealthTool(ctx: ToolContext): ToolDefinition { `Stale state: ${staleSlots.length} account(s) blocked by a stale cooldown/rate-limit (slots: ${staleSlots.join(", ")}). Run \`codex-doctor --fix\`.`, ); } + if (quotaSlots.length > 0) { + results.push( + `Quota exhausted: ${quotaSlots.length} account(s) carry an active quota-exhaustion stamp (slots: ${quotaSlots.join(", ")}). Stamps are set by the usage poller or a quota 429 and are usually real; \`codex-doctor --fix\` clears them after verification, and the next quota 429 re-establishes the block.`, + ); + } if (dupSlots.length > 0) { results.push( `Duplicates: ${dupSlots.length} disabled duplicate entry(ies) shadow a real account (slots: ${dupSlots.join(", ")}). Remove with \`codex-remove\`.`, @@ -191,6 +199,7 @@ export function createCodexHealthTool(ctx: ToolContext): ToolDefinition { unhealthyCount, skippedCount, staleRecoverableSlots: staleSlots, + quotaExhaustedSlots: quotaSlots, disabledDuplicateSlots: dupSlots, businessMemberConflictSlots: memberConflictSlots, disabledWithFreshCredentialSlots: absorbedSlots, diff --git a/test/index.test.ts b/test/index.test.ts index 7f8023b4..c64e4847 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -3432,6 +3432,30 @@ describe("OpenAIOAuthPlugin", () => { expect(result.staleRecoverableSlots).toContain(1); expect(result.disabledDuplicateSlots).toContain(2); }); + + it("reports quota exhaustion separately from stale state in health output", async () => { + mockStorage.accounts = [ + { + refreshToken: "r-quota", + email: "quota@example.com", + accountId: "org-QUOTA", + organizationId: "org-QUOTA", + accountIdSource: "org", + enabled: true, + quotaExhaustedUntil: Date.now() + 7 * 24 * 60 * 60 * 1000, + }, + ]; + const text = (await plugin.tool["codex-health"].execute()) as string; + expect(text).toContain("Quota exhausted:"); + expect(text).toContain("codex-doctor --fix"); + expect(text).not.toContain("Stale state:"); + const result = parseJsonOutput<{ + staleRecoverableSlots: number[]; + quotaExhaustedSlots: number[]; + }>(await plugin.tool["codex-health"].execute({ format: "json" })); + expect(result.staleRecoverableSlots).toEqual([]); + expect(result.quotaExhaustedSlots).toContain(1); + }); }); describe("codex-remove tool", () => { diff --git a/test/stale-state.test.ts b/test/stale-state.test.ts index a02afee2..3887064d 100644 --- a/test/stale-state.test.ts +++ b/test/stale-state.test.ts @@ -6,6 +6,7 @@ import { findDisabledAccountsWithFreshCredential, findConflictingBusinessMemberCredentials, findStaleRecoverableAccounts, + findQuotaExhaustedAccounts, type StaleStateAccount, } from "../lib/accounts/stale-state.js"; @@ -205,8 +206,13 @@ describe("findStaleRecoverableAccounts", () => { expect(findStaleRecoverableAccounts(accounts, NOW)).toEqual([0]); }); - it("flags an account blocked only by a future quota-exhaustion stamp", () => { + it("does not flag a quota-only block as stale (quota stamps are authoritative)", () => { const accounts = [{ enabled: true, quotaExhaustedUntil: FUTURE }]; + expect(findStaleRecoverableAccounts(accounts, NOW)).toEqual([]); + }); + + it("still flags an account whose cooldown is stale alongside a quota stamp", () => { + const accounts = [{ enabled: true, coolingDownUntil: FUTURE, quotaExhaustedUntil: FUTURE }]; expect(findStaleRecoverableAccounts(accounts, NOW)).toEqual([0]); }); @@ -238,6 +244,30 @@ describe("findStaleRecoverableAccounts", () => { }); }); +describe("findQuotaExhaustedAccounts", () => { + const NOW = 1_700_000_000_000; + const FUTURE = NOW + 7 * 24 * 60 * 60 * 1000; + const PAST = NOW - 3_600_000; + + it("flags an enabled account blocked only by a future quota-exhaustion stamp", () => { + const accounts = [{ enabled: true, quotaExhaustedUntil: FUTURE }]; + expect(findQuotaExhaustedAccounts(accounts, NOW)).toEqual([0]); + }); + + it("ignores an expired stamp and a disabled account", () => { + const accounts = [ + { enabled: true, quotaExhaustedUntil: PAST }, + { enabled: false, quotaExhaustedUntil: FUTURE }, + ]; + expect(findQuotaExhaustedAccounts(accounts, NOW)).toEqual([]); + }); + + it("flags a quota stamp even when a stale cooldown is also present", () => { + const accounts = [{ enabled: true, coolingDownUntil: FUTURE, quotaExhaustedUntil: FUTURE }]; + expect(findQuotaExhaustedAccounts(accounts, NOW)).toEqual([0]); + }); +}); + describe("findDisabledAccountsWithFreshCredential (issue #171)", () => { const NOW = 1_700_000_000_000; const FUTURE = NOW + 3_600_000; From 39074adf1c24ceae859fd6610a6915fd60593fb3 Mon Sep 17 00:00:00 2001 From: ndycode Date: Sun, 13 Sep 2026 23:06:16 +0800 Subject: [PATCH 21/22] docs(rotation): note the request-path hybrid last-resort override The request loop discards a hybrid last-resort account that the selection explainability marks ineligible, so an all-blocked pool waits out or fails on the block instead of retrying it. Document the override next to the last-resort contract it carves out. --- lib/accounts/rotation.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/accounts/rotation.ts b/lib/accounts/rotation.ts index e9f5fb44..08d7bf88 100644 --- a/lib/accounts/rotation.ts +++ b/lib/accounts/rotation.ts @@ -150,16 +150,25 @@ export class AccountRotation { * Health/token/freshness-weighted selection. The historical default. * * When at least one account is selectable this returns the best-scoring one. - * When NONE is — every account disabled, rate-limited or cooling down — - * `selectHybridAccount` deliberately falls back to the least-recently-used - * account instead of returning null, because retrying a blocked account - * beats refusing to send anything (a single-account pool has nowhere to fail - * over to, and a persisted block can outlive the limit that caused it). + * When NONE is — every account disabled, rate-limited, quota-exhausted or + * cooling down — `selectHybridAccount` deliberately falls back to the + * least-recently-used account instead of returning null, because retrying a + * blocked account beats refusing to send anything (a single-account pool has + * nowhere to fail over to, and a persisted block can outlive the limit that + * caused it). * * So a returned account is NOT a promise that it is selectable. Callers that * need that guarantee must consult `getSelectionExplainability`, which is * what `codex-doctor` does. {@link getCurrentOrNextForFamilySticky} and * {@link getCurrentOrNextForFamily} return null in the same situation. + * + * The request path overrides this last-resort behavior: when the fallback + * account is marked ineligible in the selection explainability, the request + * loop discards it instead of sending it upstream, so an all-blocked pool + * waits out (or fails on) the block rather than retrying it — which is also + * what allows model fallback to degrade the model when every account is + * blocked. The last-resort retry still applies to callers that do not + * re-check eligibility (for example `codex-doctor` probing). */ getCurrentOrNextForFamilyHybrid( family: ModelFamily, From 7b8c983aac18fe938e4aaa1e2a0f30da56bbbe85 Mon Sep 17 00:00:00 2001 From: ndycode Date: Sun, 13 Sep 2026 23:06:23 +0800 Subject: [PATCH 22/22] refactor(tools): drop the unused getQuotaExhaustedUntil context surface No tool consumed ctx.getQuotaExhaustedUntil; formatQuotaExhaustionEntry covers the display need and keeps its closure helper. --- index.ts | 1 - lib/tools/index.ts | 4 ---- test/tools-codex-list.test.ts | 1 - 3 files changed, 6 deletions(-) diff --git a/index.ts b/index.ts index a090c44e..26ea9900 100644 --- a/index.ts +++ b/index.ts @@ -1837,7 +1837,6 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { resolveActiveIndex, getRateLimitResetTimeForFamily, formatRateLimitEntry, - getQuotaExhaustedUntil, formatQuotaExhaustionEntry, buildJsonAccountIdentity, buildRoutingVisibilitySnapshot, diff --git a/lib/tools/index.ts b/lib/tools/index.ts index 99418e95..ba7ccfce 100644 --- a/lib/tools/index.ts +++ b/lib/tools/index.ts @@ -143,10 +143,6 @@ export interface ToolContext { now: number, family?: ModelFamily, ) => string | null; - getQuotaExhaustedUntil: ( - account: { quotaExhaustedUntil?: number }, - now: number, - ) => number | null; formatQuotaExhaustionEntry: ( account: { quotaExhaustedUntil?: number }, now: number, diff --git a/test/tools-codex-list.test.ts b/test/tools-codex-list.test.ts index ece3cb63..11b09567 100644 --- a/test/tools-codex-list.test.ts +++ b/test/tools-codex-list.test.ts @@ -44,7 +44,6 @@ function buildCtx(options: { v2Enabled?: boolean } = {}): ToolContext { resolveActiveIndex: () => 0, formatCommandAccountLabel, formatRateLimitEntry: () => null, - getQuotaExhaustedUntil: () => null, formatQuotaExhaustionEntry: () => null, buildJsonAccountIdentity: ( index: number,