fix(rotation): break stale-recovery deadlock on transient account state

lib/accounts.ts

@@ -65,6 +65,7 @@ export {
 	getQuotaKey,
 	clampNonNegativeInt,
 	clearExpiredRateLimits,
+	clearAllRateLimits,
 	isRateLimitedForQuotaKey,
 	isRateLimitedForFamily,
 	formatWaitTime,
@@ -91,6 +92,7 @@ import {
 	clampNonNegativeInt,
 	getQuotaKey,
 	clearExpiredRateLimits,
+	clearAllRateLimits,
 	isRateLimitedForFamily,
 	formatWaitTime,
 	type RateLimitReason,
@@ -668,6 +670,35 @@ export class AccountManager {
 		resetAllCircuitBreakers();
 	}
 
+	/**
+	 * Wipe per-account transient state — active cooldowns and all rate-limit
+	 * reset windows — across every managed account, then schedule a debounced
+	 * persist of the cleared pool.
+	 *
+	 * `resetVolatileRuntimeState` only clears process-global singletons (rotation
+	 * trackers, circuit breakers); the cooldown timestamps and `rateLimitResetTimes`
+	 * maps live on the `ManagedAccount` objects and are serialized to disk by
+	 * `buildStorageSnapshot`. The stale-runtime recovery path reloads accounts
+	 * from that snapshot, so without this the same transient state that wedged the
+	 * pool is restored verbatim and recovery never escapes it (issue #606).
+	 *
+	 * The in-memory clear is immediate (it is what unblocks the live pool). The
+	 * disk write is debounced via `saveToDiskDebounced`, so a caller that needs
+	 * the cleared state to survive a restart should `await flushPendingSave()`
+	 * afterwards rather than rely on the debounce window completing.
+	 */
+	clearAccountTransientState(): void {
+		if (this.accounts.length === 0) return;
+		// Snapshot the reference list so a concurrent mutation of `this.accounts`
+		// (e.g. removeAccount) cannot reshape the array mid-iteration.
+		for (const account of [...this.accounts]) {
+			this.clearAccountCooldown(account);
+			clearAllRateLimits(account);
+			account.lastRateLimitReason = undefined;
+		}
+		this.saveToDiskDebounced();
+	}
+
 	setActiveIndex(index: number): ManagedAccount | null {
 		if (!Number.isFinite(index)) return null;
 		if (index < 0 || index >= this.accounts.length) return null;

lib/accounts/rate-limits.ts

@@ -53,6 +53,20 @@ export function clearExpiredRateLimits(entity: RateLimitedEntity): void {
 	}
 }
 
+/**
+ * Remove every rate-limit reset entry from `entity`, regardless of whether the
+ * window has expired. Unlike {@link clearExpiredRateLimits}, this drops
+ * still-future entries too. Used by the stale-runtime recovery path to give a
+ * reloaded account pool a clean slate so transient state persisted to disk
+ * cannot keep the pool wedged (issue #606).
+ */
+export function clearAllRateLimits(entity: RateLimitedEntity): void {
+	const keys = Object.keys(entity.rateLimitResetTimes);
+	for (const key of keys) {
+		delete entity.rateLimitResetTimes[key];
+	}
+}
+
 export function isRateLimitedForQuotaKey(entity: RateLimitedEntity, key: QuotaKey): boolean {
 	const resetTime = entity.rateLimitResetTimes[key];
 	return resetTime !== undefined && nowMs() < resetTime;

lib/runtime-rotation-proxy.ts

@@ -982,10 +982,15 @@ async function handleRequestInner(
 					exhaustionReason === "no-account" &&
 					(policyDecision?.blockedAccountIndexes.size ?? 0) === 0 &&
 					![...accountSkipReasons.values()].some(
-						(reason) =>
-							reason === "rate-limited" ||
-							reason.startsWith("cooling-down") ||
-							reason === "policy-blocked",
+						// Only policy blocks still suppress stale-state recovery: a policy
+						// decision is external and won't change across a disk reload, so
+						// reloading cannot help. "rate-limited" and "cooling-down*" are
+						// transient states that recovery is *designed* to escape — they are
+						// persisted to disk (buildStorageSnapshot) and so survive a reload,
+						// which previously deadlocked the pool against the very recovery that
+						// would clear them. recoverStaleRuntimeState now wipes that transient
+						// state after reloading, so let those reasons through (issue #606).
+						(reason) => reason === "policy-blocked",
 					)
 				) {
 					reloadedAfterNoAccount = true;

lib/runtime/rotation-proxy-state.ts

@@ -99,6 +99,30 @@ export async function recoverStaleRuntimeState(
 		recordRuntimeReset("pool-exhausted-no-account");
 		const reloaded = await AccountManager.loadFromDisk();
 		reloaded.setRoutingMutexMode(state.routingMutexMode);
+		// Wipe per-account cooldowns and rate-limit windows on the freshly
+		// reloaded pool. `resetVolatileRuntimeState` above only cleared global
+		// singletons (trackers, circuit breakers); the per-account transient
+		// state is serialized to disk, so loadFromDisk restores the same state
+		// that wedged the pool. Clearing it here gives recovery a real clean
+		// slate before any request can pick up the manager (issue #606). Runs
+		// before the `state.activeAccountManager` assignment so no concurrent
+		// request can observe the reloaded manager with stale state.
+		//
+		// This also drops still-future rate-limit windows from genuine upstream
+		// 429s, not just stale ones from a dead prior process. That is the
+		// intended trade-off: recovery only runs at full pool exhaustion, where
+		// the alternative is a hard 503 anyway, and the 1s reload dedupe bounds
+		// re-probing to ~1/sec — the upstream simply re-429s and re-populates the
+		// window until the next exhaustion. Availability is preferred over
+		// honoring backoff in this already-degraded state.
+		reloaded.clearAccountTransientState();
+		// Force the cleared snapshot to disk now rather than waiting out the
+		// debounce window inside clearAccountTransientState. If the process
+		// exited during that window the next startup would reload the wedged
+		// snapshot; flushing here makes the "next reload starts clean" guarantee
+		// durable across a restart, not just best-effort. Recovery is rare
+		// (full-pool exhaustion), so the extra synchronous write is cheap.
+		await reloaded.flushPendingSave();
 		state.activeAccountManager = reloaded;
 		state.knownAccountManagers.add(reloaded);
 		state.lastStaleRuntimeReloadAt = Date.now();

test/accounts.test.ts

@@ -1447,6 +1447,155 @@ describe("AccountManager", () => {
 		});
 	});
 
+	describe("clearAccountTransientState", () => {
+		it("clears active cooldowns on every account", () => {
+			const now = Date.now();
+			const stored = {
+				version: 3 as const,
+				activeIndex: 0,
+				accounts: [
+					{ refreshToken: "token-1", addedAt: now, lastUsed: now },
+					{ refreshToken: "token-2", addedAt: now, lastUsed: now },
+				],
+			};
+
+			const manager = new AccountManager(undefined, stored);
+			const first = manager.getAccountByIndex(0)!;
+			const second = manager.getAccountByIndex(1)!;
+			manager.markAccountCoolingDown(first, 60_000, "network-error");
+			manager.markAccountCoolingDown(second, 60_000, "server-error");
+
+			manager.clearAccountTransientState();
+
+			expect(manager.isAccountCoolingDown(first)).toBe(false);
+			expect(manager.isAccountCoolingDown(second)).toBe(false);
+			expect(first.coolingDownUntil).toBeUndefined();
+			expect(first.cooldownReason).toBeUndefined();
+			expect(second.coolingDownUntil).toBeUndefined();
+			expect(second.cooldownReason).toBeUndefined();
+		});
+
+		it("clears all rate-limit reset windows, including ones still in the future", () => {
+			const now = Date.now();
+			const stored = {
+				version: 3 as const,
+				activeIndex: 0,
+				accounts: [{ refreshToken: "token-1", addedAt: now, lastUsed: now }],
+			};
+
+			const manager = new AccountManager(undefined, stored);
+			const account = manager.getCurrentAccount()!;
+			manager.markRateLimitedWithReason(account, 60_000, "codex", "quota");
+			manager.markRateLimitedWithReason(
+				account,
+				60_000,
+				"codex",
+				"tokens",
+				"gpt-5.2",
+			);
+			expect(Object.keys(account.rateLimitResetTimes).length).toBeGreaterThan(0);
+			expect(account.lastRateLimitReason).toBe("tokens");
+
+			manager.clearAccountTransientState();
+
+			expect(account.rateLimitResetTimes).toEqual({});
+			expect(account.lastRateLimitReason).toBeUndefined();
+		});
+
+		it("clears cooldown and rate-limit state together on the same account", () => {
+			const now = Date.now();
+			const stored = {
+				version: 3 as const,
+				activeIndex: 0,
+				accounts: [{ refreshToken: "token-1", addedAt: now, lastUsed: now }],
+			};
+
+			const manager = new AccountManager(undefined, stored);
+			const account = manager.getCurrentAccount()!;
+			// A real stuck account from #606 carries both states at once; prove the
+			// method clears both in one pass rather than short-circuiting.
+			manager.markAccountCoolingDown(account, 60_000, "server-error");
+			manager.markRateLimitedWithReason(account, 60_000, "codex", "quota");
+			expect(manager.isAccountCoolingDown(account)).toBe(true);
+			expect(Object.keys(account.rateLimitResetTimes).length).toBeGreaterThan(0);
+
+			manager.clearAccountTransientState();
+
+			expect(account.coolingDownUntil).toBeUndefined();
+			expect(account.cooldownReason).toBeUndefined();
+			expect(account.rateLimitResetTimes).toEqual({});
+			expect(account.lastRateLimitReason).toBeUndefined();
+		});
+
+		it("clears mixed per-account state without throwing on clean accounts", () => {
+			const now = Date.now();
+			const stored = {
+				version: 3 as const,
+				activeIndex: 0,
+				accounts: [
+					{ refreshToken: "token-1", addedAt: now, lastUsed: now },
+					{ refreshToken: "token-2", addedAt: now, lastUsed: now },
+					{ refreshToken: "token-3", addedAt: now, lastUsed: now },
+				],
+			};
+
+			const manager = new AccountManager(undefined, stored);
+			const coolingDown = manager.getAccountByIndex(0)!;
+			const rateLimited = manager.getAccountByIndex(1)!;
+			const clean = manager.getAccountByIndex(2)!;
+			manager.markAccountCoolingDown(coolingDown, 60_000, "network-error");
+			manager.markRateLimitedWithReason(rateLimited, 60_000, "codex", "quota");
+
+			expect(() => manager.clearAccountTransientState()).not.toThrow();
+
+			expect(coolingDown.coolingDownUntil).toBeUndefined();
+			expect(rateLimited.rateLimitResetTimes).toEqual({});
+			expect(clean.coolingDownUntil).toBeUndefined();
+			expect(clean.rateLimitResetTimes).toEqual({});
+		});
+
+		it("persists the cleared pool so a reload does not restore stale state", async () => {
+			const { saveAccounts } = await import("../lib/storage.js");
+			const mockSaveAccounts = vi.mocked(saveAccounts);
+			mockSaveAccounts.mockClear();
+
+			const now = Date.now();
+			const stored = {
+				version: 3 as const,
+				activeIndex: 0,
+				accounts: [{ refreshToken: "token-1", addedAt: now, lastUsed: now }],
+			};
+
+			const manager = new AccountManager(undefined, stored);
+			const account = manager.getCurrentAccount()!;
+			manager.markAccountCoolingDown(account, 60_000, "server-error");
+			manager.markRateLimitedWithReason(account, 60_000, "codex", "quota");
+
+			manager.clearAccountTransientState();
+			// Flush so we assert against the actual persisted snapshot, not just
+			// that a save was scheduled — this is the #606 durability guarantee.
+			await manager.flushPendingSave();
+
+			expect(mockSaveAccounts).toHaveBeenCalledTimes(1);
+			const persisted = mockSaveAccounts.mock.calls[0]?.[0];
+			expect(persisted?.accounts[0]?.coolingDownUntil).toBeUndefined();
+			expect(persisted?.accounts[0]?.cooldownReason).toBeUndefined();
+			expect(persisted?.accounts[0]?.rateLimitResetTimes ?? {}).toEqual({});
+		});
+
+		it("is a no-op when no accounts are loaded", () => {
+			const manager = new AccountManager(undefined, {
+				version: 3 as const,
+				activeIndex: 0,
+				accounts: [],
+			});
+			const saveSpy = vi.spyOn(manager, "saveToDiskDebounced");
+
+			expect(() => manager.clearAccountTransientState()).not.toThrow();
+			expect(saveSpy).not.toHaveBeenCalled();
+		});
+	});
+
 	describe("auth failure tracking", () => {
 		it("increments consecutive auth failures", () => {
 			const now = Date.now();