diff --git a/lib/forecast.ts b/lib/forecast.ts index 9283d4e1b..6db6d1486 100644 --- a/lib/forecast.ts +++ b/lib/forecast.ts @@ -258,11 +258,36 @@ export function evaluateForecastAccount( overlay?.accountSkipReasons?.[String(index)] ?? overlay?.lastPoolExhaustionSkipReasons?.[String(index)] ?? null; + // Time-bounded overlay reasons ("rate-limited", "cooling-down:...") are + // persisted to runtime-observability.json on pool exhaustion and only ever + // cleared by an explicit runtime reset, never on a subsequent successful + // request. That leaves a stale reason on disk after the underlying window + // expires, so the forecast would keep marking a working account as + // unavailable. Cross-reference the time-aware disk state before applying: + // drop the overlay reason when the condition it describes is no longer + // active. Each reason validates only against its own backing disk state + // ("rate-limited" -> rateLimitResetTimes, "cooling-down" -> coolingDownUntil) + // so we never substitute a misleading reason string. Non-time-bounded + // reasons ("circuit-open", "token-exhausted", "policy-blocked") have no disk + // expiry to check and are always applied. + const coolingDownActive = + typeof account.coolingDownUntil === "number" && + account.coolingDownUntil > now; + const isStaleOverlayReason = + overlayReason === "rate-limited" + ? rateLimitResetAt === null + : overlayReason?.startsWith("cooling-down") + ? !coolingDownActive + : false; if (overlay?.policyBlockedIndexes?.includes(index)) { availability = "unavailable"; riskScore += 95; reasons.push("runtime policy blocked account"); - } else if (overlayReason && overlayReason !== "already-attempted") { + } else if ( + overlayReason && + overlayReason !== "already-attempted" && + !isStaleOverlayReason + ) { availability = "unavailable"; riskScore += overlayReason === "circuit-open" diff --git a/test/forecast.test.ts b/test/forecast.test.ts index ad4547def..fb82495c6 100644 --- a/test/forecast.test.ts +++ b/test/forecast.test.ts @@ -182,8 +182,8 @@ describe("forecast helpers", () => { expect(overlaid.reasons).toContain("runtime skip: circuit-open"); }); - it.each(["rate-limited", "cooling-down:server-error", "workspace-disabled"])( - "marks runtime skip reason %s as unavailable", + it.each(["circuit-open", "token-exhausted", "workspace-disabled"])( + "marks non-time-bounded runtime skip reason %s as unavailable", (reason) => { const now = 1_700_000_000_000; const result = evaluateForecastAccount({ @@ -205,6 +205,212 @@ describe("forecast helpers", () => { }, ); + it("ignores a stale rate-limited overlay when no rate limit is active on disk", () => { + const now = 1_700_000_000_000; + const account = { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // Expired entry: clearExpiredRateLimits-equivalent semantics mean this + // is no longer an active rate limit. + rateLimitResetTimes: { codex: now - 30_000 }, + }; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain("runtime skip: rate-limited"); + }); + + it("ignores a stale rate-limited overlay when rateLimitResetTimes is absent", () => { + const now = 1_700_000_000_000; + const account = { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // No rateLimitResetTimes field at all: the limit was cleared (runtime + // reset or a successful request) after the overlay was written. + }; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain("runtime skip: rate-limited"); + }); + + it("ignores a stale overlay reason resolved from accountSkipReasons (precedence path)", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // No active rate limit or cooldown on disk. + }, + runtimeOverlay: { + // accountSkipReasons takes precedence over + // lastPoolExhaustionSkipReasons in the resolver; the staleness guard + // must apply to whichever key wins. + accountSkipReasons: { "0": "rate-limited" }, + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain("runtime skip: rate-limited"); + expect(result.reasons).not.toContain( + "runtime skip: cooling-down:server-error", + ); + }); + + it("applies an active overlay reason resolved from accountSkipReasons (precedence path)", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + rateLimitResetTimes: { codex: now + 30_000 }, + }, + runtimeOverlay: { + accountSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: rate-limited"); + }); + + it("applies a rate-limited overlay when the rate limit is still active on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + rateLimitResetTimes: { codex: now + 30_000 }, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: rate-limited"); + }); + + it("applies a rate-limited overlay when a model-scoped limit is active on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + rateLimitResetTimes: { "codex:5h": now + 30_000 }, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: rate-limited"); + }); + + it("ignores a stale cooling-down overlay when cooldown has elapsed on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + coolingDownUntil: now - 1, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain( + "runtime skip: cooling-down:server-error", + ); + }); + + it("ignores a stale cooling-down overlay when coolingDownUntil is absent", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // No coolingDownUntil field at all: cooldown cleared after the + // overlay was written. + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain( + "runtime skip: cooling-down:server-error", + ); + }); + + it("applies a cooling-down overlay when cooldown is still active on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + coolingDownUntil: now + 60_000, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: cooling-down:server-error"); + }); + it("recommends the best ready account", () => { const now = 1_700_000_000_000; const results = evaluateForecastAccounts([