From ff9c9f40f1d9e593662cf6b11684b24134fc442e Mon Sep 17 00:00:00 2001 From: Neil Daquioag Date: Wed, 3 Jun 2026 22:03:04 +0800 Subject: [PATCH 1/3] fix(forecast): ignore stale time-bounded runtime overlay reasons (#507) forecast --live marked working accounts as unavailable because the runtime overlay in runtime-observability.json persists a skip reason ("rate-limited", "cooling-down:...") on pool exhaustion and only clears it on an explicit runtime reset, never on a subsequent successful request. After the underlying window expired, the stale reason kept the forecast reporting the account as unavailable even though the proxy (which rebuilds skip reasons fresh per request) still routed to it successfully. doctor's forecast-runtime-alignment warning surfaced the same stale state via the shared forecast evaluation. Cross-reference time-bounded overlay reasons against the time-aware disk state before applying them: - "rate-limited" is dropped when getRateLimitResetTimeForFamily returns null (no active reset on disk for codex or a model-scoped key) - "cooling-down:..." is dropped when coolingDownUntil is absent or has elapsed Each reason validates only against its own backing disk state, so the fix never substitutes a misleading reason string. Non-time-bounded reasons ("circuit-open", "token-exhausted", "policy-blocked") have no disk expiry and remain unconditional. The check reuses the rateLimitResetAt value and the single now timestamp already computed in the function, so no extra disk read or clock skew is introduced. Tests: split the runtime-skip parametrized test into non-time-bounded reasons (still unconditional) and add coverage for stale vs active rate-limited and cooling-down overlays, including a model-scoped (codex:5h) active rate limit. --- lib/forecast.ts | 27 +++++++++- test/forecast.test.ts | 114 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/lib/forecast.ts b/lib/forecast.ts index 9283d4e1b..6db6d1486 100644 --- a/lib/forecast.ts +++ b/lib/forecast.ts @@ -258,11 +258,36 @@ export function evaluateForecastAccount( overlay?.accountSkipReasons?.[String(index)] ?? overlay?.lastPoolExhaustionSkipReasons?.[String(index)] ?? null; + // Time-bounded overlay reasons ("rate-limited", "cooling-down:...") are + // persisted to runtime-observability.json on pool exhaustion and only ever + // cleared by an explicit runtime reset, never on a subsequent successful + // request. That leaves a stale reason on disk after the underlying window + // expires, so the forecast would keep marking a working account as + // unavailable. Cross-reference the time-aware disk state before applying: + // drop the overlay reason when the condition it describes is no longer + // active. Each reason validates only against its own backing disk state + // ("rate-limited" -> rateLimitResetTimes, "cooling-down" -> coolingDownUntil) + // so we never substitute a misleading reason string. Non-time-bounded + // reasons ("circuit-open", "token-exhausted", "policy-blocked") have no disk + // expiry to check and are always applied. + const coolingDownActive = + typeof account.coolingDownUntil === "number" && + account.coolingDownUntil > now; + const isStaleOverlayReason = + overlayReason === "rate-limited" + ? rateLimitResetAt === null + : overlayReason?.startsWith("cooling-down") + ? !coolingDownActive + : false; if (overlay?.policyBlockedIndexes?.includes(index)) { availability = "unavailable"; riskScore += 95; reasons.push("runtime policy blocked account"); - } else if (overlayReason && overlayReason !== "already-attempted") { + } else if ( + overlayReason && + overlayReason !== "already-attempted" && + !isStaleOverlayReason + ) { availability = "unavailable"; riskScore += overlayReason === "circuit-open" diff --git a/test/forecast.test.ts b/test/forecast.test.ts index ad4547def..ec0be29ed 100644 --- a/test/forecast.test.ts +++ b/test/forecast.test.ts @@ -182,8 +182,8 @@ describe("forecast helpers", () => { expect(overlaid.reasons).toContain("runtime skip: circuit-open"); }); - it.each(["rate-limited", "cooling-down:server-error", "workspace-disabled"])( - "marks runtime skip reason %s as unavailable", + it.each(["circuit-open", "token-exhausted", "workspace-disabled"])( + "marks non-time-bounded runtime skip reason %s as unavailable", (reason) => { const now = 1_700_000_000_000; const result = evaluateForecastAccount({ @@ -205,6 +205,116 @@ describe("forecast helpers", () => { }, ); + it("ignores a stale rate-limited overlay when no rate limit is active on disk", () => { + const now = 1_700_000_000_000; + const account = { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // Expired entry: clearExpiredRateLimits-equivalent semantics mean this + // is no longer an active rate limit. + rateLimitResetTimes: { codex: now - 30_000 }, + }; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain("runtime skip: rate-limited"); + }); + + it("applies a rate-limited overlay when the rate limit is still active on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + rateLimitResetTimes: { codex: now + 30_000 }, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: rate-limited"); + }); + + it("applies a rate-limited overlay when a model-scoped limit is active on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + rateLimitResetTimes: { "codex:5h": now + 30_000 }, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: rate-limited"); + }); + + it("ignores a stale cooling-down overlay when cooldown has elapsed on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + coolingDownUntil: now - 1, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain( + "runtime skip: cooling-down:server-error", + ); + }); + + it("applies a cooling-down overlay when cooldown is still active on disk", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + coolingDownUntil: now + 60_000, + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: cooling-down:server-error"); + }); + it("recommends the best ready account", () => { const now = 1_700_000_000_000; const results = evaluateForecastAccounts([ From 65db170a82d684222756f0b1711661b2a56eac04 Mon Sep 17 00:00:00 2001 From: Neil Daquioag Date: Wed, 3 Jun 2026 22:14:37 +0800 Subject: [PATCH 2/3] test(forecast): cover missing rate-limit/cooldown disk fields in overlay staleness Add regression cases for the absent-field branch of the stale-overlay guard, raised in review: rateLimitResetTimes / coolingDownUntil entirely undefined (limit cleared by runtime reset or successful request after the overlay was written), not just an expired entry. - "rate-limited" overlay with no rateLimitResetTimes -> account ready, reason dropped (getRateLimitResetTimeForFamily returns null on !times) - "cooling-down:..." overlay with no coolingDownUntil -> account ready, reason dropped --- test/forecast.test.ts | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/forecast.test.ts b/test/forecast.test.ts index ec0be29ed..d809fba5a 100644 --- a/test/forecast.test.ts +++ b/test/forecast.test.ts @@ -229,6 +229,29 @@ describe("forecast helpers", () => { expect(result.reasons).not.toContain("runtime skip: rate-limited"); }); + it("ignores a stale rate-limited overlay when rateLimitResetTimes is absent", () => { + const now = 1_700_000_000_000; + const account = { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // No rateLimitResetTimes field at all: the limit was cleared (runtime + // reset or a successful request) after the overlay was written. + }; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain("runtime skip: rate-limited"); + }); + it("applies a rate-limited overlay when the rate limit is still active on disk", () => { const now = 1_700_000_000_000; const result = evaluateForecastAccount({ @@ -294,6 +317,30 @@ describe("forecast helpers", () => { ); }); + it("ignores a stale cooling-down overlay when coolingDownUntil is absent", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // No coolingDownUntil field at all: cooldown cleared after the + // overlay was written. + }, + runtimeOverlay: { + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain( + "runtime skip: cooling-down:server-error", + ); + }); + it("applies a cooling-down overlay when cooldown is still active on disk", () => { const now = 1_700_000_000_000; const result = evaluateForecastAccount({ From c547ff851057970c29ece9e91eb7b1f476ad403d Mon Sep 17 00:00:00 2001 From: Neil Daquioag Date: Wed, 3 Jun 2026 22:21:54 +0800 Subject: [PATCH 3/3] test(forecast): cover accountSkipReasons precedence in overlay staleness Raised in re-review: the resolver reads accountSkipReasons before lastPoolExhaustionSkipReasons (forecast.ts ?? chain), but existing tests only exercised the latter key. - stale reason via accountSkipReasons (no disk-backed limit/cooldown) -> account ready, reason dropped on the precedence path - active reason via accountSkipReasons (future rateLimitResetTimes) -> account unavailable, reason applied --- test/forecast.test.ts | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/forecast.test.ts b/test/forecast.test.ts index d809fba5a..fb82495c6 100644 --- a/test/forecast.test.ts +++ b/test/forecast.test.ts @@ -252,6 +252,55 @@ describe("forecast helpers", () => { expect(result.reasons).not.toContain("runtime skip: rate-limited"); }); + it("ignores a stale overlay reason resolved from accountSkipReasons (precedence path)", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + // No active rate limit or cooldown on disk. + }, + runtimeOverlay: { + // accountSkipReasons takes precedence over + // lastPoolExhaustionSkipReasons in the resolver; the staleness guard + // must apply to whichever key wins. + accountSkipReasons: { "0": "rate-limited" }, + lastPoolExhaustionSkipReasons: { "0": "cooling-down:server-error" }, + }, + }); + + expect(result.availability).toBe("ready"); + expect(result.reasons).not.toContain("runtime skip: rate-limited"); + expect(result.reasons).not.toContain( + "runtime skip: cooling-down:server-error", + ); + }); + + it("applies an active overlay reason resolved from accountSkipReasons (precedence path)", () => { + const now = 1_700_000_000_000; + const result = evaluateForecastAccount({ + index: 0, + now, + isCurrent: false, + account: { + refreshToken: "refresh-1", + addedAt: now - 10_000, + lastUsed: now - 10_000, + rateLimitResetTimes: { codex: now + 30_000 }, + }, + runtimeOverlay: { + accountSkipReasons: { "0": "rate-limited" }, + }, + }); + + expect(result.availability).toBe("unavailable"); + expect(result.reasons).toContain("runtime skip: rate-limited"); + }); + it("applies a rate-limited overlay when the rate limit is still active on disk", () => { const now = 1_700_000_000_000; const result = evaluateForecastAccount({