Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion lib/codex-manager/commands/best.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import type { ForecastAccountResult } from "../../forecast.js";
import { type CodexQuotaSnapshot, describeCodexProbeFailure } from "../../quota-probe.js";
import { resolveNormalizedModel } from "../../request/helpers/model-map.js";
import {
getModelProfile,
type ModelFamily,
resolveNormalizedModel,
} from "../../request/helpers/model-map.js";
import type { AccountStorageV3 } from "../../storage.js";
import type { TokenFailure, TokenResult } from "../../types.js";
import { DEFAULT_LIVE_PROBE_MODEL } from "../quota-cache-helpers.js";
Expand Down Expand Up @@ -123,6 +127,8 @@ export interface BestCommandDeps {
now: number;
refreshFailure?: TokenFailure;
liveQuota?: CodexQuotaSnapshot;
family?: ModelFamily;
model?: string | null;
}>,
) => ForecastAccountResult[];
recommendForecastAccount: (results: ForecastAccountResult[]) => {
Expand Down Expand Up @@ -282,13 +288,22 @@ export async function runBestCommand(
}
}

// Only an explicit --model moves the recommendation off the codex family;
// see the note in the forecast command. `best` exists to pick the account
// for wrapper traffic, which is codex-family.
const forecastFamily = options.modelProvided
? getModelProfile(probeModel).promptFamily
: undefined;
const forecastModel = options.modelProvided ? probeModel : undefined;
const forecastInputs = storage.accounts.map((account, index) => ({
index,
account,
isCurrent: index === deps.resolveActiveIndex(storage, "codex"),
now,
refreshFailure: refreshFailures.get(index),
liveQuota: liveQuotaByIndex.get(index),
family: forecastFamily,
model: forecastModel,
}));
const forecastResults = deps.evaluateForecastAccounts(forecastInputs);
const recommendation = deps.recommendForecastAccount(forecastResults);
Expand Down
31 changes: 30 additions & 1 deletion lib/codex-manager/commands/forecast.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ import {
} from "../forecast-report-shared.js";
import type { QuotaCacheData } from "../../quota-cache.js";
import { type CodexQuotaSnapshot, describeCodexProbeFailure } from "../../quota-probe.js";
import { DEFAULT_PROBE_MODEL, resolveNormalizedModel } from "../../request/helpers/model-map.js";
import {
DEFAULT_PROBE_MODEL,
getModelProfile,
type ModelFamily,
resolveNormalizedModel,
} from "../../request/helpers/model-map.js";
import { type AccountMetadataV3, type AccountStorageV3 } from "../../storage.js";
import type { TokenFailure, TokenResult } from "../../types.js";

Expand All @@ -23,6 +28,11 @@ interface ForecastCliOptions {
json: boolean;
explain: boolean;
model: string;
/**
* Whether --model was actually passed. The default probe model is not a
* codex-family model, so its family must NOT govern a bare invocation.
*/
modelProvided: boolean;
runtimeOverlay: boolean;
}

Expand Down Expand Up @@ -85,6 +95,8 @@ export interface ForecastCommandDeps {
quotaCache?: QuotaCacheData | null;
allAccounts?: readonly AccountMetadataV3[];
runtimeOverlay?: RuntimeForecastOverlay | null;
family?: ModelFamily;
model?: string | null;
}>,
) => ForecastAccountResult[];
summarizeForecast: (results: ForecastAccountResult[]) => {
Expand Down Expand Up @@ -150,6 +162,7 @@ function parseForecastArgs(
json: false,
explain: false,
model: DEFAULT_PROBE_MODEL,
modelProvided: false,
runtimeOverlay: true,
};

Expand Down Expand Up @@ -178,6 +191,7 @@ function parseForecastArgs(
return { ok: false, message: "Missing value for --model" };
}
options.model = value;
options.modelProvided = true;
i += 1;
continue;
}
Expand All @@ -187,6 +201,7 @@ function parseForecastArgs(
return { ok: false, message: "Missing value for --model" };
}
options.model = value;
options.modelProvided = true;
continue;
}
return { ok: false, message: `Unknown option: ${arg}` };
Expand Down Expand Up @@ -358,6 +373,18 @@ export async function runForecastCommand(
}
}

// Only an explicit --model moves the forecast off the codex family. The
// default probe model is gpt-5.6-sol, whose family is gpt-5.2, so keying a
// bare `forecast` on it would evaluate every account against a family no
// wrapper request uses - /codex/responses buckets into codex.
//
// probeModel, not requestedModel: rate-limit records are keyed by the
// normalized model the proxy routes on. Resolved once rather than per
// account: getModelProfile re-parses the model string on every call.
const forecastFamily = options.modelProvided
? getModelProfile(requestedModel).promptFamily
: undefined;
const forecastModel = options.modelProvided ? probeModel : undefined;
const forecastInputs = storage.accounts.map((account, index) => ({
index,
account,
Expand All @@ -368,6 +395,8 @@ export async function runForecastCommand(
quotaCache,
allAccounts: storage.accounts,
runtimeOverlay,
family: forecastFamily,
model: forecastModel,
}));
const forecastResults = deps.evaluateForecastAccounts(forecastInputs);
const summary = deps.summarizeForecast(forecastResults);
Expand Down
16 changes: 16 additions & 0 deletions lib/codex-manager/commands/report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ interface ReportCliOptions {
json: boolean;
explain: boolean;
model: string;
/** Whether --model was actually passed; see ForecastCliOptions.modelProvided. */
modelProvided: boolean;
maxAccounts?: number;
maxProbes?: number;
cachedOnly: boolean;
Expand Down Expand Up @@ -144,6 +146,7 @@ function parseReportArgs(args: string[]): ParsedArgsResult<ReportCliOptions> {
json: false,
explain: false,
model: DEFAULT_PROBE_MODEL,
modelProvided: false,
cachedOnly: false,
};

Expand Down Expand Up @@ -172,6 +175,7 @@ function parseReportArgs(args: string[]): ParsedArgsResult<ReportCliOptions> {
return { ok: false, message: "Missing value for --model" };
}
options.model = value;
options.modelProvided = true;
i += 1;
continue;
}
Expand All @@ -181,6 +185,7 @@ function parseReportArgs(args: string[]): ParsedArgsResult<ReportCliOptions> {
return { ok: false, message: "Missing value for --model" };
}
options.model = value;
options.modelProvided = true;
continue;
}
if (arg === "--max-accounts") {
Expand Down Expand Up @@ -462,6 +467,15 @@ export async function runReportCommand(
}
}

// Only an explicit --model moves the report off the codex family; see the
// note in the forecast command. promptFamily is reused from the inspection
// rather than re-resolved per account.
const forecastFamily = options.modelProvided
? modelInspection.promptFamily
: undefined;
const forecastModel = options.modelProvided
? modelInspection.normalized
: undefined;
const forecastResults = storage
? evaluateForecastAccounts(
storage.accounts.map((account, index) => ({
Expand All @@ -474,6 +488,8 @@ export async function runReportCommand(
quotaCache,
allAccounts: storage.accounts,
runtimeOverlay: runtimeSnapshot,
family: forecastFamily,
model: forecastModel,
})),
)
: [];
Expand Down
41 changes: 34 additions & 7 deletions lib/forecast.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ import {
isQuotaCacheEntryExhausted,
quotaUsedPercentIsExhausted,
} from "./quota-readiness.js";
import { getRateLimitResetTimeForFamily } from "./runtime/account-status.js";
import type { ModelFamily } from "./request/helpers/model-map.js";
import {
getRateLimitResetTimeForFamily,
getRateLimitResetTimeForModel,
} from "./runtime/account-status.js";
import type { AccountMetadataV3 } from "./storage.js";
import type { TokenFailure } from "./types.js";

Expand All @@ -27,6 +31,18 @@ export interface ForecastAccountInput {
quotaCache?: QuotaCacheData | null;
allAccounts?: readonly AccountMetadataV3[];
runtimeOverlay?: RuntimeForecastOverlay | null;
/**
* Prompt family whose per-family rate-limit records gate this forecast.
* Callers with a model in hand resolve it via getModelProfile; the codex
* default preserves the historical behavior for model-less surfaces.
*/
family?: ModelFamily;
/**
* Normalized model the forecast is about, when the caller has one. Selection
* keys token/concurrency limits under `family:<model>`, so without it a
* sibling model's record would be read as gating this one.
*/
model?: string | null;
}

export interface RuntimeForecastOverlay {
Expand Down Expand Up @@ -242,11 +258,18 @@ export function evaluateForecastAccount(
appendWaitReason(reasons, "cooldown remaining", remaining);
}

const rateLimitResetAt = getRateLimitResetTimeForFamily(
account,
now,
"codex",
);
// With a model in hand, gate on exactly the keys selection consults for that
// family/model pair, and on the LATEST of them: a sibling model's record does
// not gate this request, and while both the family-wide and model-scoped keys
// are active the account stays skipped until the later one expires.
//
// Without one (status, fix) no model key can be singled out, so keep the
// family-wide union - the conservative answer, and the behavior those
// surfaces have always had.
const forecastFamily = input.family ?? "codex";
const rateLimitResetAt = input.model
? getRateLimitResetTimeForModel(account, now, forecastFamily, input.model)
: getRateLimitResetTimeForFamily(account, now, forecastFamily);
if (typeof rateLimitResetAt === "number") {
const remaining = Math.max(0, rateLimitResetAt - now);
waitMs = Math.max(waitMs, remaining);
Expand Down Expand Up @@ -298,7 +321,11 @@ export function evaluateForecastAccount(
// drop the overlay reason when the condition it describes is no longer
// active. Each reason validates only against its own backing disk state
// ("rate-limited" -> rateLimitResetTimes, "cooling-down" -> coolingDownUntil)
// so we never substitute a misleading reason string. Non-time-bounded
// so we never substitute a misleading reason string. The rate-limited
// cross-check inherits rateLimitResetAt's scope above: a record under one of
// the keys that actually gates this family/model request keeps the reason,
// while a record for another family - or another model in the same family -
// neither sustains it nor gates this request. Non-time-bounded
// reasons ("circuit-open", "token-exhausted", "policy-blocked") have no disk
// expiry to check and are always applied.
const coolingDownActive =
Expand Down
41 changes: 41 additions & 0 deletions lib/runtime/account-status.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { getQuotaKey } from "../accounts/rate-limits.js";
import type { ModelFamily } from "../prompts/codex.js";

export function resolveActiveIndex(
Expand Down Expand Up @@ -50,3 +51,43 @@ export function formatRateLimitEntry(
if (remaining <= 0) return null;
return `resets in ${formatWaitTime(remaining)}`;
}

/**
* When a request for `family`/`model` stops being rate limited: the LATEST
* active bound among exactly the two keys selection consults — the family-wide
* key and `family:<model>` (see `isRateLimitedForFamily`). Null when neither is
* active.
*
* Deliberately narrower and later than `getRateLimitResetTimeForFamily`, whose
* earliest-reset-across-every-`family:*`-key answer feeds wait displays and
* model-less callers:
*
* - narrower, because `markRateLimitedWithReason` keys token/concurrency limits
* under `family:<model>`, and a sibling model's record does not gate this
* request — folding it in reports a delay the runtime proxy would not impose;
* - later, because the account stays skipped while EITHER key is active, so the
* earliest reset understates the wait when both are set.
*
* Requires a model by construction: a caller without one cannot know which
* model key applies and should keep the family-wide union above.
*/
export function getRateLimitResetTimeForModel(
account: { rateLimitResetTimes?: Record<string, number | undefined> },
now: number,
family: ModelFamily,
model: string,
): number | null {
const times = account.rateLimitResetTimes;
if (!times) return null;

let latest: number | null = null;
const consider = (value: number | undefined): void => {
if (typeof value !== "number" || !Number.isFinite(value)) return;
if (value <= now) return;
if (latest === null || value > latest) latest = value;
};

consider(times[getQuotaKey(family)]);
consider(times[getQuotaKey(family, model)]);
return latest;
}
74 changes: 74 additions & 0 deletions test/codex-manager-best-command.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ import {
} from "../lib/codex-manager/commands/best.js";
import { CodexUnavailableError } from "../lib/errors.js";
import { CODEX_UNAVAILABLE_PROBE_NOTE } from "../lib/quota-probe.js";
import { DEFAULT_LIVE_PROBE_MODEL } from "../lib/codex-manager/quota-cache-helpers.js";
import {
getModelProfile,
resolveNormalizedModel,
} from "../lib/request/helpers/model-map.js";
import type { AccountStorageV3 } from "../lib/storage.js";

function createAccount(
Expand Down Expand Up @@ -137,6 +142,75 @@ describe("runBestCommand", () => {
);
});

it("threads the probe model's family and id into forecast evaluation", async () => {
const evaluateForecastAccounts = vi.fn((inputs) => {
void inputs;
return [
{
index: 0,
label: "1. best@example.com",
isCurrent: true,
availability: "ready",
riskScore: 0,
riskLevel: "low",
waitMs: 0,
reasons: [],
},
] as const;
});
const deps = createDeps({
evaluateForecastAccounts,
parseBestArgs: vi.fn(() => ({
ok: true as const,
options: {
live: false,
json: true,
model: DEFAULT_LIVE_PROBE_MODEL,
modelProvided: false,
} satisfies BestCliOptions,
})),
});

await expect(runBestCommand(["--json"], deps)).resolves.toBe(0);
const defaulted = evaluateForecastAccounts.mock.calls.at(-1)?.[0] as
| Array<{ family?: string; model?: string | null }>
| undefined;
// `best` picks the account for wrapper traffic, which is codex-family.
// DEFAULT_LIVE_PROBE_MODEL is gpt-5.6-sol, whose family is gpt-5.2, so a
// bare invocation must leave both unset and fall back to codex rather
// than rank accounts against a family no wrapper request uses.
expect(getModelProfile(DEFAULT_LIVE_PROBE_MODEL).promptFamily).not.toBe(
"codex",
);
expect(defaulted?.[0]?.family).toBeUndefined();
expect(defaulted?.[0]?.model).toBeUndefined();

const explicitDeps = createDeps({
evaluateForecastAccounts,
parseBestArgs: vi.fn(() => ({
ok: true as const,
options: {
live: true,
json: true,
// The bare alias, NOT the canonical id: resolveNormalizedModel
// maps it to "gpt-5.6-sol", so this proves the normalized id
// is what reaches evaluation rather than the raw flag value.
model: "gpt-5.6",
modelProvided: true,
} satisfies BestCliOptions,
})),
});
await expect(
runBestCommand(["--json", "--live", "--model", "gpt-5.6"], explicitDeps),
).resolves.toBe(0);
const explicit = evaluateForecastAccounts.mock.calls.at(-1)?.[0] as
| Array<{ family?: string; model?: string | null }>
| undefined;
expect(explicit?.[0]?.family).toBe(getModelProfile("gpt-5.6").promptFamily);
expect(resolveNormalizedModel("gpt-5.6")).not.toBe("gpt-5.6");
expect(explicit?.[0]?.model).toBe(resolveNormalizedModel("gpt-5.6"));
});

it("emits json output when no accounts are configured", async () => {
const deps = createDeps({
loadAccounts: vi.fn(async () => ({
Expand Down
Loading