fix(codex): rotate pool accounts on model capacity

docs-site/src/content/docs/reference/cli/providers-accounts.md

@@ -164,6 +164,11 @@ Direct mode keeps using the caller-owned/native main credential. Usage-based pro
 recovery may later select another eligible Pool account. Those recovery paths remain active when
 usage-based switching is off. OpenCodex replays the conversation after an account change, but the
 provider-side prompt cache may be cold. Unknown providers or ids exit 1.
+Before substantive output, an upstream model-capacity rejection rotates through each eligible Pool
+account once without persisting a cooldown. A rejected account does not create or refresh thread
+affinity; only the account whose response is accepted becomes the thread binding. Exhaustion returns
+the first capacity error, and the next request starts with a fresh eligible set and no new affinity from
+the rejected attempts. Direct mode, exact account selectors, and failures after output do not rotate.
 On a **401/403**, App login clears that account's process-local affinity and requires reauthentication.
 On a **429**, opencodex honors `Retry-After`, starts the account cooldown, clears affinity, and may
 rotate the request to another eligible Pool account. These failure transitions remain active with

docs-site/src/content/docs/reference/configuration/providers.md

@@ -150,6 +150,16 @@ alternate account in the same request, even when usage-based proactive switching
 changes preserve and replay the conversation context, but provider-side prompt-cache reuse across
 accounts is not guaranteed and the cache may need to warm again.
 
+Before any text, reasoning, tool call, or other model output reaches the client, a structured
+`server_is_overloaded` / `slow_down` model-capacity rejection (or the standard
+`Selected model is at capacity. Please try a different model.` response) tries each remaining
+eligible Pool account once. This exclusion is request-local: rejected capacity attempts do not write
+account cooldown or health. Thread affinity is not created or refreshed until a non-capacity response
+is accepted; after rotation, only the accepted account is bound. If every account returns capacity,
+the first response is returned and the next request starts with a fresh eligible set, without any new
+affinity from the rejected attempts. Direct mode and exact account selectors never use this rotation,
+and capacity after substantive output is never replayed.
+
 On a **401/403**, App login clears that account's process-local affinity and requires reauthentication.
 On a **429**, opencodex honors `Retry-After`, starts the account cooldown, clears affinity, and may
 rotate the request to another eligible Pool account. These failure transitions remain active with

src/codex/auth-context.ts

@@ -21,6 +21,8 @@ import {
   tryAcquireCodexQuotaProbeLease,
   tryAcquireCodexQuotaScopeProbeLease,
   pickAlternateCodexAccount,
+  pickAlternateCodexAccountExcluding,
+  previewCodexAccountForRequestDetailed,
   resolveCodexAccountForThreadDetailed,
 } from "./routing";
 import type { CodexCooldownSource, CodexQuotaScope } from "./routing";
@@ -227,10 +229,14 @@ export function shouldMarkAccountNeedsReauthForCodexAuthFailure(cause: unknown):
 
 export interface ResolveCodexAuthContextOptions {
   excludeAccountId?: string;
+  /** Request-local exclusion set for bounded multi-account recovery. */
+  excludeAccountIds?: ReadonlySet<string>;
   /** Resolve exactly this account without consulting or mutating Pool selection. */
   accountId?: string;
   /** Final native model selected for this request, used to select its quota group. */
   modelId?: string;
+  /** Preview without selection-state writes until the upstream response is accepted. */
+  deferSelectionCommit?: boolean;
   /** Short reservation converted to turn ownership before native `__main__` token materialization. */
   beginCodexAccountSelection?: () => CodexAccountSelectionAdmission | undefined;
   /** Test-only native credential read seams. */
@@ -253,7 +259,9 @@ export async function resolveCodexAuthContext(
 ): Promise<CodexAuthContext> {
   const writerGeneration = captureConfigGeneration();
   const fixedAccountId = options.accountId;
-  if (fixedAccountId !== undefined && options.excludeAccountId !== undefined) {
+  const hasExclusions = options.excludeAccountId !== undefined
+    || (options.excludeAccountIds?.size ?? 0) > 0;
+  if (fixedAccountId !== undefined && hasExclusions) {
     throw new Error("Codex auth context cannot select and exclude an account simultaneously");
   }
   // An explicit namespace binding is stronger than the provider's default mode. It must use the
@@ -284,19 +292,31 @@ export async function resolveCodexAuthContext(
     const threadId = headers.get("x-codex-parent-thread-id");
     const resolution = fixedAccountId !== undefined
       ? { status: "selected" as const, accountId: fixedAccountId }
-      : options.excludeAccountId
+      : hasExclusions
       ? (() => {
-          const selected = pickAlternateCodexAccount(
-            config,
-            options.excludeAccountId!,
-            Date.now(),
-            quotaScope,
-            selectionOptions,
-          );
+          const selected = options.excludeAccountIds
+            ? pickAlternateCodexAccountExcluding(
+              config,
+              options.excludeAccountIds,
+              [...options.excludeAccountIds].at(-1)!,
+              Date.now(),
+              quotaScope,
+              selectionOptions,
+              false,
+            )
+            : pickAlternateCodexAccount(
+              config,
+              options.excludeAccountId!,
+              Date.now(),
+              quotaScope,
+              selectionOptions,
+            );
           return selected
             ? { status: "selected" as const, accountId: selected }
             : { status: "none" as const };
         })()
+      : options.deferSelectionCommit
+      ? previewCodexAccountForRequestDetailed(threadId, config, Date.now(), quotaScope, selectionOptions)
       : resolveCodexAccountForThreadDetailed(threadId, config, Date.now(), quotaScope, selectionOptions);
     if (resolution.status === "expired") throw new CodexThreadAffinityExpiredError(resolution.accountId);
     const selected = resolution.status === "selected" ? resolution.accountId : null;
@@ -309,7 +329,7 @@ export async function resolveCodexAuthContext(
       // temporary fence rather than misclassifying that credential as invalid.
       // A configured pool retry/exclusion that finds no alternate preserves its
       // ordinary pool-auth failure instead of being mislabeled as a main fence.
-      if (nativeMainTrafficBlocked && !options.excludeAccountId) {
+      if (nativeMainTrafficBlocked && !hasExclusions) {
         throw new CodexMainProfileDrainingError();
       }
       throw new CodexPoolAuthenticationError();

src/codex/pool-rotation.ts

@@ -210,6 +210,32 @@ export function peekRoundRobinAccount(
   return pickRoundRobinFromState(eligibleIds, stickyLimit, scratch, false);
 }
 
+/**
+ * Record the account whose response was actually accepted after a deferred pick.
+ * Completion order is authoritative: concurrent requests each contribute one
+ * smooth-weighted success for the account that served them.
+ */
+export function commitRoundRobinAccountSuccess(
+  poolKey: string,
+  eligibleIds: readonly string[],
+  accountId: string,
+  stickyLimit: number,
+): boolean {
+  if (!eligibleIds.includes(accountId)) return false;
+  const state = getOrCreateState(poolKey);
+  if (state.activeKey !== accountId) {
+    delete state.activeKey;
+    state.successes = 0;
+    const total = eligibleIds.length;
+    for (const id of eligibleIds) {
+      state.currentWeights.set(id, (state.currentWeights.get(id) ?? 0) + 1);
+    }
+    state.currentWeights.set(accountId, (state.currentWeights.get(accountId) ?? 0) - total);
+  }
+  notePoolRotationSuccess(poolKey, accountId, stickyLimit);
+  return true;
+}
+
 export function notePoolRotationSuccess(
   poolKey: string,
   accountId: string,

src/codex/routing.ts

@@ -8,6 +8,7 @@ import { isCodexAccountUsable, type CodexAccountUsabilityOptions } from "./accou
 import { isAccountNeedsReauth, markAccountNeedsReauth } from "./account-runtime-state";
 import {
   POOL_KEY_CODEX,
+  commitRoundRobinAccountSuccess,
   normalizeAccountPoolStickyLimit,
   normalizeAccountPoolStrategy,
   notePoolRotationFailure,
@@ -900,16 +901,25 @@ function bindThreadAffinity(
   pruneLruThreadAffinities();
 }
 
+type CodexAccountExclusion = string | ReadonlySet<string> | undefined;
+
+/** Match a single legacy exclusion or a request-local exclusion set. */
+function isExcludedCodexAccount(exclusion: CodexAccountExclusion, accountId: string): boolean {
+  return typeof exclusion === "string"
+    ? exclusion === accountId
+    : exclusion?.has(accountId) === true;
+}
+
 function getEligiblePoolAccounts(
   config: OcxConfig,
-  excludeId?: string,
+  exclusion?: CodexAccountExclusion,
   now = Date.now(),
   quotaScope?: CodexQuotaScope,
   selectionOptions?: CodexAccountUsabilityOptions,
 ): readonly string[] {
   const ids = (config.codexAccounts ?? [])
     .filter(account => isSelectableCodexPoolAccount(account)
-      && account.id !== excludeId
+      && !isExcludedCodexAccount(exclusion, account.id)
       && !isCodexAccountPaused(config, account.id)
       && !isAccountNeedsReauth(account.id))
     .filter(account => getCodexQuotaHealthSnapshot(account.id, quotaScope, now) === null)
@@ -919,7 +929,7 @@ function getEligiblePoolAccounts(
   // The main Codex account is not stored in config.codexAccounts; include it as a
   // first-class rotation candidate when its read-only token is usable (Option A).
   if (
-    excludeId !== MAIN_CODEX_ACCOUNT_ID
+    !isExcludedCodexAccount(exclusion, MAIN_CODEX_ACCOUNT_ID)
     && !isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID)
     && !isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)
     && getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope, now) === null
@@ -1151,21 +1161,48 @@ export function pickAlternateCodexAccount(
   now = Date.now(),
   quotaScope?: CodexQuotaScope,
   selectionOptions?: CodexAccountUsabilityOptions,
+): string | null {
+  return pickAlternateCodexAccountExcluding(
+    config,
+    new Set([excludeId]),
+    excludeId,
+    now,
+    quotaScope,
+    selectionOptions,
+  );
+}
+
+/** Strategy-aware alternate that never revisits an account already tried by this request. */
+export function pickAlternateCodexAccountExcluding(
+  config: OcxConfig,
+  excludedIds: ReadonlySet<string>,
+  afterId: string,
+  now = Date.now(),
+  quotaScope?: CodexQuotaScope,
+  selectionOptions?: CodexAccountUsabilityOptions,
+  commitRoundRobin = true,
 ): string | null {
   const strategy = normalizeAccountPoolStrategy(config.accountPoolStrategy);
   // The exclusion is passed into eligibility rather than post-filtered off its
   // result: when the excluded account is the only healthy member of the top
   // tier, the tier walk must be free to descend instead of selecting that tier
   // and then handing back an empty list.
   if (strategy === "round-robin") {
-    const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
-    return pickRoundRobinAccount(codexPoolKeyForScope(quotaScope), eligible, stickyLimitForConfig(config));
+    const eligible = getEligiblePoolAccounts(config, excludedIds, now, quotaScope, selectionOptions);
+    const poolKey = codexPoolKeyForScope(quotaScope);
+    const stickyLimit = stickyLimitForConfig(config);
+    return commitRoundRobin
+      ? pickRoundRobinAccount(poolKey, eligible, stickyLimit)
+      : peekRoundRobinAccount(poolKey, eligible, stickyLimit);
   }
   if (strategy === "fill-first") {
-    const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
-    return pickNextFillFirstCodexAccount(config, excludeId, eligible, now, selectionOptions);
+    const eligible = getEligiblePoolAccounts(config, excludedIds, now, quotaScope, selectionOptions);
+    return pickNextFillFirstCodexAccount(config, afterId, eligible, now, selectionOptions);
   }
-  return pickLowestUsageCodexAccount(config, excludeId, now, quotaScope, selectionOptions);
+  return pickLowestUsageAmong(
+    config,
+    getEligiblePoolAccounts(config, excludedIds, now, quotaScope, selectionOptions),
+  );
 }
 
 /** Effective active: automatic runtime cursor, else operator/persisted selection. */
@@ -1465,6 +1502,80 @@ export function previewCodexAccountForRequest(
   return active;
 }
 
+/** Preview Pool selection; expired-affinity cleanup matches live resolve. */
+export function previewCodexAccountForRequestDetailed(
+  threadId: string | null,
+  config: OcxConfig,
+  now = Date.now(),
+  quotaScope?: CodexQuotaScope,
+  selectionOptions?: CodexAccountUsabilityOptions,
+): CodexThreadResolution {
+  const entry = threadId ? getThreadAffinity(threadId, quotaScope) : undefined;
+  if (entry && isThreadAffinityExpired(entry, now)) {
+    deleteThreadAffinity(threadId!, quotaScope);
+    return { status: "expired", accountId: entry.accountId };
+  }
+  const accountId = previewCodexAccountForRequest(threadId, config, now, quotaScope, selectionOptions);
+  return accountId ? { status: "selected", accountId } : { status: "none" };
+}
+
+/**
+ * Commit a deferred Pool selection exactly once, after its response is accepted.
+ * Capacity-rejected and unavailable attempts remain request-local exclusions.
+ */
+export function commitCodexAcceptedAccountSelection(
+  config: OcxConfig,
+  threadId: string | null,
+  accountId: string,
+  modelId: string | undefined,
+  excludedAccountIds: ReadonlySet<string>,
+  now = Date.now(),
+): void {
+  const quotaScope = codexQuotaScopeForModel(modelId);
+  if (!isCodexAccountSelectable(config, accountId, now, quotaScope)) return;
+  if (!isIndependentCodexQuotaScope(quotaScope)) releaseDrainedCodexAccountPin(config);
+
+  const existingAffinity = threadId ? getThreadAffinity(threadId, quotaScope) : undefined;
+  const reusedAffinity = existingAffinity?.accountId === accountId
+    && !isThreadAffinityExpired(existingAffinity, now)
+    && isThreadAffinityGenerationLive(existingAffinity)
+    && !shouldFailover(config, accountId, now);
+
+  const strategy = normalizeAccountPoolStrategy(config.accountPoolStrategy);
+  if (reusedAffinity) {
+    existingAffinity.lastUsedAt = now;
+    if (strategy === "quota") {
+      const threshold = config.autoSwitchThreshold ?? 80;
+      const usage = threshold > 0
+        ? computeCodexUsageScore(getAccountQuota(accountId), getPoolAccountPlan(config, accountId))
+        : 0;
+      const overThreshold = threshold > 0 && !isUnknownUsage(usage) && usage >= threshold;
+      if (overThreshold || now - existingAffinity.lastReevalAt >= CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS) {
+        existingAffinity.lastReevalAt = now;
+      }
+    }
+    return;
+  }
+
+  if (strategy === "round-robin") {
+    const eligible = getEligiblePoolAccounts(config, excludedAccountIds, now, quotaScope);
+    const committed = commitRoundRobinAccountSuccess(
+      codexPoolKeyForScope(quotaScope),
+      eligible,
+      accountId,
+      stickyLimitForConfig(config),
+    );
+    if (!committed) return;
+    if (!isIndependentCodexQuotaScope(quotaScope)) rememberActiveCodexAccount(config, accountId);
+  } else if (strategy === "fill-first") {
+    if (!isIndependentCodexQuotaScope(quotaScope)) rememberActiveCodexAccount(config, accountId);
+  } else if (!isIndependentCodexQuotaScope(quotaScope)) {
+    setActiveCodexAccount(config, accountId);
+  }
+
+  if (threadId) bindThreadAffinity(threadId, accountId, now, quotaScope);
+}
+
 export function resolveCodexAccountForThreadDetailed(
   threadId: string | null,
   config: OcxConfig,

src/lib/upstream-retry.ts

@@ -244,6 +244,10 @@ export interface ResetRetryOptions {
 export interface TransientRetryOptions extends ResetRetryOptions {
   /** Test seam: per-attempt slow budget override (defaults to TRANSIENT_RETRY_SLOW_ATTEMPT_MS). */
   slowAttemptMs?: number;
+  /** Optional semantic gate run before retrying a transient HTTP response. */
+  prepareTransientRetry?: (
+    response: Response,
+  ) => Promise<{ response: Response; retry: boolean }> | { response: Response; retry: boolean };
 }
 
 export type UpstreamSendRecovery = "connection-reset" | "transient-5xx";
@@ -367,6 +371,11 @@ export async function fetchWithTransientRetry(
     if (res.ok || !isTransientUpstreamStatus(res.status)) return res;
     if (opts.abortSignal?.aborted) return res;
     if (Date.now() - attemptStart > slowAttemptMs) return res;
+    if (opts.prepareTransientRetry) {
+      const prepared = await opts.prepareTransientRetry(res);
+      res = prepared.response;
+      if (!prepared.retry) return res;
+    }
     console.warn(
       `[upstream-retry] transient ${res.status}${opts.label ? ` (${opts.label})` : ""} — retrying (${attempt + 2}/${attempts})`,
     );