diff --git a/src/codex/paths.ts b/src/codex/paths.ts index 1266db6db3..362bf09f1c 100644 --- a/src/codex/paths.ts +++ b/src/codex/paths.ts @@ -29,6 +29,11 @@ export const CODEX_PROFILE_PATH = join(CODEX_HOME, "opencodex.config.toml"); export const DEFAULT_CATALOG_PATH = join(CODEX_HOME, "opencodex-catalog.json"); export const CODEX_MODELS_CACHE_PATH = join(CODEX_HOME, "models_cache.json"); +/** Runtime CODEX_HOME lookup (honors CODEX_HOME env changes after import). */ +export function getCodexHome(): string { + return resolveCodexHome(); +} + export function tomlString(value: string): string { return JSON.stringify(value); } diff --git a/src/codex/routing.ts b/src/codex/routing.ts index e67b6a44a5..c2ab45a2ea 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -235,14 +235,8 @@ export function computeQuotaCooldownUntil(meta: CodexUpstreamOutcomeMeta = {}): * Returns the lease id, or null when no probe may go out right now. */ export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): string | null { - const health = upstreamHealth.get(accountId); - if (!health) return null; - const cooldownUntil = health.cooldownUntil; - if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return null; - if (health.cooldownSource === "retry-after") return null; - if (health.probeLeaseId !== undefined) return null; - const origin = health.lastProbeAt ?? health.cooldownSince ?? cooldownUntil; - if (now - origin < CODEX_QUOTA_PROBE_INTERVAL_MS) return null; + if (!canAcquireCodexQuotaProbeLease(accountId, now)) return null; + const health = upstreamHealth.get(accountId)!; const probeLeaseId = randomUUID(); upstreamHealth.set(accountId, { ...health, @@ -253,6 +247,18 @@ export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now return probeLeaseId; } +/** Side-effect-free check mirroring {@link tryAcquireCodexQuotaProbeLease} eligibility. */ +export function canAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): boolean { + const health = upstreamHealth.get(accountId); + if (!health) return false; + const cooldownUntil = health.cooldownUntil; + if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return false; + if (health.cooldownSource === "retry-after") return false; + if (health.probeLeaseId !== undefined) return false; + const origin = health.lastProbeAt ?? health.cooldownSince ?? cooldownUntil; + return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS; +} + /** * Hand a probe lease back without recording an upstream outcome. Used by paths * that take a lease and then fail before any request reaches upstream. @@ -391,7 +397,7 @@ function getEligiblePoolAccounts(config: OcxConfig, excludeId?: string, now = Da return ids; } -function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined { +export function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined { if (accountId === MAIN_CODEX_ACCOUNT_ID) return getMainAccountPlan(); return (config.codexAccounts ?? []).find(account => !account.isMain && account.id === accountId)?.plan; } @@ -499,6 +505,74 @@ export function resolveCodexAccountForThread( return resolution.status === "selected" ? resolution.accountId : null; } +/** + * Side-effect-free preview of the Codex pool account native routing would prefer. + * Used for subagent fallback quota decisions before final auth. + * + * Does not mutate activeCodexAccountId, thread affinity, config on disk, or probe leases. + * Mirrors {@link resolveCodexAccountForThreadDetailed} account choice, including returning a + * configured cooled account so callers can evaluate probe/quota availability. + */ +export function previewCodexAccountForRequest( + threadId: string | null, + config: OcxConfig, + now = Date.now(), +): string | null { + if (threadId && threadAccountMap.has(threadId)) { + const entry = threadAccountMap.get(threadId)!; + if ( + !isThreadAffinityExpired(entry, now) + && isThreadAffinityGenerationLive(entry) + && isCodexAccountSelectable(config, entry.accountId, now) + && !shouldFailover(config, entry.accountId, now) + ) { + const threshold = config.autoSwitchThreshold ?? 80; + if (threshold > 0) { + const usage = computeCodexUsageScore( + getAccountQuota(entry.accountId), + getPoolAccountPlan(config, entry.accountId), + ); + if (usage >= threshold) { + const best = pickLowerUsageAccount(config, entry.accountId, usage, now); + if (best !== entry.accountId) return best; + } + } + return entry.accountId; + } + // Stale/unusable affinity is ignored for preview (no map mutation). + } + + let active = config.activeCodexAccountId ?? null; + if (!active) { + return pickLowestUsageCodexAccount(config, undefined, now); + } + if (!isCodexAccountSelectable(config, active, now)) { + const fallback = pickLowestUsageCodexAccount(config, active, now); + if (fallback) active = fallback; + else if (hasConfiguredPoolAccount(config, active)) return active; + else return null; + } + + const threshold = config.autoSwitchThreshold ?? 80; + if (threshold > 0) { + const usage = computeCodexUsageScore(getAccountQuota(active), getPoolAccountPlan(config, active)); + if (usage >= threshold) { + active = pickLowerUsageAccount(config, active, usage, now); + } + } + if (shouldFailover(config, active, now)) { + const best = pickLowestUsageCodexAccount(config, active, now); + if (best) active = best; + } + if (!isCodexAccountUsable(config, active)) { + return hasConfiguredPoolAccount(config, active) ? active : null; + } + if (isCodexAccountInCooldown(active, now)) { + return hasConfiguredPoolAccount(config, active) ? active : null; + } + return active; +} + export function resolveCodexAccountForThreadDetailed( threadId: string | null, config: OcxConfig, diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts new file mode 100644 index 0000000000..ec07d51db3 --- /dev/null +++ b/src/codex/subagent-model-fallback.ts @@ -0,0 +1,455 @@ +/** + * Quota-aware subagent model fallback (issue #374). + * + * codex-rs spawns children with the agent-role TOML `model` pinned; when that model's + * provider quota is exhausted the child fails immediately. This module rewrites thread_spawn + * requests at the proxy choke point to the next healthy model in a configured fallback chain. + */ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { hasOwnProvider } from "../config"; +import { isRateLimitOrQuotaFailureMessage } from "../lib/errors"; +import type { OcxParsedRequest, OcxConfig } from "../types"; +import { slugsEquivalent } from "../providers/slug-codec"; +import { CODEX_HOME, getCodexHome } from "./paths"; +import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota"; +import { + canAcquireCodexQuotaProbeLease, + computeCodexUsageScore, + getPoolAccountPlan, + isCodexAccountInCooldown, +} from "./routing"; +import { isCodexAccountUsable } from "./account-usability"; +import { slugEquals } from "../providers/slug-codec"; +import { isThreadSpawnRequest } from "../server/effort-policy"; +import { PROVIDER_REGISTRY } from "../providers/registry"; +import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; +import { routeModel, type RouteResult } from "../router"; +export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; + +type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise; +let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null; +let quotaPrimeInFlight: Promise | null = null; + +type ModelHealth = { + unavailableUntil: number; + reason: string; +}; + +const modelHealth = new Map(); +const quotaPrimedAt = new Map(); +const knownProviderIdSet = new Set(PROVIDER_REGISTRY.map(entry => entry.id.toLowerCase())); + +function tryRouteFallbackModel(config: OcxConfig, model: string): RouteResult | null { + try { + return routeModel(config, model); + } catch { + return null; + } +} + +function isPoolCodexRoute(route: RouteResult): boolean { + return route.codexAccountMode === "pool"; +} + +function healthKey(model: string, accountId: string | null, poolScoped: boolean): string { + const scopedAccountId = poolScoped ? accountId : null; + return `${scopedAccountId ?? "none"}::${model.toLowerCase()}`; +} + +function isDisabledFallbackModel(model: string, config: OcxConfig): boolean { + const disabled = config.disabledModels ?? []; + if (disabled.length === 0) return false; + if (!model.includes("/")) { + return disabled.some(stored => stored === model || slugEquals(stored, "openai", model)); + } + const slash = model.indexOf("/"); + const provider = model.slice(0, slash); + const modelId = model.slice(slash + 1); + return disabled.some(stored => stored === model || slugEquals(stored, provider, modelId)); +} + +function pollIntervalMs(config: OcxConfig): number { + const configured = config.subagentModelFallbackPollMs; + if (typeof configured !== "number" || !Number.isFinite(configured) || configured < 1_000) { + return DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS; + } + return configured; +} + +function normalizedChain(primary: string, config: OcxConfig, extra: readonly string[] = []): string[] { + const chain: string[] = []; + const seen = new Set(); + const push = (model: string | undefined) => { + if (!model || model.trim() === "") return; + const trimmed = model.trim(); + const key = trimmed.toLowerCase(); + if (seen.has(key)) return; + seen.add(key); + chain.push(trimmed); + }; + push(primary); + for (const model of extra) push(model); + for (const model of config.subagentModelFallback ?? []) push(model); + return chain; +} + +export function buildSubagentModelChain( + primary: string, + config: OcxConfig, + extraFallback: readonly string[] = [], +): string[] { + return normalizedChain(primary, config, extraFallback); +} + +function quotaThreshold(config: OcxConfig): number { + const threshold = config.autoSwitchThreshold ?? 80; + return threshold > 0 ? threshold : Number.POSITIVE_INFINITY; +} + +function activeCodexAccountId(config: OcxConfig): string | null { + return config.activeCodexAccountId ?? null; +} + +/** + * Resolve the account id used for pool-scoped quota/health checks. + * Explicit `null` means the pre-fallback preview found no usable account — do not + * substitute `activeCodexAccountId` (that active id may itself be unusable). + */ +function resolvePoolFallbackAccountId( + config: OcxConfig, + accountId?: string | null, +): string | null { + if (typeof accountId === "string") return accountId; + if (accountId === null) return null; + return activeCodexAccountId(config); +} + +function isRoutableFallbackModel(model: string, config: OcxConfig): boolean { + const slash = model.indexOf("/"); + if (slash > 0) { + const providerName = model.slice(0, slash); + if (!hasOwnProvider(config.providers, providerName)) { + // Allow well-known "vendor/model" ids (e.g. anthropic/claude-*) to flow as + // raw model ids through the default provider, but reject stale/typo prefixes. + return knownProviderIdSet.has(providerName.toLowerCase()); + } + const provider = config.providers[providerName]; + if (provider?.disabled === true) return false; + } + return true; +} + +export function isNativeModelQuotaExhausted( + model: string, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), +): boolean { + const route = tryRouteFallbackModel(config, model); + if (!route || !isPoolCodexRoute(route)) return false; + const resolvedAccountId = resolvePoolFallbackAccountId(config, accountId); + if (!resolvedAccountId) return false; + const quota = getAccountQuota(resolvedAccountId); + const usage = computeCodexUsageScore(quota, getPoolAccountPlan(config, resolvedAccountId)); + if (usage >= CODEX_UNKNOWN_USAGE_SCORE) return false; + return usage >= quotaThreshold(config); +} + +export function isModelHealthBlocked( + model: string, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), +): boolean { + const route = tryRouteFallbackModel(config, model); + const poolScoped = !!route && isPoolCodexRoute(route); + const health = modelHealth.get( + healthKey(model, resolvePoolFallbackAccountId(config, accountId), poolScoped), + ); + return !!health && health.unavailableUntil > now; +} + +export function isSubagentModelUnavailable( + model: string, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), +): boolean { + if (isDisabledFallbackModel(model, config)) return true; + if (!isRoutableFallbackModel(model, config)) return true; + const route = tryRouteFallbackModel(config, model); + if (!route || route.provider.disabled === true) return true; + if (isModelHealthBlocked(model, config, accountId, now)) return true; + if (!isPoolCodexRoute(route)) return false; + + // Pool candidates need a usable account. Derive requirement from the resolved + // route (canonical openai defaults to pool even when codexAccountMode is omitted). + const resolvedAccountId = resolvePoolFallbackAccountId(config, accountId); + if (!resolvedAccountId) return true; + if (!isCodexAccountUsable(config, resolvedAccountId)) return true; + if ( + isCodexAccountInCooldown(resolvedAccountId, now) + && !canAcquireCodexQuotaProbeLease(resolvedAccountId, now) + ) { + return true; + } + return isNativeModelQuotaExhausted(model, config, accountId, now); +} + +export function selectAvailableSubagentModel( + primary: string, + config: OcxConfig, + extraFallback: readonly string[] = [], + accountId?: string | null, + now = Date.now(), + nativeFallbackOnly = false, +): { model: string; rewritten: boolean; skipped: string[] } { + const chain = normalizedChain(primary, config, extraFallback); + const skipped: string[] = []; + for (const candidate of chain) { + if (nativeFallbackOnly) { + const route = tryRouteFallbackModel(config, candidate); + if (!route || !isCanonicalOpenAiForwardProvider(route.provider)) { + skipped.push(candidate); + continue; + } + } + if (isSubagentModelUnavailable(candidate, config, accountId, now)) { + skipped.push(candidate); + continue; + } + return { model: candidate, rewritten: !slugsEquivalent(candidate, primary), skipped }; + } + return { model: primary, rewritten: false, skipped }; +} + +export function noteSubagentModelFailure( + model: string, + message: string, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), + ttlMs?: number, +): void { + const interval = ttlMs ?? DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS; + if (!isRateLimitOrQuotaFailureMessage(message)) return; + const route = tryRouteFallbackModel(config, model); + const poolScoped = !!route && isPoolCodexRoute(route); + modelHealth.set( + healthKey(model, resolvePoolFallbackAccountId(config, accountId), poolScoped), + { + unavailableUntil: now + interval, + reason: "quota_exhausted", + }, + ); +} + +export function resetSubagentModelFallbackStateForTests(): void { + modelHealth.clear(); + quotaPrimedAt.clear(); + quotaPrimeInFlight = null; + subagentQuotaPrimeForTests = null; +} + +/** Test-only: inject the quota prime implementation used by {@link maybePrimeSubagentQuota}. */ +export function setSubagentQuotaPrimeForTests(fn: SubagentQuotaPrimeFn | null): void { + subagentQuotaPrimeForTests = fn; +} + +/** Test-only: inspect shared prime TTL / in-flight state. */ +export function getSubagentQuotaPrimeStateForTests(): { + primedAt: number; + inFlight: boolean; +} { + return { + primedAt: quotaPrimedAt.get("global") ?? 0, + inFlight: quotaPrimeInFlight !== null, + }; +} + +function rewriteParsedModel(parsed: OcxParsedRequest, model: string): void { + parsed.modelId = model; + if (parsed._rawBody && typeof parsed._rawBody === "object") { + (parsed._rawBody as { model?: string }).model = model; + } +} + +const TOML_MODEL = /^(model)\s*=\s*("(?:\\.|[^"\\])*")\s*$/; + +function parseTomlQuotedString(raw: string): string { + const trimmed = raw.trim(); + if ((trimmed.startsWith("\"") && trimmed.endsWith("\"")) + || (trimmed.startsWith("'") && trimmed.endsWith("'"))) { + return trimmed.slice(1, -1).replace(/\\"/g, "\""); + } + return trimmed; +} + +function readAgentModel(filePath: string): string | null { + try { + const content = readFileSync(filePath, "utf8"); + for (const line of content.split(/\r?\n/)) { + const match = line.match(TOML_MODEL); + if (!match) continue; + const model = parseTomlQuotedString(match[2] ?? ""); + return model.trim() === "" ? null : model.trim(); + } + } catch { + return null; + } + return null; +} + +export function readCodexAgentModel(role: string, codexHome = CODEX_HOME): string | null { + const file = join(codexHome, "agents", `${role}.toml`); + if (!existsSync(file)) return null; + return readAgentModel(file); +} + +export function resolveAgentModelFallbackForPrimary( + primary: string, + codexHome = CODEX_HOME, +): string[] { + const merged: string[] = []; + const seen = new Set(); + const push = (model: string | null | undefined) => { + if (!model || model.trim() === "") return; + const trimmed = model.trim(); + const key = trimmed.toLowerCase(); + if (seen.has(key)) return; + seen.add(key); + merged.push(trimmed); + }; + for (const role of listCodexAgentRoles(codexHome)) { + const model = readCodexAgentModel(role, codexHome); + if (!model || !slugsEquivalent(model, primary)) continue; + for (const fallback of readCodexAgentModelFallback(role, codexHome)) push(fallback); + } + return merged; +} + +/** + * Best-effort quota refresh before subagent model selection. + * Concurrent callers share one in-flight promise. The success TTL is updated only + * after a successful refresh so failures remain retryable. Errors are swallowed so + * spawn routing can continue. + */ +export function maybePrimeSubagentQuota(config: OcxConfig, now = Date.now()): Promise { + if (quotaPrimeInFlight) return quotaPrimeInFlight; + if (!shouldPrimeSubagentQuota(config, now)) return Promise.resolve(); + + quotaPrimeInFlight = (async () => { + try { + if (subagentQuotaPrimeForTests) { + await subagentQuotaPrimeForTests(config, "subagent-spawn"); + } else { + const { primeCodexPoolQuotas } = await import("./auth-api"); + await primeCodexPoolQuotas(config, "subagent-spawn"); + } + quotaPrimedAt.set("global", Date.now()); + } catch { + // Owning boundary: do not fail the spawn path when priming is unavailable. + // Leave quotaPrimedAt untouched so a later spawn can retry. + } finally { + quotaPrimeInFlight = null; + } + })(); + return quotaPrimeInFlight; +} + +export function recordSubagentQuotaFailureForThreadSpawn( + headers: Headers, + model: string, + message: string | number, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), +): void { + if (!isThreadSpawnRequest(headers)) return; + noteSubagentModelFailure(model, String(message), config, accountId, now, pollIntervalMs(config)); +} + +export function applySubagentModelFallback( + parsed: OcxParsedRequest, + headers: Headers, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), + nativeFallbackOnly = false, +): { from?: string; to?: string; skipped?: string[] } | null { + if (!isThreadSpawnRequest(headers)) return null; + const roleFallback = resolveAgentModelFallbackForPrimary(parsed.modelId, getCodexHome()); + const globalFallback = config.subagentModelFallback ?? []; + if (globalFallback.length === 0 && roleFallback.length === 0) return null; + const selection = selectAvailableSubagentModel( + parsed.modelId, + config, + roleFallback, + accountId, + now, + nativeFallbackOnly, + ); + if (!selection.rewritten) return selection.skipped.length > 0 + ? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped } + : null; + const from = parsed.modelId; + rewriteParsedModel(parsed, selection.model); + return { from, to: selection.model, skipped: selection.skipped }; +} + +export function subagentFallbackGuidanceText(config: OcxConfig): string { + const chain = config.subagentModelFallback ?? []; + if (chain.length === 0) return ""; + const quoted = chain.map(model => `"${model}"`).join(", "); + return ` Subagent model fallback chain (priority order): ${quoted}. When the primary model is quota-exhausted, opencodex rewrites thread_spawn requests to the next available model automatically.`; +} + +const TOML_STRING_ARRAY = /^(model_fallback)\s*=\s*\[(.*)\]\s*$/s; + +function parseTomlStringArray(raw: string): string[] { + const matches = [...raw.matchAll(/"((?:\\.|[^"\\])*)"/g)]; + return matches.map(match => match[1]!.replace(/\\"/g, "\"")); +} + +function parseTomlModelFallback(content: string): string[] | null { + const match = content.match(/^\s*model_fallback\s*=\s*\[(.*?)\]\s*$/ms); + if (!match) return null; + return parseTomlStringArray(match[1] ?? ""); +} + +export function readAgentModelFallback(filePath: string): string[] | null { + try { + const content = readFileSync(filePath, "utf8"); + const multiline = parseTomlModelFallback(content); + if (multiline) return multiline; + for (const line of content.split(/\r?\n/)) { + const match = line.match(TOML_STRING_ARRAY); + if (!match) continue; + return parseTomlStringArray(match[2] ?? ""); + } + } catch { + return null; + } + return null; +} + +export function readCodexAgentModelFallback(role: string, codexHome = CODEX_HOME): string[] { + const file = join(codexHome, "agents", `${role}.toml`); + if (!existsSync(file)) return []; + return readAgentModelFallback(file) ?? []; +} + +export function listCodexAgentRoles(codexHome = CODEX_HOME): string[] { + const dir = join(codexHome, "agents"); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter(name => name.endsWith(".toml")) + .map(name => name.slice(0, -".toml".length)); +} + +/** True when a new quota prime should start (no success within the poll interval). */ +export function shouldPrimeSubagentQuota(config: OcxConfig, now = Date.now()): boolean { + const last = quotaPrimedAt.get("global") ?? 0; + return now - last >= pollIntervalMs(config); +} diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 6197df8229..a316e381bf 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -190,6 +190,29 @@ export function classifyError(status: number, type: string, message: string): Oc return { message, type, code: type || null }; } +/** + * True when a provider failure should participate in rate-limit / quota health blocking. + * Reuses {@link classifyError} so generic 429 wording and quota phrases stay aligned. + */ +export function isRateLimitOrQuotaFailureMessage(message: string): boolean { + const normalized = String(message ?? "").trim(); + if (!normalized) return false; + const numericStatus = Number(normalized); + if (numericStatus === 429 || numericStatus === 402) return true; + const statusHint = Number.isInteger(numericStatus) && numericStatus > 0 ? numericStatus : 0; + const classified = classifyError(statusHint, "", normalized); + if ( + classified.type === "rate_limit_error" + || classified.code === "rate_limit_exceeded" + || classified.type === "insufficient_quota" + || classified.code === "insufficient_quota" + ) { + return true; + } + // Retained quota cue used by subagent health before classifyError covered it. + return normalized.toLowerCase().includes("usage limit"); +} + /** Best-effort parse of a retry delay embedded in an upstream error message. */ export function parseRetryAfterFromMessage(message: string): number | undefined { const patterns = [ diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index c46b77dd04..c41623ad7f 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -288,6 +288,75 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise return jsonResponse({ ok: true, applied: chosen }); } + // Priority-ordered subagent model fallback chain for quota-aware spawn routing. + if (url.pathname === "/api/subagent-model-fallback" && req.method === "GET") { + const models = await fetchAllModels(config); + const disabled = new Set(config.disabledModels ?? []); + const { listCatalogNativeSlugs } = await import("../../codex/catalog"); + const visibleRouted = [...new Set(models + .filter(m => ![...disabled].some(stored => + stored === catalogModelSlug(m) || slugEquals(stored, m.provider, m.id) + )) + .map(catalogModelSlug))]; + const available = [ + ...listCatalogNativeSlugs().filter(ns => !disabled.has(ns)), + ...visibleRouted, + ]; + return jsonResponse({ + models: config.subagentModelFallback ?? [], + pollMs: config.subagentModelFallbackPollMs ?? 60_000, + available, + }); + } + if (url.pathname === "/api/subagent-model-fallback" && req.method === "PUT") { + let body: { models?: unknown; pollMs?: unknown }; + try { + body = await req.json(); + } catch { + return jsonResponse({ error: "invalid JSON body" }, 400); + } + if (!body || typeof body !== "object" || Array.isArray(body)) { + return jsonResponse({ error: "invalid JSON body" }, 400); + } + let nextModels = config.subagentModelFallback; + let nextPollMs = config.subagentModelFallbackPollMs; + if ("models" in body) { + if (!Array.isArray(body.models)) return jsonResponse({ error: "models must be an array" }, 400); + const models: string[] = []; + for (let i = 0; i < body.models.length; i++) { + const entry = body.models[i]; + if (typeof entry !== "string" || entry.trim().length === 0) { + return jsonResponse({ + error: `models[${i}] must be a non-empty string`, + index: i, + value: entry, + }, 400); + } + models.push(entry.trim()); + } + nextModels = models.length > 0 ? models : undefined; + } + if ("pollMs" in body) { + const pollMs = body.pollMs; + if (pollMs === null || pollMs === "") nextPollMs = undefined; + else if (typeof pollMs === "number" && Number.isInteger(pollMs) && pollMs >= 5_000 && pollMs <= 600_000) { + nextPollMs = pollMs; + } else { + return jsonResponse({ error: "pollMs must be an integer between 5000 and 600000" }, 400); + } + } + if (nextModels !== undefined) config.subagentModelFallback = nextModels; + else delete config.subagentModelFallback; + if (nextPollMs !== undefined) config.subagentModelFallbackPollMs = nextPollMs; + else delete config.subagentModelFallbackPollMs; + saveConfig(config); + return jsonResponse({ + ok: true, + models: config.subagentModelFallback ?? [], + pollMs: config.subagentModelFallbackPollMs ?? 60_000, + }); + } + // Claude Code inbound settings (GUI "Claude ON" toggle + Claude page). if (url.pathname === "/api/claude-code" && req.method === "GET") { const models = await fetchAllModels(config); diff --git a/src/server/responses/collaboration.ts b/src/server/responses/collaboration.ts index e0d37d54af..907d1af533 100644 --- a/src/server/responses/collaboration.ts +++ b/src/server/responses/collaboration.ts @@ -61,6 +61,7 @@ import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } f import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { slugsEquivalent } from "../../providers/slug-codec"; +import { subagentFallbackGuidanceText } from "../../codex/subagent-model-fallback"; import { applyOpenAiVirtualModel, resolveOpenAiCompactModel } from "../../providers/openai-virtual-models"; import { isUsageDebugEnabled } from "../../usage/debug"; import { readJsonRequestBody, DecompressedBodyTooLargeError, UnsupportedContentEncodingError } from "../request-decompress"; @@ -160,6 +161,7 @@ export interface MultiAgentGuidanceOptions { injectionModel?: string; injectionEffort?: string; subagentModels?: string[]; + subagentModelFallback?: string[]; injectionPrompt?: string; } @@ -194,6 +196,7 @@ export async function multiAgentGuidanceText( injectionModel, injectionEffort, subagentModels, + subagentModelFallback, injectionPrompt, } = options; const surface = collabSurface(parsed); @@ -222,11 +225,12 @@ export async function multiAgentGuidanceText( .map(item => `${item.configured}:${item.reason}`) .join(", ")}`); } - if (!injectionModel && roster === "") return null; + const fallbackGuidance = subagentFallbackGuidanceText({ subagentModelFallback } as OcxConfig); + if (!injectionModel && roster === "" && fallbackGuidance === "") return null; if (injectionPrompt) { - return `${applyInjectionPlaceholders(injectionPrompt, injectionModel, injectionEffort, roster)}`; + return `${applyInjectionPlaceholders(injectionPrompt, injectionModel, injectionEffort, roster, fallbackGuidance)}`; } - if (!preferred && roster === "") return null; + if (!preferred && roster === "" && fallbackGuidance === "") return null; let text = "When the active spawn_agent tool supports optional \"model\" or \"reasoning_effort\" overrides, " + "use only models listed for this collaboration surface. " + "When setting either override, set fork_turns to \"none\" " @@ -237,6 +241,7 @@ export async function multiAgentGuidanceText( + (injectionEffort ? `, reasoning_effort "${injectionEffort}"` : "") + " — use it unless the user names another."; } + text += fallbackGuidance; text += roster; if (text.length > V2_GUIDANCE_CHAR_BUDGET) { // Roster is the only unbounded part — drop it before breaking the budget. @@ -256,11 +261,12 @@ export async function multiAgentGuidanceText( export const V2_GUIDANCE_CHAR_BUDGET = 700; -export function applyInjectionPlaceholders(prompt: string, model?: string, effort?: string, roster?: string): string { +export function applyInjectionPlaceholders(prompt: string, model?: string, effort?: string, roster?: string, fallback?: string): string { return prompt .replaceAll("{{model}}", model ?? "") .replaceAll("{{effort}}", effort ?? "") - .replaceAll("{{roster}}", roster ?? ""); + .replaceAll("{{roster}}", roster ?? "") + .replaceAll("{{fallback}}", fallback ?? ""); } @@ -314,4 +320,3 @@ export function injectDeveloperMessage(parsed: OcxParsedRequest, text: string): } } } - diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index e7d81f1104..824d5dda7c 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -9,7 +9,7 @@ import { parseRequest } from "../../responses/parser"; import { buildCompactV1Output, COMPACT_PROMPT, decodeCompactionSummary, extractCompactUserMessages } from "../../responses/compaction"; import { FORWARD_HEADERS, sanitizeReasoningInputContent } from "../../adapters/openai-responses"; import { expandPreviousResponseInput, previousResponseProviderState, rememberResponseState } from "../../responses/state"; -import { routeModel } from "../../router"; +import { routeModel, type RouteResult } from "../../router"; import { advanceComboAfterFailure, comboDefaultEffort, @@ -56,6 +56,7 @@ import { } from "../../codex/auth-context"; import { formatCodexProviderForLog, + previewCodexAccountForRequest, recordCodexUpstreamOutcome, type CodexUpstreamOutcome, } from "../../codex/routing"; @@ -76,6 +77,12 @@ import { registerTurn, trackStreamLifetime, unregisterTurn } from "../lifecycle" import { redactSecretString } from "../../lib/redact"; import { readBoundedResponseBody } from "../../lib/bounded-body"; import { supportedLadderFor } from "../effort-policy"; +import { isThreadSpawnRequest } from "../effort-policy"; +import { + applySubagentModelFallback, + maybePrimeSubagentQuota, + recordSubagentQuotaFailureForThreadSpawn, +} from "../../codex/subagent-model-fallback"; import { beginRequestAttempt, catalogModelSupportsServiceTier, @@ -414,6 +421,169 @@ function unreadableEncryptedAgentTaskResponse(): Response { ); } +type ResponsesAuthResolution = + | { ok: true; authCtx: CodexAuthContext; headers: Headers } + | { ok: false; response: Response }; + +/** + * Resolve Codex auth for a route. On unusable contexts, releases any probe lease + * before returning the 401 (nothing reaches upstream). + */ +async function resolveResponsesCodexAuth( + req: Request, + config: OcxConfig, + route: RouteResult, + options: HandleResponsesOptions, +): Promise { + try { + if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config); + let authCtx: CodexAuthContext; + if (route.codexAccountMode) { + authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode); + options.onCodexAuthContextResolved?.(authCtx); + } else { + authCtx = { kind: "main", accountId: null }; + options.onCodexAuthContextResolved?.(undefined); + } + if (!isCodexAuthContextUsable(authCtx, config)) { + releaseCodexAuthContextProbeLease(authCtx); + return { + ok: false, + response: formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"), + }; + } + return { + ok: true, + authCtx, + headers: headersForCodexAuthContext(req.headers, authCtx), + }; + } catch (err) { + if (err instanceof CodexAccountCooldownError) { + return { ok: false, response: formatErrorResponse(429, "rate_limit_error", "Selected Codex account is cooling down") }; + } + if (err instanceof CodexThreadAffinityExpiredError) { + return { + ok: false, + response: formatErrorResponse(409, "invalid_request_error", "Codex thread account affinity expired; start a new session"), + }; + } + if (err instanceof CodexAuthContextError) { + const safeAccountLabel = formatCodexProviderForLog(route.providerName, err.accountId, config); + console.error(`[codex-auth] Pool account ${safeAccountLabel} token failed; reauthentication required`); + return { + ok: false, + response: formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"), + }; + } + if (err instanceof CodexPoolAuthenticationError) { + return { ok: false, response: formatErrorResponse(401, "authentication_error", err.message) }; + } + if (err instanceof CodexDirectAuthenticationError) { + return { ok: false, response: formatErrorResponse(401, "authentication_error", err.message) }; + } + if (err instanceof ForwardAdmissionCredentialError) { + return { ok: false, response: formatErrorResponse(401, "authentication_error", err.message) }; + } + throw err; + } +} + +/** + * Apply every route-dependent request mutation against the final selected route. + * Must run only after subagent fallback has settled the model/provider. + */ +async function applyFinalRouteRequestNormalization(args: { + parsed: OcxParsedRequest; + route: RouteResult; + config: OcxConfig; + req: Request; + logCtx: RequestLogContext; +}): Promise { + const { parsed, route, config, req, logCtx } = args; + + // Apply the routed model id upstream: routing may strip a "/" namespace. + if (route.modelId !== parsed.modelId) { + if (parsed._rawBody && typeof parsed._rawBody === "object") { + (parsed._rawBody as { model?: string }).model = route.modelId; + } + parsed.modelId = route.modelId; + } + // Settle the wire once so logging, fast-mode, auth, and sidecars read the adapter + // this request will actually use (#404). + route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider); + logCtx.model = route.modelId; + logCtx.provider = route.providerName; + logCtx.providerAdapter = route.provider.adapter; + + // Final selected model before virtual wire-model rewriting (Pro aliases). + const finalSelectedModelId = route.modelId; + + // Virtual model rewriting: Pro aliases → base model + reasoning.mode="pro". + applyOpenAiVirtualModel(parsed, route, logCtx); + + // Fast mode override for OpenAI-routed models. + if (config.fastMode !== undefined && route.provider.adapter === "openai-responses") { + const tier = config.fastMode ? "priority" : undefined; + if (parsed._rawBody && typeof parsed._rawBody === "object") { + if (tier) (parsed._rawBody as Record).service_tier = tier; + else delete (parsed._rawBody as Record).service_tier; + } + parsed.options.serviceTier = tier; + } + + { + const guidance = await multiAgentGuidanceText(parsed, { + multiAgentGuidanceEnabled: config.multiAgentGuidanceEnabled, + injectionModel: config.injectionModel, + injectionEffort: config.injectionEffort, + subagentModels: config.subagentModels, + subagentModelFallback: config.subagentModelFallback, + injectionPrompt: config.injectionPrompt, + }); + if (guidance) { + injectDeveloperMessage(parsed, guidance); + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] ${route.modelId}: multi-agent guidance injected (surface=${collabSurface(parsed)}, guidanceEnabled=${multiAgentGuidanceEnabled(config)}, ${guidance.length} chars)`); + } + } else if (isInjectionDebugEnabled() && collabSurface(parsed) !== null) { + injectionDebugLog(`[opencodex] ${route.modelId}: collab surface=${collabSurface(parsed)}, guidance silent (effort=${parsed.options.reasoning ?? "unset"}, injectionModel=${config.injectionModel ?? "unset"})`); + } + } + + { + const { applyEffortCap, effortCapAppliesTo, supportedLadderFor } = await import("../effort-policy"); + const surface = collabSurface(parsed); + if (effortCapAppliesTo(surface, req.headers, config, parsed._compactionRequest === true)) { + const capped = applyEffortCap(parsed, req.headers, config, supportedLadderFor(route)); + if (capped) { + logCtx.requestedEffort = `${capped.from}->${capped.to}`; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] ${route.modelId}: effort cap applied (${capped.from} -> ${capped.to}, ${capped.subagent ? "sub-agent" : "main"} turn)`); + } + } + } else if (isInjectionDebugEnabled() && (config.effortCap || config.subagentEffortCap)) { + injectionDebugLog(`[opencodex] ${route.modelId}: effort cap skipped (surface=${surface ?? "none"}, v2 feature only)`); + } + } + + { + const { nativeEffortClamp, shouldApplyNativeEffortClamp } = await import("../../codex/catalog"); + const clamped = shouldApplyNativeEffortClamp(route.providerName, route.provider, finalSelectedModelId) + ? nativeEffortClamp(route.modelId, parsed.options.reasoning) + : null; + if (clamped) { + parsed.options.reasoning = clamped; + const raw = parsed._rawBody as { reasoning?: { effort?: string } } | undefined; + if (raw?.reasoning && typeof raw.reasoning === "object") raw.reasoning.effort = clamped; + logCtx.requestedEffort = `${logCtx.requestedEffort ?? "max"}->${clamped}`; + } + } + logCtx.modelSupportsServiceTier = catalogModelSupportsServiceTier( + route.modelId, + logCtx.requestedServiceTier ?? logCtx.configuredServiceTier, + ); +} + export async function handleComboResponses( @@ -706,7 +876,11 @@ export async function handleResponses( } if (parsed._compactionRequest === true) parsed._cursorIsolateConversation = true; - let route; + if (isThreadSpawnRequest(req.headers)) { + await maybePrimeSubagentQuota(config); + } + + let route: RouteResult; try { route = routeModel(config, parsed.modelId); } catch (err) { @@ -716,167 +890,72 @@ export async function handleResponses( return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); } - // The canonical ChatGPT backend can decrypt its V2 Fernet task tokens; routed - // providers cannot. Reject the raw-input classification before adapter construction - // or provider dispatch so an unreadable worker task cannot trigger a cost storm. - if (!isCanonicalOpenAiForwardProvider(route.provider) && unreadableEncryptedAgentTask) { - return unreadableEncryptedAgentTaskResponse(); - } - - // Apply the routed model id upstream: routing may strip a "/" namespace - // (e.g. "opencode-go/deepseek-v4-pro" → "deepseek-v4-pro"). Adapters read parsed.modelId, - // and the passthrough adapter serializes _rawBody, so rewrite both. - if (route.modelId !== parsed.modelId) { - if (parsed._rawBody && typeof parsed._rawBody === "object") { - (parsed._rawBody as { model?: string }).model = route.modelId; + let authCtx: CodexAuthContext = { kind: "main", accountId: null }; + let selectedForwardHeaders = req.headers; + let subagentFallbackAccountId = config.activeCodexAccountId ?? null; + let subagentQuotaFailureModel = parsed.modelId; + + // Subagent fallback must settle the final model/provider BEFORE route-dependent + // normalization (virtual models, effort caps, service tier, wire protocol). + // Preview the preferred Codex account without acquiring a probe lease or refreshing + // tokens — auth is resolved only after the final route is selected. + if (isThreadSpawnRequest(req.headers) && !options.comboAttempt) { + const threadId = req.headers.get("x-codex-parent-thread-id"); + const previewAccountId = previewCodexAccountForRequest(threadId, config); + subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null; + const fallback = applySubagentModelFallback( + parsed, + req.headers, + config, + previewAccountId, + Date.now(), + unreadableEncryptedAgentTask, + ); + if (fallback) { + (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; + (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); + } } - parsed.modelId = route.modelId; - } - // Settle the wire once, right after the native model id is known, so logging, - // fast-mode injection, auth, and sidecar decisions all read the adapter this - // request will actually use rather than the provider-wide default (#404). - route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider); - logCtx.model = route.modelId; - logCtx.provider = route.providerName; - logCtx.providerAdapter = route.provider.adapter; - - // Virtual model rewriting: Pro aliases → base model + reasoning.mode="pro". - // Must run before effort caps/native clamps so the base model gets correct limits. - applyOpenAiVirtualModel(parsed, route, logCtx); + subagentQuotaFailureModel = fallback?.to ?? parsed.modelId; - // Fast mode override: when config.fastMode is explicitly set, inject or strip - // service_tier for OpenAI-routed models. Undefined = passthrough (client decides). - if (config.fastMode !== undefined && route.provider.adapter === "openai-responses") { - const tier = config.fastMode ? "priority" : undefined; - if (parsed._rawBody && typeof parsed._rawBody === "object") { - if (tier) (parsed._rawBody as Record).service_tier = tier; - else delete (parsed._rawBody as Record).service_tier; + if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { + try { + route = routeModel(config, fallback.to); + } catch (err) { + if (err instanceof NoAvailableComboTargetsError) { + return comboUnavailableResponse(err.message); + } + return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); + } } - parsed.options.serviceTier = tier; } - // Multi-agent guidance shim: codex-rs emits its Proactive delegation developer - // message only on the v2 surface. The proxy fills the gaps: the Proactive text - // for v1 collab surfaces at the top tier (no model designation on v1), and the - // sub-agent model/roster designation plus fork_turns override rules on v2. - // The surface is judged from the request's own tool list. Runs BEFORE the - // mock-max clamp below so the synthetic top tier (ultra arrives as max on the - // codex wire) is still visible. Both request shapes are rewritten. - { - const guidance = await multiAgentGuidanceText(parsed, { - multiAgentGuidanceEnabled: config.multiAgentGuidanceEnabled, - injectionModel: config.injectionModel, - injectionEffort: config.injectionEffort, - subagentModels: config.subagentModels, - injectionPrompt: config.injectionPrompt, - }); - if (guidance) { - injectDeveloperMessage(parsed, guidance); - if (isInjectionDebugEnabled()) injectionDebugLog(`[opencodex] ${route.modelId}: multi-agent guidance injected (surface=${collabSurface(parsed)}, guidanceEnabled=${multiAgentGuidanceEnabled(config)}, ${guidance.length} chars)`); - } else if (isInjectionDebugEnabled() && collabSurface(parsed) !== null) { - injectionDebugLog(`[opencodex] ${route.modelId}: collab surface=${collabSurface(parsed)}, guidance silent (effort=${parsed.options.reasoning ?? "unset"}, injectionModel=${config.injectionModel ?? "unset"})`); - } + // Encrypted child tasks may only reach the canonical native backend. This check + // runs against the FINAL route so native-only fallback can rescue a routed primary. + if (!isCanonicalOpenAiForwardProvider(route.provider) && unreadableEncryptedAgentTask) { + return unreadableEncryptedAgentTaskResponse(); } - // Hard effort caps (effortCap / subagentEffortCap): enforcement companion to the advisory - // injection above — spawn-arg prompting cannot stop codex-rs from inheriting the parent's - // ultra-tier default on bare spawns (see src/server/effort-policy.ts). Runs BEFORE the - // mock-max clamp so a capped effort is what nativeness clamping then validates; rewrites - // both request shapes (same dual-write contract as the clamp below). - // GATE: v2 feature only (effortCapAppliesTo) — v2-surface main turns plus header-marked - // child turns admitted regardless of tool surface (depth-limited leaves carry no collab - // tools while shallower children do, so tool sniffing alone would cap siblings - // inconsistently); multiAgentMode "v1" disables caps entirely; compaction turns bypass - // caps so routed compaction matches native /v1/responses/compact (which never enters - // handleResponses). - { - const { applyEffortCap, effortCapAppliesTo, supportedLadderFor } = await import("../effort-policy"); - const surface = collabSurface(parsed); - if (effortCapAppliesTo(surface, req.headers, config, parsed._compactionRequest === true)) { - const capped = applyEffortCap(parsed, req.headers, config, supportedLadderFor(route)); - if (capped) { - logCtx.requestedEffort = `${capped.from}->${capped.to}`; - if (isInjectionDebugEnabled()) { - injectionDebugLog(`[opencodex] ${route.modelId}: effort cap applied (${capped.from} -> ${capped.to}, ${capped.subagent ? "sub-agent" : "main"} turn)`); - } - } - } else if (isInjectionDebugEnabled() && (config.effortCap || config.subagentEffortCap)) { - injectionDebugLog(`[opencodex] ${route.modelId}: effort cap skipped (surface=${surface ?? "none"}, v2 feature only)`); - } - } + await applyFinalRouteRequestNormalization({ parsed, route, config, req, logCtx }); - // Mock-max clamp: native models whose real ladder stops below max (gpt-5.5/5.4/…) - // receive `max` when the user picks Ultra (codex converts ultra->max client-side). - // Clamp to the model's highest real effort BEFORE any adapter — the ChatGPT - // passthrough serializes _rawBody verbatim, so both shapes must be rewritten. - // GUARD: judge nativeness by BOTH the originally requested id (logCtx.requestedModel) - // and the resolved provider identity. Routing strips the "/" namespace, and - // some third-party providers expose bare `defaultModel` selectors, so route.modelId - // alone can make a routed model masquerade as an off-snapshot native. Only the - // canonical built-in ChatGPT forward provider should receive the native clamp. { - const requestedModelId = logCtx.requestedModel ?? route.modelId; - const { nativeEffortClamp, shouldApplyNativeEffortClamp } = await import("../../codex/catalog"); - const clamped = shouldApplyNativeEffortClamp(route.providerName, route.provider, requestedModelId) - ? nativeEffortClamp(route.modelId, parsed.options.reasoning) - : null; - if (clamped) { - parsed.options.reasoning = clamped; - const raw = parsed._rawBody as { reasoning?: { effort?: string } } | undefined; - if (raw?.reasoning && typeof raw.reasoning === "object") raw.reasoning.effort = clamped; - logCtx.requestedEffort = `${logCtx.requestedEffort ?? "max"}->${clamped}`; - } + const finalAuth = await resolveResponsesCodexAuth(req, config, route, options); + if (!finalAuth.ok) return finalAuth.response; + authCtx = finalAuth.authCtx; + selectedForwardHeaders = finalAuth.headers; } - logCtx.modelSupportsServiceTier = catalogModelSupportsServiceTier( - route.modelId, - logCtx.requestedServiceTier ?? logCtx.configuredServiceTier, - ); - let authCtx: CodexAuthContext = { kind: "main", accountId: null }; - let selectedForwardHeaders: Headers; - try { - if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config); - if (route.codexAccountMode) { - authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode); - options.onCodexAuthContextResolved?.(authCtx); - } else { - options.onCodexAuthContextResolved?.(undefined); - } - selectedForwardHeaders = headersForCodexAuthContext(req.headers, authCtx); - } catch (err) { - if (err instanceof CodexAccountCooldownError) { - return formatErrorResponse(429, "rate_limit_error", "Selected Codex account is cooling down"); - } - if (err instanceof CodexThreadAffinityExpiredError) { - return formatErrorResponse(409, "invalid_request_error", "Codex thread account affinity expired; start a new session"); - } - if (err instanceof CodexAuthContextError) { - const safeAccountLabel = formatCodexProviderForLog(route.providerName, err.accountId, config); - console.error(`[codex-auth] Pool account ${safeAccountLabel} token failed; reauthentication required`); - return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); - } - if (err instanceof CodexPoolAuthenticationError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - if (err instanceof CodexDirectAuthenticationError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - if (err instanceof ForwardAdmissionCredentialError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - throw err; - } - if (!isCodexAuthContextUsable(authCtx, config)) { - // Nothing reaches upstream on this path, so give the probe back. - releaseCodexAuthContextProbeLease(authCtx); - return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); - } route.provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); logCtx.provider = formatCodexProviderForLog(route.providerName, codexLogAccountId(authCtx), config); // Prefer Codex pool account as the Cursor thread namespace when present. Cursor routes without // codexAccountMode still get a credential-derived scope inside the Cursor adapter. const identityScope = codexLogAccountId(authCtx); if (identityScope) parsed._cursorIdentityScope = identityScope; + subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" + ? authCtx.accountId + : config.activeCodexAccountId ?? null; // OAuth providers: swap in a fresh access token (auto-refreshed) as the Bearer key, so the // existing openai-chat / anthropic adapters authenticate with no change. @@ -1178,6 +1257,22 @@ export async function handleResponses( if (terminalBodyWillRecord) { options.setTerminalOutcomeRecorder?.((status, httpStatusOverride) => { terminalRecorder(status, httpStatusOverride); + if (status === "failed") { + const quotaFailureMessage = httpStatusOverride === 429 || httpStatusOverride === 402 + || logCtx.terminalHttpStatus === 429 + || logCtx.terminalHttpStatus === 402 + ? (httpStatusOverride ?? logCtx.terminalHttpStatus) + : undefined; + if (quotaFailureMessage !== undefined) { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + quotaFailureMessage, + config, + subagentFallbackAccountId, + ); + } + } options.onNativePassthroughTerminal?.(status); }); } else { @@ -1215,6 +1310,22 @@ export async function handleResponses( const reportNativeTerminal = recordTerminalOutcomes ? (status: ResponsesTerminalStatus, httpStatusOverride?: number) => { terminalRecorder?.(status, httpStatusOverride); + if (status === "failed") { + const quotaFailureMessage = httpStatusOverride === 429 || httpStatusOverride === 402 + || logCtx.terminalHttpStatus === 429 + || logCtx.terminalHttpStatus === 402 + ? (httpStatusOverride ?? logCtx.terminalHttpStatus) + : undefined; + if (quotaFailureMessage !== undefined) { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + quotaFailureMessage, + config, + subagentFallbackAccountId, + ); + } + } options.onNativePassthroughTerminal?.(status); } : undefined; @@ -1259,6 +1370,22 @@ export async function handleResponses( // client-cancel (no terminal seen) is finalized separately via consumeForInspection's onCancel. const reportNativeTerminal = (status: ResponsesTerminalStatus, httpStatusOverride?: number) => { terminalRecorder?.(status, httpStatusOverride); + if (status === "failed") { + const quotaFailureMessage = httpStatusOverride === 429 || httpStatusOverride === 402 + || logCtx.terminalHttpStatus === 429 + || logCtx.terminalHttpStatus === 402 + ? (httpStatusOverride ?? logCtx.terminalHttpStatus) + : undefined; + if (quotaFailureMessage !== undefined) { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + quotaFailureMessage, + config, + subagentFallbackAccountId, + ); + } + } options.onNativePassthroughTerminal?.(status); }; consumeForInspection( @@ -1640,6 +1767,15 @@ export async function handleResponses( } const errorText = await upstreamResponse.text().catch(() => "unknown error"); cleanupUpstreamAbort(); + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + upstreamResponse.status === 429 || upstreamResponse.status === 402 + ? upstreamResponse.status + : `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`, + config, + subagentFallbackAccountId, + ); // Upstreams occasionally echo request details in error bodies — scrub token-shaped // material before it reaches the client-facing error surface. return formatErrorResponse(upstreamResponse.status, "upstream_error", `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`); diff --git a/src/types.ts b/src/types.ts index d4422034b0..3b36e35d70 100644 --- a/src/types.ts +++ b/src/types.ts @@ -454,6 +454,16 @@ export interface OcxConfig { * Codex's spawn_agent only advertises the first 5 routed models, so this picks which 5 appear. */ subagentModels?: string[]; + /** + * Priority-ordered fallback models for spawned sub-agents. When the requested + * model is quota-exhausted or recently failed, opencodex rewrites the child + * turn to the next available entry before routing. + */ + subagentModelFallback?: string[]; + /** + * TTL (ms) for cached sub-agent model availability probes. Default 60_000. + */ + subagentModelFallbackPollMs?: number; injectionModel?: string; /** * Optional reasoning effort the delegation prompt tells the agent to pass in spawn_agent calls @@ -481,7 +491,8 @@ export interface OcxConfig { * tags). When set, it replaces the built-in prompt on whichever * collab surface would have fired; firing gates are unchanged. Placeholders: * `{{model}}` -> injectionModel, `{{effort}}` -> injectionEffort, `{{roster}}` -> - * the resolved sub-agent roster block ("" when nothing resolves). + * the resolved sub-agent roster block ("" when nothing resolves), `{{fallback}}` -> + * the configured subagent model fallback guidance block ("" when unset). */ injectionPrompt?: string; /** diff --git a/tests/multi-agent-compat.test.ts b/tests/multi-agent-compat.test.ts index 5b55f6274b..e83c1e7cf1 100644 --- a/tests/multi-agent-compat.test.ts +++ b/tests/multi-agent-compat.test.ts @@ -382,6 +382,19 @@ describe("multiAgentGuidanceText", () => { expect(text).not.toContain("gpt-5.6-luna"); }); + test("injectionPrompt substitutes fallback guidance via {{fallback}}", async () => { + const text = await multiAgentGuidanceText( + parsedFixture({ tools: [{ name: "spawn_agent" }] }), + { + injectionPrompt: "FALLBACK={{fallback}}", + subagentModelFallback: ["alibaba-token-plan/qwen3.8-max-preview", "kimi/k3"], + }, + ); + expect(text).toContain("FALLBACK="); + expect(text).toContain("alibaba-token-plan/qwen3.8-max-preview"); + expect(text).toContain("kimi/k3"); + }); + test("v1 ignores injectionPrompt and custom prompt does not fire a bare v2 surface", async () => { codexHomeFixture(V2_ON); const custom = "CUSTOM RULES model={{model}} effort={{effort}}{{roster}}"; diff --git a/tests/subagent-fallback-handle-responses.test.ts b/tests/subagent-fallback-handle-responses.test.ts new file mode 100644 index 0000000000..afaf12ccba --- /dev/null +++ b/tests/subagent-fallback-handle-responses.test.ts @@ -0,0 +1,943 @@ +/** + * handleResponses integration coverage for PR #391 merge blockers: + * pre-fallback account preview (no probe lease), final-route normalization, + * native effort clamp on final route, pool account preview for native fallback, + * encrypted native-only fallback, native passthrough terminal finalization. + */ +import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveCodexAccountCredential } from "../src/codex/account-store"; +import { + clearAccountQuota, + updateAccountQuota, +} from "../src/codex/quota"; +import { + CODEX_QUOTA_PROBE_INTERVAL_MS, + clearCodexUpstreamHealth, + clearThreadAccountMap, + getCodexUpstreamHealth, + previewCodexAccountForRequest, + recordCodexUpstreamOutcome, + resolveCodexAccountForThreadDetailed, +} from "../src/codex/routing"; +import { + isModelHealthBlocked, + resetSubagentModelFallbackStateForTests, +} from "../src/codex/subagent-model-fallback"; +import type { CodexAuthContext } from "../src/codex/auth-context"; +import { handleResponses } from "../src/server/responses"; +import type { OcxConfig } from "../src/types"; +import type { RequestLogContext } from "../src/server/request-log"; +import type { ResponsesTerminalStatus } from "../src/bridge"; + +setDefaultTimeout(30_000); + +const originalFetch = globalThis.fetch; +const originalNow = Date.now; +let testDir: string; +let previousOpencodexHome: string | undefined; +let previousCodexHome: string | undefined; + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-subagent-hr-")); + previousOpencodexHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; + clearThreadAccountMap(); + clearCodexUpstreamHealth(); + clearAccountQuota(); + resetSubagentModelFallbackStateForTests(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + Date.now = originalNow; + clearThreadAccountMap(); + clearCodexUpstreamHealth(); + clearAccountQuota(); + resetSubagentModelFallbackStateForTests(); + rmSync(testDir, { recursive: true, force: true }); + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; +}); + +function fernetFixture(ciphertextBytes = 16): string { + const raw = Buffer.alloc(57 + ciphertextBytes, 0x5a); + raw[0] = 0x80; + raw.writeBigUInt64BE(1_720_000_000n, 1); + const unpadded = raw.toString("base64url"); + return `${unpadded}${"=".repeat((4 - (unpadded.length % 4)) % 4)}`; +} + +const FERNET_TASK = fernetFixture(); + +function encryptedAgentInput(): unknown[] { + return [{ + type: "agent_message", + author: "/root", + recipient: "/root/worker", + content: [{ type: "encrypted_content", encrypted_content: FERNET_TASK }], + }]; +} + +function readableAgentInput(): unknown[] { + return [{ + type: "agent_message", + author: "/root", + recipient: "/root/worker", + content: [{ type: "input_text", text: "do the work" }], + }]; +} + +function spawnHeaders(extra: HeadersInit = {}): Headers { + return new Headers({ + "content-type": "application/json", + "x-openai-subagent": "collab_spawn", + authorization: "Bearer caller-codex-token", + ...Object.fromEntries(new Headers(extra)), + }); +} + +function poolNativePlusRoutedConfig(overrides: Partial = {}): OcxConfig { + return { + port: 0, + defaultProvider: "openai", + activeCodexAccountId: "pool-a", + autoSwitchThreshold: 80, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-apikey": { + adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authMode: "key", + apiKey: "sk-test", + }, + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "pool@example.test", isMain: false, chatgptAccountId: "pool_acc" }, + ], + ...overrides, + } as OcxConfig; +} + +function installPoolCredential(accountId: string, chatgptAccountId: string, now: number): void { + saveCodexAccountCredential(accountId, { + accessToken: `${accountId}_token`, + refreshToken: `${accountId}_refresh`, + expiresAt: now + 24 * 60 * 60_000, + chatgptAccountId, + }); +} + +function mockUpstream(capture: { + urls: string[]; + bodies: string[]; + auths: Array; +}): void { + globalThis.fetch = (async (input, init) => { + capture.urls.push(String(input)); + capture.bodies.push(typeof init?.body === "string" ? init.body : ""); + const headers = new Headers(init?.headers); + capture.auths.push(headers.get("authorization")); + return Response.json({ + id: "resp_test", + object: "response", + status: "completed", + model: "gpt-5.6-sol", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; +} + +function mockSseUpstream(sseBody: string, capture?: { urls: string[] }): void { + globalThis.fetch = (async (input) => { + capture?.urls.push(String(input)); + return new Response(sseBody, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }) as typeof fetch; +} + +async function postSpawn( + config: OcxConfig, + body: Record, + options: Parameters[3] = {}, + logCtx: RequestLogContext = { model: "", provider: "" }, + headers: HeadersInit = {}, +): Promise { + return handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: spawnHeaders(headers), + body: JSON.stringify(body), + }), + config, + logCtx, + options, + ); +} + +describe("subagent fallback without primary auth cooldown failure", () => { + test("cooled primary with no probe lease selects healthy routed fallback", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc", now); + const cfg = poolNativePlusRoutedConfig({ + subagentModelFallback: ["xai/grok-4.5"], + }); + updateAccountQuota("pool-a", 95, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + + const authPublications: Array = []; + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn( + cfg, + { model: "gpt-5.6-sol", input: readableAgentInput(), stream: false }, + { onCodexAuthContextResolved: (ctx) => authPublications.push(ctx) }, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + expect(response.status).not.toBe(429); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + // No auth is resolved before final route; routed final publishes undefined. + expect(authPublications).toEqual([undefined]); + }); + + test("cooled primary with no usable fallback still returns cooldown 429", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc", now); + const cfg = poolNativePlusRoutedConfig({ + subagentModelFallback: ["gpt-5.5"], + }); + updateAccountQuota("pool-a", 95, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + throw new Error("must not dispatch"); + }) as typeof fetch; + + const response = await postSpawn(cfg, { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + }); + + expect(response.status).toBe(429); + expect(fetchCalls).toBe(0); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + }); + + test("same-provider native fallback at probe window authenticates only for final route", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc", now); + const cfg = poolNativePlusRoutedConfig({ + subagentModelFallback: ["gpt-5.5"], + }); + // Health-block the primary so fallback selects another native model; keep + // account below auto-switch threshold so the probe path is exercised. + updateAccountQuota("pool-a", 20, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "429", cfg, "pool-a"); + + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; + Date.now = () => probeAt; + + let finalAuth: CodexAuthContext | undefined; + const authPublications: Array = []; + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn( + cfg, + { model: "gpt-5.6-sol", input: readableAgentInput(), stream: false }, + { + onCodexAuthContextResolved: (ctx) => { + authPublications.push(ctx); + finalAuth = ctx; + }, + }, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + expect(finalAuth).toMatchObject({ kind: "pool", accountId: "pool-a" }); + expect((finalAuth as { probeLeaseId?: string }).probeLeaseId).toBeTruthy(); + expect(authPublications).toHaveLength(1); + expect(response.status).not.toBe(429); + }); + + test("final-route auth failure does not leave a primary probe lease", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc", now); + const cfg: OcxConfig = { + port: 0, + defaultProvider: "openai", + activeCodexAccountId: "pool-a", + autoSwitchThreshold: 80, + subagentModelFallback: ["openai-direct/gpt-5.5"], + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-direct": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "pool@example.test", isMain: false, chatgptAccountId: "pool_acc" }, + ], + }; + updateAccountQuota("pool-a", 95, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + + // Omit authorization so direct-mode final auth fails — primary never leased. + const response = await handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { + "content-type": "application/json", + "x-openai-subagent": "collab_spawn", + }, + body: JSON.stringify({ + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + }), + }), + cfg, + { model: "", provider: "" }, + ); + + expect(response.status).toBe(401); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + }); +}); + +describe("subagent fallback final-route normalization", () => { + test("falls back to gpt-5.6-sol-pro and rewrites wire model + reasoning.mode", async () => { + const cfg = poolNativePlusRoutedConfig({ + activeCodexAccountId: undefined, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + "openai-apikey": { + adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authMode: "key", + apiKey: "sk-test", + }, + }, + subagentModelFallback: ["openai-apikey/gpt-5.6-sol-pro"], + fastMode: true, + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await postSpawn( + cfg, + { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "high" }, + service_tier: "default", + }, + {}, + logCtx, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.openai.com"))).toBe(true); + const body = JSON.parse(capture.bodies[0]!) as { + model?: string; + reasoning?: { effort?: string; mode?: string }; + service_tier?: string; + }; + expect(body.model).toBe("gpt-5.6-sol"); + expect(body.reasoning?.mode).toBe("pro"); + expect(body.service_tier).toBe("priority"); + expect(logCtx.provider).toContain("openai-apikey"); + expect(logCtx.model).toBe("gpt-5.6-sol-pro"); + expect(logCtx.resolvedModel).toBe("gpt-5.6-sol"); + expect(logCtx.providerAdapter).toBe("openai-responses"); + }); + + test("routed primary falling back to native gpt-5.5 clamps max effort to xhigh", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.5"], + activeCodexAccountId: undefined, + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + const logCtx: RequestLogContext = { model: "", provider: "", requestedModel: "xai/grok-4.5" }; + + const response = await postSpawn( + cfg, + { + model: "xai/grok-4.5", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "max" }, + }, + {}, + logCtx, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + const body = JSON.parse(capture.bodies[0]!) as { + model?: string; + reasoning?: { effort?: string }; + }; + expect(body.model).toBe("gpt-5.5"); + expect(body.reasoning?.effort).toBe("xhigh"); + }); + + test("routed primary falling back to native gpt-5.6 keeps real max effort", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.6-terra"], + activeCodexAccountId: undefined, + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "max" }, + }); + + expect(response.status).toBe(200); + const body = JSON.parse(capture.bodies[0]!) as { reasoning?: { effort?: string } }; + expect(body.reasoning?.effort).toBe("max"); + }); + + test("native primary falling back to routed does not receive a native clamp", async () => { + const cfg = poolNativePlusRoutedConfig({ + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + subagentModelFallback: ["xai/grok-4.5"], + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "rate limit exceeded", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "max" }, + }); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + const body = JSON.parse(capture.bodies[0]!) as { reasoning?: { effort?: string } }; + // Routed adapters own effort mapping; the native clamp must not rewrite to xhigh. + expect(body.reasoning?.effort).not.toBe("xhigh"); + }); + + test("routed primary falls back to native and preserves encrypted task passthrough", async () => { + resetSubagentModelFallbackStateForTests(); + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.6-terra"], + activeCodexAccountId: undefined, + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }); + + if (response.status !== 200) { + const body = await response.text(); + throw new Error(`expected 200, got ${response.status}: ${body}`); + } + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + expect(capture.bodies[0]).toContain(FERNET_TASK); + }); + + test("native primary falls back to routed for readable child tasks", async () => { + const cfg = poolNativePlusRoutedConfig({ + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + subagentModelFallback: ["xai/grok-4.5"], + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "rate limit exceeded", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + }); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + }); +}); + +describe("native fallback account preview", () => { + test("uses healthier pool account B when active A is above threshold", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc_a", now); + installPoolCredential("pool-b", "pool_acc_b", now); + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + activeCodexAccountId: "pool-a", + subagentModelFallback: ["gpt-5.6-terra"], + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, + ], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + }); + updateAccountQuota("pool-a", 95, undefined, 20); + updateAccountQuota("pool-b", 10, undefined, 20); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const activeBefore = cfg.activeCodexAccountId; + expect(previewCodexAccountForRequest(null, cfg, now)).toBe("pool-b"); + expect(cfg.activeCodexAccountId).toBe(activeBefore); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + expect(getCodexUpstreamHealth("pool-b")?.probeLeaseId).toBeUndefined(); + + let finalAuth: CodexAuthContext | undefined; + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn( + cfg, + { model: "xai/grok-4.5", input: readableAgentInput(), stream: false }, + { onCodexAuthContextResolved: (ctx) => { finalAuth = ctx; } }, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + expect(finalAuth).toMatchObject({ kind: "pool", accountId: "pool-b" }); + expect(capture.auths.some((auth) => auth?.includes("pool-b_token"))).toBe(true); + }); + + test("skips native fallback when every pool account is exhausted", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc_a", now); + installPoolCredential("pool-b", "pool_acc_b", now); + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + activeCodexAccountId: "pool-a", + subagentModelFallback: ["gpt-5.6-terra", "xai/grok-3"], + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, + ], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + }); + updateAccountQuota("pool-a", 95, undefined, 20); + updateAccountQuota("pool-b", 90, undefined, 20); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: readableAgentInput(), + stream: false, + }); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + expect(capture.urls.some((url) => url.includes("chatgpt.com"))).toBe(false); + }); + + test("preview selection does not mutate affinity or acquire probe leases", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc_a", now); + installPoolCredential("pool-b", "pool_acc_b", now); + const cfg = poolNativePlusRoutedConfig({ + activeCodexAccountId: "pool-a", + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, + ], + }); + updateAccountQuota("pool-a", 95, undefined, 20); + updateAccountQuota("pool-b", 10, undefined, 20); + + // Bind affinity to pool-a via normal resolution once. + const bound = resolveCodexAccountForThreadDetailed("thread-1", cfg, now); + expect(bound).toMatchObject({ status: "selected", accountId: "pool-b" }); + const activeAfterBind = cfg.activeCodexAccountId; + + const previewed = previewCodexAccountForRequest("thread-1", cfg, now); + expect(previewed).toBe("pool-b"); + expect(cfg.activeCodexAccountId).toBe(activeAfterBind); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + expect(getCodexUpstreamHealth("pool-b")?.probeLeaseId).toBeUndefined(); + }); +}); + +describe("encrypted child native-only fallback", () => { + test("rejects encrypted routed primary when only routed fallbacks exist", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["xai/grok-3"], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + }); + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + throw new Error("must not dispatch"); + }) as typeof fetch; + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }); + const json = await response.json() as { error?: { code?: string } }; + expect(response.status).toBe(400); + expect(json.error?.code).toBe("unreadable_encrypted_agent_task"); + expect(fetchCalls).toBe(0); + }); + + test("skips exhausted native candidates before rejecting encrypted routed primary", async () => { + resetSubagentModelFallbackStateForTests(); + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.6-terra", "xai/grok-3"], + activeCodexAccountId: undefined, + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-terra", "429", cfg); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }); + expect(response.status).toBe(400); + }); + + test("non-thread-spawn encrypted routed requests stay rejected without fallback", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.5"], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const response = await handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { + "content-type": "application/json", + authorization: "Bearer caller-codex-token", + }, + body: JSON.stringify({ + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }), + }), + cfg, + { model: "", provider: "" }, + ); + expect(response.status).toBe(400); + }); +}); + +describe("native passthrough terminal finalization", () => { + function failedSse(message: string, type = "rate_limit_error"): string { + return `event: response.failed\ndata: ${JSON.stringify({ + type: "response.failed", + response: { + status: "failed", + error: { type, message }, + }, + })}\n\n`; + } + + async function runStreamingSpawn( + streamMode: "legacy-tee" | "eager-relay", + sseBody: string, + ): Promise<{ + terminals: ResponsesTerminalStatus[]; + healthBlocked: boolean; + responseText: string; + }> { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc", now); + const cfg = poolNativePlusRoutedConfig({ + streamMode, + activeCodexAccountId: "pool-a", + subagentModelFallback: ["xai/grok-4.5"], + }); + updateAccountQuota("pool-a", 20, undefined, 20); + + const terminals: ResponsesTerminalStatus[] = []; + mockSseUpstream(sseBody); + + const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); + // Force win32 so eager-relay decision path is reachable via streamMode override. + Object.defineProperty(process, "platform", { value: "win32", configurable: true }); + try { + const response = await postSpawn( + cfg, + { model: "gpt-5.6-sol", input: readableAgentInput(), stream: true }, + { + onNativePassthroughTerminal: (status) => terminals.push(status), + }, + ); + const responseText = await response.text(); + // Allow inspection consumer microtasks to settle. + await Bun.sleep(20); + return { + terminals, + healthBlocked: isModelHealthBlocked("gpt-5.6-sol", cfg, "pool-a"), + responseText, + }; + } finally { + if (platformDescriptor) Object.defineProperty(process, "platform", platformDescriptor); + } + } + + for (const streamMode of ["legacy-tee", "eager-relay"] as const) { + test(`${streamMode}: 429 failed records health and invokes terminal callback`, async () => { + const result = await runStreamingSpawn(streamMode, failedSse("rate limited", "rate_limit_error")); + expect(result.terminals).toEqual(["failed"]); + expect(result.healthBlocked).toBe(true); + expect(result.responseText).toContain("response.failed"); + }); + + test(`${streamMode}: 402-style insufficient_quota records health and invokes callback`, async () => { + const result = await runStreamingSpawn( + streamMode, + failedSse("insufficient quota", "insufficient_quota"), + ); + expect(result.terminals).toEqual(["failed"]); + expect(result.healthBlocked).toBe(true); + }); + + test(`${streamMode}: generic 500 failure invokes terminal callback without health block`, async () => { + const result = await runStreamingSpawn( + streamMode, + failedSse("internal server error", "server_error"), + ); + expect(result.terminals).toEqual(["failed"]); + expect(result.healthBlocked).toBe(false); + }); + + test(`${streamMode}: completed terminal fires exactly once`, async () => { + const sse = `event: response.completed\ndata: ${JSON.stringify({ + type: "response.completed", + response: { id: "r1", status: "completed", output: [] }, + })}\n\n`; + const result = await runStreamingSpawn(streamMode, sse); + expect(result.terminals).toEqual(["completed"]); + expect(result.healthBlocked).toBe(false); + }); + } +}); diff --git a/tests/subagent-model-fallback-api.test.ts b/tests/subagent-model-fallback-api.test.ts new file mode 100644 index 0000000000..10fde15475 --- /dev/null +++ b/tests/subagent-model-fallback-api.test.ts @@ -0,0 +1,91 @@ +/** + * /api/subagent-model-fallback atomic validation (PR #391). + * Invalid chain entries must 400 without mutating the previous config. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleManagementAPI } from "../src/server/management-api"; +import type { OcxConfig } from "../src/types"; + +const savedHome = process.env.OPENCODEX_HOME; +let tempHome: string | null = null; + +afterEach(() => { + if (savedHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = savedHome; + if (tempHome) { + rmSync(tempHome, { recursive: true, force: true }); + tempHome = null; + } +}); + +function isolatedHome(): void { + tempHome = mkdtempSync(join(tmpdir(), "ocx-subagent-fallback-api-")); + process.env.OPENCODEX_HOME = tempHome; +} + +function makeConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + providers: {}, + defaultProvider: "openai", + subagentModelFallback: ["gpt-5.6-sol", "kimi/k3"], + ...overrides, + } as OcxConfig; +} + +async function put(config: OcxConfig, body: unknown): Promise { + const req = new Request("http://localhost/api/subagent-model-fallback", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + const res = await handleManagementAPI(req, new URL(req.url), config); + expect(res).not.toBeNull(); + return res!; +} + +describe("/api/subagent-model-fallback atomic validation", () => { + test("rejects one invalid entry with 400 and leaves previous config unchanged", async () => { + isolatedHome(); + const previous = ["gpt-5.6-sol", "kimi/k3"]; + const config = makeConfig({ subagentModelFallback: [...previous] }); + + const res = await put(config, { + models: ["gpt-5.6-sol", 42, "alibaba-token-plan/qwen3.8-max-preview"], + }); + expect(res.status).toBe(400); + const body = await res.json() as { error: string; index: number; value: unknown }; + expect(body.error).toBe("models[1] must be a non-empty string"); + expect(body.index).toBe(1); + expect(body.value).toBe(42); + expect(config.subagentModelFallback).toEqual(previous); + }); + + test("rejects empty-string entries without truncating the chain", async () => { + isolatedHome(); + const previous = ["gpt-5.6-sol", "kimi/k3"]; + const config = makeConfig({ subagentModelFallback: [...previous] }); + + const res = await put(config, { + models: ["gpt-5.6-sol", " ", "kimi/k3"], + }); + expect(res.status).toBe(400); + const body = await res.json() as { error: string; index: number }; + expect(body.error).toBe("models[1] must be a non-empty string"); + expect(body.index).toBe(1); + expect(config.subagentModelFallback).toEqual(previous); + }); + + test("accepts a fully valid chain after validation", async () => { + isolatedHome(); + const config = makeConfig(); + const next = ["gpt-5.6-sol", "alibaba-token-plan/qwen3.8-max-preview"]; + const res = await put(config, { models: next }); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ ok: true, models: next }); + expect(config.subagentModelFallback).toEqual(next); + }); +}); diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts new file mode 100644 index 0000000000..3e13782d0b --- /dev/null +++ b/tests/subagent-model-fallback.test.ts @@ -0,0 +1,718 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + applySubagentModelFallback, + buildSubagentModelChain, + getSubagentQuotaPrimeStateForTests, + isNativeModelQuotaExhausted, + isSubagentModelUnavailable, + maybePrimeSubagentQuota, + noteSubagentModelFailure, + readCodexAgentModelFallback, + resetSubagentModelFallbackStateForTests, + resolveAgentModelFallbackForPrimary, + selectAvailableSubagentModel, + setSubagentQuotaPrimeForTests, + subagentFallbackGuidanceText, +} from "../src/codex/subagent-model-fallback"; +import { saveCodexAccountCredential } from "../src/codex/account-store"; +import { clearAccountNeedsReauth, markAccountNeedsReauth } from "../src/codex/account-runtime-state"; +import { clearAccountQuota, updateAccountQuota } from "../src/codex/quota"; +import type { OcxConfig } from "../src/types"; + +const savedCodexHome = process.env.CODEX_HOME; +const savedOpencodexHome = process.env.OPENCODEX_HOME; +let testDir: string; + +function installPoolCredential(accountId: string, now = Date.now()): void { + saveCodexAccountCredential(accountId, { + accessToken: `${accountId}_token`, + refreshToken: `${accountId}_refresh`, + expiresAt: now + 24 * 60 * 60_000, + chatgptAccountId: `${accountId}_acc`, + }); +} + +function cfg(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + providers: { + // Omitted codexAccountMode — canonical openai defaults to pool via routeModel. + openai: { adapter: "openai-responses" }, + "alibaba-token-plan": { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + xai: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://api.x.ai/v1" }, + }, + defaultProvider: "openai", + activeCodexAccountId: "pool-a", + autoSwitchThreshold: 80, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_a_acc" }, + { id: "account-a", email: "aa@example.test", isMain: false, chatgptAccountId: "aa_acc" }, + { id: "account-b", email: "bb@example.test", isMain: false, chatgptAccountId: "bb_acc" }, + ], + subagentModelFallback: [ + "gpt-5.6-sol", + "alibaba-token-plan/qwen3.8-max-preview", + "kimi/k3", + ], + ...overrides, + }; +} + +function codexHomeFixture(): string { + const dir = mkdtempSync(join(tmpdir(), "ocx-subagent-fallback-")); + mkdirSync(join(dir, "agents"), { recursive: true }); + process.env.CODEX_HOME = dir; + return dir; +} + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-subagent-fb-")); + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; + installPoolCredential("pool-a"); + installPoolCredential("account-a"); + installPoolCredential("account-b"); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("account-a"); + clearAccountNeedsReauth("account-b"); + clearAccountNeedsReauth("main"); +}); + +afterEach(() => { + if (savedCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = savedCodexHome; + if (savedOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = savedOpencodexHome; + clearAccountQuota(); + resetSubagentModelFallbackStateForTests(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("account-a"); + clearAccountNeedsReauth("account-b"); + clearAccountNeedsReauth("main"); + rmSync(testDir, { recursive: true, force: true }); +}); + +describe("subagent model fallback chain", () => { + test("buildSubagentModelChain dedupes and preserves order", () => { + expect(buildSubagentModelChain("gpt-5.6-sol", cfg())).toEqual([ + "gpt-5.6-sol", + "alibaba-token-plan/qwen3.8-max-preview", + "kimi/k3", + ]); + expect(buildSubagentModelChain("kimi/k3", cfg())).toEqual([ + "kimi/k3", + "gpt-5.6-sol", + "alibaba-token-plan/qwen3.8-max-preview", + ]); + }); + + test("selectAvailableSubagentModel skips quota-exhausted native models", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); + expect(selected).toEqual({ + model: "alibaba-token-plan/qwen3.8-max-preview", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("selectAvailableSubagentModel scopes quota exhaustion to the selected account", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("account-a", 95, undefined, 20); + updateAccountQuota("account-b", 10, undefined, 20); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg(), [], "account-b"); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: [], + }); + }); + + test("selectAvailableSubagentModel skips cached routed failures", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted", cfg()); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); + expect(selected.model).toBe("kimi/k3"); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg())).toBe(true); + }); + + test("selectAvailableSubagentModel skips stale fallback entries that cannot route", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + subagentModelFallback: [ + "missing-provider/does-not-exist", + "kimi/k3", + ], + }), + ); + expect(selected).toEqual({ + model: "kimi/k3", + rewritten: true, + skipped: ["gpt-5.6-sol", "missing-provider/does-not-exist"], + }); + }); + + test("noteSubagentModelFailure treats numeric 429 as quota-like", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "429", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + }); + + test("noteSubagentModelFailure records generic rate-limit wording as a health block", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "Rate limit exceeded. Please try again later.", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "Too Many Requests", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "provider temporarily rate limited", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + }); + + test("noteSubagentModelFailure ignores unrelated errors", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "connection refused", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); + noteSubagentModelFailure("kimi/k3", "invalid_request_error: missing field", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); + }); + + test("await maybePrimeSubagentQuota waits for deferred refresh before selection", async () => { + resetSubagentModelFallbackStateForTests(); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let midRefreshModel = ""; + + setSubagentQuotaPrimeForTests(async () => { + midRefreshModel = selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model; + await gate; + updateAccountQuota("pool-a", 95, undefined, 20); + }); + + const priming = maybePrimeSubagentQuota(cfg()); + expect(selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model).toBe("gpt-5.6-sol"); + release(); + await priming; + expect(midRefreshModel).toBe("gpt-5.6-sol"); + expect(selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model).toBe( + "alibaba-token-plan/qwen3.8-max-preview", + ); + expect(getSubagentQuotaPrimeStateForTests().primedAt).toBeGreaterThan(0); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(false); + }); + + test("concurrent maybePrimeSubagentQuota callers share one in-flight refresh", async () => { + resetSubagentModelFallbackStateForTests(); + let calls = 0; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + setSubagentQuotaPrimeForTests(async () => { + calls += 1; + await gate; + updateAccountQuota("pool-a", 95, undefined, 20); + }); + + const a = maybePrimeSubagentQuota(cfg()); + const b = maybePrimeSubagentQuota(cfg()); + const c = maybePrimeSubagentQuota(cfg()); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(true); + release(); + await Promise.all([a, b, c]); + expect(calls).toBe(1); + expect(selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model).toBe( + "alibaba-token-plan/qwen3.8-max-preview", + ); + }); + + test("failed quota prime does not mark success TTL and allows retry", async () => { + resetSubagentModelFallbackStateForTests(); + let calls = 0; + setSubagentQuotaPrimeForTests(async () => { + calls += 1; + throw new Error("prime failed"); + }); + await maybePrimeSubagentQuota(cfg()); + expect(calls).toBe(1); + expect(getSubagentQuotaPrimeStateForTests().primedAt).toBe(0); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(false); + + setSubagentQuotaPrimeForTests(async () => { + calls += 1; + updateAccountQuota("pool-a", 95, undefined, 20); + }); + await maybePrimeSubagentQuota(cfg()); + expect(calls).toBe(2); + expect(getSubagentQuotaPrimeStateForTests().primedAt).toBeGreaterThan(0); + }); + + test("reset clears timestamp, in-flight, and health state", async () => { + resetSubagentModelFallbackStateForTests(); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + setSubagentQuotaPrimeForTests(async () => { + await gate; + throw new Error("cancelled after reset"); + }); + const priming = maybePrimeSubagentQuota(cfg()); + noteSubagentModelFailure("kimi/k3", "429", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(true); + resetSubagentModelFallbackStateForTests(); + expect(getSubagentQuotaPrimeStateForTests()).toEqual({ primedAt: 0, inFlight: false }); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); + release(); + await priming; + expect(getSubagentQuotaPrimeStateForTests()).toEqual({ primedAt: 0, inFlight: false }); + }); + + test("noteSubagentModelFailure records the configured fallback slug", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg()); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg())).toBe(true); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); + }); + + test("selectAvailableSubagentModel can require native-only fallback for encrypted tasks", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg(), + [], + "pool-a", + Date.now(), + true, + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: ["gpt-5.6-sol", "alibaba-token-plan/qwen3.8-max-preview", "kimi/k3"], + }); + }); + + test("selectAvailableSubagentModel can stay native-only for encrypted spawns", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg(), + [], + "pool-a", + Date.now(), + true, + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: ["gpt-5.6-sol", "alibaba-token-plan/qwen3.8-max-preview", "kimi/k3"], + }); + }); + + test("direct bare GPT route ignores exhausted retained pool account quota", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + autoSwitchThreshold: 80, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["kimi/k3"], + }); + expect(isSubagentModelUnavailable("gpt-5.6-sol", config, "pool-a")).toBe(false); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: [], + }); + }); + + test("another pool provider in config does not affect a direct GPT candidate", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + "openai-pool": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["kimi/k3"], + }); + expect(isNativeModelQuotaExhausted("gpt-5.6-sol", config, "pool-a")).toBe(false); + expect(isSubagentModelUnavailable("gpt-5.6-sol", config, "pool-a")).toBe(false); + }); + + test("openai-direct/gpt-5.5 is accepted as encrypted-task fallback when canonical", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-direct": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["openai-direct/gpt-5.5", "kimi/k3"], + }); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + Date.now(), + true, + ); + expect(selected).toEqual({ + model: "openai-direct/gpt-5.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("namespaced noncanonical OpenAI-compatible provider is rejected for encrypted tasks", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-compat": { + adapter: "openai-responses", + baseUrl: "https://api.example.com/v1", + authMode: "key", + apiKey: "sk-test", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["openai-compat/gpt-5.5", "kimi/k3"], + }); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + Date.now(), + true, + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: ["gpt-5.6-sol", "openai-compat/gpt-5.5", "kimi/k3"], + }); + }); + + test("pool quota affects only candidates whose resolved route uses pool mode", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-direct": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["openai-direct/gpt-5.5", "kimi/k3"], + }); + expect(isNativeModelQuotaExhausted("gpt-5.6-sol", config, "pool-a")).toBe(true); + expect(isNativeModelQuotaExhausted("openai-direct/gpt-5.5", config, "pool-a")).toBe(false); + expect(isNativeModelQuotaExhausted("kimi/k3", config, "pool-a")).toBe(false); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + ); + expect(selected).toEqual({ + model: "openai-direct/gpt-5.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("omitted openai codexAccountMode still requires a usable pool account", () => { + resetSubagentModelFallbackStateForTests(); + // Default cfg omits codexAccountMode on openai (defaults to pool via routeModel). + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ subagentModelFallback: ["xai/grok-4.5"] }), + [], + null, + ); + expect(selected).toEqual({ + model: "xai/grok-4.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("no usable pool account falls back to XAI", () => { + resetSubagentModelFallbackStateForTests(); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + activeCodexAccountId: undefined, + codexAccounts: [{ id: "main", email: "main@example.test", isMain: true }], + subagentModelFallback: ["xai/grok-4.5"], + }), + [], + null, + ); + expect(selected).toEqual({ + model: "xai/grok-4.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("reauthentication-required pool account falls back to XAI", () => { + resetSubagentModelFallbackStateForTests(); + markAccountNeedsReauth("pool-a"); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ subagentModelFallback: ["xai/grok-4.5"] }), + [], + "pool-a", + ); + expect(selected).toEqual({ + model: "xai/grok-4.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("explicit direct mode remains unaffected by missing pool accounts", () => { + resetSubagentModelFallbackStateForTests(); + const config = cfg({ + activeCodexAccountId: undefined, + codexAccounts: [{ id: "main", email: "main@example.test", isMain: true }], + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + xai: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://api.x.ai/v1" }, + }, + subagentModelFallback: ["xai/grok-4.5"], + }); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", config, [], null); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: [], + }); + }); + + test("noteSubagentModelFailure records failures under the configured fallback slug", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg(), "pool-a"); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg(), "pool-a")).toBe(true); + expect(isSubagentModelUnavailable("kimi/k3", cfg(), "pool-a")).toBe(false); + }); + + test("readCodexAgentModelFallback parses multiline TOML arrays", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [", + " \"alibaba-token-plan/qwen3.8-max-preview\",", + " \"kimi/k3\",", + "]", + "", + ].join("\n"), "utf8"); + expect(readCodexAgentModelFallback("executor", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + "kimi/k3", + ]); + }); + + test("readCodexAgentModelFallback stops at the model_fallback array terminator", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [", + " \"alibaba-token-plan/qwen3.8-max-preview\",", + "]", + "tools = [\"search\", \"edit\"]", + "", + ].join("\n"), "utf8"); + expect(readCodexAgentModelFallback("executor", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + ]); + }); + + test("selectAvailableSubagentModel skips disabled bare native fallback entries", () => { + resetSubagentModelFallbackStateForTests(); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + disabledModels: ["gpt-5.6-sol"], + subagentModelFallback: ["kimi/k3"], + }), + ); + expect(selected).toEqual({ + model: "kimi/k3", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("selectAvailableSubagentModel allows raw slash model ids without provider namespaces", () => { + resetSubagentModelFallbackStateForTests(); + // Health-block the primary model only. A raw vendor/model id still routes through the + // default provider; it must remain selectable (not rejected as an unknown namespace). + noteSubagentModelFailure("gpt-5.6-sol", "429", cfg()); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + subagentModelFallback: [ + "anthropic/claude-sonnet-4-6", + "kimi/k3", + ], + }), + ); + expect(selected).toEqual({ + model: "anthropic/claude-sonnet-4-6", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("noteSubagentModelFailure scopes routed-provider health globally", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted", cfg(), "account-a"); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg(), "account-b")).toBe(true); + }); + + test("applySubagentModelFallback rewrites parsed request model", () => { + updateAccountQuota("pool-a", 95); + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + const result = applySubagentModelFallback( + parsed as never, + new Headers({ "x-openai-subagent": "collab_spawn" }), + cfg(), + ); + expect(result).toEqual({ + from: "gpt-5.6-sol", + to: "alibaba-token-plan/qwen3.8-max-preview", + skipped: ["gpt-5.6-sol"], + }); + expect(parsed.modelId).toBe("alibaba-token-plan/qwen3.8-max-preview"); + expect((parsed._rawBody as { model?: string }).model).toBe("alibaba-token-plan/qwen3.8-max-preview"); + }); + + test("applySubagentModelFallback is a no-op for main turns", () => { + updateAccountQuota("pool-a", 95); + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + expect(applySubagentModelFallback(parsed as never, new Headers(), cfg())).toBeNull(); + expect(parsed.modelId).toBe("gpt-5.6-sol"); + }); + + test("applySubagentModelFallback can use per-agent model_fallback without global config", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"alibaba-token-plan/qwen3.8-max-preview\"]", + "", + ].join("\n"), "utf8"); + updateAccountQuota("pool-a", 95); + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + const result = applySubagentModelFallback( + parsed as never, + new Headers({ "x-openai-subagent": "collab_spawn" }), + cfg({ subagentModelFallback: undefined }), + ); + expect(result?.to).toBe("alibaba-token-plan/qwen3.8-max-preview"); + expect(resolveAgentModelFallbackForPrimary("gpt-5.6-sol", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + ]); + expect(readCodexAgentModelFallback("executor", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + ]); + }); + + test("subagentFallbackGuidanceText renders configured chain", () => { + expect(subagentFallbackGuidanceText(cfg())).toContain("gpt-5.6-sol"); + expect(subagentFallbackGuidanceText(cfg({ subagentModelFallback: undefined }))).toBe(""); + }); +});