diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 6018d7e1bd..6f51870ad3 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -263,10 +263,11 @@ A combo can also advance after an intact HTTP 400 `invalid_request_error` that s ## Default reasoning effort -`defaultEffort` fills an absent `reasoning.effort` when the combo has a non-null default and the selected target has a known, nonempty supported ladder. If the target supports the configured value, it is retained; otherwise the highest supported rung at or below it is used, or the lowest supported rung when none is lower. Unknown or empty ladders omit the default. +`defaultEffort` supplies a configured effort when the selected target has a known, nonempty supported ladder. With the default `defaultEffortMode: "fallback"`, an explicit caller effort keeps precedence. `defaultEffortMode: "force"` overrides a valid caller effort with the configured default; it requires a valid, non-null `defaultEffort` and can increase cost and latency. Force mode is an explicit operator choice through combo configuration or management. -The default-injection step preserves existing effort and other reasoning fields. Capability normalization can separately remove unsupported effort/thinking controls as described below. Supported defaults are `low`, `medium`, `high`, `xhigh`, `max`, and `ultra`; omit the field or use `null` to disable default injection. +The target's advertised ladder remains authoritative. An exact supported value is retained; otherwise the highest supported rung at or below it is selected, or the lowest supported rung when none is lower. Unknown or empty ladders never cause default injection. Force mode does not repair malformed caller effort into a valid expensive request. Other reasoning fields, including `reasoning.summary`, are preserved. +`reasoningEffortMode` remains independent of `defaultEffortMode`: explicit empty ladders remove unsupported effort/thinking controls, and adaptive unknown ladders do so as well, as described below. Strict unknown ladders preserve the caller's request without forcing a default. Supported defaults are `low`, `medium`, `high`, `xhigh`, `max`, and `ultra`; omit `defaultEffort` or set it to `null` to disable default injection in fallback mode. ### Mixed-capability groups (`reasoningEffortMode`) @@ -413,7 +414,8 @@ Combos are stored in the top-level `combos` object, keyed by combo id: | `stickyLimit` | No | `1` | Integer from 1 to 100 successful requests per round-robin selection. Applies only to round-robin. | | `cooldownMs` | No | unset → upstream fallback (5 s for request-rate 429 codes `1302`/`1305`, otherwise 60 s) | Integer from 1 to 600000. When set, applies as the per-target cooldown whenever no usable upstream `Retry-After` or Codex reset signal exists, including request-rate 429s; when unset, uses the upstream fallback. | | `waitForCooldownMs` | No | `0` | Integer from 0 to 600000. Maximum time to wait for the earliest eligible cooling target before returning `combo_unavailable`; abort cancels the wait. | -| `defaultEffort` | No | `null` | `low`, `medium`, `high`, `xhigh`, `max`, or `ultra`; applied only when the caller omits effort and the target advertises support. | +| `defaultEffort` | No | `null` | `low`, `medium`, `high`, `xhigh`, `max`, or `ultra`; resolved against each target's advertised ladder. | +| `defaultEffortMode` | No | `"fallback"` | `"fallback"` preserves explicit caller effort. `"force"` overrides valid caller effort, requires a valid non-null default, and can increase cost and latency. | | `reasoningEffortMode` | No | `"strict"` | `"strict"` intersects every known target ladder, so one target advertising no effort control empties the combo's picker. `"adaptive"` excludes those empty ladders from the published intersection. At dispatch, explicit empty or adaptive unknown ladders remove unsupported effort/thinking controls while preserving supported non-effort reasoning fields such as `reasoning.summary`; known non-empty targets keep existing effort resolution. | | `imageInput` | No | `"auto"` | `"auto"` or `"disabled"`. `"auto"` publishes image support only when every target supports images; `"disabled"` forces text-only (drops image from published modalities and rejects image-bearing requests before dispatch). | | `alias` | No | none | Optional trimmed public model id; use the alias rules above. An empty value is stored as no alias. | diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 2934c337a6..d830faafae 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -90,7 +90,8 @@ namespace, and cannot use reserved bare native families such as `gpt-*`, `o1-*`, | `stickyLimit?` | `number` | `1` | Successful requests retained in one round-robin batch. Range 1–100. Applies only to round-robin. | | `cooldownMs?` | `number` | unset → upstream fallback (5 s for request-rate 429 codes `1302`/`1305`, otherwise 60 s) | Range 1–600000. When set, applies whenever no usable upstream `Retry-After` or Codex reset signal exists, including request-rate 429s; when unset, uses the upstream fallback. Upstream signals take precedence and all cooldowns are capped at 10 minutes. | | `waitForCooldownMs?` | `number` | `0` | Maximum wait for the earliest eligible cooling target on each selection attempt before returning `combo_unavailable`. Range 0–600000; an abort cancels the wait. | -| `defaultEffort?` | `"low" \| "medium" \| "high" \| "xhigh" \| "max" \| "ultra" \| null` | unset | `defaultEffort` fills an absent `reasoning.effort` when the combo has a non-null default and the selected target has a known, nonempty supported ladder. If the target supports the configured value, it is retained; otherwise the highest supported rung at or below it is used, or the lowest supported rung when none is lower. Unknown or empty ladders omit the default. | +| `defaultEffort?` | `"low" \| "medium" \| "high" \| "xhigh" \| "max" \| "ultra" \| null` | unset | `defaultEffort` fills an absent `reasoning.effort` in fallback mode, or overrides valid caller effort in explicit force mode when the combo has a non-null default and the selected target has a known, nonempty supported ladder. If the target supports the configured value, it is retained; otherwise the highest supported rung at or below it is used, or the lowest supported rung when none is lower. Unknown or empty ladders omit the default. | +| `defaultEffortMode?` | `"fallback" \| "force"` | `"fallback"` | Preserves caller precedence by default. Explicit force requires a valid non-null default, respects target capability and can increase cost and latency. `reasoningEffortMode` remains independent. | | `reasoningEffortMode?` | `"strict" \| "adaptive"` | `"strict"` | `"strict"` intersects all known target ladders, including empty ones; `"adaptive"` excludes empty ladders. Unknown ladders are catalog wildcards in both modes. At dispatch, explicit empty ladders remove effort/thinking controls in both modes; unknown ladders do so only in adaptive. `reasoning.summary` is preserved. Known nonempty targets retain their effort resolution, and target selection/order is unchanged. | | `imageInput?` | `"auto" \| "disabled"` | `"auto"` | `"auto"` publishes image only when every target supports images; `"disabled"` forces text-only (drops image from published modalities and rejects image-bearing requests before dispatch). | | `alias?` | `string` | — | Optional public model id in place of the canonical picker slug. | diff --git a/src/cli/combo.ts b/src/cli/combo.ts index 3e0aa0d0bf..380bcd36ff 100644 --- a/src/cli/combo.ts +++ b/src/cli/combo.ts @@ -15,7 +15,8 @@ const USAGE = `Usage: ocx combo show [--json] ocx combo set --targets [--strategy ] [--sticky <1-100>] - [--effort ] [--alias ] + [--effort ] [--effort-mode ] + (force overrides valid client effort and can increase cost/latency) [--alias ] [--native-alias] [--display-name ] [--rename-from ] [--json] ocx combo remove --yes [--json]`; @@ -80,6 +81,10 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise { if (strategy !== "round-robin") throw new CliUsageError("--sticky applies only to round-robin", USAGE); } const effort = takeOption(args, "--effort"); + const effortMode = takeOption(args, "--effort-mode"); + if (effortMode !== undefined && effortMode !== "fallback" && effortMode !== "force") { + throw new CliUsageError("--effort-mode must be fallback or force", USAGE); + } const alias = takeOption(args, "--alias"); const nativeAlias = takeFlag(args, "--native-alias"); const displayName = takeOption(args, "--display-name"); @@ -91,12 +96,16 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise { targets: parseTargets(targetsRaw), }; if (effort !== undefined) combo.defaultEffort = effort === "-" ? null : effort; + if (effortMode !== undefined) combo.defaultEffortMode = effortMode; if (alias !== undefined) combo.alias = alias === "-" ? "" : alias; if (nativeAlias) combo.nativeAlias = true; if (displayName !== undefined) combo.displayName = displayName === "-" ? "" : displayName; const current = await runtimeRequest<{ combos?: ComboRow[] }>("/api/combos", {}, deps); const existing = (current.combos ?? []).find(row => row.id === (renameFrom ?? id)); if (existing?.imageInput === "disabled") combo.imageInput = "disabled"; + if (effortMode === undefined && existing?.defaultEffortMode === "force") { + combo.defaultEffortMode = effort === "-" ? "fallback" : "force"; + } const result = await runtimeRequest("/api/combos", { method: "PUT", body: JSON.stringify({ id, combo, ...(renameFrom ? { renameFrom } : {}) }), diff --git a/src/combos/request.ts b/src/combos/request.ts index 63c5ba7fca..e0fa642608 100644 --- a/src/combos/request.ts +++ b/src/combos/request.ts @@ -1,5 +1,5 @@ -import type { OcxComboDefaultEffort, OcxComboReasoningEffortMode, OcxComboTarget, OcxConfig } from "../types"; -import { resolveEffortAtOrBelow } from "../reasoning-effort"; +import type { OcxComboDefaultEffort, OcxComboDefaultEffortMode, OcxComboReasoningEffortMode, OcxComboTarget, OcxConfig } from "../types"; +import { isCodexReasoningEffort, resolveEffortAtOrBelow } from "../reasoning-effort"; import { resolveComboId } from "./types"; const warnedUnsupportedDefaults = new Set(); @@ -60,22 +60,29 @@ export function concreteComboRequestBody( defaultEffort: OcxComboDefaultEffort | null, targetReasoningEfforts: readonly string[] | undefined, reasoningEffortMode: OcxComboReasoningEffortMode = "strict", + defaultEffortMode: OcxComboDefaultEffortMode = "fallback", ): Record { const clone = structuredClone(body) as Record; clone.model = `${target.provider}/${target.model}`; + if (defaultEffortMode === "force" && (!defaultEffort || !isCodexReasoningEffort(defaultEffort))) { + throw new Error("force combo default effort requires a valid defaultEffort"); + } if (targetReasoningEfforts?.length === 0 || (reasoningEffortMode === "adaptive" && targetReasoningEfforts === undefined)) { stripUnsupportedReasoningControls(clone); } - if (!defaultEffort) return clone; + if (!defaultEffort || !isCodexReasoningEffort(defaultEffort)) return clone; const reasoning = clone.reasoning; - const needsDefault = reasoning === undefined || ( - reasoning - && typeof reasoning === "object" - && !Array.isArray(reasoning) - && !Object.prototype.hasOwnProperty.call(reasoning, "effort") - ); - if (!needsDefault) return clone; + const reasoningRecord = reasoning && typeof reasoning === "object" && !Array.isArray(reasoning) + ? reasoning as Record + : undefined; + const hasEffort = reasoningRecord !== undefined + && Object.prototype.hasOwnProperty.call(reasoningRecord, "effort"); + const callerEffort = reasoningRecord?.effort; + const validCallerEffort = typeof callerEffort === "string" && isCodexReasoningEffort(callerEffort); + const needsDefault = reasoning === undefined || (reasoningRecord !== undefined && !hasEffort); + const shouldForce = defaultEffortMode === "force" && validCallerEffort; + if (!needsDefault && !shouldForce) return clone; // Picker availability treats an unknown ladder as a wildcard, but runtime // injection stays fail-closed until this concrete target advertises support. // diff --git a/src/combos/types.ts b/src/combos/types.ts index b5c5bf697c..f4b3e26b2f 100644 --- a/src/combos/types.ts +++ b/src/combos/types.ts @@ -1,6 +1,6 @@ import { isCodexReasoningEffort } from "../reasoning-effort"; import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models"; -import type { OcxComboConfig, OcxComboDefaultEffort, OcxComboReasoningEffortMode, OcxComboStrategy, OcxComboTarget, OcxProviderConfig } from "../types"; +import type { OcxComboConfig, OcxComboDefaultEffort, OcxComboDefaultEffortMode, OcxComboReasoningEffortMode, OcxComboStrategy, OcxComboTarget, OcxProviderConfig } from "../types"; import { COMBO_NAMESPACE, isValidComboId, targetKey } from "./identifiers"; export const COMBO_DEFAULT_WAIT_FOR_COOLDOWN_MS = 0; @@ -26,6 +26,8 @@ export interface NormalizedComboConfig { cooldownMs?: number; waitForCooldownMs: number; defaultEffort: OcxComboDefaultEffort | null; + /** Client-precedence policy; `fallback` preserves legacy behavior. */ + defaultEffortMode: OcxComboDefaultEffortMode; /** Picker-ladder derivation policy; `strict` preserves the legacy intersection rule. */ reasoningEffortMode: OcxComboReasoningEffortMode; /** Disable image input; `auto` preserves the intersection derived from all targets. */ @@ -167,6 +169,21 @@ export function comboConfigIssues( message: "defaultEffort must be one of: low, medium, high, xhigh, max, ultra", }); } + if (body.defaultEffortMode !== undefined + && body.defaultEffortMode !== "fallback" + && body.defaultEffortMode !== "force") { + issues.push({ + path: ["defaultEffortMode"], + message: 'defaultEffortMode must be "fallback" or "force"', + }); + } + if (body.defaultEffortMode === "force" + && (typeof body.defaultEffort !== "string" || !isCodexReasoningEffort(body.defaultEffort))) { + issues.push({ + path: ["defaultEffort"], + message: "defaultEffort is required when defaultEffortMode is force", + }); + } if (body.imageInput !== undefined && body.imageInput !== "auto" && body.imageInput !== "disabled") { issues.push({ path: ["imageInput"], message: 'imageInput must be "auto" or "disabled"' }); } @@ -293,12 +310,16 @@ export function comboConfigError( export function normalizeComboConfig(raw: OcxComboConfig): NormalizedComboConfig { const alias = typeof raw.alias === "string" ? raw.alias.trim() : ""; const displayName = typeof raw.displayName === "string" ? raw.displayName.trim() : ""; + const defaultEffort = typeof raw.defaultEffort === "string" && isCodexReasoningEffort(raw.defaultEffort) + ? raw.defaultEffort + : null; return { strategy: raw.strategy ?? "failover", stickyLimit: raw.stickyLimit ?? 1, cooldownMs: raw.cooldownMs, waitForCooldownMs: raw.waitForCooldownMs ?? COMBO_DEFAULT_WAIT_FOR_COOLDOWN_MS, - defaultEffort: raw.defaultEffort ?? null, + defaultEffort, + defaultEffortMode: raw.defaultEffortMode === "force" && defaultEffort !== null ? "force" : "fallback", reasoningEffortMode: raw.reasoningEffortMode === "adaptive" ? "adaptive" : "strict", imageInput: raw.imageInput === "disabled" ? "disabled" : "auto", alias: alias || null, diff --git a/src/server/chat-completions.ts b/src/server/chat-completions.ts index d44ac60c74..2d268289e3 100644 --- a/src/server/chat-completions.ts +++ b/src/server/chat-completions.ts @@ -170,7 +170,9 @@ async function handleChatCompletionsWithBudget( if (chatBody.tools !== undefined) parts.push(JSON.stringify(chatBody.tools)); logCtx.usageLogInputTokens = Math.max(1, estimateTokens(parts.join("\n"), requestedModel)); } - if (!effortRow && isNativeChatRouteEligible(route, chatBody, config)) chatNativeRoute = route; + // Combos must enter the Responses routing path so child selection, forced default + // effort, failover, and per-attempt telemetry run before any native Chat send. + if (!route.combo && !effortRow && isNativeChatRouteEligible(route, chatBody, config)) chatNativeRoute = route; } catch (err) { if (err instanceof UnknownRoutingPolicyError) { logCtx.requestedModel = requestedModel; diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index 475e72db41..cdc567a6eb 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -80,17 +80,20 @@ function sparseComboConfig(combo: T): Omit & { + defaultEffortMode?: "fallback" | "force"; +}>(combo: T): Omit & { cooldownMs?: number; waitForCooldownMs?: number; imageInput?: "disabled"; reasoningEffortMode?: "adaptive"; + defaultEffortMode?: "force"; } { const { cooldownMs, waitForCooldownMs, imageInput, reasoningEffortMode, + defaultEffortMode, ...rest } = combo; return { @@ -101,6 +104,7 @@ function sparseComboConfig { + if (originalRequestedEffort === undefined) return; + const normalizedRequestedEffort = childLog.requestedEffort; + const transitionIndex = normalizedRequestedEffort?.indexOf("->") ?? -1; + childLog.requestedEffort = transitionIndex >= 0 + ? `${originalRequestedEffort}${normalizedRequestedEffort!.slice(transitionIndex)}` + : originalRequestedEffort; + recordAttemptRequestedEffort(childLog); + }; + let lastFailure: Response | null = null; // Dispatched targets, not attempted picks: it indexes the declared target list so the clamp // below can tell how many targets are still entitled to a send. @@ -407,6 +429,7 @@ export async function executeComboResponses( comboDefaultEffort(config, comboId), supportedLadderFor({ provider: targetRoute.provider, modelId: targetRoute.modelId }), combo.reasoningEffortMode, + combo.defaultEffortMode, ); const childHeaders = buildComboChildHeaders(req.headers); const childRequest = new Request(req.url, { @@ -425,6 +448,10 @@ export async function executeComboResponses( config.providers[pick.target.provider]!.adapter, ); childLog.activeAttempt = attempt; + if (originalRequestedEffort !== undefined) { + childLog.requestedEffort = originalRequestedEffort; + recordAttemptRequestedEffort(childLog); + } let attemptRetained = false; const retainCancelledAttempt = (): void => { if (attemptRetained) return; @@ -499,6 +526,7 @@ export async function executeComboResponses( onNativePassthroughCancel: callbackGate.onCancel, onResponseComplete: callbackGate.onResponseComplete, }); + restoreOriginalRequestedEffort(childLog); } catch (error) { callbackGate.discard(); if (options.abortSignal?.aborted) { diff --git a/src/types.ts b/src/types.ts index 234dbdc0d5..c2104f9d41 100644 --- a/src/types.ts +++ b/src/types.ts @@ -74,6 +74,7 @@ export type { OcxAccountPoolQuotaWindow, OcxComboStrategy, OcxComboDefaultEffort, + OcxComboDefaultEffortMode, OcxComboReasoningEffortMode, OcxComboTarget, OcxComboConfig, diff --git a/src/types/config.ts b/src/types/config.ts index bcc21c825d..17bf93218b 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1011,6 +1011,7 @@ export type OcxAccountPoolQuotaWindow = "five-hour" | "weekly" | "max-utilizatio export type OcxComboStrategy = "failover" | "round-robin" | "random" | "least-used" | "reset-window"; export type OcxComboDefaultEffort = "low" | "medium" | "high" | "xhigh" | "max" | "ultra"; +export type OcxComboDefaultEffortMode = "fallback" | "force"; /** * How a combo derives the reasoning ladder it publishes to the picker. @@ -1046,8 +1047,10 @@ export interface OcxComboConfig { cooldownMs?: number; /** Maximum wait for an eligible target cooldown to expire before failing closed. Default 0; range 0..600000, per selection attempt. */ waitForCooldownMs?: number; - /** Used when the client omits reasoning.effort. null/omitted leaves the target default unchanged. */ + /** Used as a fallback when the client omits reasoning.effort, or as an override in `force` mode. null/omitted leaves the target default unchanged. */ defaultEffort?: OcxComboDefaultEffort | null; + /** `force` makes the combo default override a valid client effort. Omitted / `fallback` preserves client precedence. */ + defaultEffortMode?: OcxComboDefaultEffortMode; /** * Picker-ladder derivation policy. Omitted / `"strict"` keeps the legacy rule where an * explicitly empty target ladder suppresses the whole combo's effort control. diff --git a/structure/runtime.md b/structure/runtime.md index bd9ebbd561..248c3011c6 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -430,3 +430,15 @@ Translated audio/file admission follows the [final-adapter input contract](adapt The combo may advance to its next eligible unattempted target before output commitment. It records no target/provider cooldown for these request-local mismatches and does not silently drop reasoning controls or raise `none` to a supported rung. Cancellation, origin/cyber-policy rejection, non-replayable post-send errors and the existing streaming commit boundary stay authoritative. Other invalid requests remain terminal. Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. + +## Combo default effort precedence + +`src/combos/request.ts` keeps `reasoningEffortMode` and `defaultEffortMode` independent. +The existing fifth argument remains the strict/adaptive capability-normalization policy; +the optional sixth argument enables fallback/force precedence. Force requires a valid +non-null default, overrides only valid caller effort on a known supported ladder, and +retains the existing unsupported-control stripping. It does not add a caller opt-in or +change target selection. `src/server/responses/core-combo.ts` applies the policy per child +and preserves the original requested effort separately from effective wire telemetry. +`src/server/chat-completions.ts` routes combos through that same child pipeline while +retaining the current config-aware native-Chat eligibility check for non-combo routes. diff --git a/tests/cli/cli-headless-parity.test.ts b/tests/cli/cli-headless-parity.test.ts index 9a11c696ac..ea02e12b29 100644 --- a/tests/cli/cli-headless-parity.test.ts +++ b/tests/cli/cli-headless-parity.test.ts @@ -526,6 +526,48 @@ describe("headless GUI parity CLI", () => { }); }); + test("combo set exposes the opt-in force-default policy", async () => { + const runtime = fakeRuntime(); + expect(await handleComboCommand([ + "set", "deep", "--targets", "ark/model-a", "--effort", "max", "--effort-mode", "force", "--json", + ], runtime.deps)).toBe(0); + expect(runtime.requests.find(request => request.method === "PUT")?.body).toMatchObject({ + id: "deep", + combo: { defaultEffort: "max", defaultEffortMode: "force" }, + }); + }); + + test("combo set sends fallback when clearing an existing forced default effort", async () => { + const runtime = fakeRuntime(req => req.method === "GET" ? { + combos: [{ + id: "deep", + defaultEffort: "max", + defaultEffortMode: "force", + targets: [{ provider: "ark", model: "old-model" }], + }], + } : undefined); + expect(await handleComboCommand([ + "set", "deep", "--targets", "ark/model-a", "--effort", "-", "--json", + ], runtime.deps)).toBe(0); + expect(runtime.requests).toEqual([ + { path: "/api/combos", method: "GET", body: null }, + { + path: "/api/combos", + method: "PUT", + body: { + id: "deep", + combo: { + strategy: "failover", + stickyLimit: 1, + targets: [{ provider: "ark", model: "model-a" }], + defaultEffort: null, + defaultEffortMode: "fallback", + }, + }, + }, + ]); + }); + test("combo set rejects --sticky outside round-robin instead of dropping it", async () => { const runtime = fakeRuntime(); const errorSpy = spyOn(console, "error").mockImplementation(() => {}); diff --git a/tests/codex-integration/combos.test.ts b/tests/codex-integration/combos.test.ts index 0bc1230e0f..2f7efb91d0 100644 --- a/tests/codex-integration/combos.test.ts +++ b/tests/codex-integration/combos.test.ts @@ -52,7 +52,7 @@ import { getConfigPath, readConfigDiagnostics, saveConfig } from "../../src/conf import { routeModel } from "../../src/router"; import { handleManagementAPI } from "../../src/server/management-api"; import { handleResponses } from "../../src/server/responses"; -import type { OcxConfig } from "../../src/types"; +import type { OcxComboConfig, OcxComboDefaultEffort, OcxConfig } from "../../src/types"; import { syncCatalogModels } from "../../src/codex/catalog"; import { injectClaudeAgentDefs } from "../../src/claude/agents-inject"; import { reconcileComboRotationState } from "../../src/combos/resolve"; @@ -392,6 +392,35 @@ describe("combo request cloning", () => { expect(concreteComboRequestBody({ model: "combo/x" }, target, "high", undefined).reasoning).toBeUndefined(); }); + test("force mode overrides only valid caller effort and resolves independently per target", () => { + const raw = { model: "combo/x", reasoning: { effort: "medium", summary: "concise" } }; + expect(concreteComboRequestBody(raw, target, "max", ["low", "high", "max"], "strict", "force").reasoning) + .toEqual({ effort: "max", summary: "concise" }); + expect(concreteComboRequestBody(raw, target, "max", ["low", "high"], "strict", "force").reasoning) + .toEqual({ effort: "high", summary: "concise" }); + expect(raw.reasoning).toEqual({ effort: "medium", summary: "concise" }); + }); + + test("force mode rejects missing or invalid direct default efforts", () => { + const raw = { model: "combo/x", reasoning: { effort: "medium", summary: "concise" } }; + for (const defaultEffort of [null, "turbo" as OcxComboDefaultEffort]) { + expect(() => concreteComboRequestBody(raw, target, defaultEffort, ["low", "high"], "strict", "force")) + .toThrow("force combo default effort requires a valid defaultEffort"); + } + }); + + test("force mode fails closed for malformed and unknown capabilities and strips unsupported effort", () => { + expect(concreteComboRequestBody( + { model: "combo/x", reasoning: { effort: "banana" } }, target, "max", ["max"], "strict", "force", + ).reasoning).toEqual({ effort: "banana" }); + expect(concreteComboRequestBody( + { model: "combo/x", reasoning: { effort: "medium" } }, target, "max", undefined, "strict", "force", + ).reasoning).toEqual({ effort: "medium" }); + expect(concreteComboRequestBody( + { model: "combo/x" }, target, "max", [], "strict", "force", + ).reasoning).toBeUndefined(); + }); + /** * #3108: a combo configured for `max` routed to a target whose ladder tops out lower * sent NO effort at all, so the provider default applied and the turn ran at `none` — @@ -420,6 +449,22 @@ describe("combo request cloning", () => { ).reasoning).toEqual({ summary: "concise", effort: "high" }); }); + test("forced default effort composes with existing strict and adaptive capability modes", () => { + const raw = { model: "combo/x", reasoning: { effort: "medium", summary: "concise" }, thinking_budget: 8192 }; + for (const mode of ["strict", "adaptive"] as const) { + expect(concreteComboRequestBody(raw, target, "max", ["low", "high"], mode, "force").reasoning) + .toEqual({ effort: "high", summary: "concise" }); + const unsupported = concreteComboRequestBody(raw, target, "max", [], mode, "force"); + expect(unsupported.reasoning).toEqual({ summary: "concise" }); + expect(unsupported.thinking_budget).toBeUndefined(); + } + expect(concreteComboRequestBody(raw, target, "max", undefined, "adaptive", "force").reasoning) + .toEqual({ summary: "concise" }); + expect(concreteComboRequestBody(raw, target, "max", undefined, "strict", "force").reasoning) + .toEqual({ effort: "medium", summary: "concise" }); + expect(raw.reasoning).toEqual({ effort: "medium", summary: "concise" }); + }); + test("debug-warns once per unsupported or unknown combo default", () => { const debug = spyOn(console, "debug").mockImplementation(() => {}); concreteComboRequestBody({ model: "combo/x" }, target, "high", []); @@ -1495,8 +1540,10 @@ describe("combo validation and normalization", () => { })).toEqual({ strategy: "failover", stickyLimit: 1, + cooldownMs: undefined, waitForCooldownMs: 0, defaultEffort: "high", + defaultEffortMode: "fallback", reasoningEffortMode: "strict", imageInput: "auto", alias: null, @@ -1530,6 +1577,20 @@ describe("combo validation and normalization", () => { expect(comboDefaultEffort(corrupt, "free")).toBeNull(); }); + test("direct normalization rejects force mode without a valid default effort", () => { + const corruptConfigs = [ + { defaultEffortMode: "force", targets: [{ provider: "a", model: "m1" }] }, + { defaultEffort: null, defaultEffortMode: "force", targets: [{ provider: "a", model: "m1" }] }, + { defaultEffort: "turbo", defaultEffortMode: "force", targets: [{ provider: "a", model: "m1" }] }, + ] as unknown as OcxComboConfig[]; + for (const corrupt of corruptConfigs) { + expect(normalizeComboConfig(corrupt)).toMatchObject({ + defaultEffort: null, + defaultEffortMode: "fallback", + }); + } + }); + test("inherited combo names are unknown across getters, effort, and routing", () => { const config = baseConfig(); for (const id of ["constructor", "toString"]) { diff --git a/tests/helpers/combo-forced-effort-cases.ts b/tests/helpers/combo-forced-effort-cases.ts new file mode 100644 index 0000000000..c99e9394d8 --- /dev/null +++ b/tests/helpers/combo-forced-effort-cases.ts @@ -0,0 +1,123 @@ +import { expect, test } from "bun:test"; +import { managementFetch as fetch } from "./management-auth"; +import { saveConfig } from "../../src/config"; +import { startServer } from "../../src/server"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; + +interface Harness { + serve(handler: (request: Request) => Response | Promise): Server; + baseUrl(server: Server): string; + chatSuccess(text: string, model?: string): Response; + chatStream(text: string): Response; + provider(adapter: string, url: string, apiKey: string, extra?: Partial): OcxProviderConfig; + comboConfig(providers: OcxConfig["providers"], targets?: Array<{ provider: string; model: string }>, + extra?: Partial[string]>): OcxConfig; + post(config: OcxConfig, raw?: Record): Promise; + latestAttemptReceipts(config: OcxConfig): Promise<{ log: unknown; usage: unknown }>; +} + +/** Register inside the parent describe: its isolated homes, mocks and cleanup still apply. */ +export function registerComboForcedEffortCases({ + serve, baseUrl, chatSuccess, chatStream, provider, comboConfig, post, latestAttemptReceipts, +}: Harness): void { + test("force-default raises Hermes-like medium to max while fallback keeps medium", async () => { + const efforts: unknown[] = []; + const upstream = serve(async request => { + const body = await request.json() as Record; + efforts.push(body.reasoning_effort); + return chatSuccess("forced", "m1"); + }); + const providers = { + a: provider("openai-chat", baseUrl(upstream), "key-a", { + reasoningEfforts: ["low", "medium", "high", "max"], + }), + }; + const forced = comboConfig(providers, undefined, { + defaultEffort: "max", + defaultEffortMode: "force", + }); + expect((await post(forced, { reasoning: { effort: "medium" } })).status).toBe(200); + const fallback = comboConfig(providers, undefined, { defaultEffort: "max" }); + expect((await post(fallback, { reasoning: { effort: "medium" } })).status).toBe(200); + expect(efforts).toEqual(["max", "medium"]); + }); + + for (const chatEffort of [ + { name: "reasoning_effort", body: { reasoning_effort: "medium" } }, + { name: "reasoning.effort", body: { reasoning: { effort: "medium" } } }, + ] as const) { + test(`Chat ${chatEffort.name} force-default routes through the combo and records normalized wire telemetry`, async () => { + const upstreamBodies: Array<{ provider: string; body: Record }> = []; + const a = serve(async request => { + upstreamBodies.push({ provider: "a", body: await request.json() as Record }); + return chatStream("forced chat"); + }); + const config = comboConfig({ + a: provider("openai-chat", baseUrl(a), "key-a", { + reasoningEfforts: ["low", "medium", "high", "max"], + }), + }, undefined, { + defaultEffort: "max", + defaultEffortMode: "force", + }); + saveConfig(config); + const server = startServer(0); + try { + const response = await fetch(new URL("/v1/chat/completions", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "combo/free", + messages: [{ role: "user", content: "hello" }], + stream: false, + ...chatEffort.body, + }), + }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("forced chat"); + expect(upstreamBodies).toEqual([ + { provider: "a", body: expect.objectContaining({ model: "m1", reasoning_effort: "max" }) }, + ]); + + const { log, usage } = await latestAttemptReceipts(config); + for (const receipt of [log, usage]) { + expect(receipt).toMatchObject({ + provider: "combo", + model: "combo/free", + requestedEffort: "medium", + effectiveEffort: "max", + reasoningWireField: "reasoning_effort", + reasoningWireValue: "max", + routeDecision: { routeKind: "combo" }, + attempts: [{ + provider: "a", + model: "m1", + requestedEffort: "medium", + effectiveEffort: "max", + reasoningWireField: "reasoning_effort", + reasoningWireValue: "max", + }], + }); + } + } finally { + await server.stop(true); + } + }); + } + + test("backup noReasoningModels removes the fresh combo default", async () => { + const a = serve(() => Response.json({ error: { message: "retry" } }, { status: 503 })); + let backupBody: Record | undefined; + const b = serve(async request => { + backupBody = await request.json() as Record; + return chatSuccess("no reasoning", "m2"); + }); + const config = comboConfig({ + a: provider("openai-chat", baseUrl(a), "key-a"), + b: provider("openai-chat", baseUrl(b), "key-b", { noReasoningModels: ["m2"] }), + }, undefined, { defaultEffort: "high" }); + expect((await post(config)).status).toBe(200); + expect(backupBody).not.toHaveProperty("reasoning_effort"); + }); + +} diff --git a/tests/routing/combo-management-api.test.ts b/tests/routing/combo-management-api.test.ts index 85f6be0ff6..ba6bd7e2d0 100644 --- a/tests/routing/combo-management-api.test.ts +++ b/tests/routing/combo-management-api.test.ts @@ -488,6 +488,47 @@ describe("combo management API", () => { }); }); + test("defaultEffortMode force round-trips sparsely and invalid policy never mutates config", async () => { + await withTempHome(async () => { + const config = baseConfig({ combos: undefined }); + saveConfig(config); + const forced = await comboApi(config, "PUT", "/api/combos", { + id: "forced", + combo: { ...VALID_COMBO, defaultEffort: "max", defaultEffortMode: "force" }, + }); + expect(forced?.status).toBe(200); + expect(await responseJson(forced)).toMatchObject({ + combo: { defaultEffort: "max", defaultEffortMode: "force" }, + }); + expect(config.combos?.forced).toMatchObject({ defaultEffort: "max", defaultEffortMode: "force" }); + + const missingDefault = await comboApi(config, "PUT", "/api/combos", { + id: "bad", combo: { ...VALID_COMBO, defaultEffortMode: "force" }, + }); + expect(missingDefault?.status).toBe(400); + expect(config.combos?.bad).toBeUndefined(); + + const fallback = await comboApi(config, "PUT", "/api/combos", { + id: "forced", + combo: { ...VALID_COMBO, defaultEffort: "max", defaultEffortMode: "fallback" }, + }); + expect(fallback?.status).toBe(200); + expect(config.combos?.forced).not.toHaveProperty("defaultEffortMode"); + + const restoreForce = await comboApi(config, "PUT", "/api/combos", { + id: "forced", + combo: { ...VALID_COMBO, defaultEffort: "max", defaultEffortMode: "force" }, + }); + expect(restoreForce?.status).toBe(200); + const guiRoundTrip = await comboApi(config, "PUT", "/api/combos", { + id: "forced", + combo: { ...VALID_COMBO, defaultEffort: "high" }, + }); + expect(guiRoundTrip?.status).toBe(200); + expect(config.combos?.forced).toMatchObject({ defaultEffort: "high", defaultEffortMode: "force" }); + }); + }); + test("PUT stores aliases and GET exposes the public model", async () => { await withTempHome(async () => { const config = baseConfig({ combos: undefined }); diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index 041ed847c0..bea04074a4 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -1,3 +1,4 @@ +import { registerComboForcedEffortCases } from "../helpers/combo-forced-effort-cases"; import { sessionLaneIdFromRequest } from "../../src/server/request-log-conversation"; import { afterEach, beforeEach, describe, expect, mock, setDefaultTimeout, test } from "bun:test"; import { logsFromApiBody } from "../helpers/logs-api"; @@ -3059,19 +3060,8 @@ describe("server combo failover 030 activation matrix", () => { expect(bodies.map(row => row.body.reasoning_effort)).toEqual(["low", "low"]); }); - test("backup noReasoningModels removes the fresh combo default", async () => { - const a = serve(() => Response.json({ error: { message: "retry" } }, { status: 503 })); - let backupBody: Record | undefined; - const b = serve(async request => { - backupBody = await request.json() as Record; - return chatSuccess("no reasoning", "m2"); - }); - const config = comboConfig({ - a: provider("openai-chat", baseUrl(a), "key-a"), - b: provider("openai-chat", baseUrl(b), "key-b", { noReasoningModels: ["m2"] }), - }, undefined, { defaultEffort: "high" }); - expect((await post(config)).status).toBe(200); - expect(backupBody).not.toHaveProperty("reasoning_effort"); + registerComboForcedEffortCases({ + serve, baseUrl, chatSuccess, chatStream, provider, comboConfig, post, latestAttemptReceipts, }); test("bare third-party defaultModel keeps max off the native clamp path", async () => {