diff --git a/docs/configuration.md b/docs/configuration.md index c45c6a6..c6fec2e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -152,13 +152,24 @@ rungs any other family's listing ladders (DeepSeek, OpenRouter, any other provid such a rung that does not answer leaves that family shorter. A family Code requires is laddered on whatever of it answers, short below three rungs, and one none of whose models answers is left out as a family with no account is; a selection that needs it is refused -where it is reviewed. `deriveCatalog` returns the document and every other offered model -with its reason; the document's compiled catalog names its short families. It refuses only a -family Code requires whose ladder would put a model above one that costs no more and thinks deeper -(`code_ladder_regression`, naming both), a benchmark that does not answer the charge exactly, -and nothing laddered at all (`code_probe_insufficient_ladder`). Only the returned `confirm` +where it is reviewed. A model whose probe got no answer the benchmark could read (OMP's +`unresolved` or `unmatched`) is left out rather than discarding every model measured beside +it: as `inconclusive`, or as it would be had it answered where its answer could not change +the list (`superseded` by a newer version that answered, `separate_quota`). `deriveCatalog` returns the document +and every other offered model with its reason; the document's compiled catalog names its +short families. It refuses only a family Code requires whose ladder would put a model above +one that costs no more and thinks deeper (`code_ladder_regression`, naming both), a benchmark +that does not answer the charge exactly, and nothing laddered at all +(`code_probe_insufficient_ladder`). Only the returned `confirm` spends: one benchmark job per provider, then derive, stage, review, promote and save -the selection narrowed to what the verified catalog hosts. Every step re-observes the +the selection narrowed to what the verified catalog hosts. An `inconclusive` model may be +left out only where nothing already chosen needs it. When it is in the list in use as a +verification with this exact account pool made it, or belongs to a family the wanted lane +names (one provider, or OpenAI and Anthropic for the mixed lane) that the new list would not +have at all, the run stops at derive (`code_models_unanswered`, naming the models) before the +selection is narrowed, with the list and the saved selection unchanged, since an outage or a +rate limit reads the same as a model the account cannot use. A list verified with other +accounts, or never verified, holds no such claim. Every step re-observes the revision, the exact account pool, OMP defaults and OMP itself, and stops, naming itself, when one moved, on a refusal or on cancellation. A stop never undoes: probe jobs stay in OMP's history and a staged catalog stays staged. A confirmation is used once, and a failed @@ -780,15 +791,19 @@ verifying, a staged list's changes (field by field, a model added or dropped, th value struck), or, after a verification in this panel, the models it left out, each with its reason in a word: a model the accounts' plan, their settings or this client excludes (`client_blocked`) reads **not served**, one no account finds (`not_found`) **not -reachable**, a model in OMP's `spark` quota class **retired** and any other class of its -own **own quota**. Ids, prices, speeds, thinking ranges and reasons +reachable**, one whose check got no clear answer (`inconclusive`) **no answer**, a model in +OMP's `spark` quota class **retired** and any other class of its own **own quota**. Ids, +prices, speeds, thinking ranges and reasons appear only in the readout line while a cell, a provider or a left-out model is pointed or focused. One next action sits where the launch sits and follows the step: **verify models**, **checking models…** (drawn with the checking hold the charge keeps), **confirm charge** (a single deliberate press of the exact charge shown, as in the launch line), **verifying…** with the requests measured, then **back to code** with **verify again** beside it, the -same Verify a current verification can always renew, its charge confirmed the same way. +same Verify a current verification can always renew, its charge confirmed the same way; +a verification again that stopped reads **verification stopped** beside it, with its +reason in the readout, since what it would have changed (the list, or for a stop while +saving the profile, the team) stays as it was. A staged list offers **use staged list**, which reviews the staged list at its revision and promotes exactly that review with the list's own default team, and **discard**. An unsaved edit made on the list a write would replace holds it, saying what the write does diff --git a/plugins/code/generator/models-sheet.tsx b/plugins/code/generator/models-sheet.tsx index b7afa64..6781bb6 100644 --- a/plugins/code/generator/models-sheet.tsx +++ b/plugins/code/generator/models-sheet.tsx @@ -284,12 +284,19 @@ export function ModelsSheet({ host, model, machine, onBack, backRef, onPlace }: : { value: "use staged list", text: draft?.provenance ? "replaces the list in use with this verified one" : "replaces the list in use; it needs verifying after" }, press: () => { if (writeRefusal) return writeRefusal; useStaged(); return null; } }; } - case "verified": + case "verified": { // The head already says when the list was verified; beside the action, only its alternative: verifying again, through - // the same gate as a first verification, whose charge still waits on its own confirmation. - return { label: "back to code", parts: refused, fixes: verifyGate.open ? [{ label: "verify again", run: () => model.actions.verify() }] : [], - said: failure ? { value: "back to code", text: failure, warn: true } : { value: "back to code", text: "the generator routes on this list" }, + // the same gate as a first verification, whose charge still waits on its own confirmation. A verification again that + // stopped says so, as a first one does: what it would have changed (the list, or for a stop while saving the profile, + // the team) stays as it was, which the head alone cannot show. + const stopped = verification.failure; + const note: readonly Part[] = stopped ? [[stopped.cancelled ? "verification cancelled" : "verification stopped", stopped.cancelled ? null : "attention"]] : []; + return { label: "back to code", parts: [...note, ...refused], fixes: verifyGate.open ? [{ label: "verify again", run: () => model.actions.verify() }] : [], + said: failure ? { value: "back to code", text: failure, warn: true } + : stopped && !stopped.cancelled ? { value: "back to code", text: stopped.reason, warn: true } + : { value: "back to code", text: "the generator routes on this list" }, press: () => { leave(onBack); return null; } }; + } case "refused": return { label: "enable in setup", parts: [["Discovery is not enabled", "attention"], ["verifying needs it", null]], fixes: [], said: { value: "enable in setup", text: `Setup turns on model discovery for ${where}` }, press: () => { leave(() => onPlace("setup")); return null; } }; case "unverified": { diff --git a/plugins/code/generator/sheets-model.ts b/plugins/code/generator/sheets-model.ts index e82d5ab..5470917 100644 --- a/plugins/code/generator/sheets-model.ts +++ b/plugins/code/generator/sheets-model.ts @@ -93,6 +93,7 @@ const EXCLUSION_WORDS: Readonly Promise; @@ -35,6 +36,10 @@ const messages: Readonly> = { code_invalid_catalog: "The model list could not be used.", code_preview_changed: "The reviewed model list changed. Review it again.", code_starter_candidate_limit: "The bundled model list is too large to verify at once.", + code_probe_missing_probe: "OMP's results do not match the models it was asked to check. Verify again.", + code_probe_insufficient_ladder: "No model answered its check, so there is no list to build.", + code_ladder_regression: "Among the models that answered, a higher tier thinks less than a lower one that costs no more.", + code_models_unanswered: "Models the list in use or this team needs gave no clear answer, so the list and team stayed as they were. Verify again later, or choose a team or accounts without them.", code_verification_cancelled: "Verification cancelled.", code_verification_changed: "The machine, workspace or edit access changed, so the verification stopped.", code_verification_confirmed: "That charge was already confirmed.", @@ -89,6 +94,8 @@ const agentRefusals: Readonly> = { /** Refusals whose detail is a list of identifiers, with the label that introduces it; the list stays exactly as written. */ const listedDetails: Readonly> = { code_model_unpublished: "Not listed:", + code_models_unanswered: "No answer:", + code_ladder_regression: "In", }; /** How a browser's dispatch reports a refused door (machine-web.ts `codeWorkflow`): `.: . No approval or readiness is assumed.` */ const DOOR_DENIAL = /^[a-z0-9_-]+(?:\.[A-Za-z0-9_-]+)+: ([^]*?)(?:\. No approval or readiness is assumed\.)?$/; @@ -117,6 +124,21 @@ export function failureWords(message: string): string { return Object.hasOwn(listedDetails, token!) ? `${words} ${listedDetails[token!]} ${detail}` : `${words} ${detail.charAt(0).toUpperCase()}${detail.slice(1)}`; } +/** + * The left-out models a verification may not lose merely because they gave no clear answer + * (`inconclusive`): one the list in use has, and every one of a family the wanted selection's lane + * names that the new list (`families`) would not have at all. An outage, a rate limit or a wording + * OMP cannot read would otherwise shorten the list in use, or move the team off a family it chose, + * which only a provider's own refusal of a model may do. `inUse` is empty where no verification + * with this exact pool vouched for the list in use: evidence from other accounts, or a list never + * verified, holds nothing these accounts could keep. + */ +function unansweredNeeds(exclusions: readonly Exclusion[], inUse: ReadonlySet, wanted: Selection | null, families: readonly string[]): string[] { + const lost = wanted ? laneFamilies(wanted.lane).filter(family => !families.includes(family)) : []; + return exclusions.filter(exclusion => exclusion.reason === "inconclusive" && + (inUse.has(probeAddress(exclusion)) || lost.includes(providerPolicy(exclusion.provider).family))).map(probeAddress); +} + /** The steps of a model verification, in the order they run; a failure names the one it stopped at. */ export type VerificationStep = "observe" | "initialize" | "inventory" | "draft" | "benchmark" | "derive" | "stage" | "review" | "promote" | "select"; /** Requests a provider's benchmark has answered out of those it was charged. */ @@ -521,6 +543,11 @@ export function createCodeWorkflowClient(dispatch: Dispatch) { // before the start: one republished while the inventory runs is then a moved OMP, refused before the charge is answered. const [defaults, pool, catalogRevision] = await run.call(() => Promise.all([omp("readDefaults", {}), observePool(target.containerId, revision), readCatalogRevision()])); + // The models of the list in use, when it was verified with this exact pool: what a model that merely gives no clear + // answer may not remove. Spark's retired rung (tier 0) is never routed, so nothing needs it. + const active = record.active; + const inUse = new Set(active && active.provenance?.poolIdentityDigest === pool.poolIdentityDigest + ? active.document.models.filter(model => model.tier !== 0).map(probeAddress) : []); // The start is checked by the job's own wait, so a stop that lands now cancels the job rather than abandoning it. const job = await run.call(() => omp("startInventory", { ...target, expectedDefaultsRevision: defaults.revision, accountPool: pool.accountPool }), false); evidence.inventoryJobId = job.jobId; @@ -546,11 +573,11 @@ export function createCodeWorkflowClient(dispatch: Dispatch) { confirm: (selection, confirmOptions = options) => { if (confirmed) return Promise.reject(new WorkflowError("code_verification_confirmed")); confirmed = true; - return confirmVerification(charge, selection, verificationRun(confirmOptions, { ...evidence, benchmarkJobIds: [] })); + return confirmVerification(charge, inUse, selection, verificationRun(confirmOptions, { ...evidence, benchmarkJobIds: [] })); }, }; } - async function confirmVerification(charge: ChargeReview, wanted: Selection | null, run: VerificationRun): Promise { + async function confirmVerification(charge: ChargeReview, inUse: ReadonlySet, wanted: Selection | null, run: VerificationRun): Promise { const { target } = charge; run.enter("benchmark"); await unchangedCharge(charge, run); @@ -575,6 +602,13 @@ export function createCodeWorkflowClient(dispatch: Dispatch) { inventoryObservedAt: charge.inventory.observedAt, startedAt: Math.min(...receipts.map(receipt => receipt.startedAt)), completedAt: Math.max(...receipts.map(receipt => receipt.completedAt)), results: receipts.flatMap(receipt => receipt.results) }; const derived = await run.call(() => code("deriveCatalog", { inventory: charge.inventory, benchmark, budget: charge.budget })); + // A model with no clear answer stays out of the list only where nothing already chosen needs it, so a stop + // here keeps the list in use and the saved team exactly as they were. It is judged before the selection is + // narrowed, since a team whose family gave no answer could not be formed from this list either. + const families = [...new Set(derived.document.models.map(model => providerPolicy(model.provider).family))]; + const unanswered = unansweredNeeds(derived.exclusions, inUse, wanted, families); + if (unanswered.length > 0) + throw run.fail(`code_models_unanswered: ${unanswered.slice(0, 8).join(", ")}${unanswered.length > 8 ? ` and ${unanswered.length - 8} more` : ""}`); // Narrowed before anything is written, so a selection the verified catalog cannot serve stops // here with nothing staged, rather than after a promotion saved its default instead. let selection: Selection; diff --git a/plugins/domain/probe.test.ts b/plugins/domain/probe.test.ts index cbf0f32..59c54ce 100644 --- a/plugins/domain/probe.test.ts +++ b/plugins/domain/probe.test.ts @@ -61,7 +61,7 @@ function receiptOf(models: Row[]): InventoryReceipt { return { schemaVersion: 1, kind: "inventory", ompVersion: OMP_VERSION, observedAt: 100, models }; } /** A benchmark of every candidate: reachable, except the ids given another verdict. */ -function observed(inv: InventoryReceipt, verdicts: Record = {}, +function observed(inv: InventoryReceipt, verdicts: Record = {}, options: ScaffoldOptions = { budget: "any" }) { const input = benchmarkCandidates(inv, options), receipt = parseBenchmarkObservation(report(input), input, 101, 200); return { ...receipt, results: receipt.results.map(result => verdicts[result.id] @@ -232,15 +232,37 @@ describe("pure typed scaffolding", () => { expect([tiers(short, "anthropic"), compileCatalog(short).short]).toEqual([[[1, "claude-sonnet-5"], [2, "claude-opus-5"]], ["anthropic"]]); expect(compileCatalog(derived(inv).document).short).toEqual([]); }); - test("missing, stale, wrong API and inconclusive probes refuse promotion", () => { + test("missing, stale and wrong-API probes refuse promotion", () => { const inv = fullInventory(), input = benchmarkCandidates(inv), receipt = parseBenchmarkObservation(report(input), input, 101, 200); expect(() => catalogFromObservations(inv, { ...receipt, results: receipt.results.slice(1) })).toThrow("probe_missing_probe"); expect(() => catalogFromObservations(inv, { ...receipt, inventoryObservedAt: 99 })).toThrow("probe_missing_probe"); expect(() => catalogFromObservations(inv, { ...receipt, results: receipt.results.map((result, i) => i ? result : { ...result, api: "other" }) })).toThrow("probe_missing_probe"); - for (const status of ["unresolved", "unmatched"]) { - expect(() => catalogFromObservations(inv, { ...receipt, results: receipt.results.map((result, i) => i ? result : { ...result, status, tokensPerSecond: null, timeToFirstTokenMs: null }) })).toThrow("probe_inconclusive_probe"); + }); + test("an inconclusive probe leaves only its own model out, named, as a refused one is", () => { + const inv = fullInventory(); + inv.models.push({ ...inv.models[0]!, provider: "deepseek", api: "openai-completions", id: "deepseek-v4" }); + const refused = catalogFromObservations(inv, observed(inv, { "gpt-pro-6": "client_blocked", "deepseek-v4": "not_found" })); + for (const status of ["unresolved", "unmatched"] as const) { + // The run leaves one model of a provider that otherwise answered without an answer, and the whole of a provider that could not be checked at all. + const derived = catalogFromObservations(inv, observed(inv, { "gpt-pro-6": status, "deepseek-v4": status })); + expect(derived.document).toEqual(refused.document); + expect(derived.exclusions).toEqual(refused.exclusions.map(exclusion => + exclusion.id === "gpt-pro-6" || exclusion.id === "deepseek-v4" ? { ...exclusion, reason: "inconclusive" } : exclusion)); } }); + test("a model with no clear answer is named as it would be had it answered wherever its answer could not change the list", () => { + const inv = fullInventory(); + inv.models.push({ ...inv.models.find(model => model.id === "claude-opus-5")!, id: "claude-opus-6" }); + inv.models.push({ ...inv.models[0]!, provider: "openai-codex", api: "openai-codex-responses", id: "gpt-spark-6", inputCostPerMillion: 0.01, quotaTier: "future-resource" }); + // A newer version that answered replaces it whatever it answers, and a quota Code does not spend never takes a rung. + const replaced = catalogFromObservations(inv, observed(inv, { "claude-opus-5": "unresolved", "gpt-spark-6": "unmatched" })); + expect(replaced.exclusions).toContainEqual({ provider: "anthropic", id: "claude-opus-5", reason: "superseded" }); + expect(replaced.exclusions).toContainEqual({ provider: "openai-codex", id: "gpt-spark-6", reason: "separate_quota", quotaTier: "future-resource" }); + // The newest version is the one whose answer decides the rung, so it is left out as inconclusive and its predecessor takes the rung. + const newest = catalogFromObservations(inv, observed(inv, { "claude-opus-6": "unresolved" })); + expect(newest.exclusions).toContainEqual({ provider: "anthropic", id: "claude-opus-6", reason: "inconclusive" }); + expect(tiers(newest.document, "anthropic")).toEqual([[1, "claude-haiku-5"], [2, "claude-sonnet-5"], [3, "claude-opus-5"]]); + }); test("every old version is probed before supersession, blocked newest cannot displace callable older", () => { const inv = fullInventory(); inv.models.push({ ...inv.models.find(model => model.id === "claude-opus-5")!, id: "claude-opus-6" }); diff --git a/plugins/domain/probe.ts b/plugins/domain/probe.ts index 469210c..4a0478f 100644 --- a/plugins/domain/probe.ts +++ b/plugins/domain/probe.ts @@ -33,15 +33,17 @@ export type ScaffoldOptions = z.infer; * `superseded`: a newer version of its family is used instead. `unstable_id`: a rolling alias, * preview or experiment, never probed or laddered. `not_found` and `client_blocked`: the * benchmark's own verdicts; a blocked model is the operator's to fix, so it is named, as the old - * scaffold's warnings named it. `regression`: as a rung it would think less than a cheaper rung - * below it. `separate_quota`: it draws a quota of its own, which Code does not spend, named - * by OMP's class for it (`quotaTier`), so a reader can tell Spark's retired class from another. + * scaffold's warnings named it. `inconclusive`: its probe got no answer the benchmark could read + * (OMP's `unresolved` or `unmatched`), so it is left out until a later verification settles it. + * `regression`: as a rung it would think less than a cheaper rung below it. `separate_quota`: it + * draws a quota of its own, which Code does not spend, named by OMP's class for it (`quotaTier`), + * so a reader can tell Spark's retired class from another. * A model that is merely not chosen (a third middle, a model priced above the top, a model whose * id names none of its provider's tier names) has no such reason and is not listed. */ const excludedIdentity = { provider: ProbeIdentitySchema.shape.provider, id: ProbeIdentitySchema.shape.id }; export const ExclusionSchema = z.discriminatedUnion("reason", [ - z.strictObject({ ...excludedIdentity, reason: z.enum(["superseded", "unstable_id", "not_found", "client_blocked", "regression"]) }), + z.strictObject({ ...excludedIdentity, reason: z.enum(["superseded", "unstable_id", "not_found", "client_blocked", "inconclusive", "regression"]) }), z.strictObject({ ...excludedIdentity, reason: z.literal("separate_quota"), quotaTier: QuotaTierSchema }), ]); export type Exclusion = z.infer; @@ -411,7 +413,15 @@ export function inventoryDraft(inventoryValue: unknown, budget: Selection["budge return { schemaVersion: 1, kind: "draft", inventoryObservedAt: inventory.observedAt, benchmark: benchmarkInput(inventory, models), exclusions: [...unstableExclusions(inventory, options.budget), ...unprobed].sort((a, b) => compare(probeAddress(a), probeAddress(b))) }; } -/** Every probed candidate (`probeSet`) must have an exact probe, before superseding older versions. */ +/** + * Every probed candidate (`probeSet`) must have an exact probe, before superseding older versions. + * Only a reachable model enters the catalog. One whose probe was inconclusive is left out and named, + * like a refused one, rather than refusing the whole catalog: a provider that could not be checked + * (a key without credit, a timeout, a wording OMP does not classify) must not discard every model + * measured beside it. Whether leaving it out may replace what is already chosen is the caller's to + * judge (workflow.ts `unansweredNeeds`), so it is named `superseded` or `separate_quota` instead + * wherever a newer version that answered, or its own quota class, keeps it out whatever it answers. + */ export function catalogFromObservations(inventoryValue: unknown, benchmarkValue: unknown, optionsValue: unknown = { budget: "any" }): DerivedCatalog { const inventory = parse(InventoryReceiptSchema, inventoryValue), benchmark = parse(BenchmarkReceiptSchema, benchmarkValue); const options = parse(ScaffoldOptionsSchema, optionsValue, "invalid_input"); @@ -424,13 +434,19 @@ export function catalogFromObservations(inventoryValue: unknown, benchmarkValue: for (const candidate of candidates) { const fact = facts.get(probeAddress(candidate)); if (!fact || fact.api !== candidate.api || fact.key !== candidate.key) throw new ProbeError("missing_probe"); - if (fact.status === "unmatched" || fact.status === "unresolved") throw new ProbeError("inconclusive_probe"); } + const reachable = offered.filter(model => facts.get(probeAddress(model))!.status === "reachable"); + // What replaces a model in the ladder (`scaffold`): a newer version of its family that answered and draws no quota of its own. + const laddered = reachable.filter(model => !separateClass(model)); const refused: Exclusion[] = []; for (const model of offered) { const status = facts.get(probeAddress(model))!.status; if (status === "not_found" || status === "client_blocked") refused.push(exclusion(model, status)); + // A model with no clear answer is named as it would be had it answered wherever its answer could not change + // the list: one drawing a quota of its own, and one a newer, answering version replaces anyway. + else if (status === "unresolved" || status === "unmatched") refused.push(separateClass(model) + ? { provider: model.provider, id: model.id, reason: "separate_quota", quotaTier: model.quotaTier } + : exclusion(model, supersede([...laddered, model]).superseded.includes(model) ? "superseded" : "inconclusive")); } - return scaffold(offered.filter(model => facts.get(probeAddress(model))!.status === "reachable"), - [...unstableExclusions(inventory, options.budget), ...unprobed, ...refused], facts); + return scaffold(reachable, [...unstableExclusions(inventory, options.budget), ...unprobed, ...refused], facts); } diff --git a/plugins/domain/routing.ts b/plugins/domain/routing.ts index bf35f25..ab0e406 100644 --- a/plugins/domain/routing.ts +++ b/plugins/domain/routing.ts @@ -46,13 +46,20 @@ const weights: Record = { const thinkingCost: Record = { minimal: 0.6, low: 0.8, medium: 1, high: 1.3, xhigh: 1.6, max: 2 }; const thinkingSpeed: Record = { minimal: 1.4, low: 1.2, medium: 1, high: 0.8, xhigh: 0.65, max: 0.5 }; +/** The two families the mixed lane blends; it exists only where a catalog has both. */ +const MIXED_FAMILIES: readonly string[] = Object.freeze(["openai", "anthropic"]); +/** The families a lane names: its own, or the two the mixed lane blends. */ +export function laneFamilies(lane: Lane): readonly string[] { + return lane.kind === "mixed" ? MIXED_FAMILIES : [lane.family]; +} + function lanesFor(catalog: CompiledCatalog): Lane[] { const lanes: Lane[] = []; for (const family of catalog.families) { lanes.push({ kind: "provider", family, blend: "only" }); if (catalog.families.length > 1 && familyPolicy(family).crossTo !== null) lanes.push({ kind: "provider", family, blend: "led" }); } - if (catalog.families.includes("openai") && catalog.families.includes("anthropic")) lanes.push({ kind: "mixed" }); + if (MIXED_FAMILIES.every(family => catalog.families.includes(family))) lanes.push({ kind: "mixed" }); return lanes; } @@ -99,7 +106,7 @@ function selectionFacts(catalog: CompiledCatalog, input: Selection): { selection * refuses a capability the lead family lacks. */ export function defaultSelection(catalog: CompiledCatalog): Selection { - const lane: Lane = catalog.families.includes("openai") && catalog.families.includes("anthropic") + const lane: Lane = MIXED_FAMILIES.every(family => catalog.families.includes(family)) ? { kind: "mixed" } : { kind: "provider", family: catalog.families[0]!, blend: "only" }; const admitted: Selection = { lane, capability: 1, thinking: "medium", advisor: "glance", spark: false, priority: false, diff --git a/plugins/test/workflow.test.ts b/plugins/test/workflow.test.ts index 26e7def..912b60b 100644 --- a/plugins/test/workflow.test.ts +++ b/plugins/test/workflow.test.ts @@ -47,7 +47,7 @@ function verificationFixture() { const hooks: Partial void | Promise>> = {}; const states = new Map(); const exitCodes = new Map(); - const verdicts: Record = {}; + const verdicts: Record = {}; const ctx: CodeContext = { now: () => 10_000, emit() {}, outsideScope: async () => null, auth: { principal: { id: "writer", kind: "human", name: "Writer", color: "#123456" }, @@ -311,6 +311,49 @@ test("a family Code requires that the accounts reach only in part is verified sh expect(verified.configuration.selection).toEqual({ ...preview, lane: { kind: "provider", family: "anthropic", blend: "only" }, capability: 3, spark: false }); }); +test("a model with no clear answer is left out of a first list, but never shortens a list these accounts verified", async () => { + // A first list: a provider that could not be checked at all is left out and named, and what answered is saved. + const f = verificationFixture(); + f.verdicts["deepseek-flash"] = "unresolved"; f.verdicts["deepseek-pro"] = "unmatched"; + const first = await (await f.workflow.verifyModels(target, { expectedRevision: 0, budget: "any" })).confirm(preview); + expect(first.configuration.active?.document.models.map(model => model.id)).toEqual(["claude-haiku-5", "claude-sonnet-5", "claude-opus-5"]); + expect(first.exclusions.filter(exclusion => exclusion.reason === "inconclusive").map(exclusion => exclusion.id)).toEqual(["deepseek-flash", "deepseek-pro"]); + // Verifying again while a model of the list in use gives no clear answer stops before anything is staged; the list and team stay. + f.verdicts["claude-opus-5"] = "unresolved"; + const before = await f.configuration(); + const again = await stopOf((await f.workflow.verifyModels(target, { expectedRevision: before.revision, budget: "any" })).confirm(first.configuration.selection)); + expect([again.step, again.reason]).toEqual(["derive", "code_models_unanswered: anthropic/claude-opus-5"]); + expect(await f.configuration()).toEqual(before); + // Another pool's verification vouches for nothing these accounts could keep, so the same non-answer only leaves the model out. + f.omp.accounts.accounts.push(f.slot("anthropic", 3)); + const otherPool = await (await f.workflow.verifyModels(target, { expectedRevision: before.revision, budget: "any" })).confirm(first.configuration.selection); + expect(otherPool.configuration.active?.document.models.map(model => model.id)).toEqual(["claude-haiku-5", "claude-sonnet-5"]); + // So does a list in use that no verification made, such as an imported one. + const imported = await f.workflow.code("stageCatalog", { containerId: target.containerId, expectedRevision: otherPool.configuration.revision, + document: first.configuration.active!.document }); + const review = await f.workflow.code("reviewCatalog", { containerId: target.containerId, expectedRevision: imported.revision, source: "draft" }); + const unverified = await f.workflow.code("promoteCatalog", { containerId: target.containerId, expectedRevision: imported.revision, source: "draft", reviewDigest: review.reviewDigest }); + expect(unverified.active?.provenance).toBeNull(); + const replaced = await (await f.workflow.verifyModels(target, { expectedRevision: unverified.revision, budget: "any" })).confirm(unverified.selection); + expect(replaced.configuration.active?.document.models.map(model => model.id)).toEqual(["claude-haiku-5", "claude-sonnet-5"]); +}); + +test("a first list stops rather than move the team off a family it chose that gave no clear answer at all", async () => { + // The mixed lane names Claude: with no Claude model answering, saving would leave the team on another family. + const mixed = verificationFixture(); + for (const id of ["claude-haiku-5", "claude-sonnet-5", "claude-opus-5"]) mixed.verdicts[id] = "unresolved"; + const blended = await stopOf((await mixed.workflow.verifyModels(target, { expectedRevision: 0, budget: "any" })).confirm(preview)); + expect([blended.step, blended.reason]).toEqual(["derive", "code_models_unanswered: anthropic/claude-haiku-5, anthropic/claude-opus-5, anthropic/claude-sonnet-5"]); + expect(mixed.calls).not.toContain("atyrode.code.stageCatalog"); + // A team that chose DeepSeek alone stops the same way when DeepSeek does not answer. + const chosen = verificationFixture(); + chosen.verdicts["deepseek-flash"] = "unresolved"; chosen.verdicts["deepseek-pro"] = "unresolved"; + const deepseekOnly: Selection = { ...preview, lane: { kind: "provider", family: "deepseek", blend: "only" } }; + const refused = await stopOf((await chosen.workflow.verifyModels(target, { expectedRevision: 0, budget: "any" })).confirm(deepseekOnly)); + expect([refused.step, refused.reason]).toEqual(["derive", "code_models_unanswered: deepseek/deepseek-flash, deepseek/deepseek-pro"]); + expect(chosen.calls).not.toContain("atyrode.code.stageCatalog"); +}); + test("a derivation or selection the verified catalog cannot serve stops before anything is staged", async () => { // Nothing the accounts offer answers, so there is no catalog to stage. const blocked = verificationFixture();