From 9aeb3d4e983f5c7e514e0b57c84ee5b4cdff0b7e Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:24:49 +0900 Subject: [PATCH 1/4] feat: wire bounded Yibuapi execution with frozen cost reservations Signed-off-by: GUOGUO Agent-assisted-by: Codex (author) --- .github/workflows/skill-eval-pr.yml | 16 +- docs/skill-eval-yibuapi-preflight.md | 258 +++++++++------------- src/inferrsi/harness/skill_evaluation.py | 6 + src/inferrsi/infra/executors/local.py | 5 +- src/inferrsi/model/anthropic_agent.py | 196 ++++++++++++++--- tests/test_anthropic_agent.py | 2 +- tests/test_yibuapi_runtime.py | 263 +++++++++++++++++++++++ tools/skill_eval_compare.py | 38 +++- tools/skill_eval_pr.py | 62 +++++- 9 files changed, 653 insertions(+), 193 deletions(-) create mode 100644 tests/test_yibuapi_runtime.py diff --git a/.github/workflows/skill-eval-pr.yml b/.github/workflows/skill-eval-pr.yml index 7c44b8256..e77f32a91 100644 --- a/.github/workflows/skill-eval-pr.yml +++ b/.github/workflows/skill-eval-pr.yml @@ -38,6 +38,9 @@ on: budget: required: false description: Required for execute; approval of the complete Budget JSON in the plan + provider_config: + required: false + description: Optional exact public model/client/pricing/spend-limit JSON; never credentials source_run: description: Execution workflow run ID, required for publish-only retries @@ -53,6 +56,8 @@ env: jobs: plan: + outputs: + client_kind: ${{ steps.prepare.outputs.client_kind }} if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 10 @@ -108,13 +113,20 @@ jobs: with: name: skill-eval-plan path: prepared - - name: Existing client preflight (requires separately provisioned model access) + - name: Existing Codex client preflight + if: needs.plan.outputs.client_kind == 'codex' run: codex --version - name: Recheck live revision and authorization before execution env: GITHUB_TOKEN: ${{ github.token }} run: python -m tools.skill_eval_pr authorize --output prepared - - name: Execute existing comparison without publication credentials + - name: Execute existing Codex comparison without publication credentials + if: needs.plan.outputs.client_kind == 'codex' + run: python -m tools.skill_eval_pr execute --output prepared + - name: Execute configured Yibuapi comparison without publication credentials + if: needs.plan.outputs.client_kind == 'anthropic-messages-agent-v1' + env: + SKILL_EVAL_YIBUAPI_KEY: ${{ secrets.SKILL_EVAL_YIBUAPI_KEY }} run: python -m tools.skill_eval_pr execute --output prepared - name: Package retained results even after interruption if: always() diff --git a/docs/skill-eval-yibuapi-preflight.md b/docs/skill-eval-yibuapi-preflight.md index b9603b148..a40ede361 100644 --- a/docs/skill-eval-yibuapi-preflight.md +++ b/docs/skill-eval-yibuapi-preflight.md @@ -1,154 +1,104 @@ -# Yibuapi Skill Eval access preflight - -Refs #101. This is a proposed integration design, not a configured execution -environment or evidence of a successful model call. The owner approved a single -smoke with a total CNY 10 limit including preflight, without automatic retries. -Effective pricing, enforceable provider-side spending control, credential setup -and the newly frozen exact plan remain prerequisites. - -## Confirmed interface and remaining compatibility question - -The provider's [model configuration example](https://yibuapi.apifox.cn/9557607m0) -names `claude-opus-5-5` under an Anthropic provider at -`https://yibuapi.com/v1`. Its [Messages reference](https://yibuapi.apifox.cn/6777246m0) -documents `POST /v1/messages`, `x-api-key`, an `anthropic-version` header, -`tool_use` / `tool_result`, stop reasons and token/cache usage. -Another [function example](https://yibuapi.apifox.cn/382404323e0) uses Bearer auth -and contains an OpenAI-shaped response example. These documents do not establish -that every model/group implements the same protocol correctly. - -A credential-free public read of [the pricing catalog](https://yibuapi.com/api/pricing) -on 2026-10-27 returned this exact model with endpoint types `anthropic` and -`openai`, model ratio 2, completion ratio 5, cache ratio 0.05 and cache-creation -ratio 1.25. Enabled groups include default, claude-code and claude-code-pro -(public group ratio 2), plus other groups with different ratios. These are -catalog ratios, not independently established CNY-per-token prices or a -confirmation of the owner's effective group. The [pricing page](https://yibuapi.com/pricing) -and [provider usage guide](https://yibuapi.apifox.cn/8994238m0) identify model/group -rates, but the static page read did not expose currency/unit conversion or hard -quota enforcement semantics. - -No model endpoint, key validation or paid preflight was called. Model-specific -tool round trips, output schema behavior, usage completeness and actual billing -remain unverified. Generic Responses support does not prove Codex agent, -streaming or output-schema compatibility. - -## Smallest proposed client boundary - -Keep the current CodexClient command, config isolation and trace parser unchanged -for existing plans. Introduce an opt-in Anthropic Messages agent client, reusing -the existing process deadline, trial directories, verifier, receipts, paired -schedule and retained-result publisher. Do not replace an agent trial with a -single plain chat completion. - -The agent would expose only bounded list/read operations over the prepared -workspace and a structured answer submission tool. A successful task.json read -and subsequent tool-result round trip must precede final submission. The -workspace contains only the validated case and that arm's selected bundle. -There are no shell, write, web, subprocess, plugin, memory or external-agent tools. - -Use non-streaming Messages first. Each request is sent once; HTTP failures, -uncertain responses and timeouts terminate the attempt without replay or model -fallback. Enforce request count, serialized input bytes, complete-file read -bytes, output-token limits, tool count and the existing whole-trial deadline. -Reject symlinks, traversal, unexposed paths, malformed/ambiguous protocol data, -unexpected model identities, unsupported schemas and premature answers. An -oversized resource fails explicitly rather than being silently truncated. - -Retain a distinct agent trace: bounded request hashes/lengths, normalized tool -reads and hashes, terminal status, provider-reported model, token/cache -observations and errors. Do not label provider events as Codex trace events. -Normalize Anthropic total input from uncached input plus cache read/creation; -missing usage stays unknown and stops further paid requests. Preserve an -interrupted attempt as consumed and incomplete evidence as INCONCLUSIVE. - -## Frozen identity and authorization - -A new plan must bind client kind/protocol version and source hash, provider, -endpoint/auth scheme/API version, model and reported-model policy, selected -provider group, every agent bound, sampling/thinking policy and the complete -monetary limit/control evidence. Both absent/old and new arms use the same -identity. No mapping from Codex `xhigh` to this model is assumed. - -Keep exact case/split/schema/grader and both bundle identities, holdout -separation, verifier and artifact/source-run provenance. A changed client, -settings, renderer/adapter hash or trusted implementation requires a new freeze -and exact approval. Previous Codex plans/approvals cannot authorize this model. -Do not rewrite or relabel retained results. - -Default mapping and hosted execution remain unchanged by this design. Current -hosted execution checks only `codex --version`; conditional provider preflight, -credential injection and new client selection need their own reviewed -implementation before any hosted provider run. - -## Proposed credential destination and payload - -After action-time security approval, the owner would provision a dedicated, -single-smoke provider key as GitHub Actions Secret `SKILL_EVAL_YIBUAPI_KEY`. -Map it to the same environment name only in the model execution step. The key -is sent only in the authentication header to -`https://yibuapi.com/v1/messages`; it is never a command argument, plan field, -trace, artifact or comment. Do not read or reuse existing auth/config stores. - -Provider/model/group/endpoint/settings belong in the frozen plan, not mutable -ambient overrides. Keep GitHub publication credentials out of the model process. -Do not configure the Secret, key, workflow access or repository settings merely -because the CNY 10 spending intent was approved. - -The transmitted payload is the evaluation prompt, validated task.json case -inputs/contract, exposed selected skill/resource bytes when the agent reads -them, output schema/tool declarations, and the bounded conversation/tool -results. Exclude held-out expected answers, verifier/check implementation, -unrelated repository/user files, credential stores and key material. - -## Setup and spending prerequisites - -1. Confirm the exact model/group permits direct Messages tool use; obtain the - effective input/output/cache prices with currency, units and any minimum, - request, long-context or group charges. Do not infer prices from ratios alone. -2. Establish an enforceable total limit including preflight and both arms. - Confirm whether a dedicated provider-key quota rejects before an over-limit - request, can overdraw during settlement, or uses preauthorization. A finite - label, observed-token stop, CLI estimate or operator attestation alone is not - proof of a hard monetary cap. If strict provider enforcement is unavailable, - require a documented conservative worst-case reservation from verified prices - and bounded requests before proceeding. -3. Obtain explicit security approval for the dedicated key and Secret exposure; - the owner provisions them without sending the key in chat or committing it. - No unlimited quota, automatic recharge or unrelated key use for this smoke. -4. Review and merge the bounded client implementation and any separate summary - renderer change. Prepare one new exact plan with the approved total limit, - group/pricing/control evidence and finite request/input/output ceilings. -5. Only after readiness and exact-plan approval, perform the single smoke. - Prefer the first approved baseline trial as the compatibility observation - rather than adding an unbudgeted paid test. Retain failure/partial evidence; - do not retry automatically or expand into multiple addition/update campaigns. - -Open blockers are effective price units/group, enforceable total-cap semantics, -owner-provisioned isolated credentials, offline-reviewed client wiring and the -new exact execution plan. No whole-#101/#88 or efficacy acceptance follows from -this preflight design. - -## Offline implementation boundary - -The opt-in comparison `client` settings select the bounded Messages read/submit -contract. Preparation freezes provider/protocol/endpoint/model, request/output/ -read/tool ceilings, source hash and maximum API-request count across both arms. -The model effort must be `not_applicable`; the Codex default is unchanged. - -This implementation slice is offline-only. The core loop accepts an explicit -transport for mocked tests, and HTTP construction is tested with fake keys. -Comparison execution refuses this provider before creating start/trial evidence, -even with an injected client. There is no live worker, environment-approval -shortcut, credential lookup/injection or hosted workflow change. Reviewed -pricing/cap enforcement and runtime credential wiring remain a separate required -implementation before live execution can be enabled. Preparation or green offline -CI is not provider compatibility, monetary control or real-evaluation acceptance. - -Only complete UTF-8 reads of exposed workspace files and structured answer -submission are supported. A task-read/tool-result round trip is mandatory; -plaintext answers, shell/network tools and general coding actions are unsupported. -Schema support is limited to the existing audit/extract vocabulary. The normalized -agent trace records request hashes/lengths, reads/hashes, provider model, usage and -terminal errors without provider text or key material. Interrupted/unknown usage -remains unverified, with known partial observations retained separately. +# Direct Yibuapi skill evaluation + +The opt-in Messages client uses the existing Python interpreter, comparison +schedule, fresh workspace, timeout/process cleanup and trusted verifier. It +exposes only read_file and submit_answer. Codex remains the default when no +provider configuration is supplied. No external agent executable is required. + +## Configuration and cost reservation + +Supply public JSON as the workflow_dispatch provider_config input, unchanged +between planning, execution and any retained-publication retry. It has exactly +four top-level fields: + +| Field | Required content | +| --- | --- | +| model | name and effort; effort is not_applicable, with no Codex xhigh mapping | +| client | kind, provider, endpoint, api_version and all five integer bounds below | +| pricing | exact model/group/currency/units, all effective rates, fees, full context bound, evidence and expiry | +| spend_limit | currency and approved finite amount as a decimal string | + +The client identity is kind=anthropic-messages-agent-v1, provider=yibuapi, +endpoint=https://yibuapi.com/v1/messages and api_version=2023-06-01. Its required +positive integer bounds are max_requests (at least two), max_output_tokens, +max_request_bytes, max_read_bytes and max_tool_calls. These are issue-scoped +execution choices; approve them in the exact frozen plan. max_requests counts +every attempted Messages request in a trial, including failed/uncertain requests. +No additional compatibility probe is sent: the first baseline request is the +compatibility observation. + +pricing requires currency (USD or CNY), unit=per_million_tokens, model, group, +rates, request_fee, max_input_tokens, rate_evidence and expires_at. rates contains +explicit nonnegative decimal strings for input, output, cache_read and cache_write. +Use final effective group prices, including every applicable billing tier and +the highest possible cache-write rate; do not infer a write price from a read +price or a group multiplier. request_fee is the verified maximum per-request fee, +including error/uncertain requests. Zero is explicit, not an unknown default. +No FX conversion is inferred. + +rate_evidence is the reviewed repository Issue-comment record linking the +provider's effective units/group/rates/fees and their verification. +expires_at is that record's Unix expiry. Software validates the frozen record +and arithmetic; the operator/reviewer must verify the external billing facts. + +The live model registry currently supports claude-opus-5-5 with its complete +1,000,000-token context bound from the +[model specification](https://platform.claude.com/docs/en/models/opus-5-5/overview). +max_input_tokens must be at least this full bound. Reservation uses the whole +model context for every possible request, including tool/system framing, rather +than an unverified characters-to-tokens estimate: + + per request = full input ceiling * max(input, cache_read, cache_write) / 1e6 + + output ceiling * output rate / 1e6 + request_fee + total = sum(trials * max_requests * per-request cost over all comparisons) + +Decimal amounts round upward; the complete reservation must fit spend_limit. +No cache discount, successful response or later token stop reduces the upfront +allowance. Both arms, all selected cases and all possible loop requests count. +Any unknown/expired rate, currency mismatch or insufficient total prevents +authorization. The allowance is conditional on the verified model/context and +provider billing contract; it is not proof that a provider will honor false or +changed external facts. + +## Credential setup and execution + +After explicit approval of the credential exposure, the owner provisions GitHub +Actions Secret SKILL_EVAL_YIBUAPI_KEY. The workflow exposes it only to the +conditional Yibuapi model-execution step. plan, authorize and publish do not +receive the key. The model child receives a filtered environment without GitHub +or Actions publication/runtime credentials; the verifier receives no provider +key. Never place a key in provider_config, command arguments, Git, logs or chat. + +1. Prepare action=plan with the exact provider_config and target PR. Inspect the + frozen model/group/rates, context/request bounds, paired cases and reservation. +2. Approve the new exact plan SHA, trusted version, base/head and full Budget JSON. + Execute with the same provider_config and all approval inputs. +3. Inspect retained outcomes and feedback. Failed/uncertain requests stop without + retry or model fallback. Use action=publish with the original source_run and + identical configuration for publication-only recovery. + +Price readiness is checked before authorization/history checks; key presence is +checked before the local execution claim and first request. plan cannot inspect +a Secret it does not receive. GitHub dispatch history still records a failed +attempt, so missing-key/failed executions must not be blindly rerun. Preserve +one-shot history and prepare a newly reviewed plan after investigating a failure. +An expired rate record blocks new execution, but retained historical results can +still be rebuilt/regraded/published without a model call. + +Opus 5.5 uses adaptive thinking and does not support forced tool choice. The +client uses auto, preserves returned thinking/signature blocks opaquely for the +tool round trip, and never logs their content. See the +[model protocol notes](https://platform.claude.com/docs/en/models/opus-5-5/overview). +A task-read/tool-result round trip and schema-valid submit_answer are required. +Unknown usage, malformed responses or exceeded bounds stop the paid path. +The publisher independently regrades supported outputs with trusted code. + +Only selected public task inputs, exposed selected-arm skill/resources, tool +schemas and their bounded conversation/results are sent to Yibuapi. Held-out +expected answers, verifier implementation, unrelated files and authentication +stores are excluded. The key goes only in the fixed endpoint's authentication +header. No paid preflight, model fallback or automatic retry is part of setup. + +Offline tests use fake keys/transports and retained-publication fixtures. +Passing them establishes software behavior; actual provider compatibility and +addition/update efficacy require the separately approved real run and evidence. diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index 561d6f8c3..22073e572 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -197,11 +197,14 @@ def run_trial( # An injected client may tighten the frozen limit, but cannot extend it. timeout = client.timeout_seconds if timeout_seconds is None else min( client.timeout_seconds, timeout_seconds) + agent_environment = getattr(client, "environment", None) + options = {"environment": agent_environment()} if agent_environment else {} execution = execute_command( command, workspace, trial_dir / "agent", min(deadline, time.monotonic() + timeout), + **options, ) atomic_write_json(trial_dir / "execution.json", execution) try: @@ -229,11 +232,14 @@ def run_trial( except ValueError: evaluation_error = "invalid_output_json" if evaluation_error is None: + verifier_environment = getattr(client, "verifier_environment", None) + options = {"environment": verifier_environment()} if verifier_environment else {} verification = execute_command( (sys.executable, str(verifier), str(workspace / "task.json"), str(final)), workspace, trial_dir / "verification", min(deadline, time.monotonic() + 30), + **options, ) result_path = trial_dir / "verification/result.json" if verification["status"] not in ("completed", "failed"): diff --git a/src/inferrsi/infra/executors/local.py b/src/inferrsi/infra/executors/local.py index 049a5fd7f..c72787cdd 100644 --- a/src/inferrsi/infra/executors/local.py +++ b/src/inferrsi/infra/executors/local.py @@ -8,7 +8,8 @@ from ..processes import _stop_tree, _WindowsJob -def execute_command(command: tuple[str, ...], cwd: Path, trial_dir: Path, deadline: float) -> dict: +def execute_command(command: tuple[str, ...], cwd: Path, trial_dir: Path, deadline: float, *, + environment: dict | None = None) -> dict: trial_dir.mkdir(parents=True) result_path = trial_dir / "result.json" argv = [arg.replace("{result}", str(result_path)) for arg in command] @@ -31,7 +32,7 @@ def execute_command(command: tuple[str, ...], cwd: Path, trial_dir: Path, deadli status="timeout", error="whole-run deadline exhausted before command start" ) else: - environment = os.environ.copy() + environment = os.environ.copy() if environment is None else environment.copy() environment["INFERRSI_RESULT_PATH"] = str(result_path) options = ( {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} diff --git a/src/inferrsi/model/anthropic_agent.py b/src/inferrsi/model/anthropic_agent.py index b137cd8cb..4d13362f9 100644 --- a/src/inferrsi/model/anthropic_agent.py +++ b/src/inferrsi/model/anthropic_agent.py @@ -1,17 +1,16 @@ -"""Bounded Messages read/submit agent for offline contract validation. - -The comparison controller may freeze this opt-in identity. Live execution stays -unavailable until verified pricing/cap controls and credential wiring are reviewed. -No credentials are discovered here; transport receives an explicit runtime key. -""" +"""Bounded direct Messages agent with frozen per-call cost reservations.""" +import argparse import hashlib import json import math +import os import re +import sys import time import urllib.request from dataclasses import dataclass +from decimal import ROUND_CEILING, Decimal from pathlib import Path, PurePosixPath from urllib.error import HTTPError @@ -19,8 +18,9 @@ ENDPOINT = "https://yibuapi.com/v1/messages" SOURCE = "src/inferrsi/model/anthropic_agent.py" MAX_RESPONSE = 1024 * 1024 -LIVE_BLOCKER = ("Live provider execution requires reviewed pricing/cap controls " - "and credential wiring") +# Full context bound from https://platform.claude.com/docs/en/models/opus-5-5/overview +MODEL_CONTEXT = {"claude-opus-5-5": 1000000} +KEY_ENV = "SKILL_EVAL_YIBUAPI_KEY" def canonical(value): @@ -99,31 +99,123 @@ def matches_schema(value, schema): or all(matches_schema(v, schema["items"]) for v in value)) +def money(value): + if not isinstance(value, str) or not re.fullmatch(r"[0-9]{1,9}(?:[.][0-9]{1,8})?", value): + raise ValueError("Money and rates must be explicit finite nonnegative decimal strings") + return Decimal(value) + + +def evidence_reference(value): + return isinstance(value, str) and re.fullmatch( + r"https://github[.]com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/issues/[0-9]+" + r"#issuecomment-[0-9]+", value) is not None + + +def reservation(model, spec, pricing, spend_limit, trials, *, now=None): + validate_spec({"name": model, "effort": "not_applicable"}, spec) + required = {"currency", "unit", "model", "group", "rates", "request_fee", + "max_input_tokens", "rate_evidence", "expires_at"} + if not isinstance(pricing, dict) or set(pricing) != required: + raise ValueError("Live provider pricing/cap settings are incomplete") + if (pricing["currency"] not in ("CNY", "USD") or pricing["unit"] != "per_million_tokens" + or pricing["model"] != model or not isinstance(pricing["group"], str) + or not pricing["group"] or len(pricing["group"]) > 100 + or any(ord(c) < 32 for c in pricing["group"])): + raise ValueError("Pricing must bind currency, units, exact model and effective group") + if not evidence_reference(pricing["rate_evidence"]): + raise ValueError("A reviewed effective-rate/group evidence reference is required") + rates = pricing["rates"] + if not isinstance(rates, dict) or set(rates) != { + "input", "output", "cache_read", "cache_write"}: + raise ValueError("Every input/output/cache rate must be explicit") + amounts = {name: money(rate) for name, rate in rates.items()} + fee = money(pricing["request_fee"]) + if (model not in MODEL_CONTEXT or type(pricing["max_input_tokens"]) is not int + or pricing["max_input_tokens"] < MODEL_CONTEXT[model] + or pricing["max_input_tokens"] > 2 ** 31): + raise ValueError("A verified full model-context input-token ceiling is required") + if (type(pricing["expires_at"]) is not int + or pricing["expires_at"] <= (time.time() if now is None else now)): + raise ValueError("Verified pricing/input-bound record expired") + if (not isinstance(spend_limit, dict) or set(spend_limit) != {"currency", "amount"} + or spend_limit["currency"] != pricing["currency"] + or money(spend_limit["amount"]) <= 0 + or type(trials) is not int or trials <= 0): + raise ValueError("A finite same-currency total limit and trial count are required") + # Charge all possible input as the most expensive cache/input class. No cache discount + # or successful completion is assumed; uncertain/error requests consume their allowance. + per_request = ((pricing["max_input_tokens"] * max(amounts["input"], amounts["cache_read"], + amounts["cache_write"]) + + spec["max_output_tokens"] * amounts["output"]) / Decimal(1000000) + fee) + per_request = per_request.quantize(Decimal("0.00000001"), rounding=ROUND_CEILING) + total_requests = trials * spec["max_requests"] + total = (per_request * total_requests).quantize(Decimal("0.01"), rounding=ROUND_CEILING) + if total > money(spend_limit["amount"]): + raise ValueError("Worst-case cost reservation exceeds approved total limit") + return {"currency": pricing["currency"], "amount": str(total), "limit": spend_limit["amount"], + "requests": total_requests, "per_request": str(per_request), + "pricing_sha256": hashlib.sha256(canonical(pricing).encode()).hexdigest(), + "group": pricing["group"]} + + +def execution_environment(environment, *, with_key=True): + # Called only at runtime. Neither plan/authorize nor tests discover existing credentials. + safe = {k: environment[k] for k in ("PATH", "SystemRoot", "WINDIR", "TEMP", "TMP", "LANG") + if k in environment} + safe["PYTHONUTF8"] = "1" + if with_key: + key = environment.get(KEY_ENV) + if (not isinstance(key, str) or not key or not key.isascii() + or any(c.isspace() for c in key)): + raise ValueError("Provider credential is missing or malformed") + safe[KEY_ENV] = key + return safe + + @dataclass(frozen=True) class AnthropicAgentClient: model: str spec: dict timeout_seconds: float + pricing: dict | None = None + spend_limit: dict | None = None def frozen_identity(self): validate_spec({"name": self.model, "effort": "not_applicable"}, self.spec) - return {"settings": json.loads(canonical(self.spec)), - "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), - "runtime_state": "offline_only"} - - @staticmethod - def require_live_readiness(): - # No environment variable or self-declared quota can authorize unknown spend. - raise ValueError(LIVE_BLOCKER) + value = {"settings": json.loads(canonical(self.spec)), + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "runtime_state": "unconfigured", + "request_policy": {"tool_choice": "auto", "thinking": "model_default", + "sampling": "model_default", "retries": 0}} + if self.pricing is not None or self.spend_limit is not None: + self.require_live_readiness(now=0) + value.update(pricing=json.loads(canonical(self.pricing)), + spend_limit=json.loads(canonical(self.spend_limit)), + runtime_state="configured") + return value + + def require_live_readiness(self, trials=1, *, now=None): + return reservation(self.model, self.spec, self.pricing, self.spend_limit, trials, now=now) + + def environment(self): + return execution_environment(os.environ) + + def verifier_environment(self): + return execution_environment(os.environ, with_key=False) def command(self, workspace, final, schema, prompt): self.require_live_readiness() + settings = {"model": self.model, "client": self.spec, "pricing": self.pricing, + "spend_limit": self.spend_limit} + return (sys.executable, "-I", str(Path(__file__).resolve()), + "--workspace", str(workspace), "--final", str(final), "--schema", str(schema), + "--settings", canonical(settings), "--prompt", prompt, + "--timeout", str(self.timeout_seconds)) @staticmethod def trace_metrics(path): return trace_metrics(path) - class AgentError(Exception): """Only fixed reason codes cross the evidence boundary.""" @@ -204,7 +296,23 @@ def exposed_files(workspace): return paths -def run_agent(workspace, schema, prompt, model, spec, timeout, emit, *, send): +def valid_content_block(block): + if not isinstance(block, dict): + return False + kind = block.get("type") + if kind == "tool_use": + return True # The read/submit dispatcher validates identity and input below. + if kind == "text": + return isinstance(block.get("text"), str) + if kind == "thinking": + return isinstance(block.get("thinking"), str) and isinstance(block.get("signature"), str) + if kind == "redacted_thinking": + return isinstance(block.get("data"), str) + return False + + +def run_agent(workspace, schema, prompt, model, spec, timeout, emit, *, send, + input_token_ceiling=None): """Run only the read/submit contract. Tests inject transport; no implicit live path.""" validate_spec({"name": model, "effort": "not_applicable"}, spec) validate_schema(schema) @@ -219,8 +327,9 @@ def run_agent(workspace, schema, prompt, model, spec, timeout, emit, *, send): {"name": "submit_answer", "description": "Submit a structured answer after reading task.json.", "input_schema": schema}] - messages = [{"role": "user", "content": prompt + "\nExposed paths: " - + canonical(sorted(files))}] + messages = [{"role": "user", "content": prompt + + "\nUse read_file to read task.json; finish with submit_answer. " + "Plain text is not a submission. Exposed paths: " + canonical(sorted(files))}] deadline, task_read_at, tool_count = time.monotonic() + timeout, None, 0 totals = {"input_tokens": 0, "output_tokens": 0, "cached_input_tokens": 0} try: @@ -229,7 +338,7 @@ def run_agent(workspace, schema, prompt, model, spec, timeout, emit, *, send): if remaining <= 0: raise AgentError("timeout") payload = {"model": model, "max_tokens": spec["max_output_tokens"], "stream": False, - "tools": tools, "messages": messages} + "tools": tools, "tool_choice": {"type": "auto"}, "messages": messages} raw = canonical(payload).encode() if len(raw) > spec["max_request_bytes"]: raise AgentError("request_bytes_limit") @@ -242,11 +351,11 @@ def run_agent(workspace, schema, prompt, model, spec, timeout, emit, *, send): or response.get("role") != "assistant" or response.get("model") != model or response.get("stop_reason") != "tool_use" or not isinstance(response.get("content"), list) - or any(not isinstance(c, dict) or c.get("type") not in ("text", "tool_use") - or (c.get("type") == "text" and not isinstance(c.get("text"), str)) - for c in response["content"])): + or any(not valid_content_block(c) for c in response["content"])): raise AgentError("incompatible_response") usage = token_usage(response.get("usage")) + if input_token_ceiling is not None and usage["input_tokens"] > input_token_ceiling: + raise AgentError("input_token_limit") if usage["output_tokens"] > spec["max_output_tokens"]: raise AgentError("output_token_limit") for name in totals: @@ -336,3 +445,42 @@ def trace_metrics(path): "completed_turns": 1 if valid else 0, "api_requests": len(requests), "client_kind": KIND, "provider_model": responses[-1]["provider_model"] if responses else None} + + +def main(argv=None, *, environment=None, transport=None, emit=None): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("workspace", "final", "schema", "settings", "prompt"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--timeout", type=float, required=True) + args = parser.parse_args(argv) + emit = emit or (lambda event: print(canonical(event), flush=True)) + try: + settings = json.loads(args.settings, object_pairs_hook=unique_object) + if set(settings) != {"model", "client", "pricing", "spend_limit"}: + raise ValueError("Unknown worker setting") + client = AnthropicAgentClient(settings["model"], settings["client"], args.timeout, + settings["pricing"], settings["spend_limit"]) + reserved = client.require_live_readiness() + runtime = execution_environment(os.environ if environment is None else environment) + key = runtime[KEY_ENV] + sender = transport or send_message + + def send(payload, spec, timeout): + return sender(payload, spec, key, timeout) + + schema = json.loads(Path(args.schema).read_text(encoding="utf-8")) + emit({"type": "agent.reservation", **reserved}) + answer = run_agent(Path(args.workspace), schema, args.prompt, client.model, + client.spec, args.timeout, emit, send=send, + input_token_ceiling=client.pricing["max_input_tokens"]) + Path(args.final).write_text(canonical(answer) + "\n", encoding="utf-8") + return 0 + except AgentError: + return 1 # run_agent already retained a fixed failure code. + except Exception: + emit({"type": "agent.failed", "reason": "readiness_or_worker_error"}) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_anthropic_agent.py b/tests/test_anthropic_agent.py index eb67cb7ba..b20bd8612 100644 --- a/tests/test_anthropic_agent.py +++ b/tests/test_anthropic_agent.py @@ -298,7 +298,7 @@ def test_opt_in_plan_freezes_identity_and_cannot_execute_or_fall_back(tmp_path, assert plan["client"]["settings"] == spec assert plan["client"]["source_sha256"] == sha256(compare.ROOT / agent.SOURCE) assert plan["controllers"][agent.SOURCE] == sha256(compare.ROOT / agent.SOURCE) - assert plan["client"]["runtime_state"] == "offline_only" + assert plan["client"]["runtime_state"] == "unconfigured" report = compare.report(output) assert report["client"] == plan["client"] assert agent.KIND in (output / "report.md").read_text(encoding="utf-8") diff --git a/tests/test_yibuapi_runtime.py b/tests/test_yibuapi_runtime.py new file mode 100644 index 000000000..f652d574b --- /dev/null +++ b/tests/test_yibuapi_runtime.py @@ -0,0 +1,263 @@ +import copy +import json +import sys +from pathlib import Path +from decimal import Decimal + +import pytest +import yaml + +from inferrsi.harness import skill_evaluation as harness +from inferrsi.infra.artifacts import write_json +from inferrsi.model import anthropic_agent as agent +from tests.fixtures.skill_eval_compare_support import DATA +from tests.fixtures.skill_eval_pr_support import SimulatedGitHub, approval, setup +from tests.test_anthropic_agent import ANSWER, MODEL, SCHEMA_SMALL, read, submit +from tools import skill_eval_compare as compare +from tools import skill_eval_pr as integration +from tools import skill_eval_verifier as verifier + +FAKE_KEY = "fake-provider-key-offline-only" + + +def configuration(): + # Synthetic prices exercise arithmetic; these are not the owner's provider rates. + return {"model": {"name": MODEL, "effort": "not_applicable"}, + "client": {"kind": agent.KIND, "provider": "yibuapi", "endpoint": agent.ENDPOINT, + "api_version": "2023-06-01", "max_requests": 3, "max_output_tokens": 256, + "max_request_bytes": 32768, "max_read_bytes": 16384, "max_tool_calls": 8}, + "pricing": {"currency": "USD", "unit": "per_million_tokens", "model": MODEL, + "group": "offline-fixture", "rates": {"input": "2", "output": "4", + "cache_read": "0.1", "cache_write": "5"}, "request_fee": "0", + "max_input_tokens": 1000000, "expires_at": 4102444800, + "rate_evidence": + "https://github.com/example/fixture/issues/1#issuecomment-1"}, + "spend_limit": {"currency": "USD", "amount": "100"}} + + +def client(profile): + return agent.AnthropicAgentClient(MODEL, profile["client"], 30, + profile["pricing"], profile["spend_limit"]) + + +def test_full_context_reservation_uses_highest_cache_rate_and_all_attempts(): + p = configuration() + value = agent.reservation(MODEL, p["client"], p["pricing"], p["spend_limit"], 2) + expected = (Decimal(1000000) * 5 + Decimal(256) * 4) / 1000000 * 6 + assert value["requests"] == 6 and value["currency"] == "USD" + assert Decimal(value["amount"]) >= expected + assert value["group"] == p["pricing"]["group"] + p["spend_limit"]["amount"] = "1" + with pytest.raises(ValueError, match="exceeds"): + client(p).require_live_readiness(2) + + +@pytest.mark.parametrize("change", [ + lambda p: p["pricing"].pop("unit"), + lambda p: p["pricing"]["rates"].pop("cache_write"), + lambda p: p["pricing"]["rates"].update(input="NaN"), + lambda p: p["pricing"].update(max_input_tokens=999999), + lambda p: p["pricing"].update(expires_at=1), + lambda p: p["pricing"].update(currency="CNY"), + lambda p: p["pricing"].update(group=""), +]) +def test_missing_unknown_stale_or_understated_cost_facts_fail_closed(change): + p = configuration() + change(p) + with pytest.raises(ValueError): + client(p).require_live_readiness(2) + + +def test_worker_argv_environment_and_verifier_keep_credentials_separate(tmp_path): + p = configuration() + c = client(p) + command = c.command(tmp_path, tmp_path / "answer", tmp_path / "schema", "public prompt") + assert command[:2] == (sys.executable, "-I") + assert FAKE_KEY not in " ".join(command) and "codex" not in command + values = {"PATH": "fixture", agent.KEY_ENV: FAKE_KEY, "GITHUB_TOKEN": "fake-github", + "ACTIONS_RUNTIME_TOKEN": "fake-runtime", "OTHER_SECRET": "fake-other"} + model = agent.execution_environment(values) + grading = agent.execution_environment(values, with_key=False) + assert model[agent.KEY_ENV] == FAKE_KEY and agent.KEY_ENV not in grading + assert all(k not in model and k not in grading for k in + ("GITHUB_TOKEN", "ACTIONS_RUNTIME_TOKEN", "OTHER_SECRET")) + with pytest.raises(ValueError, match="credential"): + agent.execution_environment({}) + + +def test_worker_runs_read_submit_with_fake_transport_and_no_key_in_evidence(tmp_path): + p = configuration() + workspace = tmp_path / "workspace" + workspace.mkdir() + write_json(workspace / "task.json", {"input": "public fixture"}) + schema, final = tmp_path / "schema.json", tmp_path / "answer.json" + write_json(schema, SCHEMA_SMALL) + events, calls = [], [] + + def transport(payload, spec, key, timeout): + assert key == FAKE_KEY + calls.append(copy.deepcopy(payload)) + return read() if len(calls) == 1 else submit() + + args = ["--workspace", str(workspace), "--final", str(final), "--schema", str(schema), + "--settings", agent.canonical(p), "--prompt", "public fixture", "--timeout", "30"] + assert agent.main(args, environment={agent.KEY_ENV: FAKE_KEY}, + transport=transport, emit=events.append) == 0 + assert json.loads(final.read_text()) == ANSWER and len(calls) == 2 + assert FAKE_KEY not in agent.canonical(events) and FAKE_KEY not in agent.canonical(calls) + before = len(calls) + assert agent.main(args, environment={}, transport=transport, emit=events.append) == 1 + assert len(calls) == before + + +def install_fake_worker(monkeypatch): + sent = [] + + def environment(self): + return {agent.KEY_ENV: FAKE_KEY} + + def execute(command, cwd, folder, deadline, **options): + folder.mkdir(parents=True) + (folder / "stderr.log").write_text("", encoding="utf-8") + if folder.name == "agent": + assert options["environment"] == {agent.KEY_ENV: FAKE_KEY} + events = [] + + def transport(payload, spec, key, timeout): + sent.append(copy.deepcopy(payload)) + assert key == FAKE_KEY + if len(payload["messages"]) == 1: + return read() + return submit(DATA["cases"][0]["answer"]) + + status = agent.main(command[3:], environment=options["environment"], + transport=transport, emit=events.append) + (folder / "stdout.log").write_text( + "".join(agent.canonical(e) + "\n" for e in events), encoding="utf-8") + else: + assert agent.KEY_ENV not in options["environment"] + checks = verifier.verify(json.loads((cwd / "task.json").read_text()), + json.loads(Path(command[-1]).read_text(encoding="utf-8"))) + write_json(folder / "result.json", checks) + (folder / "stdout.log").write_text("", encoding="utf-8") + status = 0 if checks["passed"] else 1 + return {"status": "completed" if status == 0 else "failed", + "returncode": status, "elapsed_seconds": 0} + + monkeypatch.setattr(agent.AnthropicAgentClient, "environment", environment) + monkeypatch.setattr(agent.AnthropicAgentClient, "verifier_environment", lambda self: {}) + monkeypatch.setattr(harness, "execute_command", execute) + return sent + + +@pytest.mark.parametrize("mode", ["addition", "update"]) +def test_configured_paired_execute_retained_feedback_and_one_shot(tmp_path, monkeypatch, mode): + repo, mapping, identity = setup(tmp_path, mode, ids=("one",)) + p, output = configuration(), tmp_path / "run" + plan = integration.prepare(repo, mapping, identity, output, provider=p) + frozen = compare.load_plan(output / "selected") + assert frozen["client"]["runtime_state"] == "configured" + assert plan["reservation"]["requests"] == 6 + assert integration.budget(plan)["provider_config_sha256"] == integration.fingerprint(p) + sent = install_fake_worker(monkeypatch) + integration.execute(output, approval(plan)) + results = integration.reports(output) + assert len(sent) == 4 and results[0]["counts"]["unchanged_correct"] == 1 + assert results[0]["status"]["execution_state"] == "completed" + with pytest.raises(FileExistsError): + integration.execute(output, approval(plan)) + assert len(sent) == 4 + data = integration.result_archive(output).read_bytes() + published = tmp_path / "publisher" + integration.prepare(repo, mapping, identity, published, provider=p) + api = SimulatedGitHub(plan, data) + integration.publish_retained(api, published, 10, 11, 1) + assert len(api.comments) == 1 and '"unchanged_correct":1' in api.comments[0]["body"] + for path in output.rglob("*"): + if path.is_file(): + assert FAKE_KEY.encode() not in path.read_bytes() + + +def test_missing_key_and_stale_rates_precede_local_claim(tmp_path, monkeypatch): + repo, mapping, identity = setup(tmp_path, ids=("one",)) + p, output = configuration(), tmp_path / "run" + plan = integration.prepare(repo, mapping, identity, output, provider=p) + + def missing(self): + raise ValueError("Provider credential is missing") + + monkeypatch.setattr(agent.AnthropicAgentClient, "environment", missing) + with pytest.raises(ValueError, match="credential"): + integration.execute(output, approval(plan)) + assert not (output / "execution-claim.json").exists() + assert not list((output / "selected/results").iterdir()) + p["pricing"]["expires_at"] = 1 + expired = integration.prepare(repo, mapping, identity, tmp_path / "expired", provider=p) + with pytest.raises(ValueError, match="expired"): + approval(expired) + + +def test_paid_path_stops_on_unknown_usage_and_never_retries(tmp_path, monkeypatch): + repo, mapping, identity = setup(tmp_path, ids=("one",)) + p, output = configuration(), tmp_path / "run" + plan = integration.prepare(repo, mapping, identity, output, provider=p) + monkeypatch.setattr(agent.AnthropicAgentClient, "environment", lambda self: {}) + calls = [] + + def stopped(folder, digest, **kwargs): + calls.append(kwargs) + assert kwargs["stop_on_unknown_usage"] is True + return {"status": {"execution_state": "stopped", "stop_reason": "unknown_usage"}} + + monkeypatch.setattr(compare, "execute", stopped) + integration.execute(output, approval(plan)) + assert len(calls) == 1 + with pytest.raises(FileExistsError): + integration.execute(output, approval(plan)) + assert len(calls) == 1 + + +def test_secret_only_exists_on_conditional_model_execution_step(): + workflow = yaml.safe_load((compare.ROOT / integration.WORKFLOW).read_text(encoding="utf-8")) + grants = [] + for job, definition in workflow["jobs"].items(): + for step in definition["steps"]: + if agent.KEY_ENV in step.get("env", {}): + grants.append((job, step)) + assert len(grants) == 1 and grants[0][0] == "execute" + step = grants[0][1] + assert agent.KIND in step["if"] and "skill_eval_pr execute" in step["run"] + assert set(step["env"]) == {agent.KEY_ENV} + assert "secrets.SKILL_EVAL_YIBUAPI_KEY" in step["env"][agent.KEY_ENV] + assert agent.KEY_ENV not in workflow.get("env", {}) and "GITHUB_TOKEN" not in step["env"] + codex = next(s for s in workflow["jobs"]["execute"]["steps"] + if s.get("run") == "codex --version") + assert "codex" in codex["if"] + + +def test_thinking_blocks_are_preserved_opaquely_in_tool_round_trip(tmp_path): + p = configuration() + write_json(tmp_path / "task.json", {"input": "public fixture"}) + messages = [] + + def send(payload, spec, timeout): + messages.append(copy.deepcopy(payload)) + if len(messages) == 1: + value = read() + value["content"].insert(0, {"type": "thinking", "thinking": "", + "signature": "opaque-fixture-signature"}) + return value + return submit() + + assert agent.run_agent(tmp_path, SCHEMA_SMALL, "fixture", MODEL, p["client"], 30, + lambda e: None, send=send) == ANSWER + assert messages[1]["messages"][1]["content"][0]["signature"] == "opaque-fixture-signature" + + +@pytest.mark.parametrize("block", [ + {"type": "thinking", "thinking": None, "signature": "opaque"}, + {"type": "thinking", "thinking": "", "signature": None}, + {"type": "redacted_thinking", "data": None}, +]) +def test_malformed_thinking_blocks_fail_before_replay(block): + assert not agent.valid_content_block(block) diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index e7be428fe..e57871c8c 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -4,6 +4,7 @@ import math import re import shutil +import sys import subprocess import tempfile import time @@ -167,7 +168,7 @@ def validate_default_audit(task: dict) -> None: def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> None: if set(config) - {"mode", "skill", "skill_root", "cases", "schema", "grader", - "snapshots", "model", "limits", "client"}: + "snapshots", "model", "limits", "client", "pricing", "spend_limit"}: raise ValueError("Unknown configuration field") if config["mode"] not in ("addition", "update"): raise ValueError("mode must be addition or update") @@ -189,6 +190,8 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> raise ValueError("model name and effort are required") if "client" in config: validate_spec(config["model"], config["client"]) + elif "pricing" in config or "spend_limit" in config: + raise ValueError("Provider pricing requires an explicit client") limits = config["limits"] if set(limits) != {"repeats", "max_task_calls", "timeout_seconds", "campaign_seconds", "token_stop_new_trials"}: @@ -342,13 +345,16 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: } if "client" in config: client = AnthropicAgentClient(config["model"]["name"], config["client"], - config["limits"]["timeout_seconds"]) + config["limits"]["timeout_seconds"], + config.get("pricing"), config.get("spend_limit")) plan["client"] = client.frozen_identity() plan["controllers"][AGENT_SOURCE] = sha256(ROOT / AGENT_SOURCE) plan["max_api_requests"] = len(plan["schedule"]) * config["client"]["max_requests"] + if "pricing" in config or "spend_limit" in config: + plan["reservation"] = client.require_live_readiness(len(plan["schedule"]), now=0) plan["isolation"] = ("Fresh workspaces; read/submit tools expose selected files only. " "No shell, network tool, external agent CLI or global config. " - "Live provider execution is unavailable in this offline slice.") + "Provider worker and verifier use separate filtered environments.") write_json(output / "plan/freeze.json", plan) return sha256(output / "plan/freeze.json") @@ -376,6 +382,12 @@ def trial_directory(output: Path, entry: dict) -> Path: def resolve_client(client): runtime = {"implementation": type(client).__module__ + "." + type(client).__qualname__, "executable": None, "executable_sha256": None, "version": None} + if isinstance(client, AnthropicAgentClient): + client.require_live_readiness() + runtime.update(version=sys.version.split()[0], client_kind=client.spec["kind"], + executable=Path(sys.executable).name, + executable_sha256=sha256(Path(sys.executable)), + source_sha256=sha256(ROOT / AGENT_SOURCE)) if isinstance(client, CodexClient): executable = shutil.which(client.executable) if executable is None: @@ -405,6 +417,16 @@ def receipt_error(raw_record: bytes, receipt_path: Path, label: str) -> str | No return None +def provider_client(plan): + identity = plan["client"] + client = AnthropicAgentClient(plan["model"]["name"], identity["settings"], + plan["limits"]["timeout_seconds"], identity.get("pricing"), + identity.get("spend_limit")) + if identity != client.frozen_identity(): + raise ValueError("Frozen provider identity no longer matches runtime") + return client + + def execute(output: Path, authorization: str, *, client=None, campaign_deadline: float | None = None, stop_on_unknown_usage: bool = False) -> dict: @@ -414,8 +436,13 @@ def execute(output: Path, authorization: str, *, client=None, if any((output / "results").iterdir()): raise ValueError("Execution requires fresh results; prepare a new output directory") if "client" in plan: - # No Codex fallback or injected-client bypass of incomplete live setup. - AnthropicAgentClient.require_live_readiness() + configured = provider_client(plan) + configured.require_live_readiness(len(plan["schedule"])) + if client is not None: + raise ValueError("A paid provider plan cannot substitute an injected client") + configured.environment() # Missing keys fail before start/trial evidence. + client = configured + stop_on_unknown_usage = True if client is None: client = CodexClient(plan["model"]["name"], plan["model"]["effort"], plan["limits"]["timeout_seconds"]) @@ -614,6 +641,7 @@ def read_report(output: Path) -> dict: if "client" in plan: result["client"] = plan["client"] result["max_api_requests"] = plan["max_api_requests"] + result["reservation"] = plan.get("reservation") return result diff --git a/tools/skill_eval_pr.py b/tools/skill_eval_pr.py index 5b5011676..1f3fad68c 100644 --- a/tools/skill_eval_pr.py +++ b/tools/skill_eval_pr.py @@ -14,6 +14,7 @@ from urllib.error import HTTPError from inferrsi.infra.artifacts import sha256, write_json +from inferrsi.model.anthropic_agent import KIND, money, reservation from tools import skill_eval_compare as compare from tools.check_knowledge_model_policy import GitHubHTTPError from tools.skill_eval_verifier import verify @@ -49,12 +50,18 @@ def blob_exists(repo, revision, path): return bool(git(repo, "ls-tree", revision, "--", path).strip()) -def prepare(repo, mapping_path, identity, output): +def prepare(repo, mapping_path, identity, output, *, provider=None): # Read only trusted configuration and raw Git objects, never PR Python or graders. for key in ("base", "head", "version"): if not re.fullmatch(SHA, identity[key]): raise ValueError("Expected immutable revision: " + key) mapping = compare.read_json(mapping_path) + if provider is not None: + if set(provider) != {"model", "client", "pricing", "spend_limit"}: + raise ValueError("Unknown or incomplete provider configuration") + reservation(provider["model"]["name"], provider["client"], provider["pricing"], + provider["spend_limit"], 1, now=0) + mapping.update(provider) changes = changed_files(repo, identity["base"], identity["head"]) output.mkdir(parents=True, exist_ok=False) coverage, selected = [], set() @@ -85,6 +92,8 @@ def prepare(repo, mapping_path, identity, output): "snapshots": {"baseline": {"git_revision": identity["base"]}, "candidate": {"git_revision": identity["head"]}}, "model": mapping["model"], "limits": mapping["limits"]} + if provider is not None: + config.update({key: provider[key] for key in ("client", "pricing", "spend_limit")}) config_path = output / (item["skill"] + ".json") write_json(config_path, config) destination = output / item["skill"] @@ -111,13 +120,35 @@ def prepare(repo, mapping_path, identity, output): "comparisons": comparisons, "model": mapping["model"], "limits": mapping["limits"], "estimated_calls": sum(c.get("calls", 0) for c in comparisons), "retention_days": RETENTION_DAYS} + if provider is not None: + plan["provider_config"] = provider + plan["reservation"] = provider_reservation(plan, now=0) write_json(output / "pr-plan.json", plan) return plan +def provider_reservation(plan, *, now=None): + provider = plan["provider_config"] + selected = [c for c in plan["comparisons"] if "plan_sha256" in c] + if sum(c["calls"] for c in selected) != plan["estimated_calls"] or not selected: + raise ValueError("Provider plan requires supported paired trials") + allocated = [reservation(provider["model"]["name"], provider["client"], + provider["pricing"], provider["spend_limit"], c["calls"], now=now) + for c in selected] + amount = sum(money(c["amount"]) for c in allocated) + if amount > money(provider["spend_limit"]["amount"]): + raise ValueError("All-comparison reservation exceeds approved total") + return {**allocated[0], "amount": str(amount), + "requests": sum(c["requests"] for c in allocated)} + + def budget(plan): - return {"estimated_calls": plan["estimated_calls"], "limits_per_comparison": plan["limits"], - "comparisons": sum("plan_sha256" in c for c in plan["comparisons"])} + value = {"estimated_calls": plan["estimated_calls"], "limits_per_comparison": plan["limits"], + "comparisons": sum("plan_sha256" in c for c in plan["comparisons"])} + if "provider_config" in plan: + value.update(reservation=plan["reservation"], + provider_config_sha256=fingerprint(plan["provider_config"])) + return value def execution_id(plan): @@ -139,6 +170,8 @@ def authorize(plan, request, actor, permission, current): raise ValueError("PR changed; prepare and authorize a new plan") if not plan["estimated_calls"]: raise ValueError("No supported comparisons selected") + if "provider_config" in plan and provider_reservation(plan) != plan["reservation"]: + raise ValueError("Provider reservation changed") return {**expected, "actor": actor, "execution_id": execution_id(plan)} @@ -159,6 +192,17 @@ def execute(output, approval, *, client=None): plan = compare.read_json(output / "pr-plan.json") if approval["execution_id"] != execution_id(plan): raise ValueError("Approval belongs to another plan") + if "provider_config" in plan: + if client is not None: + raise ValueError("A paid provider plan cannot substitute an injected client") + if provider_reservation(plan) != plan["reservation"]: + raise ValueError("Provider reservation changed") + for item in plan["comparisons"]: + if "plan_sha256" in item: + frozen = compare.load_plan(output / item["skill"]) + provider = compare.provider_client(frozen) + provider.require_live_readiness(len(frozen["schedule"])) + provider.environment() # Before the exclusive local consumption record. # Exclusive creation also protects local/offline callers sharing retained storage. with (output / "execution-claim.json").open("x", encoding="utf-8") as handle: handle.write(canonical(approval)) @@ -166,7 +210,10 @@ def execute(output, approval, *, client=None): if "plan_sha256" in item: # Artifact downloads omit empty directories required by the executor. (output / item["skill"] / "results").mkdir(exist_ok=True) - compare.execute(output / item["skill"], item["plan_sha256"], client=client) + result = compare.execute(output / item["skill"], item["plan_sha256"], client=client, + stop_on_unknown_usage="provider_config" in plan) + if "provider_config" in plan and result["status"]["execution_state"] != "completed": + break def result_archive(output): @@ -293,6 +340,8 @@ def summary(plan, results, evidence_url, expires_at, now): lines.append("\nFrozen exam hashes: `" + canonical(item["exam"]) + "`\n") else: lines.append(f"| {item['skill']} | unsupported snapshot | — | 0 |") + if "provider_config" in plan: + lines += ["", "Provider/settings: `" + canonical(plan["provider_config"]) + "`"] lines += ["", "", "", "
", f"Uncovered / unsupported paths ({len(gaps)})", ""] # A collapsed HTML container still requires data escaping for untrusted Git paths. @@ -564,9 +613,12 @@ def main(): f"refs/pull/{pr}/head"], cwd=ROOT, check=True) version = git(ROOT, "rev-parse", "HEAD").decode().strip() plan = prepare(ROOT, MAPPING, {"repository": repository, "pr": pr, - "version": version, **current}, output) + "version": version, **current}, output, + provider=json.loads(inputs["provider_config"]) + if inputs.get("provider_config") else None) with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as handle: handle.write(f"pr={pr}\nplan_sha256={fingerprint(plan)}\n") + handle.write("client_kind=" + (KIND if "provider_config" in plan else "codex") + "\n") return plan = compare.read_json(output / "pr-plan.json") if args.mode == "authorize": From 5986afb4e853a83c25da8b7b9ae3833a6361b105 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:26:42 +0900 Subject: [PATCH 2/4] style: normalize provider runtime import ordering Signed-off-by: GUOGUO Agent-assisted-by: Codex (author) --- tests/test_yibuapi_runtime.py | 2 +- tools/skill_eval_compare.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_yibuapi_runtime.py b/tests/test_yibuapi_runtime.py index f652d574b..fab584e4f 100644 --- a/tests/test_yibuapi_runtime.py +++ b/tests/test_yibuapi_runtime.py @@ -1,8 +1,8 @@ import copy import json import sys -from pathlib import Path from decimal import Decimal +from pathlib import Path import pytest import yaml diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index e57871c8c..b64ac89c0 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -4,8 +4,8 @@ import math import re import shutil -import sys import subprocess +import sys import tempfile import time from pathlib import Path From 59d1e3dfe5df48a4ab0d76597161f8ad8a31a254 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:31:13 +0900 Subject: [PATCH 3/4] test: use actual worker command settings in runtime fixture Signed-off-by: GUOGUO Agent-assisted-by: Codex (author) --- tests/test_yibuapi_runtime.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_yibuapi_runtime.py b/tests/test_yibuapi_runtime.py index fab584e4f..075018db9 100644 --- a/tests/test_yibuapi_runtime.py +++ b/tests/test_yibuapi_runtime.py @@ -99,8 +99,7 @@ def transport(payload, spec, key, timeout): calls.append(copy.deepcopy(payload)) return read() if len(calls) == 1 else submit() - args = ["--workspace", str(workspace), "--final", str(final), "--schema", str(schema), - "--settings", agent.canonical(p), "--prompt", "public fixture", "--timeout", "30"] + args = client(p).command(workspace, final, schema, "public fixture")[3:] assert agent.main(args, environment={agent.KEY_ENV: FAKE_KEY}, transport=transport, emit=events.append) == 0 assert json.loads(final.read_text()) == ANSWER and len(calls) == 2 From ffb5ce1e167cce33fb931dff27f705fc1b0a9212 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:34:01 +0900 Subject: [PATCH 4/4] fix: round every attempted request reservation upward Signed-off-by: GUOGUO Agent-assisted-by: Codex (author) --- src/inferrsi/model/anthropic_agent.py | 2 +- tests/test_yibuapi_runtime.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/inferrsi/model/anthropic_agent.py b/src/inferrsi/model/anthropic_agent.py index 4d13362f9..08246cdc2 100644 --- a/src/inferrsi/model/anthropic_agent.py +++ b/src/inferrsi/model/anthropic_agent.py @@ -147,7 +147,7 @@ def reservation(model, spec, pricing, spend_limit, trials, *, now=None): per_request = ((pricing["max_input_tokens"] * max(amounts["input"], amounts["cache_read"], amounts["cache_write"]) + spec["max_output_tokens"] * amounts["output"]) / Decimal(1000000) + fee) - per_request = per_request.quantize(Decimal("0.00000001"), rounding=ROUND_CEILING) + per_request = per_request.quantize(Decimal("0.01"), rounding=ROUND_CEILING) total_requests = trials * spec["max_requests"] total = (per_request * total_requests).quantize(Decimal("0.01"), rounding=ROUND_CEILING) if total > money(spend_limit["amount"]): diff --git a/tests/test_yibuapi_runtime.py b/tests/test_yibuapi_runtime.py index 075018db9..7c8272c8c 100644 --- a/tests/test_yibuapi_runtime.py +++ b/tests/test_yibuapi_runtime.py @@ -45,6 +45,8 @@ def test_full_context_reservation_uses_highest_cache_rate_and_all_attempts(): value = agent.reservation(MODEL, p["client"], p["pricing"], p["spend_limit"], 2) expected = (Decimal(1000000) * 5 + Decimal(256) * 4) / 1000000 * 6 assert value["requests"] == 6 and value["currency"] == "USD" + one = client(p).require_live_readiness(1) + assert Decimal(value["amount"]) == Decimal(one["amount"]) * 2 assert Decimal(value["amount"]) >= expected assert value["group"] == p["pricing"]["group"] p["spend_limit"]["amount"] = "1"