From 41e3930939d3b3899d6fe9cd7b9baccbf9b02b04 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Fri, 13 Nov 2026 18:37:29 -0500 Subject: [PATCH] Use native API credits with bounded token accounting --- AGENTS.md | 7 +- README.md | 19 ++- docs/native-provider-credits.md | 127 ++++++++++++++++++ pyproject.toml | 1 + scripts/run_improvement_cycle.py | 78 ++++++++++-- src/backintel/bench/improvement.py | 43 +++++-- src/backintel/bench/native_costs.py | 104 +++++++++++++++ src/backintel/bench/run.py | 22 +++- src/backintel/bench/usage.py | 60 ++++++++- src/backintel/models.py | 50 +++++++- src/backintel/settings.py | 4 +- tests/test_campaign_routing.py | 46 +++++++ tests/test_improvement_cycle.py | 50 +++++++- tests/test_native_accounting.py | 188 +++++++++++++++++++++++++++ tests/test_native_providers.py | 191 ++++++++++++++++++++++++++++ uv.lock | 2 + 16 files changed, 945 insertions(+), 47 deletions(-) create mode 100644 docs/native-provider-credits.md create mode 100644 src/backintel/bench/native_costs.py create mode 100644 tests/test_campaign_routing.py create mode 100644 tests/test_native_accounting.py create mode 100644 tests/test_native_providers.py diff --git a/AGENTS.md b/AGENTS.md index 27dbe01..f2d6fbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,8 +50,11 @@ infrastructure last. The agent's value is measured, not asserted. aggregates into `results/scorecard.md`. - Models (`models.py`): `openrouter//` (OpenAI, Anthropic, Google only, per the technology-selection policy), `chatgpt-plan/` (operator's ChatGPT plan after - `backintel chatgpt login`), `openai/`. The judge is a different model family from - the analyst. + `backintel chatgpt login`), `openai/`, and `anthropic/` for native APIs. + Default analyst/coding worker: `openai/gpt-6-luna`; default judge: `anthropic/claude-haiku-5-5`. + The judge is a different model family from the analyst. Native API token usage is not a + per-request invoice; do not fabricate dollar charges or bypass budget admission. + Read `docs/native-provider-credits.md`. ## Working rules diff --git a/README.md b/README.md index dcdb558..6f95bef 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ not a product version. Code is available under the [MIT license](LICENSE). Two entry points share the same source boundaries: - **`ask`** runs the benchmark analyst and returns a structured answer with evidence, method - and caveats. Its default is `openrouter/openai/gpt-6-luna`. + and caveats. Its default is `openai/gpt-6-luna` through the native OpenAI API. - **`analyze`** runs an opt-in coding specialist with a persistent per-run workspace and artifact export. Its default is `chatgpt-plan/gpt-6.1-sol` with medium reasoning. @@ -150,8 +150,12 @@ access to IntelIP's Doppler project is **not required** by the application. | Setting | Purpose | | --- | --- | | `BACKINTEL_ANALYSIS_MODEL` | Override the specialist default; explicit `--model` takes precedence | -| `BACKINTEL_MODEL` | Benchmark analyst model | -| `BACKINTEL_JUDGE_MODEL` | Method judge; default `openrouter/anthropic/claude-haiku-5.5` | +| `BACKINTEL_MODEL` | Benchmark analyst model; default `openai/gpt-6-luna` | +| `BACKINTEL_JUDGE_MODEL` | Method judge; default `anthropic/claude-haiku-5-5` | +| `OPENAI_API_KEY` | Native OpenAI API key from the organization receiving API credits | +| `ANTHROPIC_API_KEY` | Native Anthropic API key from the organization receiving API credits | +| `OPENAI_ORG_ID`, `OPENAI_PROJECT_ID` | Optional native OpenAI account/project selectors | +| `ANTHROPIC_WORKSPACE_ID` | Workspace selector when required by the Anthropic key | | `OPENROUTER_API_KEY` | Required when using the OpenRouter route | | `LANGSMITH_API_KEY`, `LANGSMITH_PROJECT`, `LANGSMITH_TRACING` | Optional tracing and experiment configuration | | `BACKINTEL_DATA_DIR` | Data directory; defaults to the checkout's `data/` | @@ -163,6 +167,15 @@ per-request API dollar charge. There is no paid-API fallback. An OpenRouter spec requires both `--max-cost-usd` and `--reservation-usd`; these are application admission limits, not a provider-enforced billing cap. See [model selection and usage](docs/specialist-default.md). +Native API calls consume the corresponding provider account's API credits. They are +separate from the specialist's ChatGPT plan connection. Explicit model overrides still +select OpenRouter; there is no automatic fallback. Native responses retain token usage, +but do not supply a per-request invoice amount: `cost_usd` stays unknown. Fresh native +improvement campaigns use separately labeled, versioned conservative token-cost +estimates for budget admission, with incomplete usage stopping later calls. Historical +OpenRouter accounting stays unchanged. See +[native provider setup](docs/native-provider-credits.md) for activation and experiment boundaries. + Olist acquisition additionally needs your own Kaggle access. Dataset permissions are separate from this repository's code license; see the [benchmark specification](docs/benchmark.md). diff --git a/docs/native-provider-credits.md b/docs/native-provider-credits.md new file mode 100644 index 0000000..6b71b2a --- /dev/null +++ b/docs/native-provider-credits.md @@ -0,0 +1,127 @@ +# Native API credits + +The owner requested native OpenAI and Anthropic API routing on 2026-11-13 to use +the API credits on those accounts. This is a provider change for future runs, not +a change to historical experiment records or permission to spend the entire balance. + +| Role | Default route | Credential | +| --- | --- | --- | +| Benchmark analyst (`ask`, benchmark harness) | `openai/gpt-6-luna` | `OPENAI_API_KEY` | +| Optional improvement coding worker | Configured analyst model, unless explicitly overridden | Same native key by default | +| Method judge | `anthropic/claude-haiku-5-5` | `ANTHROPIC_API_KEY` | +| Report specialist (`analyze`) | `chatgpt-plan/gpt-6.1-sol`, medium reasoning | Existing BackIntel OAuth cache | + +The native routes explicitly target `api.openai.com` and `api.anthropic.com`. +Anthropic uses the installed `langchain-anthropic` integration and native Messages +API, including structured output. It does not pass through the OpenAI compatibility +layer. Both routes honor completion-token, timeout and retry limits. Missing native +credentials fail visibly; there is no fallback to OpenRouter or the plan connection. +OpenRouter remains an explicit route for reproducing historical comparisons. + +## Account setup + +Use Doppler project `backintel`, configuration `dev`. Store the two native keys +there, never in source, `.env`, chat or a VM mount. A browser sign-in alone does not +configure an API key. The key must belong to the organization that receives the +credit grant; a key from a different organization does not inherit that benefit. + +Optional selectors are `OPENAI_ORG_ID`, `OPENAI_PROJECT_ID`, and +`ANTHROPIC_WORKSPACE_ID` (required for some Anthropic account-scoped keys). +Reuse existing keys when available. Verify with each provider's read-only model +catalog before inference, including access to the exact model IDs above. + +Once credentials are verified, set the existing Doppler model overrides to: + +```text +BACKINTEL_MODEL=openai/gpt-6-luna +BACKINTEL_JUDGE_MODEL=anthropic/claude-haiku-5-5 +``` + +Explicit environment values take precedence over built-in defaults. Changing the +code alone does not override an existing OpenRouter selection in Doppler. + +## Accounting and experiments + +Provider credits, gross API usage and cash paid are different quantities. A credit +benefit confirmation is not proof of the current remaining balance. Native responses +provide token counts, not the account's net dollar debit after credits. + +The usage tracker preserves native input/output tokens and reports unknown `cost_usd` +with `cost_complete=false`. `NativeRunBudget` separately admits calls against a +**conservative usage-cost estimate**, with `estimated_cost_usd`, `estimate_complete` +and an accounting version. It never relabels that estimate as an invoice or a credit +balance. Native budgets always report `unknown_charge=true`; continuation instead +requires complete estimated accounting (`accounting_uncertain=false`). The original +`RunBudget` still requires provider-reported charges for OpenRouter. + +The qualified policy is frozen from official standard-service prices on **2026-11-13**: + +| Native model | Input per million tokens | Output per million tokens | +| --- | ---: | ---: | +| `openai/gpt-6-luna` | $0.50 | $0.75 | +| `anthropic/claude-haiku-5-5` | $1.00 | $2.50 | + +These deliberately use each model's highest standard long-context cache-write input +rate and long-context output rate for every request. They overestimate ordinary short +prompts and cache hits. All cached input and all output, including reasoning tokens, +are included exactly once; discounts and credits are excluded. Each call rounds up +to one billionth of a dollar. The policy, source URLs and model limits live in +`src/backintel/bench/native_costs.py`; its hash is frozen into comparison plans. + +The native factory pins OpenAI `service_tier=default` and Anthropic +`service_tier=standard_only` at their global endpoints. This policy covers these two +model IDs and dated snapshots, text and local tools, up to their declared context and +output limits. Unknown models, missing or inconsistent token counts, audio usage, +unexpected service tiers and Anthropic paid server-tool requests cannot settle an +estimate. An unsuccessful or unaccounted request retains its reservation and stops +later calls. Per-call reservation overruns also stop continuation. These remain +application admission limits, not provider-enforced billing caps. + +Prepare a fresh improvement campaign with native routes in both arms. Preparation +locks the accounting basis and policy hash; execution and comparison reject changed +or incomplete accounting. Mixed OpenRouter/native campaigns are rejected. The old +OpenRouter dollar-complete campaign cannot be resumed by swapping keys/model IDs. +Keep historical manifests, reported charges and unpriced-call reservations unchanged. + +Retain the original **$5 combined reliability allowance** across analyst, worker and +judge calls, subtracting historical reported usage and outstanding reservations before +setting the new run's `--max-cost-usd`. Revalidate the judge through the native adapter +before the 92-response campaign. Available account credit does not increase that +allowance. SDK mock tests establish adapter/accounting behavior; they do not establish +live judge quality or actual account billing. + +## References + +- [OpenAI API credits for ChatGPT subscribers](https://developers.openai.com/api/docs/guides/chatgpt-api-credits) +- [Anthropic Haiku 5.5 and API credit announcement](https://www.anthropic.com/claude-haiku-5-5) +- [Native Haiku 5.5 model ID](https://platform.claude.com/docs/en/models/overview) +- [LangChain ChatAnthropic integration](https://docs.langchain.com/oss/python/integrations/chat/anthropic) +- [Anthropic OpenAI compatibility limitations](https://platform.claude.com/docs/en/cli-sdks-libraries/libraries/openai-sdk) +- [OpenAI native prices](https://developers.openai.com/api/docs/pricing) +- [Luna long-context rates and limits](https://developers.openai.com/api/docs/models/gpt-6-luna) +- [Haiku 5.5 prices and limits](https://platform.claude.com/docs/en/models/haiku-5-5/overview) + +The integration reuses the already locked LangChain/Anthropic packages, with +`langchain-anthropic` now declared as a direct dependency. This preserves the +selected OpenAI/Anthropic model families and uses their own endpoints. + +## Live adapter evidence — 2026-11-13 + +Native Anthropic completed six authored evaluator controls with complete token estimates +and no budget uncertainty or reservation breach. Conservative usage estimate: **$0.028152**. +The judge matched only four of six frozen criteria, so this proves the native transport +and accounting path, not judge reliability. The larger campaign remains held pending that +separate quality issue. [Retained LangSmith experiment](https://smith.langchain.com/o/90b24546-f3e2-4f48-af72-b06e73b27b49/projects/p/e2832945-3a35-43b1-b445-44f7b6f7c80a). + +Native OpenAI then completed one structured-output adapter smoke with 55 input and 13 +output tokens. Its conservative estimate was **$0.00003725**. This was a transport check, +not an analytical question. Combined new estimates: **$0.02818925**. Actual invoice +amounts and remaining credit balances are unknown. Evidence is retained in the canonical +checkout under `data/output/judge-native-validation-20261113/` and +`data/output/native-openai-smoke-20261113/`; accounting version `a0cc73ec381e4778`. + +The native-only working tree passed 232 tests with 15 optional skips. Integration with +current main and the PR8 evaluator passed 247 tests with 15 optional skips, Ruff lint and +format, Vulture and whitespace checks. The six-call trial used the same model-factory, +usage-tracker and native-accounting bytes. Shared Doppler selectors have not been changed; +activate native routes only after the native-capable implementation is adopted. diff --git a/pyproject.toml b/pyproject.toml index 031934e..2379a17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "deepagents==0.7.35", "langchain>=1.4,<2", "langchain-openai>=1.7,<2", + "langchain-anthropic>=1.7.10,<2", "langsmith>=0.14.17,<0.15", "pydantic>=2,<3", "polars>=2.1,<3", diff --git a/scripts/run_improvement_cycle.py b/scripts/run_improvement_cycle.py index ebe28be..1d848b0 100644 --- a/scripts/run_improvement_cycle.py +++ b/scripts/run_improvement_cycle.py @@ -17,6 +17,48 @@ from pathlib import Path RUNTIME_FILES = ("src/backintel/agent.py", "src/backintel/tools.py", "src/backintel/contracts.py") +MODEL_ROUTING = "first_party_only" + + +def accounting_policy(model: str, judge_model: str) -> dict: + """Freeze one accounting basis for every model role, before any calls.""" + from backintel.bench.native_costs import NATIVE_POLICY, NATIVE_RATES, native_accounting_version + + if all(m.startswith("openrouter/") for m in (model, judge_model)): + return {"basis": "provider_reported", "version": None} + if all(m in NATIVE_RATES for m in (model, judge_model)): + return {"basis": "native_estimate", "version": native_accounting_version(), "policy": NATIVE_POLICY} + raise ValueError("Campaign requires qualified native models or OpenRouter models in every role") + + +def campaign_budget(plan, limit_usd, max_calls, reservation_usd): + from backintel.bench.usage import NativeRunBudget, RunBudget + + basis = plan.get("accounting", {}).get("basis", "provider_reported") + if basis not in {"provider_reported", "native_estimate"}: + raise ValueError("Unknown campaign accounting basis") + cls = NativeRunBudget if basis == "native_estimate" else RunBudget + return cls(limit_usd, max_calls, reservation_usd) + + +def configure_stage_routing(policy: str) -> None: + """Pin models inside this isolated stage process, leaving application defaults alone.""" + from backintel.bench import improvement, judge, run + from backintel.models import build_model + + if policy != MODEL_ROUTING: + raise ValueError("Prepared model routing must be first_party_only") + + def routed_model(model_id: str, **kwargs): + if model_id.startswith("openrouter/"): + kwargs["first_party_only"] = True + return build_model(model_id, **kwargs) + + # Every model role in the stage uses the same routing, including per-store analyst + # reconstruction. The standalone CLI starts a new process for each stage. + run.build_model = routed_model + judge.build_model = routed_model + improvement.build_model = routed_model def write_new(path: Path, value) -> None: @@ -140,6 +182,7 @@ def prepare(args) -> None: if getattr(args, "scenario_manifest", None): raise ValueError("Explicit scenario manifests are for frozen current-runtime comparisons") settings = Settings.from_env() + accounting = accounting_policy(settings.model, settings.judge_model) scenario_manifest = getattr(args, "scenario_manifest", None) if scenario_manifest: document, scenarios = load_scenario_manifest(scenario_manifest, settings) @@ -182,6 +225,8 @@ def prepare(args) -> None: tuple(sorted(wanted)), trials=args.trials, require_improvement=require_improvement, + cost_basis=accounting["basis"], + accounting_version=accounting["version"], ) write_new(args.output / "policy.json", policy.document()) manifest = { @@ -194,6 +239,8 @@ def prepare(args) -> None: "trials": args.trials, "model": settings.model, "judge_model": settings.judge_model, + "model_routing": MODEL_ROUTING, + "accounting": accounting, "data_dir": str(settings.data_dir), "worker_files": worker_files, "brief_hash": brief_hash, @@ -221,6 +268,8 @@ def load_plan(output: Path) -> dict: raise ValueError("Prepared manifest or runner changed") if sha(Path(improvement.__file__)) != plan["gate_hash"]: raise ValueError("Frozen comparison implementation changed") + if plan.get("accounting") != accounting_policy(plan["model"], plan["judge_model"]): + raise ValueError("Frozen accounting policy changed") candidate_source = plan.get("candidate_source", "worker") if candidate_source not in {"worker", "current"}: raise ValueError("Unknown candidate source") @@ -252,10 +301,10 @@ def stage(args) -> None: from backintel.bench.acceptance import build_acceptance, verify_acceptance_data from backintel.bench.improvement import apply_candidate, propose_candidate, read_records from backintel.bench.run import Harness, reference_version, run_langsmith, scorer_version, source_version - from backintel.bench.usage import RunBudget from backintel.settings import Settings, repo_root plan = load_plan(args.output) + configure_stage_routing(plan.get("model_routing")) candidate_source = plan.get("candidate_source", "worker") if args.stage == "worker" and candidate_source != "worker": raise ValueError("Frozen current-runtime comparisons have no tuning worker") @@ -277,7 +326,7 @@ def stage(args) -> None: worker = json.loads((args.output / "worker-result.json").read_text()) if worker["status"] != "proposed" or source_version() != worker["source_version"]: raise ValueError("Candidate differs from the actual worker proposal") - budget = RunBudget(args.max_cost_usd, args.max_calls, args.reservation_usd) + budget = campaign_budget(plan, args.max_cost_usd, args.max_calls, args.reservation_usd) stage_dir = args.output / args.stage / "cycle-results" stage_dir.mkdir(parents=True, exist_ok=False) if args.stage == "worker": @@ -336,6 +385,9 @@ def verify_data(): git_sha=plan["baseline_revision"], baseline_runtime_revision=plan["baseline_revision"], candidate_source=candidate_source, + model_routing=plan["model_routing"], + cost_basis=plan["accounting"]["basis"], + accounting_version=plan["accounting"]["version"], ) report = run_langsmith(harness, scenarios, out_dir=stage_dir, max_concurrency=1) reports.append(report) @@ -354,8 +406,13 @@ def verify_data(): with (stage_dir / "records.jsonl").open("x") as handle: for record in records: handle.write(json.dumps(record, default=str) + "\n") + complete_field = ( + "total_estimate_complete" + if plan["accounting"]["basis"] == "native_estimate" + else "total_cost_complete" + ) valid = len(records) == len(scenarios) * plan["trials"] and all( - r.get("evaluation_status") == "valid" and r["outputs"]["metrics"].get("total_cost_complete") + r.get("evaluation_status") == "valid" and r["outputs"]["metrics"].get(complete_field) for r in records ) report = { @@ -371,18 +428,19 @@ def verify_data(): def execute(args) -> None: from backintel.bench.improvement import ComparisonPolicy, compare_records, read_records - from backintel.bench.usage import RunBudget if args.max_cost_usd is None: raise ValueError("--execute requires an explicitly approved --max-cost-usd") plan = load_plan(args.output) - RunBudget(args.max_cost_usd, args.max_calls, args.reservation_usd) # Validate before any subprocess. + campaign_budget(plan, args.max_cost_usd, args.max_calls, args.reservation_usd) + native = plan.get("accounting", {}).get("basis") == "native_estimate" write_new( args.output / "execution-start.json", { "limit_usd": args.max_cost_usd, "max_calls": args.max_calls, "reservation_usd": args.reservation_usd, + "accounting": plan.get("accounting", {"basis": "provider_reported"}), "note": "operator-invoked paid run; reserved/unknown charges halt continuation", }, ) @@ -420,12 +478,13 @@ def execute(args) -> None: raise RuntimeError(f"{name} stage failed; inspect its retained log before further spending") report = json.loads((args.output / f"{name}-result.json").read_text()) budget = report["budget"] - spent += budget["reported_spent_usd"] + spent += budget["estimated_spent_usd" if native else "reported_spent_usd"] calls += budget["calls"] write_new( args.output / f"allowance-after-{name}.json", { - "reported_spent_usd": spent, + "reported_spent_usd": None if native else spent, + "estimated_spent_usd": spent if native else None, "calls": calls, "remaining_usd": args.max_cost_usd - spent, "stage_budget": budget, @@ -433,7 +492,7 @@ def execute(args) -> None: ) if ( report["status"] not in {"completed", "proposed"} - or budget["unknown_charge"] + or budget["accounting_uncertain" if native else "unknown_charge"] or budget["reservation_breached"] ): raise RuntimeError(f"{name} is incomplete or accounting uncertain; no subsequent stage will run") @@ -442,7 +501,8 @@ def execute(args) -> None: read_records(Path(json.loads((args.output / "candidate-result.json").read_text())["records"])), ComparisonPolicy.from_document(json.loads((args.output / "policy.json").read_text())), ) - result["cycle_reported_cost_usd"] = spent + result["cycle_reported_cost_usd"] = None if native else spent + result["cycle_estimated_cost_usd"] = spent if native else None result["cycle_model_calls"] = calls write_new(args.output / "comparison.json", result) print(json.dumps(result, indent=2)) diff --git a/src/backintel/bench/improvement.py b/src/backintel/bench/improvement.py index d7f5394..eb5a254 100644 --- a/src/backintel/bench/improvement.py +++ b/src/backintel/bench/improvement.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field +from backintel.bench.native_costs import native_accounting_version, valid_amount from backintel.bench.usage import RunBudget, UsageTracker from backintel.models import build_model @@ -50,8 +51,14 @@ class ComparisonPolicy: min_method_score: float | None = None max_candidate_cost_usd: float | None = None max_case_latency_s: float | None = None + cost_basis: str = "provider_reported" + accounting_version: str | None = None def __post_init__(self): + if self.cost_basis not in {"provider_reported", "native_estimate"}: + raise ValueError("Unknown cost accounting basis") + if self.cost_basis == "native_estimate" and self.accounting_version != native_accounting_version(): + raise ValueError("Native comparison must freeze the qualified accounting version") if not self.case_ids or len(set(self.case_ids)) != len(self.case_ids): raise ValueError("Expected case IDs must be nonempty and unique") if not set(self.mandatory_ids) <= set(self.case_ids) or not self.mandatory_ids: @@ -88,6 +95,9 @@ def from_document(cls, document: dict) -> ComparisonPolicy: def compare_records(baseline: list[dict], candidate: list[dict], policy: ComparisonPolicy) -> dict: expected = {(case, trial) for case in policy.case_ids for trial in range(policy.trials)} invalid, regressions, improvements, failures = [], [], [], [] + native = policy.cost_basis == "native_estimate" + cost_field = "total_estimated_cost_usd" if native else "total_reported_cost_usd" + complete_field = "total_estimate_complete" if native else "total_cost_complete" def index(records, label): found = {} @@ -117,14 +127,15 @@ def index(records, label): ): invalid.append(f"{label}/{key}: missing or invalid {field}") metrics = row.get("outputs", {}).get("metrics", {}) - cost = metrics.get("total_reported_cost_usd") - if cost is not None and ( - not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0 - ): + cost = metrics.get(cost_field) + if cost is not None and not valid_amount(cost): invalid.append(f"{label}/{key}: invalid reported cost") - if policy.require_complete_cost and ( - cost is None or not metrics.get("total_cost_complete", False) + if native and ( + metrics.get("accounting_version") != policy.accounting_version + or metrics.get("billing_mode") != "native_api_estimate" ): + invalid.append(f"{label}/{key}: native accounting identity differs from policy") + if policy.require_complete_cost and (cost is None or not metrics.get(complete_field, False)): invalid.append(f"{label}/{key}: analyst/judge accounting is incomplete") if scores.get("fully_supported") == 1 and ( row.get("outputs", {}).get("error") @@ -158,16 +169,22 @@ def index(records, label): def summary(rows): metrics = [r.get("outputs", {}).get("metrics", {}) for r in rows] costs = [m.get("total_reported_cost_usd") for m in metrics] + estimates = [m.get("total_estimated_cost_usd") for m in metrics] latency = [m.get("latency_s") for m in metrics] return { "responses": len(rows), "supported": sum(r.get("scores", {}).get("fully_supported") == 1 for r in rows), - "reported_cost_usd": sum( - c for c in costs if isinstance(c, (int, float)) and math.isfinite(c) and c >= 0 - ), + "reported_cost_usd": sum(c for c in costs if valid_amount(c)) + if any(valid_amount(c) for c in costs) + else None, "cost_complete": bool(rows) and all(m.get("total_cost_complete", False) for m in metrics) and all(isinstance(c, (int, float)) and math.isfinite(c) and c >= 0 for c in costs), + "estimated_cost_usd": sum(c for c in estimates if valid_amount(c)) if native else None, + "estimate_complete": native + and bool(rows) + and all(m.get("total_estimate_complete", False) for m in metrics) + and all(valid_amount(c) for c in estimates), "latency_s": sum( t for t in latency if isinstance(t, (int, float)) and math.isfinite(t) and t >= 0 ), @@ -180,7 +197,9 @@ def summary(rows): old_summary, new_summary = summary(baseline), summary(candidate) if policy.max_candidate_cost_usd is not None and ( - not new_summary["cost_complete"] or new_summary["reported_cost_usd"] > policy.max_candidate_cost_usd + not new_summary["estimate_complete" if native else "cost_complete"] + or new_summary["estimated_cost_usd" if native else "reported_cost_usd"] + > policy.max_candidate_cost_usd ): failures.append({"reason": "candidate cost exceeds policy or is incomplete"}) if policy.require_improvement and not improvements: @@ -190,6 +209,8 @@ def summary(rows): if invalid else ("reject" if regressions or failures else "accept_for_review"), "policy_hash": policy.document()["policy_hash"], + "cost_basis": policy.cost_basis, + "accounting_version": policy.accounting_version, "baseline_records_hash": fingerprint(baseline), "candidate_records_hash": fingerprint(candidate), "baseline": old_summary, @@ -243,7 +264,7 @@ def propose_candidate(files: dict[str, str], failure_brief: str, *, model_id: st {"role": "system", "content": instruction}, {"role": "user", "content": json.dumps({"failure_brief": failure_brief, "editable_files": files})}, ] - tracker = UsageTracker(budget) + tracker = UsageTracker(budget, model_id=model_id) model = build_model(model_id, max_retries=0) try: patch = model.with_structured_output(CandidatePatch).invoke(prompt, config={"callbacks": [tracker]}) diff --git a/src/backintel/bench/native_costs.py b/src/backintel/bench/native_costs.py new file mode 100644 index 0000000..7fc1cf9 --- /dev/null +++ b/src/backintel/bench/native_costs.py @@ -0,0 +1,104 @@ +"""Conservative native text-token accounting, never an invoice or credit balance. + +All input tokens use the highest standard long-context cache-write rate, so caching +discounts and short-context pricing cannot make the allowance undercount. All output +tokens, including reasoning, use the long-context output rate. Frozen 2026-11-13. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from decimal import ROUND_CEILING, Decimal + +NATIVE_RATES = { + "openai/gpt-6-luna": { + "input_per_million": "0.50", # 12h cache write, long context + "output_per_million": "0.75", + "context_limit": 1_050_000, + "output_limit": 128_000, + }, + "anthropic/claude-haiku-5-5": { + "input_per_million": "1.00", # 1h cache write, >100k prompt + "output_per_million": "2.50", + "context_limit": 1_000_000, + "output_limit": 128_000, + }, +} +NATIVE_POLICY = { + "basis": "conservative_native_text_tokens", + "date": "2026-11-13", + "rates": NATIVE_RATES, + "sources": [ + "https://developers.openai.com/api/docs/pricing", + "https://developers.openai.com/api/docs/models/gpt-6-luna", + "https://platform.claude.com/docs/en/models/haiku-5-5/overview", + ], + "scope": "native standard service, global endpoints, text and local tools only; no discounts", + "actual_charge": "unknown until provider billing reconciliation; credits are not zero-cost usage", + "rounding": "round each call upward to USD 0.000000001", +} + + +def native_accounting_version() -> str: + return hashlib.sha256(json.dumps(NATIVE_POLICY, sort_keys=True).encode()).hexdigest()[:16] + + +def _count(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def native_estimate(model_id: str, message) -> float | None: + """Use complete standardized totals; reject unqualified models/usage/service tiers.""" + if model_id not in NATIVE_RATES: + raise ValueError(f"No qualified native accounting for {model_id!r}") + usage = getattr(message, "usage_metadata", None) or {} + meta = getattr(message, "response_metadata", None) or {} + inp, out, total = (usage.get(k) for k in ("input_tokens", "output_tokens", "total_tokens")) + if not all(_count(v) for v in (inp, out, total)) or total != inp + out: + return None + rates = NATIVE_RATES[model_id] + if total > rates["context_limit"] or out > rates["output_limit"]: + return None + if meta.get("service_tier") not in {None, "default", "standard"}: + return None + name = model_id.partition("/")[2] + actual_model = meta.get("model_name", meta.get("model")) + if not isinstance(actual_model, str) or not re.fullmatch( + re.escape(name) + r"(?:-\d{4}-\d{2}-\d{2})?", actual_model + ): + return None + for field, limit in (("input_token_details", inp), ("output_token_details", out)): + for key, value in (usage.get(field) or {}).items(): + if not _count(value) or value > limit or ("audio" in key and value): + return None + # LangChain adds both cache reads and writes to Anthropic's standardized input + # total. Check against the raw response so a missing cache count isn't assumed zero. + if model_id.startswith("anthropic/"): + raw = meta.get("usage") or {} + counts = [ + raw.get(k) for k in ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") + ] + if not all(_count(v) for v in counts) or sum(counts) != inp or raw.get("output_tokens") != out: + return None + if raw.get("service_tier") not in {None, "standard"}: + return None + server_tools = raw.get("server_tool_use") or {} + if any(v not in (None, 0) for v in server_tools.values()): + return None + value = ( + Decimal(inp) * Decimal(rates["input_per_million"]) + + Decimal(out) * Decimal(rates["output_per_million"]) + ) / Decimal(1_000_000) + return float(value.quantize(Decimal("0.000000001"), rounding=ROUND_CEILING)) + + +def valid_amount(value) -> bool: + return ( + not isinstance(value, bool) + and isinstance(value, (int, float)) + and math.isfinite(value) + and value >= 0 + ) diff --git a/src/backintel/bench/run.py b/src/backintel/bench/run.py index 664620e..b398b25 100644 --- a/src/backintel/bench/run.py +++ b/src/backintel/bench/run.py @@ -27,7 +27,7 @@ from backintel.bench.judge import judge_answer, judge_version from backintel.bench.oracle import load_answers from backintel.bench.scoring import abstention_correct, evidence_reproduces, score_correct -from backintel.bench.usage import BudgetExceeded, RunBudget, UsageTracker +from backintel.bench.usage import BudgetExceeded, NativeRunBudget, RunBudget, UsageTracker from backintel.contracts import AnalystAnswer, Scenario from backintel.models import build_model, usage_from_message from backintel.providers.chatgpt_plan import PlanUsageStopped @@ -207,7 +207,7 @@ def answer(self, question: str, scenario: Scenario | None = None) -> dict: "tools": {}, } answer = error = failure_class = None - usage = UsageTracker(self.budget) + usage = UsageTracker(self.budget, model_id=self.model_id) result = {} try: for state in agent.stream( @@ -233,7 +233,7 @@ def answer(self, question: str, scenario: Scenario | None = None) -> dict: if isinstance(exc, BudgetExceeded) else "infrastructure" if isinstance(exc, (TimeoutError, ConnectionError, PlanUsageStopped)) - or type(exc).__module__.startswith(("openai", "httpx", "requests")) + or type(exc).__module__.startswith(("openai", "anthropic", "httpx", "requests")) or _provider_error_envelope(exc) else "agent" ) @@ -264,6 +264,9 @@ def answer(self, question: str, scenario: Scenario | None = None) -> dict: ) if failure_class == "infrastructure" and not metrics["llm_calls"]: metrics.update(usage_complete=False, cost_complete=False, cost_usd=None) + if isinstance(self.budget, NativeRunBudget) and not metrics["usage_complete"]: + metrics["estimate_complete"] = False + self.budget.uncertain = True metrics["latency_s"] = round(time.monotonic() - started, 2) return { "answer": answer.model_dump() if answer else None, @@ -302,7 +305,7 @@ def score(self, scenario: Scenario, outputs: dict) -> dict[str, dict]: "abstention_correct": abstention_correct(scenario, answer), } if self.judge: - judge_usage = UsageTracker(self.budget) + judge_usage = UsageTracker(self.budget, model_id=self.settings.judge_model) judge_started = time.monotonic() try: judged = judge_answer( @@ -346,7 +349,7 @@ def record(self, scenario: Scenario, outputs: dict, scores: dict) -> dict: outputs = {**outputs, "metrics": dict(outputs.get("metrics") or {})} judge_usage = scores.get("judge_method", {}).get("metrics") if not self.judge: - judge_usage = UsageTracker().report() + judge_usage = UsageTracker(self.budget, model_id=self.model_id).report() if judge_usage is not None: outputs["metrics"]["judge"] = judge_usage analyst = outputs["metrics"] @@ -355,6 +358,15 @@ def record(self, scenario: Scenario, outputs: dict, scores: dict) -> dict: outputs["metrics"]["total_cost_complete"] = bool( analyst.get("cost_complete") and judge_usage.get("cost_complete") ) + if analyst.get("billing_mode") == "native_api_estimate": + outputs["metrics"]["total_estimated_cost_usd"] = analyst.get( + "estimated_cost_usd", 0 + ) + judge_usage.get("estimated_cost_usd", 0) + outputs["metrics"]["total_estimate_complete"] = bool( + analyst.get("estimate_complete") + and judge_usage.get("estimate_complete") + and analyst.get("accounting_version") == judge_usage.get("accounting_version") + ) judge_invalid = self.judge and scores.get("judge_method", {}).get("score") is None if judge_invalid: outputs["evaluation_error"] = scores.get("judge_method", {}).get( diff --git a/src/backintel/bench/usage.py b/src/backintel/bench/usage.py index 2fd4caf..531b3e2 100644 --- a/src/backintel/bench/usage.py +++ b/src/backintel/bench/usage.py @@ -14,6 +14,12 @@ from langchain_core.callbacks import BaseCallbackHandler +from backintel.bench.native_costs import ( + NATIVE_RATES, + native_accounting_version, + native_estimate, + valid_amount, +) from backintel.models import message_token_usage, normalize_token_usage @@ -125,14 +131,36 @@ def report(self) -> dict: } +class NativeRunBudget(RunBudget): + """Reuse call/reservation admission with conservative estimates, not invoice costs.""" + + def report(self) -> dict: + result = super().report() + result.pop("reported_spent_usd") + result.pop("unknown_charge") + return { + **result, + "billing_mode": "native_api_estimate", + "accounting_version": native_accounting_version(), + "estimated_spent_usd": self.spent_usd, + "reported_spent_usd": None, + "unknown_charge": True, + "accounting_uncertain": self.uncertain, + "scope": "conservative token-cost admission; not provider invoice or hard billing cap", + } + + class UsageTracker(BaseCallbackHandler): """Use one tracker per answer/judge call; optional shared budget spans the experiment.""" raise_error = True run_inline = True - def __init__(self, budget: RunBudget | PlanRunBudget | None = None): + def __init__(self, budget: RunBudget | PlanRunBudget | None = None, *, model_id: str | None = None): self.budget = budget + self.model_id = model_id + if isinstance(budget, NativeRunBudget) and model_id not in NATIVE_RATES: + raise ValueError(f"No qualified native accounting for {model_id!r}") self.started: set[UUID] = set() self.finished: dict[UUID, dict | None] = {} self._lock = Lock() @@ -147,9 +175,11 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): def on_llm_end(self, response, *, run_id, **kwargs): value = normalize_token_usage((response.llm_output or {}).get("token_usage")) + native_message = None for generations in response.generations: for generation in generations: message = getattr(generation, "message", None) + native_message = message reported = message_token_usage(message) if reported: # LLM output may be empty for Responses or omit standard token counts. @@ -158,6 +188,10 @@ def on_llm_end(self, response, *, run_id, **kwargs): break if value: break + if isinstance(self.budget, NativeRunBudget): + single = len(response.generations) == 1 and len(response.generations[0]) == 1 + value["estimated_cost_usd"] = native_estimate(self.model_id, native_message) if single else None + value.pop("cost", None) # The native response is not an invoice. self._finish(run_id, value or None) def on_llm_error(self, error, *, run_id, **kwargs): @@ -170,9 +204,11 @@ def _finish(self, run_id, value): return self.finished[run_id] = value if self.budget and run_id in self.started: - self.budget.settle( - value if isinstance(self.budget, PlanRunBudget) else value.get("cost") if value else None - ) + if isinstance(self.budget, PlanRunBudget): + self.budget.settle(value) + else: + field = "estimated_cost_usd" if isinstance(self.budget, NativeRunBudget) else "cost" + self.budget.settle(value.get(field) if value else None) def report(self) -> dict: values = [v for v in self.finished.values() if v is not None] @@ -205,4 +241,20 @@ def report(self) -> dict: cost_applicable=False, scope="ChatGPT subscription token usage; no per-request dollar charge reported", ) + if isinstance(self.budget, NativeRunBudget): + estimates = [v.get("estimated_cost_usd") for v in values] + complete = ( + not incomplete + and len(estimates) == report["llm_calls"] + and all(valid_amount(v) for v in estimates) + ) + report.update( + billing_mode="native_api_estimate", + accounting_version=native_accounting_version(), + estimated_cost_usd=sum(v for v in estimates if valid_amount(v)), + estimate_complete=complete, + cost_usd=None, + cost_complete=False, + scope="conservative native token costs; actual provider charges remain unknown", + ) return report diff --git a/src/backintel/models.py b/src/backintel/models.py index e5966a6..ae72478 100644 --- a/src/backintel/models.py +++ b/src/backintel/models.py @@ -4,6 +4,7 @@ Google models are allowed, per the technology-selection policy of 2026-10-30. - `chatgpt-plan/`: the operator's ChatGPT plan after `backintel chatgpt login`. - `openai/`: OpenAI directly with OPENAI_API_KEY. +- `anthropic/`: Anthropic directly with ANTHROPIC_API_KEY. """ from __future__ import annotations @@ -11,11 +12,14 @@ import math import os +from langchain_anthropic import ChatAnthropic from langchain_core.language_models import BaseChatModel from langchain_openai import ChatOpenAI from langchain_openai.chat_models.base import _construct_lc_result_from_responses_api, _is_pydantic_class OPENROUTER_URL = "https://openrouter.ai/api/v1" +OPENAI_URL = "https://api.openai.com/v1" +ANTHROPIC_URL = "https://api.anthropic.com" ALLOWED_OPENROUTER_VENDORS = ("openai", "anthropic", "google") @@ -108,7 +112,7 @@ async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs): def parse_model_id(model_id: str) -> tuple[str, str]: provider, _, name = model_id.partition("/") if not name: - raise ValueError("Model ids look like '/', e.g. 'openrouter/openai/gpt-6-luna'.") + raise ValueError("Model ids look like '/', e.g. 'openai/gpt-6-luna'.") return provider, name @@ -186,9 +190,47 @@ def build_model( if provider == "openai": key = env.get("OPENAI_API_KEY") if not key: - raise RuntimeError("OPENAI_API_KEY is not set.") - return ChatOpenAI(model=name, api_key=key, max_retries=max_retries, timeout=timeout_s, **transport) - raise ValueError(f"Unknown provider {provider!r}; use openrouter, chatgpt-plan or openai.") + raise RuntimeError("OPENAI_API_KEY is not set (run under Doppler).") + return ChatOpenAI( + model=name, + api_key=key, + base_url=OPENAI_URL, + organization=env.get("OPENAI_ORG_ID"), + default_headers=( + {"OpenAI-Project": env["OPENAI_PROJECT_ID"]} if env.get("OPENAI_PROJECT_ID") else {} + ), + max_retries=max_retries, + max_tokens=int(env.get("BACKINTEL_MAX_COMPLETION_TOKENS", "8192")), + timeout=timeout_s, + service_tier="default", + **({"temperature": temperature} if temperature is not None else {}), + **transport, + ) + if provider == "anthropic": + if responses_api: + raise ValueError("Anthropic uses its native Messages API, not the OpenAI Responses API.") + if reasoning_effort not in {None, "low", "medium", "high", "xhigh"}: + raise ValueError("Anthropic reasoning effort must be low, medium, high or xhigh.") + key = env.get("ANTHROPIC_API_KEY") + if not key: + raise RuntimeError("ANTHROPIC_API_KEY is not set (run under Doppler).") + headers = {} + if workspace := env.get("ANTHROPIC_WORKSPACE_ID"): + headers["anthropic-workspace-id"] = workspace + return ChatAnthropic( + model=name, + api_key=key, + base_url=ANTHROPIC_URL, + max_retries=max_retries, + max_tokens=int(env.get("BACKINTEL_MAX_COMPLETION_TOKENS", "8192")), + timeout=timeout_s, + default_headers=headers, + disable_streaming=True, + model_kwargs={"service_tier": "standard_only"}, + **({"temperature": temperature} if temperature is not None else {}), + **({"effort": reasoning_effort} if reasoning_effort is not None else {}), + ) + raise ValueError(f"Unknown provider {provider!r}; use openai, anthropic, chatgpt-plan or openrouter.") def usage_from_message(message) -> dict: diff --git a/src/backintel/settings.py b/src/backintel/settings.py index 3b42112..7a765f5 100644 --- a/src/backintel/settings.py +++ b/src/backintel/settings.py @@ -6,8 +6,8 @@ from dataclasses import dataclass, field from pathlib import Path -DEFAULT_MODEL = "openrouter/openai/gpt-6-luna" -DEFAULT_JUDGE_MODEL = "openrouter/anthropic/claude-haiku-5.5" +DEFAULT_MODEL = "openai/gpt-6-luna" +DEFAULT_JUDGE_MODEL = "anthropic/claude-haiku-5-5" DEFAULT_ANALYSIS_MODEL = "chatgpt-plan/gpt-6.1-sol" diff --git a/tests/test_campaign_routing.py b/tests/test_campaign_routing.py new file mode 100644 index 0000000..6ffcc19 --- /dev/null +++ b/tests/test_campaign_routing.py @@ -0,0 +1,46 @@ +"""The frozen comparison uses first-party routing for every OpenRouter role.""" + +import importlib.util +from pathlib import Path + +import pytest + +from backintel.bench import improvement, judge, run +from backintel.models import build_model + + +def runner(): + path = Path(__file__).parents[1] / "scripts/run_improvement_cycle.py" + spec = importlib.util.spec_from_file_location("campaign_routing_runner", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_stage_pins_analyst_judge_and_worker_without_changing_factory(monkeypatch): + monkeypatch.setenv("OPENROUTER_API_KEY", "offline-test-no-requests") + for module in (run, judge, improvement): + monkeypatch.setattr(module, "build_model", module.build_model) + cycle = runner() + cycle.configure_stage_routing(cycle.MODEL_ROUTING) + for module, model_id, vendor in ( + (run, "openrouter/openai/gpt-6-luna", "openai"), + (judge, "openrouter/anthropic/claude-haiku-5.5", "anthropic"), + (improvement, "openrouter/openai/gpt-6-luna", "openai"), + ): + model = module.build_model(model_id, max_retries=0) + assert model.extra_body["provider"] == { + "only": [vendor], + "allow_fallbacks": False, + "require_parameters": True, + } + assert model.max_retries == 0 + ordinary = build_model("openrouter/openai/gpt-6-luna", max_retries=0) + assert "provider" not in ordinary.extra_body + + +def test_stage_rejects_missing_or_unknown_routing_before_model_construction(): + cycle = runner() + for policy in (None, "fallback_allowed"): + with pytest.raises(ValueError, match="first_party_only"): + cycle.configure_stage_routing(policy) diff --git a/tests/test_improvement_cycle.py b/tests/test_improvement_cycle.py index 5b07a85..ba6b8a7 100644 --- a/tests/test_improvement_cycle.py +++ b/tests/test_improvement_cycle.py @@ -10,6 +10,7 @@ import pytest from backintel.bench.improvement import ComparisonPolicy, fingerprint +from backintel.bench.native_costs import native_accounting_version from backintel.bench.run import reference_version, scorer_version, source_version from backintel.contracts import Scenario @@ -147,13 +148,17 @@ def test_manifest_binds_scenarios_to_exact_store_bytes_and_reference_hash(tmp_pa cycle.load_scenario_manifest(manifest, settings) -def test_current_mode_invokes_only_fresh_baseline_candidate_and_shares_allowance(tmp_path, monkeypatch): +@pytest.mark.parametrize("native", [False, True]) +def test_current_mode_invokes_only_fresh_baseline_candidate_and_shares_allowance( + tmp_path, monkeypatch, native +): cycle = runner() plan = { "candidate_source": "current", "baseline": str(tmp_path / "baseline"), "candidate": str(tmp_path / "candidate"), "data_dir": str(tmp_path / "data"), + "accounting": {"basis": "native_estimate" if native else "provider_reported"}, } identity = dict( dataset="frozen", @@ -163,7 +168,16 @@ def test_current_mode_invokes_only_fresh_baseline_candidate_and_shares_allowance judge_model="judge", judge_version="rubric1", ) - cycle.write_new(tmp_path / "policy.json", ComparisonPolicy(identity, ("case",), ("case",)).document()) + cycle.write_new( + tmp_path / "policy.json", + ComparisonPolicy( + identity, + ("case",), + ("case",), + cost_basis=plan["accounting"]["basis"], + accounting_version=native_accounting_version() if native else None, + ).document(), + ) monkeypatch.setattr(cycle, "load_plan", lambda output: plan) invoked = [] @@ -188,8 +202,12 @@ def subprocess_run(command, **kwargs): "outputs": { "answer": {"status": "answered"}, "metrics": { - "total_reported_cost_usd": 0.02, - "total_cost_complete": True, + "total_reported_cost_usd": None if native else 0.02, + "total_cost_complete": not native, + "total_estimated_cost_usd": 0.02 if native else None, + "total_estimate_complete": native, + "billing_mode": "native_api_estimate" if native else "api", + "accounting_version": native_accounting_version() if native else None, "latency_s": 1, "llm_calls": 2, }, @@ -202,9 +220,11 @@ def subprocess_run(command, **kwargs): "status": "completed", "records": str(records), "budget": { - "reported_spent_usd": 0.02, + "reported_spent_usd": None if native else 0.02, + "estimated_spent_usd": 0.02 if native else None, "calls": 2, - "unknown_charge": False, + "unknown_charge": native, + "accounting_uncertain": False, "reservation_breached": False, }, }, @@ -216,6 +236,22 @@ def subprocess_run(command, **kwargs): assert invoked == [("baseline", 0.1), ("candidate", 0.08)] result = json.loads((tmp_path / "comparison.json").read_text()) assert result["verdict"] == "accept_for_review" - assert result["cycle_reported_cost_usd"] == 0.04 + assert result["cycle_reported_cost_usd"] == (None if native else 0.04) + assert result["cycle_estimated_cost_usd"] == (0.04 if native else None) assert result["cycle_model_calls"] == 4 assert not (tmp_path / "worker.log").exists() + + +def test_campaign_accounting_rejects_mixed_or_unqualified_models_before_calls(): + cycle = runner() + from backintel.bench.usage import NativeRunBudget + + policy = cycle.accounting_policy("openai/gpt-6-luna", "anthropic/claude-haiku-5-5") + assert policy["version"] == native_accounting_version() + assert isinstance(cycle.campaign_budget({"accounting": policy}, 0.1, 2, 0.01), NativeRunBudget) + for model, judge in ( + ("openai/gpt-6-luna", "openrouter/anthropic/claude-haiku-5.5"), + ("openai/unqualified", "anthropic/claude-haiku-5-5"), + ): + with pytest.raises(ValueError, match="qualified native"): + cycle.accounting_policy(model, judge) diff --git a/tests/test_native_accounting.py b/tests/test_native_accounting.py new file mode 100644 index 0000000..389088d --- /dev/null +++ b/tests/test_native_accounting.py @@ -0,0 +1,188 @@ +"""Native allowance estimates stay separate from invoices and fail closed.""" + +from copy import deepcopy +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, LLMResult + +from backintel.bench.improvement import ComparisonPolicy, compare_records +from backintel.bench.native_costs import native_accounting_version, native_estimate +from backintel.bench.run import Harness +from backintel.bench.usage import BudgetExceeded, NativeRunBudget, UsageTracker +from backintel.contracts import Scenario + +MODEL = "openai/gpt-6-luna" + + +def message(): + return SimpleNamespace( + usage_metadata={ + "input_tokens": 100, + "output_tokens": 20, + "total_tokens": 120, + "input_token_details": {"cache_read": 50}, + "output_token_details": {"reasoning": 15}, + }, + response_metadata={"model_name": "gpt-6-luna-2026-11-01", "service_tier": "default"}, + ) + + +def finish(tracker, value=None): + value = value or message() + run_id = uuid4() + tracker.on_chat_model_start({}, [], run_id=run_id) + tracker.on_llm_end( + LLMResult(generations=[[ChatGeneration(message=AIMessage(content="ok", **vars(value)))]]), + run_id=run_id, + ) + + +def test_estimate_counts_cache_and_reasoning_once_and_never_claims_invoice(): + budget = NativeRunBudget(0.1, 3, 0.01) + tracker = UsageTracker(budget, model_id=MODEL) + finish(tracker) + finish(tracker) + assert tracker.report()["estimated_cost_usd"] == pytest.approx(0.00013) + assert tracker.report()["estimate_complete"] + assert tracker.report()["cost_usd"] is None + assert not tracker.report()["cost_complete"] + assert budget.report()["estimated_spent_usd"] == pytest.approx(0.00013) + assert budget.report()["reported_spent_usd"] is None + assert budget.report()["unknown_charge"] + assert not budget.report()["accounting_uncertain"] + assert budget.can_admit() + + +@pytest.mark.parametrize( + "section,field,value", + [ + ("usage_metadata", "input_tokens", None), + ("usage_metadata", "input_tokens", True), + ("usage_metadata", "output_tokens", -1), + ("usage_metadata", "total_tokens", 121), + ("usage_metadata", "input_token_details", {"audio": 1}), + ("usage_metadata", "output_token_details", {"reasoning": 21}), + ("response_metadata", "model_name", "gpt-6-luna-unqualified"), + ("response_metadata", "model_name", "gpt-6-sol"), + ("response_metadata", "service_tier", "priority"), + ], +) +def test_unqualified_usage_cannot_be_estimated(section, field, value): + response = message() + getattr(response, section)[field] = value + assert native_estimate(MODEL, response) is None + + +def test_native_accounting_stops_on_missing_usage_error_or_oversized_response(): + with pytest.raises(ValueError, match="No qualified"): + UsageTracker(NativeRunBudget(0.1, 3, 0.01), model_id="openai/unqualified") + for kind in ("missing", "error", "too_large"): + budget = NativeRunBudget(0.1, 3, 0.01) + tracker = UsageTracker(budget, model_id=MODEL) + if kind == "error": + run_id = uuid4() + tracker.on_chat_model_start({}, [], run_id=run_id) + tracker.on_llm_error(RuntimeError("HTTP request failed"), run_id=run_id) + else: + value = message() + if kind == "missing": + value.response_metadata = {} + else: + value.usage_metadata.update(input_tokens=1_050_000, total_tokens=1_050_020) + finish(tracker, value) + assert not tracker.report()["estimate_complete"] + assert budget.report()["accounting_uncertain"] + assert budget.report()["reserved_usd"] == 0.01 + with pytest.raises(BudgetExceeded): + tracker.on_chat_model_start({}, [], run_id=uuid4()) + + +def test_anthropic_requires_complete_raw_cache_counts_and_rejects_server_tool_fees(): + response = SimpleNamespace( + usage_metadata={"input_tokens": 120, "output_tokens": 20, "total_tokens": 140}, + response_metadata={ + "model": "claude-haiku-5-5", + "usage": { + "input_tokens": 100, + "output_tokens": 20, + "cache_read_input_tokens": 10, + "cache_creation_input_tokens": 10, + }, + }, + ) + assert native_estimate("anthropic/claude-haiku-5-5", response) == pytest.approx(0.00017) + for usage in ( + {"input_tokens": 120, "output_tokens": 20}, + {**response.response_metadata["usage"], "cache_creation_input_tokens": 11}, + {**response.response_metadata["usage"], "server_tool_use": {"web_search_requests": 1}}, + ): + changed = deepcopy(response) + changed.response_metadata["usage"] = usage + assert native_estimate("anthropic/claude-haiku-5-5", changed) is None + + +def test_native_record_and_comparison_require_the_frozen_estimate_basis(): + budget = NativeRunBudget(0.1, 3, 0.01) + analyst = UsageTracker(budget, model_id=MODEL) + judge = UsageTracker(budget, model_id=MODEL) + finish(analyst) + finish(judge) + identity = dict( + dataset="test", + data_version="data", + reference_version="ref", + scorer_version="scorer", + judge_model=MODEL, + judge_version="judge", + ) + harness = Harness(SimpleNamespace(), None, None, MODEL, None, {}, judge=True, run_metadata=identity) + scenario = Scenario(id="case", dataset="test", level=1, question="q", answer_type="number", expected=1) + scores = { + name: {"score": 1, "comment": "fixture"} + for name in ("correct", "fully_supported", "abstention_correct", "evidence_reproduces") + } + scores["judge_method"] = {"score": 1, "comment": "fixture", "metrics": judge.report()} + row = harness.record(scenario, {"metrics": analyst.report()}, scores) + metrics = row["outputs"]["metrics"] + assert metrics["total_estimated_cost_usd"] == pytest.approx(0.00013) + assert metrics["total_estimate_complete"] + assert metrics["total_reported_cost_usd"] is None + assert not metrics["total_cost_complete"] + policy = ComparisonPolicy( + identity, + ("case",), + ("case",), + require_improvement=False, + cost_basis="native_estimate", + accounting_version=native_accounting_version(), + max_candidate_cost_usd=0.01, + ) + result = compare_records([row], [row], policy) + assert result["verdict"] == "accept_for_review" + assert result["candidate"]["reported_cost_usd"] is None + assert not result["candidate"]["cost_complete"] + for key, value in ( + ("accounting_version", "different"), + ("total_estimate_complete", False), + ("total_estimated_cost_usd", None), + ): + invalid = deepcopy(row) + invalid["outputs"]["metrics"][key] = value + assert compare_records([row], [invalid], policy)["verdict"] == "invalid_comparison" + + +def test_native_message_fallback_cannot_report_complete_zero_cost(): + class MissingCallbacks: + def stream(self, *args, **kwargs): + yield {"messages": [AIMessage(content="untracked", **vars(message()))]} + + budget = NativeRunBudget(0.1, 3, 0.01) + harness = Harness( + SimpleNamespace(), None, None, MODEL, MissingCallbacks(), {}, judge=False, budget=budget + ) + outputs = harness.answer("q") + assert not outputs["metrics"]["estimate_complete"] + assert not budget.can_admit() diff --git a/tests/test_native_providers.py b/tests/test_native_providers.py new file mode 100644 index 0000000..c055264 --- /dev/null +++ b/tests/test_native_providers.py @@ -0,0 +1,191 @@ +"""Native provider contracts; all HTTP is intercepted, with no account calls.""" + +import json + +import anthropic +import httpx +import httpx2 +import pytest +from langsmith import tracing_context +from openai import OpenAI + +from backintel.bench.usage import BudgetExceeded, NativeRunBudget, RunBudget, UsageTracker +from backintel.models import build_model +from backintel.settings import Settings + + +def test_native_defaults_and_explicit_override(): + settings = Settings.from_env({}) + assert settings.model == "openai/gpt-6-luna" + assert settings.judge_model == "anthropic/claude-haiku-5-5" + assert settings.analysis_model == "chatgpt-plan/gpt-6.1-sol" + explicit = Settings.from_env({"BACKINTEL_MODEL": "openrouter/openai/gpt-6-luna"}) + assert explicit.model == "openrouter/openai/gpt-6-luna" + + +@pytest.mark.parametrize("provider,key", [("openai", "OPENAI_API_KEY"), ("anthropic", "ANTHROPIC_API_KEY")]) +def test_native_credentials_do_not_fall_back_to_openrouter(provider, key): + with pytest.raises(RuntimeError, match=key): + build_model(f"{provider}/test-not-invoked", env={"OPENROUTER_API_KEY": "test"}) + + +@pytest.mark.parametrize("native_accounting", [False, True]) +def test_openai_native_endpoint_limits_and_usage(monkeypatch, native_accounting): + monkeypatch.setenv("OPENAI_BASE_URL", "https://unwanted.example/v1") + model = build_model( + "openai/gpt-6-luna", + env={ + "OPENAI_API_KEY": "test", + "BACKINTEL_MAX_COMPLETION_TOKENS": "256", + "OPENAI_PROJECT_ID": "test-project", + }, + first_party_only=True, + max_retries=0, + timeout_s=30, + ) + assert str(model.root_client.base_url) == "https://api.openai.com/v1/" + assert model.max_retries == 0 + assert model.default_headers["OpenAI-Project"] == "test-project" + requests = [] + + def handle(request): + assert request.url.host == "api.openai.com" + assert request.url.path == "/v1/chat/completions" + payload = json.loads(request.content) + requests.append(payload) + assert payload["max_completion_tokens"] == 256 + assert payload["service_tier"] == "default" + assert "provider" not in payload + return httpx.Response( + 200, + json={ + "id": "chatcmpl-native-test", + "object": "chat.completion", + "created": 1, + "model": "gpt-6-luna", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "OK"}, "finish_reason": "stop"} + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 2, + "total_tokens": 12, + "prompt_tokens_details": {"cached_tokens": 5}, + "completion_tokens_details": {"reasoning_tokens": 1}, + }, + }, + ) + + budget_type = NativeRunBudget if native_accounting else RunBudget + tracker = UsageTracker(budget_type(0.05, 2, 0.01), model_id="openai/gpt-6-luna") + with ( + OpenAI( + api_key="test", + http_client=httpx.Client(transport=httpx.MockTransport(handle)), + base_url="https://api.openai.com/v1", + ) as client, + tracing_context(enabled=False), + ): + monkeypatch.setattr(model, "client", client.chat.completions) + assert model.invoke("Check native routing", config={"callbacks": [tracker]}).content == "OK" + if not native_accounting: + with pytest.raises(BudgetExceeded): + model.invoke("Do not send", config={"callbacks": [tracker]}) + assert len(requests) == 1 + assert tracker.report()["input_tokens"] == 10 + assert tracker.report()["usage_complete"] + assert tracker.report()["cost_usd"] is None + assert not tracker.report()["cost_complete"] + assert tracker.budget.can_admit() is native_accounting + if native_accounting: + assert tracker.report()["estimate_complete"] + assert tracker.report()["estimated_cost_usd"] == pytest.approx(0.0000065) + + +@pytest.mark.parametrize("native_accounting", [False, True]) +def test_anthropic_native_structured_output_and_usage(monkeypatch, native_accounting): + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://unwanted.example") + model = build_model( + "anthropic/claude-haiku-5-5", + env={ + "ANTHROPIC_API_KEY": "test", + "BACKINTEL_MAX_COMPLETION_TOKENS": "256", + "ANTHROPIC_WORKSPACE_ID": "test-workspace", + }, + max_retries=0, + timeout_s=30, + first_party_only=True, + ) + assert str(model._client.base_url) == "https://api.anthropic.com" + assert model.max_retries == 0 + assert model.default_headers["anthropic-workspace-id"] == "test-workspace" + schema = { + "title": "test_rubric", + "description": "Test grading", + "type": "object", + "properties": {"rating": {"type": "integer"}}, + "required": ["rating"], + } + requests = [] + + def handle(request): + assert request.url.host == "api.anthropic.com" + assert request.url.path == "/v1/messages" + payload = json.loads(request.content) + requests.append(payload) + assert payload["max_tokens"] == 256 + assert payload["service_tier"] == "standard_only" + assert payload["model"] == "claude-haiku-5-5" + assert payload["tools"][0]["name"] == "test_rubric" + assert "provider" not in payload + return httpx2.Response( + 200, + json={ + "id": "msg_native_test", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "stop_reason": "tool_use", + "stop_sequence": None, + "content": [ + {"type": "tool_use", "id": "toolu_test", "name": "test_rubric", "input": {"rating": 2}} + ], + "usage": { + "input_tokens": 20, + "output_tokens": 8, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 5, + }, + }, + ) + + budget_type = NativeRunBudget if native_accounting else RunBudget + tracker = UsageTracker(budget_type(0.05, 2, 0.01), model_id="anthropic/claude-haiku-5-5") + with ( + anthropic.Anthropic( + api_key="test", + base_url="https://api.anthropic.com", + http_client=httpx2.Client(transport=httpx2.MockTransport(handle)), + ) as client, + tracing_context(enabled=False), + ): + monkeypatch.setattr(model, "_client", client) + result = model.with_structured_output(schema).invoke("Grade this", config={"callbacks": [tracker]}) + assert result == {"rating": 2} + assert len(requests) == 1 + assert tracker.report()["input_tokens"] == 35 + assert tracker.report()["output_tokens"] == 8 + assert tracker.report()["usage_complete"] + assert tracker.report()["cost_usd"] is None + assert not tracker.report()["cost_complete"] + assert tracker.budget.can_admit() is native_accounting + if native_accounting: + assert tracker.report()["estimate_complete"] + assert tracker.report()["estimated_cost_usd"] == pytest.approx(0.000055) + + +def test_anthropic_does_not_silently_ignore_openai_options(): + with pytest.raises(ValueError, match="Messages API"): + build_model("anthropic/claude-haiku-5-5", responses_api=True) + with pytest.raises(ValueError, match="reasoning effort"): + build_model("anthropic/claude-haiku-5-5", reasoning_effort="minimal") diff --git a/uv.lock b/uv.lock index f7f604d..2da3085 100644 --- a/uv.lock +++ b/uv.lock @@ -53,6 +53,7 @@ dependencies = [ { name = "duckdb" }, { name = "httpx" }, { name = "langchain" }, + { name = "langchain-anthropic" }, { name = "langchain-openai" }, { name = "langsmith" }, { name = "openpyxl" }, @@ -78,6 +79,7 @@ requires-dist = [ { name = "duckdb", specifier = ">=2.0.0" }, { name = "httpx", specifier = ">=0.28,<1" }, { name = "langchain", specifier = ">=1.4,<2" }, + { name = "langchain-anthropic", specifier = ">=1.7.10,<2" }, { name = "langchain-openai", specifier = ">=1.7,<2" }, { name = "langsmith", specifier = ">=0.14.17,<0.15" }, { name = "openpyxl", specifier = ">=3.1,<4" },