diff --git a/docs/skill-eval-yibuapi-preflight.md b/docs/skill-eval-yibuapi-preflight.md index a40ede36..fa69f72c 100644 --- a/docs/skill-eval-yibuapi-preflight.md +++ b/docs/skill-eval-yibuapi-preflight.md @@ -8,8 +8,8 @@ provider configuration is supplied. No external agent executable is required. ## Configuration and cost reservation Supply public JSON as the workflow_dispatch provider_config input, unchanged -between planning, execution and any retained-publication retry. It has exactly -four top-level fields: +between planning, execution and any retained-publication retry. It has four +required top-level fields and one narrowly scoped optional override: | Field | Required content | | --- | --- | @@ -17,6 +17,7 @@ four top-level fields: | client | kind, provider, endpoint, api_version and all five integer bounds below | | pricing | exact model/group/currency/units, all effective rates, fees, full context bound, evidence and expiry | | spend_limit | currency and approved finite amount as a decimal string | +| limits (optional) | exactly token_stop_new_trials as a finite positive integer | The client identity is kind=anthropic-messages-agent-v1, provider=yibuapi, endpoint=https://yibuapi.com/v1/messages and api_version=2023-06-01. Its required @@ -27,6 +28,20 @@ every attempted Messages request in a trial, including failed/uncertain requests No additional compatibility probe is sent: the first baseline request is the compatibility observation. +For a paid pair, optional limits.token_stop_new_trials overrides only that +mapping limit. Other limit overrides, zero/negative values, booleans and strings +are rejected. Omitting limits preserves the existing mapping and Codex defaults. +The effective limit is frozen in each comparison, the PR plan, and the complete +Budget JSON; changing it requires a new exact plan and approval. Monetary +reservation, request/tool/read/output/deadline bounds and unknown-usage stops +remain unchanged. + +Token usage is cumulative across both arms and checked before every new trial. +If a completed baseline reaches the threshold, candidate is not started. Choose +explicit finite headroom appropriate to the whole authorized pair; it is neither +a per-arm limit nor a hard provider token/money cap. Do not reuse an older plan +or retry a consumed paid run after changing headroom. + pricing requires currency (USD or CNY), unit=per_million_tokens, model, group, rates, request_fee, max_input_tokens, rate_evidence and expires_at. rates contains explicit nonnegative decimal strings for input, output, cache_read and cache_write. diff --git a/tests/test_yibuapi_runtime.py b/tests/test_yibuapi_runtime.py index 7c8272c8..fff90c88 100644 --- a/tests/test_yibuapi_runtime.py +++ b/tests/test_yibuapi_runtime.py @@ -111,7 +111,7 @@ def transport(payload, spec, key, timeout): assert len(calls) == before -def install_fake_worker(monkeypatch): +def install_fake_worker(monkeypatch, *, usage=None): sent = [] def environment(self): @@ -127,9 +127,11 @@ def execute(command, cwd, folder, deadline, **options): def transport(payload, spec, key, timeout): sent.append(copy.deepcopy(payload)) assert key == FAKE_KEY - if len(payload["messages"]) == 1: - return read() - return submit(DATA["cases"][0]["answer"]) + result = (read() if len(payload["messages"]) == 1 + else submit(DATA["cases"][0]["answer"])) + if usage is not None: + result["usage"] = copy.deepcopy(usage) + return result status = agent.main(command[3:], environment=options["environment"], transport=transport, emit=events.append) @@ -262,3 +264,113 @@ def send(payload, spec, timeout): ]) def test_malformed_thinking_blocks_fail_before_replay(block): assert not agent.valid_content_block(block) + + +@pytest.mark.parametrize("headroom,expected_trials", [ + (None, 1), (20000, 1), (30200, 1), (100000, 2), +]) +def test_explicit_token_headroom_controls_candidate_after_large_baseline( + tmp_path, monkeypatch, headroom, expected_trials): + repo, mapping, identity = setup(tmp_path, ids=("one",), token_stop_new_trials=20000) + original = compare.read_json(mapping) + profile = configuration() + if headroom is not None: + profile["limits"] = {"token_stop_new_trials": headroom} + output = tmp_path / "run" + plan = integration.prepare(repo, mapping, identity, output, provider=profile) + frozen = compare.load_plan(output / "selected") + effective = 20000 if headroom is None else headroom + assert plan["limits"] == {**original["limits"], "token_stop_new_trials": effective} + assert frozen["limits"] == plan["limits"] + assert integration.budget(plan)["limits_per_comparison"] == plan["limits"] + assert compare.read_json(mapping) == original + sent = install_fake_worker(monkeypatch, usage={ + "input_tokens": 15000, "output_tokens": 100, + "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}) + integration.execute(output, approval(plan)) + report = integration.reports(output)[0] + status = report["status"] + assert status["recorded_trials"] == expected_trials + assert len(sent) == 2 * expected_trials + assert report["conditions"]["baseline"]["input_tokens"] == 30000 + assert report["conditions"]["baseline"]["output_tokens"] == 200 + if expected_trials == 2: + assert status["execution_state"] == "completed" and status["stop_reason"] is None + assert report["counts"]["unchanged_correct"] == 1 + assert status["observed_tokens"] == 60400 + else: + assert status["execution_state"] == "stopped" and status["stop_reason"] == "token_budget" + assert report["counts"]["missing_pair"] == 1 + assert report["conditions"]["candidate"]["missing_trials"] == 1 + with pytest.raises(FileExistsError): + integration.execute(output, approval(plan)) + assert len(sent) == 2 * expected_trials + + +def test_headroom_is_bound_to_plan_and_full_budget_without_changing_money(tmp_path): + repo, mapping, identity = setup(tmp_path, ids=("one",), token_stop_new_trials=20000) + profile = configuration() + profile["limits"] = {"token_stop_new_trials": 100000} + first_out = tmp_path / "first" + first = integration.prepare(repo, mapping, identity, first_out, provider=profile) + old = approval(first) + changed_profile = copy.deepcopy(profile) + changed_profile["limits"]["token_stop_new_trials"] = 200000 + second_out = tmp_path / "second" + second = integration.prepare(repo, mapping, identity, second_out, provider=changed_profile) + assert first["comparisons"][0]["plan_sha256"] != second["comparisons"][0]["plan_sha256"] + assert integration.fingerprint(first) != integration.fingerprint(second) + assert integration.budget(first) != integration.budget(second) + assert first["reservation"] == second["reservation"] + assert first["estimated_calls"] == second["estimated_calls"] == 2 + assert first["provider_config"]["client"] == second["provider_config"]["client"] + request = {key: old[key] for key in ("plan_sha256", "version", "base", "head", "budget")} + with pytest.raises(ValueError, match="exact plan"): + integration.authorize(second, request, "fixture-operator", "write", + {"base": identity["base"], "head": identity["head"]}) + + +@pytest.mark.parametrize("limits", [ + {}, None, [], {"token_stop_new_trials": 0}, {"token_stop_new_trials": -1}, + {"token_stop_new_trials": True}, {"token_stop_new_trials": 1.5}, + {"token_stop_new_trials": "100000"}, {"timeout_seconds": 600}, + {"token_stop_new_trials": 100000, "repeats": 2}, +]) +def test_provider_headroom_rejects_invalid_or_unrelated_limit_overrides(tmp_path, limits): + repo, mapping, identity = setup(tmp_path, ids=("one",)) + profile = configuration() + profile["limits"] = limits + output = tmp_path / "invalid" + with pytest.raises(ValueError, match="token_stop_new_trials"): + integration.prepare(repo, mapping, identity, output, provider=profile) + assert not output.exists() + + +def test_high_headroom_preserves_unknown_usage_and_money_stops(tmp_path, monkeypatch): + repo, mapping, identity = setup(tmp_path, ids=("one",), token_stop_new_trials=20000) + profile = configuration() + profile["limits"] = {"token_stop_new_trials": 100000} + poor = copy.deepcopy(profile) + poor["spend_limit"]["amount"] = "1" + with pytest.raises(ValueError, match="reservation exceeds"): + integration.prepare(repo, mapping, identity, tmp_path / "too-poor", provider=poor) + assert not (tmp_path / "too-poor").exists() + output = tmp_path / "unknown" + plan = integration.prepare(repo, mapping, identity, output, provider=profile) + sent = install_fake_worker(monkeypatch, usage={"input_tokens": 15000, "output_tokens": 100}) + integration.execute(output, approval(plan)) + report = integration.reports(output)[0] + assert len(sent) == report["status"]["recorded_trials"] == 1 + assert report["status"]["execution_state"] == "stopped" + assert report["counts"]["missing_pair"] == 1 + assert report["pairs"][0]["baseline"]["verdict"] == "INCONCLUSIVE" + + +def test_codex_plan_limits_are_unchanged_without_provider_override(tmp_path): + repo, mapping, identity = setup(tmp_path, ids=("one",), token_stop_new_trials=20000) + original = compare.read_json(mapping) + output = tmp_path / "codex" + plan = integration.prepare(repo, mapping, identity, output) + assert plan["limits"] == original["limits"] + assert compare.load_plan(output / "selected")["limits"] == original["limits"] + assert "provider_config" not in plan diff --git a/tools/skill_eval_pr.py b/tools/skill_eval_pr.py index 1f3fad68..357b1e1e 100644 --- a/tools/skill_eval_pr.py +++ b/tools/skill_eval_pr.py @@ -57,11 +57,20 @@ def prepare(repo, mapping_path, identity, output, *, provider=None): raise ValueError("Expected immutable revision: " + key) mapping = compare.read_json(mapping_path) if provider is not None: - if set(provider) != {"model", "client", "pricing", "spend_limit"}: + required = {"model", "client", "pricing", "spend_limit"} + if (not isinstance(provider, dict) or not required <= provider.keys() + or provider.keys() - required - {"limits"}): raise ValueError("Unknown or incomplete provider configuration") + overrides = provider.get("limits", {}) + if "limits" in provider and ( + not isinstance(overrides, dict) or set(overrides) != {"token_stop_new_trials"} + or type(overrides["token_stop_new_trials"]) is not int + or overrides["token_stop_new_trials"] <= 0): + raise ValueError("Provider limits support one positive integer token_stop_new_trials") reservation(provider["model"]["name"], provider["client"], provider["pricing"], provider["spend_limit"], 1, now=0) - mapping.update(provider) + mapping.update({key: provider[key] for key in required}) + mapping["limits"] = {**mapping["limits"], **overrides} changes = changed_files(repo, identity["base"], identity["head"]) output.mkdir(parents=True, exist_ok=False) coverage, selected = [], set()