From adeede68798eb83c0de04d0f20b0edcf5c53ff59 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:20:49 +0900 Subject: [PATCH 1/2] feat: add bounded diagnostic evaluation and daily admission Keep paid activation disabled pending audited ledger migration and frozen experiment configuration. Agent-assisted-by: OpenAI Codex (runtime model/effort not independently exposed) Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> --- docs/skill-eval-comparison.md | 44 ++++ docs/skill-eval-daily-admission.md | 106 ++++++++ docs/skill-eval-pr.md | 14 + docs/skill-eval-quarantine.md | 58 ++++ src/inferrsi/harness/skill_eval_diagnostic.py | 131 +++++++++ .../fixtures/skill_eval_diagnostic_cases.json | 242 +++++++++++++++++ tests/test_skill_eval_auto.py | 20 +- tests/test_skill_eval_auto_daily.py | 210 +++++++++++++++ tests/test_skill_eval_daily_budget.py | 214 +++++++++++++++ tests/test_skill_eval_diagnostic.py | 214 +++++++++++++++ tests/test_skill_eval_pr_diagnostic.py | 67 +++++ tests/test_skill_eval_quarantine.py | 104 ++++++++ tests/test_skill_eval_snapshot.py | 174 ++++++++++++ tools/skill_eval_auto.py | 60 ++++- tools/skill_eval_budget.py | 248 ++++++++++++++++++ tools/skill_eval_budget_git.py | 65 ++++- tools/skill_eval_compare.py | 59 ++++- tools/skill_eval_pr.py | 20 +- tools/skill_eval_quarantine.py | 89 +++++++ tools/skill_eval_snapshot.py | 159 +++++++++++ 20 files changed, 2263 insertions(+), 35 deletions(-) create mode 100644 docs/skill-eval-daily-admission.md create mode 100644 docs/skill-eval-quarantine.md create mode 100644 src/inferrsi/harness/skill_eval_diagnostic.py create mode 100644 tests/fixtures/skill_eval_diagnostic_cases.json create mode 100644 tests/test_skill_eval_auto_daily.py create mode 100644 tests/test_skill_eval_daily_budget.py create mode 100644 tests/test_skill_eval_diagnostic.py create mode 100644 tests/test_skill_eval_pr_diagnostic.py create mode 100644 tests/test_skill_eval_quarantine.py create mode 100644 tests/test_skill_eval_snapshot.py create mode 100644 tools/skill_eval_quarantine.py create mode 100644 tools/skill_eval_snapshot.py diff --git a/docs/skill-eval-comparison.md b/docs/skill-eval-comparison.md index c83529baa..41b0aa4c9 100644 --- a/docs/skill-eval-comparison.md +++ b/docs/skill-eval-comparison.md @@ -259,3 +259,47 @@ python -m tests.fixtures.skill_eval_compare_demo --mode update --output /daily", + "genesis_commit": "", + "checkpoint_commit": "" + } + } +} +``` + +The checked-in schema-1 configuration is deliberately unchanged. Missing daily +configuration/genesis, unknown obligations, corrupt history, expired rates or +unavailable live identity block paid execution, never imply zero spend. Free +planning and retained evidence do not require a new paid grant. + +An authorized rollout must first quiesce all older paid writers and reconcile +**all** historical charges, existing PR reservations and uncertain/in-flight +work, including historical PR 159 and 160 run records. Preserve their existing +PR anchors, history and evidence byte-for-byte; do not reset/rebase their budgets. +Import conservative historical daily intervals and outstanding upper bounds into +an explicitly audited global genesis. Missing provider bills are not proof of +zero cost. Unknown upper exposure prevents activation. Pin the audited genesis +and independent checkpoint in reviewed configuration; no initializer, zero +bootstrap, ref write or config activation is supplied here. + +## Durable admission and failure boundaries + +The order is: verify both histories; commit a **new** daily reservation; commit a +**new** PR reservation; obtain a one-shot daily claim on the admission day; obtain +a one-shot PR claim; recheck all live authority; launch once in this process. +Any duplicate, conflict, lost response or crash stops that invocation. Partial +admission burns capacity. Observing an earlier committed reservation/claim never +recovers a launch grant. A genuinely new attempt needs a new full frozen binding +and new reservations; a workflow rerun or label toggle cannot reset old ledgers. + +Before launch and every provider request, the existing ephemeral broker invokes +the controller to re-read current policy/label/configuration and both exact +claimed bindings, and verify the global ledger is not quarantined. The trusted +wall clock is sampled for admission, again for daily claim, and after live reads +for each grant. A changed Shanghai day denies all subsequent requests; no +cross-midnight continuation or automatic day rollover exists. A request already +in flight cannot be recalled. Its full unresolved bound remains counted on the +next day, so midnight does not free its capacity. + +The existing trusted provider client enforces the frozen request count, payload +and output bounds and requires a fresh broker grant immediately before every +HTTP request; transport retries do not get a replay grant. The reservation covers +all calls using the full verified model-context ceiling, highest applicable +input/cache rate, maximum output and request fees, rounded upward. If those +upper-bound assumptions cannot be established, do not run. + +## Conservative terminal policy and quarantine + +The controller performs **no automatic terminal settlement or refunds**, even +when its worker returns successfully. Worker failure, cancellation, crashes and +missing/partial actual usage retain the entire PR and global bound. Unresolved +global reservations remain charged on every later day; exhaustion stops future +work instead of resetting at midnight. This deliberately trades availability for +safety in the initial bounded pilot. + +A future settlement path must prove provider work and every retry are terminal, +sample a fresh trusted closing day **after** that proof, and retain unknown-cost +bounds. Never use a cached admission day as a settlement day. Do not interpret a +successful subprocess return or artifact's claimed usage as that proof. + +Evidence of an exceeded upper bound or unknowable exposure requires persistent +global quarantine before further activity; ordinary unavailable actual billing +under a still-valid conservative bound is different. Quarantine is monotonic: +no automatic clearing/reset. The controller checks the persisted quarantine, but +does not itself ingest provider billing or invent evidence of terminal usage. +Activation therefore still requires the reviewed pricing/bounds, audited +history, compatible storage permissions and a trusted operational path to +record any externally discovered over-bound exposure. Git administrator rollback +limitations remain as documented in [the accounting core](skill-eval-budget-core.md). + +## Offline validation + +```sh +PYTHONPATH=src:. python -m unittest tests.test_skill_eval_daily_budget tests.test_skill_eval_auto_daily -v +``` + +The controller matrix covers all four durable admission writes under conflicts, +uncertain successful writes and crashes; replay after partial progress; rollover +before daily claim, after both claims and between provider requests; claim +revocation and worker failure after launch. Tests use only local in-memory state +and fake credentials, not provider requests, GitHub writes or paid experiments. +Full pytest validation requires the development dependencies; these unit tests +do not establish provider billing or evaluation efficacy. diff --git a/docs/skill-eval-pr.md b/docs/skill-eval-pr.md index a611fc7ad..547cf3002 100644 --- a/docs/skill-eval-pr.md +++ b/docs/skill-eval-pr.md @@ -361,3 +361,17 @@ model, no provider selection and no executable comparison freezes. It never silently falls back to another model/provider. These blocked plans are rejected by PR authorization and execution as well as the automatic controller. Direct standalone comparison-harness behavior is unchanged. + +A trusted mapping item may explicitly set `adapter: "kv-admission-diagnostic-v1"` +to use the synthetic diagnostic contract documented in +[skill comparisons](skill-eval-comparison.md#synthetic-kv-admission-diagnostic-adapter). +This is not inferred from PR content. It requires an existing baseline bundle +and preserves exact PR base/head comparisons; unknown adapters or an addition +remain unsupported. Report regeneration rechecks the frozen built-in adapter, +not a performance-audit grader applied to diagnostic answers. + +Mapping a skill's changed files selects the comparison; it does not establish +that a small case covers every mechanism in those files. Unmapped generated +projections, navigation documents and tests remain visible coverage gaps. +Source-only review dispositions must be explicit and must not be represented as +model-evaluated passes. The maintained mapping is not expanded automatically. diff --git a/docs/skill-eval-quarantine.md b/docs/skill-eval-quarantine.md new file mode 100644 index 000000000..3faeea53c --- /dev/null +++ b/docs/skill-eval-quarantine.md @@ -0,0 +1,58 @@ +# Persisting an incident quarantine + +This is a trusted operator/controller stop operation, not paid admission or a +billing ingestion interface. Run only from the current reviewed implementation +with its trusted schema-2 daily anchor configuration and the existing GitHub +credential injected by the authorized environment: + +```sh +PYTHONPATH=src:. python -m tools.skill_eval_quarantine \ + --repository \ + --evidence https://github.com///issues/ +``` + +The evidence must be a sanitized public GitHub issue/PR reference, optionally a +comment/review permalink. Do not pass raw bills, provider errors, credentials or +private payloads. No provider credential is needed or read. No model, dispatch, +ledger initialization, reset, settlement, clearing, or repository-setting API +is called. This CLI itself performs real Git ledger writes when invoked against +a live client; tests and delivery of its source do not authorize invocation. + +Use this operation when a trusted incident establishes exposure above the +reserved upper bound or makes that upper bound unknowable. A missing actual +bill under a still-valid conservative bound does not by itself establish either. +No expiry/pricing/label check can prevent this safety-only stop: it loads only +the trusted repository identity and daily anchor from root configuration, then +the Git adapter verifies repository identity and full pinned ledger lineage. + +The writer retries at most three verified read/compare-and-swap cycles, with a +final read-only verification. A concurrent admission is preserved and the stop +is recomputed against the newer history. An unknown/lost write receipt is +reconciled **only into a verified stop**, never into a launch grant. Existing +quarantine is sufficient; repeated calls preserve its first evidence and do not +rewrite history. Subsequent admission and per-request checks reject quarantine. +Already-issued or in-flight provider requests cannot be recalled by this write. + +Exit zero and `{"state":"quarantine_verified",...}` mean fresh authenticated +readback established the durable stop. Merely receiving a successful CAS is not +enough. Nonzero exit / `Global quarantine NOT verified` means the operation is +incomplete: keep paid execution stopped, resolve storage/identity/anchor access, +and rerun this monotonic stop until verified. Do not restart paid work because +retries were exhausted, and do not change/reset anchors to bypass failure. +If storage is inaccessible, this CLI cannot claim it stopped other processes; +that requires the separately authorized operational response. + +There is no unquarantine command. Resuming after an incident requires separately +reviewed reconciliation; deleting a ledger/ref or lowering a checkpoint is not +reconciliation. This operation does not prove provider billing, undo spend or +provide administrator-resistant rollback protection. + +Offline tests: + +```sh +PYTHONPATH=src:. python -m unittest tests.test_skill_eval_quarantine -v +``` + +They cover concurrent admission, lost successful write receipt, repeat-stop +idempotence, conflict exhaustion, failed readback, expired pricing, rejection of +raw/private evidence inputs, and nonzero CLI failure without a success report. diff --git a/src/inferrsi/harness/skill_eval_diagnostic.py b/src/inferrsi/harness/skill_eval_diagnostic.py new file mode 100644 index 000000000..1c0a9506b --- /dev/null +++ b/src/inferrsi/harness/skill_eval_diagnostic.py @@ -0,0 +1,131 @@ +"""Offline, synthetic KV admission diagnosis contract; not an engine simulator. + +Only observations cross the task boundary. The oracle stays in the validator, +not in the task or selected knowledge bundle. No provider or execution hooks. +""" +import json +import os +import re +import sys +from pathlib import Path + +ADAPTER = "kv-admission-diagnostic-v1" +PROMPT = ( + "Read task.json and diagnose why the synthetic request cannot run. Knowledge is fallible " + "guidance; observations are authoritative. Cite only decisive event IDs. For insufficient " + "evidence cite all observed receive, identity and allocator events, or the lookup event " + "only if none of those exists. Choose one next read-only " + "check, and decide whether these observations authorize reusing KV now. Return only the " + "required JSON. Inspect only this workspace; do not modify inputs, use network, execute " + "repository code, run another agent, commit or publish. No service-performance claim is asked." +) +DIAGNOSES = ("readiness", "capacity", "invalid_reuse", "insufficient_evidence") +CHECKS = ("check_receive_completion", "inspect_allocator_accounting", + "check_transfer_integrity", "collect_missing_events") +SCHEMA = { + "type": "object", "additionalProperties": False, + "required": ["diagnosis", "evidence", "next_check", "reuse_authorized"], + "properties": { + "diagnosis": {"type": "string", "enum": list(DIAGNOSES)}, + "evidence": {"type": "array", "items": {"type": "string"}}, + "next_check": {"type": "string", "enum": list(CHECKS)}, + "reuse_authorized": {"type": "boolean"}, + }, +} +VALUES = { + "cache_lookup": {"hit", "miss"}, + "receive": {"pending", "complete", "failed"}, + "identity": {"match", "mismatch"}, + "allocator": {"denied", "granted", "not_attempted"}, +} + + +def validate_task(task): + if not isinstance(task, dict) or set(task) != { + "id", "split", "request", "provenance", "events" + }: + raise ValueError("Diagnostic task requires only id/split/request/provenance/events") + if any(type(task[k]) is not str or not task[k].strip() + for k in ("id", "split", "request", "provenance")): + raise ValueError("Diagnostic task metadata must be nonempty strings") + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._-]*", task["id"]): + raise ValueError("Diagnostic task ID must be a safe path component") + if task["split"] not in ("development", "holdout", "discovery"): + raise ValueError("Unknown case split") + if not isinstance(task["events"], list) or not task["events"]: + raise ValueError("Diagnostic observations must be a nonempty list") + ids, kinds = set(), set() + for event in task["events"]: + if not isinstance(event, dict) or set(event) != {"id", "kind", "value"}: + raise ValueError("Diagnostic event requires only id/kind/value") + if any(type(value) is not str for value in event.values()): + raise ValueError("Diagnostic event fields must be strings") + if not re.fullmatch(r"E[0-9]+", event["id"]) or event["id"] in ids: + raise ValueError("Diagnostic event IDs must be unique E-number identifiers") + if event["kind"] not in VALUES or event["value"] not in VALUES[event["kind"]]: + raise ValueError("Unknown diagnostic observation") + if event["kind"] in kinds: + raise ValueError("Use a single observation per kind at the decision boundary") + ids.add(event["id"]) + kinds.add(event["kind"]) + + +def expected(task): + """Finite-state decision table with conservative unknown-state handling. + + An explicitly failed transfer or identity mismatch forbids reuse regardless + of allocation. A pending receive is a readiness barrier, not capacity proof. + Capacity attribution needs completed, matching KV and explicit denial. + Anything else is insufficient to identify why admission is blocked. + """ + validate_task(task) + states = {e["kind"]: e["value"] for e in task["events"]} + ids = {e["kind"]: e["id"] for e in task["events"]} + invalid = {k for k, v in (("receive", "failed"), ("identity", "mismatch")) + if states.get(k) == v} + if invalid: + return "invalid_reuse", {ids[k] for k in invalid}, "check_transfer_integrity" + if states.get("receive") == "pending": + return "readiness", {ids["receive"]}, "check_receive_completion" + if (states.get("receive"), states.get("identity"), states.get("allocator")) == ( + "complete", "match", "denied" + ): + return "capacity", {ids[k] for k in ("receive", "identity", "allocator")}, ( + "inspect_allocator_accounting") + relevant = {event_id for kind, event_id in ids.items() if kind != "cache_lookup"} + return "insufficient_evidence", relevant or set(ids.values()), "collect_missing_events" + + +def verify(task, answer): + diagnosis, required, next_check = expected(task) + valid = (isinstance(answer, dict) and set(answer) == set(SCHEMA["required"]) + and type(answer["diagnosis"]) is str and answer["diagnosis"] in DIAGNOSES + and type(answer["next_check"]) is str and answer["next_check"] in CHECKS + and type(answer["reuse_authorized"]) is bool + and isinstance(answer["evidence"], list) + and all(type(v) is str for v in answer["evidence"])) + if not valid: + return {"passed": False, "assertions": {"schema": False}} + cited = set(answer["evidence"]) + checks = { + "schema": True, + "diagnosis": answer["diagnosis"] == diagnosis, + "citation": cited == required + and len(cited) == len(answer["evidence"]), + "safe_next_check": answer["next_check"] == next_check, + # These blocked-request observations never authorize executing reuse. + "no_invariant_violation": answer["reuse_authorized"] is False, + } + return {"passed": all(checks.values()), "assertions": checks} + + +def main(): + task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + answer = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + result = verify(task, answer) + Path(os.environ["INFERRSI_RESULT_PATH"]).write_text(json.dumps(result), encoding="utf-8") + raise SystemExit(0 if result["passed"] else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/skill_eval_diagnostic_cases.json b/tests/fixtures/skill_eval_diagnostic_cases.json new file mode 100644 index 000000000..1ecbd8d4c --- /dev/null +++ b/tests/fixtures/skill_eval_diagnostic_cases.json @@ -0,0 +1,242 @@ +[ + { + "id": "case-01", + "split": "development", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "pending" + }, + { + "id": "E3", + "kind": "allocator", + "value": "not_attempted" + } + ] + }, + { + "id": "case-02", + "split": "development", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "complete" + }, + { + "id": "E3", + "kind": "identity", + "value": "match" + }, + { + "id": "E4", + "kind": "allocator", + "value": "denied" + } + ] + }, + { + "id": "case-03", + "split": "development", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "failed" + } + ] + }, + { + "id": "case-04", + "split": "development", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + } + ] + }, + { + "id": "case-05", + "split": "holdout", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "pending" + }, + { + "id": "E3", + "kind": "identity", + "value": "match" + }, + { + "id": "E4", + "kind": "allocator", + "value": "denied" + } + ] + }, + { + "id": "case-06", + "split": "holdout", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "miss" + }, + { + "id": "E2", + "kind": "receive", + "value": "complete" + }, + { + "id": "E3", + "kind": "identity", + "value": "match" + }, + { + "id": "E4", + "kind": "allocator", + "value": "denied" + } + ] + }, + { + "id": "case-07", + "split": "holdout", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "complete" + }, + { + "id": "E3", + "kind": "identity", + "value": "mismatch" + }, + { + "id": "E4", + "kind": "allocator", + "value": "denied" + } + ] + }, + { + "id": "case-08", + "split": "holdout", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "complete" + }, + { + "id": "E3", + "kind": "allocator", + "value": "denied" + } + ] + }, + { + "id": "case-09", + "split": "holdout", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "complete" + }, + { + "id": "E3", + "kind": "identity", + "value": "match" + } + ] + }, + { + "id": "case-10", + "split": "holdout", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "failed" + }, + { + "id": "E3", + "kind": "identity", + "value": "match" + }, + { + "id": "E4", + "kind": "allocator", + "value": "granted" + } + ] + } +] diff --git a/tests/test_skill_eval_auto.py b/tests/test_skill_eval_auto.py index e9b48b783..810649649 100644 --- a/tests/test_skill_eval_auto.py +++ b/tests/test_skill_eval_auto.py @@ -27,10 +27,14 @@ def config(): "rate_evidence": "https://github.com/example/repo/issues/1#issuecomment-1", "expires_at": 4102444800}, "spend_limit": {"currency": "USD", "amount": "200"}} - return {"schema_version": 1, "repository_id": 123, "label": "skill-eval:execute", + return {"schema_version": 2, "repository_id": 123, "label": "skill-eval:execute", "operators": ["operator"], "expires_at": 4102444800, "provider": provider, "provider_sha256": pr.fingerprint(provider), "review_evidence": "offline fixture", - "anchors": {}} + "anchors": {}, + "daily_budget": {"currency": "USD", "amount": "1000", + "timezone": "Asia/Shanghai", "anchor": { + "ref": "refs/heads/skill-eval-budget/123/daily", + "genesis_commit": "a" * 40, "checkpoint_commit": "a" * 40}}} @pytest.fixture @@ -195,7 +199,19 @@ def compare_and_swap(self, before, after): self.ledger = after return WriteResult.COMMITTED + from tools.skill_eval_budget import DailyKey, DailyLedger, DailySnapshot + + class GlobalStore(Store): + def __init__(self): + self.ledger = DailyLedger(DailyKey(123), "audited offline history") + + def read_verified(self, key): + assert key == self.ledger.key + return DailySnapshot(str(self.ledger.revision), self.ledger) + store, api = Store(), API(plan, config) + global_store = GlobalStore() + monkeypatch.setattr(auto, "daily_store", lambda *args: (global_store, global_store.ledger.key)) config_path = tmp_path / "config.json" config_path.write_text(json.dumps(config)) (tmp_path / "pr-plan.json").write_text(json.dumps(plan)) diff --git a/tests/test_skill_eval_auto_daily.py b/tests/test_skill_eval_auto_daily.py new file mode 100644 index 000000000..effe6b2bf --- /dev/null +++ b/tests/test_skill_eval_auto_daily.py @@ -0,0 +1,210 @@ +"""Offline controller fault matrix; stdlib only, no provider or remote calls.""" +import contextlib +import copy +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from tools import skill_eval_auto as auto +from tools import skill_eval_budget as b + +DAY = "2026-10-29" +NEXT = "2026-10-30" +KEY = b.PRKey(123, 7) +GLOBAL = b.DailyKey(123) +BUDGET = {"currency": "USD", "amount": "1000", "timezone": "Asia/Shanghai", + "anchor": {"ref": "refs/heads/skill-eval-budget/123/daily", + "genesis_commit": "a" * 40, "checkpoint_commit": "a" * 40}} + + +class Store: + def __init__(self, ledger, snapshot, name, events, fault=None): + self.ledger, self.snapshot, self.name = ledger, snapshot, name + self.events, self.fault, self.writes = events, fault, 0 + + def read_verified(self, key): + assert key == self.ledger.key + return self.snapshot(str(self.ledger.revision), self.ledger) + + def compare_and_swap(self, before, after): + self.writes += 1 + self.events.append((self.name, after.entries[-1].phase)) + assert before.ledger == self.ledger + fault = self.fault and self.fault[0] == self.writes + if fault and self.fault[1] == "conflict": + return b.WriteResult.CONFLICT + self.ledger = after + if fault and self.fault[1] == "crash": + raise RuntimeError("crash after durable write") + return b.WriteResult.UNKNOWN if fault else b.WriteResult.COMMITTED + + +class ControllerTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.output = Path(self.temp.name) + self.plan = {"pr": 7, "reservation": {"amount": "10", "requests": 2}} + (self.output / "pr-plan.json").write_text(json.dumps(self.plan)) + self.config = {"schema_version": 2, "repository_id": 123, + "daily_budget": copy.deepcopy(BUDGET)} + self.events = [] + self.pr_store = Store(b.Ledger(KEY, b.Genesis("offline audited history", 0)), + b.Snapshot, "pr", self.events) + self.global_store = Store(b.DailyLedger(GLOBAL, "offline audited history"), + b.DailySnapshot, "daily", self.events) + self.day = DAY + self.callback = None + self.launch = Mock() + + @contextlib.contextmanager + def broker(self, check, max_requests, *, context): + self.callback = check + yield {} + + def execute(self, run=42): + with contextlib.ExitStack() as stack: + for target, value in ( + ("configuration", lambda *a, **k: self.config), + ("eligible", lambda *a: None), + ("live_label", lambda *a: {"actor": "offline", "label_event_id": 1}), + ("ledger_store", lambda *a: (self.pr_store, KEY)), + ("daily_store", lambda *a: (self.global_store, GLOBAL)), + ("shanghai_day", lambda: self.day)): + stack.enter_context(patch.object(auto, target, value)) + stack.enter_context(patch.object(auto.pr, "execution_id", return_value="offline")) + stack.enter_context(patch.dict(auto.os.environ, + {"SKILL_EVAL_YIBUAPI_KEY": "offline-fixture"})) + stack.enter_context(patch("inferrsi.model.request_guard.authorization_broker", + self.broker)) + auto.controlled_execute(Mock(), self.output, run_id=run, attempt=1, + launch=self.launch) + + def test_both_new_reservations_and_claims_precede_launch(self): + def launch(*args, **kwargs): + self.assertEqual(self.events, [("daily", "reserved"), ("pr", "reserved"), + ("daily", "claimed"), ("pr", "claimed")]) + self.callback() + self.launch.side_effect = launch + self.execute() + self.assertEqual(self.pr_store.ledger.charged, 10_000_000) + self.assertEqual(self.global_store.ledger.charged_on(NEXT), 10_000_000) + + def test_every_write_conflict_unknown_or_crash_blocks_and_retry_never_launches(self): + for name in ("pr_store", "global_store"): + for write in (1, 2): + for failure in ("conflict", "unknown", "crash"): + with self.subTest(store=name, write=write, failure=failure): + self.setUp() + store = getattr(self, name) + store.fault = (write, failure) + with self.assertRaises((b.BudgetError, RuntimeError)): + self.execute() + self.launch.assert_not_called() + store.fault = None + # A conflict on the very first global write leaves no state, + # so a new invocation can genuinely reserve. All partial + # or ambiguous commits are non-replayable. + if name == "global_store" and write == 1 and failure == "conflict": + continue + with self.assertRaises(b.BudgetError): + self.execute() + self.launch.assert_not_called() + + def test_midnight_after_claim_blocks_launch(self): + original = self.pr_store.compare_and_swap + def write(before, after): + result = original(before, after) + if after.entries[-1].phase == "claimed": + self.day = NEXT + return result + self.pr_store.compare_and_swap = write + with self.assertRaisesRegex(b.BudgetError, "day ended"): + self.execute() + self.launch.assert_not_called() + + def test_midnight_before_daily_claim_blocks_launch(self): + original = self.pr_store.compare_and_swap + def write(before, after): + result = original(before, after) + self.day = NEXT + return result + self.pr_store.compare_and_swap = write + with self.assertRaisesRegex(b.BudgetError, "admission day"): + self.execute() + self.launch.assert_not_called() + + def test_midnight_next_request_denied_and_crash_retains_bounds(self): + def launch(*args, **kwargs): + self.callback() + self.day = NEXT + with self.assertRaisesRegex(b.BudgetError, "day ended"): + self.callback() + raise RuntimeError("worker crashed after first request") + self.launch.side_effect = launch + with self.assertRaises(RuntimeError): + self.execute() + self.assertEqual(self.pr_store.ledger.charged, 10_000_000) + self.assertEqual(self.global_store.ledger.charged_on(NEXT), 10_000_000) + with self.assertRaises(b.BudgetError): + self.execute() + self.assertEqual(self.launch.call_count, 1) + + def test_missing_global_claim_revokes_next_request(self): + def launch(*args, **kwargs): + self.callback() + binding = self.global_store.ledger.entries[0].binding + self.global_store.ledger = b.daily_advance( + self.global_store.ledger, KEY, binding, "finished", DAY, actual_cost=1) + with self.assertRaisesRegex(b.BudgetError, "global claim"): + self.callback() + self.launch.side_effect = launch + self.execute() + + def test_quarantine_revokes_next_request(self): + def launch(*args, **kwargs): + self.callback() + self.global_store.ledger = b.daily_quarantine( + self.global_store.ledger, "offline externally verified unknown exposure") + with self.assertRaisesRegex(b.BudgetError, "quarantin"): + self.callback() + self.launch.side_effect = launch + self.execute() + with self.assertRaises(b.BudgetError): + self.execute(run=43) + self.assertEqual(self.launch.call_count, 1) + + def test_missing_global_history_never_writes_pr_reservation(self): + self.global_store.read_verified = Mock(side_effect=b.BudgetError("missing genesis")) + with self.assertRaisesRegex(b.BudgetError, "missing genesis"): + self.execute() + self.assertEqual(self.events, []) + self.launch.assert_not_called() + + def test_configuration_migration_preserves_legacy_read_only(self): + legacy = json.loads(auto.CONFIG.read_text()) + path = self.output / "config.json" + path.write_text(json.dumps(legacy)) + self.assertEqual(auto.configuration(path, now=0), legacy) + with self.assertRaises(b.BudgetError): + auto.daily_anchor(legacy) + migrated = {**legacy, "schema_version": 2, "daily_budget": copy.deepcopy(BUDGET)} + migrated["daily_budget"]["anchor"]["ref"] = ( + f"refs/heads/skill-eval-budget/{legacy['repository_id']}/daily") + path.write_text(json.dumps(migrated)) + self.assertEqual(auto.configuration(path, now=0), migrated) + self.assertEqual(migrated["anchors"], legacy["anchors"]) + + def test_daily_schema_is_explicit_no_legacy_bootstrap(self): + self.assertEqual(auto.daily_anchor(self.config).key, GLOBAL) + for mutation in ({"schema_version": 1}, {"daily_budget": None}, + {"daily_budget": {**BUDGET, "amount": "2000"}}, + {"daily_budget": {**BUDGET, "timezone": "UTC"}}): + with self.subTest(mutation=mutation), self.assertRaises(b.BudgetError): + auto.daily_anchor({**self.config, **mutation}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_skill_eval_daily_budget.py b/tests/test_skill_eval_daily_budget.py new file mode 100644 index 000000000..4569a1d46 --- /dev/null +++ b/tests/test_skill_eval_daily_budget.py @@ -0,0 +1,214 @@ +"""Offline standard-library tests; no credentials, network, or provider calls.""" + +import base64 +import copy +import hashlib +import json +import unittest +from dataclasses import replace +from datetime import datetime, timezone + +from tools import skill_eval_budget as b +from tools.skill_eval_budget_git import DailyLedgerAnchor, GitHubDailyStore + +KEY = b.DailyKey(123) +PR = b.PRKey(123, 1) +DAY = "2026-10-29" +NEXT = "2026-10-30" +REF = "refs/heads/skill-eval-budget/123/daily" + + +def binding(n, amount=200_000_000): + return b.Binding(f"{n:064x}", f"{n}.1", amount) + + +class API: + def __init__(self, ledger=None): + self.objects = {} + self.tip = None + self.before_patch = None + self.lose_response = False + self.genesis = self.append(ledger or b.DailyLedger(KEY, "reviewed legacy audit"), []) + self.anchor = DailyLedgerAnchor(KEY, REF, self.genesis, self.genesis) + + def put(self, value): + sha = hashlib.sha1(json.dumps(value, sort_keys=True).encode()).hexdigest() + self.objects[sha] = {**value, "sha": sha} + return sha + + def tree(self, content): + raw = content.encode() + blob = self.put({"encoding": "base64", "size": len(raw), + "content": base64.b64encode(raw).decode()}) + return self.put({"truncated": False, "tree": [{"path": "ledger.json", + "mode": "100644", "type": "blob", "sha": blob}]}) + + def append(self, ledger, parents): + self.tip = self.put({"tree": {"sha": self.tree(b.encode_daily(ledger))}, + "parents": [{"sha": p} for p in parents]}) + return self.tip + + def ref(self): + if self.tip is None: + raise OSError("404") + return {"ref": REF, "object": {"type": "commit", "sha": self.tip}} + + def request(self, method, path, data=None): + if method == "GET": + if path == "": + return {"id": 123} + if path.startswith("/git/ref/"): + return self.ref() + return copy.deepcopy(self.objects[path.rsplit("/", 1)[1]]) + if path == "/git/trees": + return {"sha": self.tree(data["tree"][0]["content"])} + if path == "/git/commits": + return {"sha": self.put({"tree": {"sha": data["tree"]}, + "parents": [{"sha": p} for p in data["parents"]], + "message": data["message"]})} + assert method == "PATCH" and path == "/git/refs/heads/skill-eval-budget/123/daily" + assert data["force"] is False + if self.before_patch: + callback, self.before_patch = self.before_patch, None + callback() + if self.objects[data["sha"]]["parents"] != [{"sha": self.tip}]: + raise OSError("422") + self.tip = data["sha"] + if self.lose_response: + raise TimeoutError("lost successful response") + return self.ref() + + +class DailyBudgetTests(unittest.TestCase): + def test_timezone_boundary(self): + self.assertEqual(b.shanghai_day(datetime(2026, 10, 29, 15, 59, tzinfo=timezone.utc)), DAY) + self.assertEqual(b.shanghai_day(datetime(2026, 10, 29, 16, tzinfo=timezone.utc)), NEXT) + with self.assertRaises(b.BudgetError): + b.shanghai_day(datetime(2026, 10, 29)) + + def test_outstanding_rollover_and_immutable_settlement(self): + ledger = b.daily_reserve(b.DailyLedger(KEY, "audit"), PR, binding(1), DAY) + self.assertEqual(ledger.charged_on(NEXT), 200_000_000) + self.assertEqual(b.daily_reserve(ledger, PR, binding(1), NEXT), ledger) + with self.assertRaises(b.BudgetError): + b.daily_advance(ledger, PR, binding(1), "claimed", NEXT) + ledger = b.daily_advance(ledger, PR, binding(1), "claimed", DAY) + done = b.daily_advance(ledger, PR, binding(1), "finished", NEXT, actual_cost=1) + b.validate_daily_successor(ledger, done) + self.assertEqual(done.charged_on(DAY), 200_000_000) + self.assertEqual(done.charged_on(NEXT), 200_000_000) + self.assertEqual(done.charged_on("2026-10-31"), 0) + with self.assertRaises(b.BudgetError): + b.daily_advance(done, PR, binding(1), "claimed", NEXT) + + def test_racing_different_pr_admission_cannot_overbook(self): + ledger = b.DailyLedger(KEY, "audit") + for n in range(1, 5): + ledger = b.daily_reserve(ledger, b.PRKey(123, n), binding(n), DAY) + # Audited imported genesis carries existing outstanding reservations. + ledger = b.DailyLedger(KEY, "import audit", ledger.entries) + api = API(ledger) + a, other = GitHubDailyStore(api, api.anchor), GitHubDailyStore(api, api.anchor) + snap = a.read_verified(KEY) + one = b.daily_reserve(snap.ledger, b.PRKey(123, 5), binding(5), DAY) + two = b.daily_reserve(snap.ledger, b.PRKey(123, 6), binding(6), DAY) + api.before_patch = lambda: self.assertIs( + other.compare_and_swap(snap, two), b.WriteResult.COMMITTED) + self.assertIs(a.compare_and_swap(snap, one), b.WriteResult.UNKNOWN) + latest = a.read_verified(KEY) + with self.assertRaises(b.BudgetError): + b.daily_reserve(latest.ledger, b.PRKey(123, 5), binding(5), NEXT) + self.assertIs(a.compare_and_swap(snap, one), b.WriteResult.CONFLICT) + + def test_claim_race_and_unknown_outcome_never_grant_twice(self): + for lose_response in (False, True): + api = API() + a, other = GitHubDailyStore(api, api.anchor), GitHubDailyStore(api, api.anchor) + snap = a.read_verified(KEY) + a.compare_and_swap(snap, b.daily_reserve(snap.ledger, PR, binding(1), DAY)) + if lose_response: + api.lose_response = True + else: + api.before_patch = lambda: b.daily_claim_once(other, PR, binding(1), DAY) + with self.assertRaises(b.BudgetError): + b.daily_claim_once(a, PR, binding(1), DAY) + api.lose_response = False + with self.assertRaises(b.BudgetError): + b.daily_claim_once(other, PR, binding(1), DAY) + + def test_strict_schema_and_history(self): + ledger = b.DailyLedger(KEY, "audit") + self.assertEqual(b.decode_daily(b.encode_daily(ledger)), ledger) + for amount in (None, True, -1, 1.5): + with self.assertRaises(b.BudgetError): + binding(1, amount) + with self.assertRaises(b.BudgetError): + payload = b.encode_daily(ledger).replace('"revision":0', '"revision":0,"revision":0') + b.decode_daily(payload) + api = API() + store = GitHubDailyStore(api, api.anchor) + snap = store.read_verified(KEY) + store.compare_and_swap(snap, b.daily_reserve(snap.ledger, PR, binding(1), DAY)) + api.tip = api.genesis + with self.assertRaises(b.BudgetError): + store.read_verified(KEY) + api.tip = None + with self.assertRaises(b.BudgetError): + store.read_verified(KEY) + + def test_quarantine_persists_and_blocks_all_execution_without_refund(self): + api = API() + store = GitHubDailyStore(api, api.anchor) + snap = store.read_verified(KEY) + reserved = b.daily_reserve(snap.ledger, PR, binding(1), DAY) + store.compare_and_swap(snap, reserved) + snap = store.read_verified(KEY) + stopped = b.daily_quarantine(snap.ledger, "audit: upper-bound exposure unknown") + self.assertIs(store.compare_and_swap(snap, stopped), b.WriteResult.COMMITTED) + fresh = GitHubDailyStore(api, api.anchor).read_verified(KEY).ledger + self.assertEqual(fresh, stopped) + self.assertEqual(fresh.charged_on(NEXT), 200_000_000) + self.assertEqual(b.daily_quarantine(fresh, "another observation"), fresh) + for action in ( + fresh.require_active, + lambda: b.daily_reserve(fresh, b.PRKey(123, 9), binding(9), NEXT), + lambda: b.daily_reserve(fresh, PR, binding(1), NEXT), + lambda: b.daily_claim_once(store, PR, binding(1), DAY), + lambda: b.validate_daily_successor( + fresh, replace(fresh, quarantine=None, revision=fresh.revision + 1)), + lambda: b.validate_daily_successor( + snap.ledger, replace(stopped, entries=())), + ): + with self.assertRaises(b.BudgetError): + action() + + def test_quarantine_race_and_unknown_write_remain_fail_closed(self): + api = API() + store, other = GitHubDailyStore(api, api.anchor), GitHubDailyStore(api, api.anchor) + snap = store.read_verified(KEY) + stopped = b.daily_quarantine(snap.ledger, "audit: observed bound exceeded") + api.before_patch = lambda: other.compare_and_swap(snap, stopped) + reserved = b.daily_reserve(snap.ledger, PR, binding(1), DAY) + self.assertIs(store.compare_and_swap(snap, reserved), b.WriteResult.UNKNOWN) + with self.assertRaises(b.BudgetError): + store.read_verified(KEY).ledger.require_active() + api = API() + store = GitHubDailyStore(api, api.anchor) + snap = store.read_verified(KEY) + api.lose_response = True + self.assertIs(store.compare_and_swap(snap, b.daily_quarantine(snap.ledger, "audit")), + b.WriteResult.UNKNOWN) + with self.assertRaises(b.BudgetError): + GitHubDailyStore(api, api.anchor).read_verified(KEY).ledger.require_active() + + def test_existing_pr_cap_is_not_reset(self): + pr = b.reserve(b.Ledger(PR, b.Genesis("legacy audit", 100_000_000)), + binding(1, 100_000_000)) + with self.assertRaises(b.BudgetError): + b.reserve(pr, binding(2, 1)) + with self.assertRaises(b.BudgetError): + b.daily_reserve(b.DailyLedger(KEY, "audit"), b.PRKey(999, 1), binding(1), DAY) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_skill_eval_diagnostic.py b/tests/test_skill_eval_diagnostic.py new file mode 100644 index 000000000..cf5638459 --- /dev/null +++ b/tests/test_skill_eval_diagnostic.py @@ -0,0 +1,214 @@ +import copy +import itertools +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from inferrsi.harness import skill_eval_diagnostic as diagnostic +from inferrsi.infra.artifacts import sha256, write_json +from tests.fixtures.skill_eval_compare_support import FixedOutputClient, configuration, git +from tools import skill_eval_compare as compare + +CASES = json.loads( + (Path(__file__).parent / 'fixtures/skill_eval_diagnostic_cases.json').read_text()) +# Independently authored expected responses: never emitted into model workspaces. +ANSWERS = [ + ('readiness', ['E2'], 'check_receive_completion'), + ('capacity', ['E2', 'E3', 'E4'], 'inspect_allocator_accounting'), + ('invalid_reuse', ['E2'], 'check_transfer_integrity'), + ('insufficient_evidence', ['E1'], 'collect_missing_events'), + ('readiness', ['E2'], 'check_receive_completion'), + ('capacity', ['E2', 'E3', 'E4'], 'inspect_allocator_accounting'), + ('invalid_reuse', ['E3'], 'check_transfer_integrity'), + ('insufficient_evidence', ['E2', 'E3'], 'collect_missing_events'), + ('insufficient_evidence', ['E2', 'E3'], 'collect_missing_events'), + ('invalid_reuse', ['E2'], 'check_transfer_integrity'), +] + + +def answer(index): + diagnosis, evidence, check = ANSWERS[index] + return dict(diagnosis=diagnosis, evidence=evidence.copy(), next_check=check, + reuse_authorized=False) + + +class DiagnosticTests(unittest.TestCase): + def test_counterfactual_cases(self): + for i, task in enumerate(CASES): + with self.subTest(task=task['id']): + self.assertEqual(diagnostic.expected(task)[0], ANSWERS[i][0]) + self.assertTrue(diagnostic.verify(task, answer(i))['passed']) + self.assertEqual(set(task), {'id', 'split', 'request', 'provenance', 'events'}) + self.assertNotIn('expected', json.dumps(task)) + + def test_exhaustive_state_table(self): + kinds = tuple(diagnostic.VALUES) + count = 0 + for values in itertools.product(*(tuple(diagnostic.VALUES[k]) + (None,) for k in kinds)): + states = dict(zip(kinds, values)) + events = [dict(id=f'E{i}', kind=k, value=v) + for i, (k, v) in enumerate(states.items()) if v is not None] + if not events: + continue + count += 1 + actual = diagnostic.expected({**CASES[0], 'events': events})[0] + if states['identity'] == 'mismatch' or states['receive'] == 'failed': + self.assertEqual(actual, 'invalid_reuse') + elif states['receive'] == 'pending': + self.assertEqual(actual, 'readiness') + elif all(states[k] == v for k, v in + [('receive', 'complete'), ('identity', 'match'), ('allocator', 'denied')]): + self.assertEqual(actual, 'capacity') + else: + self.assertEqual(actual, 'insufficient_evidence') + self.assertEqual(count, 143) + + def test_reject_wrong_or_unsafe_answer(self): + for field, value in [ + ('evidence', ['E999']), ('evidence', []), ('evidence', ['E2', 'E2']), + ('evidence', ['E1', 'E2', 'E3']), + ('diagnosis', 'capacity'), ('next_check', 'inspect_allocator_accounting'), + ('reuse_authorized', True), ('reuse_authorized', 0), ('diagnosis', []), + ]: + with self.subTest(field=field, value=value): + self.assertFalse(diagnostic.verify(CASES[0], {**answer(0), field: value})['passed']) + + def test_insufficient_evidence_accepts_concise_relevant_citations(self): + result = dict(diagnosis='insufficient_evidence', evidence=['E2', 'E3'], + next_check='collect_missing_events', reuse_authorized=False) + self.assertTrue(diagnostic.verify(CASES[7], result)['passed']) + self.assertFalse(diagnostic.verify(CASES[7], {**result, 'evidence': ['E3']})['passed']) + + def test_strict_input_boundary(self): + for mutation in ('answer_key', 'duplicate_id', 'duplicate_kind', 'unknown'): + task = copy.deepcopy(CASES[0]) + if mutation == 'answer_key': + task['expected_diagnosis'] = 'readiness' + elif mutation == 'duplicate_id': + task['events'][1]['id'] = task['events'][0]['id'] + elif mutation == 'duplicate_kind': + task['events'].append({**task['events'][0], 'id': 'E99'}) + else: + task['events'][0]['value'] = 'unknown' + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + diagnostic.validate_task(task) + + def test_prepare_and_adapter_boundary(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path, repo = configuration(root, ids=('one',), max_task_calls=20) + config = json.loads(path.read_text()) + config['adapter'] = diagnostic.ADAPTER + write_json(path, config) + write_json(root / 'cases.json', CASES) + out = root / 'output' + compare.prepare(path, repo, out) + plan = json.loads((out / 'plan/freeze.json').read_text()) + self.assertEqual(compare.load_plan(out), plan) + self.assertEqual(plan['adapter'], diagnostic.ADAPTER) + self.assertEqual(plan['prompt'], diagnostic.PROMPT) + self.assertEqual(json.loads((out / 'plan/schema.json').read_text()), diagnostic.SCHEMA) + self.assertEqual(plan['exam']['grader'], sha256(Path(diagnostic.__file__))) + self.assertTrue(all(plan['arms'][role]['bundle'] for role in compare.ROLES)) + self.assertTrue(plan['bundle_changes']) + self.assertEqual(len(plan['schedule']), 20) + self.assertEqual(json.loads((out / 'plan/tasks.json').read_text()), CASES) + for change in [{'mode': 'addition'}, {'grader': 'x'}, {'schema': 'x'}, + {'adapter': 'unknown'}]: + with self.subTest(change=change), self.assertRaises(ValueError): + compare.validate({**config, **change}, CASES) + + def test_fake_client_paired_trial_and_report(self): + # This is plumbing coverage using prewritten responses, not a model experiment. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path, repo = configuration(root, ids=('one',)) + config = json.loads(path.read_text()) + config['adapter'] = diagnostic.ADAPTER + write_json(path, config) + write_json(root / 'cases.json', CASES[:1]) + out = root / 'output' + authorization = compare.prepare(path, repo, out) + client = FixedOutputClient(root) + write_json(client.answer, answer(0)) + compare.execute(out, authorization, client=client) + report = compare.read_report(out) + self.assertEqual(report['pairs'][0]['outcome'], 'unchanged_correct') + for role in compare.ROLES: + self.assertEqual(report['pairs'][0][role]['grader_trust'], 'reverified') + self.assertEqual(len(client.calls), 2) + for call in client.calls: + self.assertTrue(all('verify.py' not in path for path in call['inputs'])) + + def test_selected_closure_keeps_legacy_blanket_guard(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path, repo = configuration(root, ids=('one',)) + unrelated = repo / 'library/shared/not-selected.py' + unrelated.write_text('raise AssertionError("must never execute")\n') + git(repo, 'add', str(unrelated)) + git(repo, 'update-index', '--chmod=+x', 'library/shared/not-selected.py') + git(repo, 'commit', '--quiet', '-m', 'Unrelated executable fixture') + revision = git(repo, 'rev-parse', 'HEAD') + with self.assertRaisesRegex(ValueError, 'Executable'): + compare.snapshot(repo, revision, 'library', root / 'blanket') + config = json.loads(path.read_text()) + config['adapter'] = diagnostic.ADAPTER + config['snapshots']['candidate']['git_revision'] = revision + write_json(path, config) + write_json(root / 'cases.json', CASES[:1]) + out = root / 'output' + compare.prepare(path, repo, out) + manifest = json.loads((out / 'plan/snapshots/candidate.json').read_text()) + self.assertNotIn('library/shared/not-selected.py', manifest['entries']) + self.assertIn('library/shared/notes.md', manifest['entries']) + self.assertEqual(compare.load_plan(out)['arms']['candidate']['git_revision'], revision) + + def test_legacy_audit_unchanged(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path, repo = configuration(root, ids=('one',)) + config = json.loads(path.read_text()) + with self.assertRaises(ValueError): + compare.validate(config, CASES[:1], default_grader=False) + out = root / 'output' + compare.prepare(path, repo, out) + plan = compare.load_plan(out) + self.assertEqual(plan['prompt'], compare.PROMPT) + self.assertEqual(json.loads((out / 'plan/schema.json').read_text()), compare.SCHEMA) + self.assertEqual(plan['exam']['grader'], sha256(compare.ROOT / + 'tools/skill_eval_verifier.py')) + + def test_standalone_frozen_verifier(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + verifier = root / 'verify.py' + verifier.write_bytes(Path(diagnostic.__file__).read_bytes()) + write_json(root / 'task.json', CASES[0]) + write_json(root / 'answer.json', answer(0)) + process = subprocess.run( + [sys.executable, str(verifier), str(root / 'task.json'), + str(root / 'answer.json')], cwd=root, capture_output=True, + env={'INFERRSI_RESULT_PATH': str(root / 'result.json')}, + ) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertTrue(json.loads((root / 'result.json').read_text())['passed']) + + def test_retained_answer_reverified_not_trusted(self): + task = CASES[0] + result = answer(0) + trial = dict(task_id=task['id'], verdict='PASS', output=json.dumps(result), + checks=diagnostic.verify(task, result)) + plan = {'exam': {'grader': sha256(Path(diagnostic.__file__))}} + compare.verify_retained_grader(plan, [trial], {task['id']: task}) + self.assertEqual(trial['grader_trust'], 'reverified') + trial['output'] = json.dumps({**result, 'reuse_authorized': True}) + with self.assertRaisesRegex(ValueError, 'disagrees'): + compare.verify_retained_grader(plan, [trial], {task['id']: task}) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_skill_eval_pr_diagnostic.py b/tests/test_skill_eval_pr_diagnostic.py new file mode 100644 index 000000000..58964e290 --- /dev/null +++ b/tests/test_skill_eval_pr_diagnostic.py @@ -0,0 +1,67 @@ +"""Offline PR mapping plumbing; no provider or maintained mapping changes.""" +import tempfile +import unittest +from pathlib import Path + +from inferrsi.harness import skill_eval_diagnostic as diagnostic +from inferrsi.infra.artifacts import write_json +from tests.fixtures.skill_eval_compare_support import FixedOutputClient +from tests.fixtures.skill_eval_pr_support import setup +from tests.test_skill_eval_diagnostic import CASES, answer +from tools import skill_eval_compare as compare +from tools import skill_eval_pr as integration + + +class PRDiagnosticTests(unittest.TestCase): + def prepare(self, root, mode='update', adapter=diagnostic.ADAPTER): + repo, path, identity = setup(root, mode, ids=('one',)) + mapping = compare.read_json(path) + mapping['skills'][0]['adapter'] = adapter + write_json(path, mapping) + write_json(root / 'cases.json', CASES[:1]) + output = root / 'planned' + plan = integration.prepare(repo, path, identity, output) + return plan, output, identity + + def test_explicit_diagnostic_mapping_preserves_exact_pair_and_gaps(self): + with tempfile.TemporaryDirectory() as directory: + plan, output, identity = self.prepare(Path(directory)) + self.assertEqual(plan['estimated_calls'], 2) + frozen = compare.load_plan(output / 'selected') + self.assertEqual(frozen['adapter'], diagnostic.ADAPTER) + self.assertEqual(frozen['mode'], 'update') + for role, revision in [('baseline', identity['base']), ('candidate', identity['head'])]: + self.assertEqual(frozen['arms'][role]['git_revision'], revision) + self.assertIsNotNone(frozen['arms'][role]['bundle']) + self.assertEqual([e for e in plan['coverage'] if e['coverage'] != 'covered'], + [{'path': 'exam.txt', 'status': 'M', 'coverage': 'uncovered'}]) + self.assertEqual(list((output / 'selected/results').iterdir()), []) + + def test_pr_report_rechecks_diagnostic_not_audit_schema(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + plan, output, _ = self.prepare(root) + client = FixedOutputClient(root) + write_json(client.answer, answer(0)) + item = plan['comparisons'][0] + compare.execute(output / 'selected', item['plan_sha256'], client=client) + result = integration.reports(output)[0] + self.assertEqual(result['pairs'][0]['outcome'], 'unchanged_correct') + + def test_unknown_adapter_cannot_fall_back_to_audit(self): + with tempfile.TemporaryDirectory() as directory: + plan, _, _ = self.prepare(Path(directory), adapter='untrusted-kind') + self.assertEqual(plan['estimated_calls'], 0) + self.assertIn('Unknown experiment adapter', plan['comparisons'][0]['unsupported']) + self.assertTrue(any(e['coverage'] == 'unsupported' for e in plan['coverage'])) + + def test_addition_cannot_claim_incremental_old_new_diagnosis(self): + with tempfile.TemporaryDirectory() as directory: + plan, _, _ = self.prepare(Path(directory), mode='addition') + self.assertEqual(plan['estimated_calls'], 0) + self.assertIn('paired old/new bundles', plan['comparisons'][0]['unsupported']) + self.assertTrue(any(e['coverage'] == 'unsupported' for e in plan['coverage'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_skill_eval_quarantine.py b/tests/test_skill_eval_quarantine.py new file mode 100644 index 000000000..4ae0f5499 --- /dev/null +++ b/tests/test_skill_eval_quarantine.py @@ -0,0 +1,104 @@ +"""Offline incident-writer races and uncertain durable outcomes; stdlib only.""" +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from tests.test_skill_eval_daily_budget import API, DAY, KEY, PR, binding +from tools import skill_eval_budget as b +from tools import skill_eval_quarantine as q +from tools.skill_eval_budget_git import GitHubDailyStore + +EVIDENCE = "https://github.com/example/repo/issues/1#issuecomment-2" + + +class QuarantineTests(unittest.TestCase): + def test_concurrent_admission_is_preserved_then_stopped(self): + api = API() + store = GitHubDailyStore(api, api.anchor) + other = GitHubDailyStore(api, api.anchor) + def admission(): + snapshot = other.read_verified(KEY) + self.assertIs(other.compare_and_swap(snapshot, b.daily_reserve( + snapshot.ledger, PR, binding(1), DAY)), b.WriteResult.COMMITTED) + api.before_patch = admission + stopped = q.persist_quarantine(store, KEY, EVIDENCE) + self.assertEqual(stopped.ledger.quarantine, EVIDENCE) + self.assertEqual(len(stopped.ledger.entries), 1) + with self.assertRaises(b.BudgetError): + b.daily_claim_once(other, PR, binding(1), DAY) + + def test_lost_write_receipt_reconciles_only_into_stop(self): + api = API() + api.lose_response = True + stopped = q.persist_quarantine(GitHubDailyStore(api, api.anchor), KEY, EVIDENCE) + self.assertEqual(stopped.ledger.quarantine, EVIDENCE) + self.assertEqual(stopped.ledger.revision, 1) + + def test_repeated_stop_preserves_original_evidence_without_new_write(self): + api = API() + store = GitHubDailyStore(api, api.anchor) + first = q.persist_quarantine(store, KEY, EVIDENCE) + tip = api.tip + self.assertEqual(q.persist_quarantine(store, KEY, EVIDENCE), first) + self.assertEqual(q.persist_quarantine(store, KEY, + "https://github.com/example/repo/issues/2"), first) + self.assertEqual(api.tip, tip) + + def test_success_receipt_without_verified_readback_is_failure(self): + api = API() + store = GitHubDailyStore(api, api.anchor) + snapshot = store.read_verified(KEY) + # Even COMMITTED is not enough for incident completion without readback. + store.read_verified = Mock(side_effect=[snapshot] + [OSError("offline")] * 3) + store.compare_and_swap = Mock(return_value=b.WriteResult.COMMITTED) + with self.assertRaisesRegex(b.BudgetError, "NOT verified"): + q.persist_quarantine(store, KEY, EVIDENCE) + self.assertEqual(store.read_verified.call_count, 4) + + def test_conflicts_exhaust_bound_without_false_success(self): + api = API() + store = GitHubDailyStore(api, api.anchor) + store.compare_and_swap = Mock(return_value=b.WriteResult.CONFLICT) + with self.assertRaisesRegex(b.BudgetError, "NOT verified"): + q.persist_quarantine(store, KEY, EVIDENCE) + self.assertEqual(store.compare_and_swap.call_count, 3) + self.assertIsNone(store.read_verified(KEY).ledger.quarantine) + + def test_expired_pricing_does_not_block_emergency_stop(self): + api = API() + config = {"schema_version": 2, "repository_id": 123, "expires_at": 1, + "provider": {"pricing": "not needed for monotonic stop"}, + "daily_budget": {"currency": "USD", "amount": "1000", + "timezone": "Asia/Shanghai", "anchor": { + "ref": api.anchor.ref, "genesis_commit": api.genesis, + "checkpoint_commit": api.genesis}}} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "config.json" + path.write_text(json.dumps(config)) + self.assertEqual(q.quarantine(api, EVIDENCE, config_path=path).ledger.quarantine, + EVIDENCE) + + def test_invalid_evidence_does_not_access_storage(self): + store = Mock() + for evidence in ("raw billing error", EVIDENCE + "?token=private", "", None): + with self.assertRaises(b.BudgetError): + q.persist_quarantine(store, KEY, evidence) + store.read_verified.assert_not_called() + + def test_cli_failure_never_reports_verified(self): + with patch("sys.argv", ["quarantine", "--repository", "example/repo", + "--evidence", EVIDENCE]), \ + patch.object(q, "GitHub", return_value=Mock()), \ + patch.dict(q.os.environ, {"GITHUB_TOKEN": "offline-fixture"}), \ + patch.object(q, "quarantine", side_effect=b.BudgetError("unavailable")), \ + patch("builtins.print") as output: + with self.assertRaises(SystemExit) as error: + q.main() + self.assertEqual(error.exception.code, 1) + output.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_skill_eval_snapshot.py b/tests/test_skill_eval_snapshot.py new file mode 100644 index 000000000..2b81916ef --- /dev/null +++ b/tests/test_skill_eval_snapshot.py @@ -0,0 +1,174 @@ +import hashlib +import json +import subprocess +import tempfile +import unittest +from pathlib import Path + +from inferrsi.knowledge_operations import KnowledgeReader +from tools.skill_eval_compare import snapshot +from tools.skill_eval_snapshot import selected_snapshot + + +def git(repo, *args): + return subprocess.check_output(["git", *args], cwd=repo).decode().strip() + + +def fixture(repo, *, indexed=True): + git(repo, "init", "-q") + root = repo / "library" + directory = root / ("methods/selected" if indexed else "selected") + directory.mkdir(parents=True) + (directory / "SKILL.md").write_text( + "---\nname: selected\ndescription: Synthetic test skill\n---\nRead resources.\n") + (directory / "metadata.json").write_text(json.dumps({"references": ["../shared/ref.md"]})) + (directory / "resource.bin").write_bytes(b"\x00\xff\r\n") + (directory / "__pycache__").mkdir() + (directory / "__pycache__/retained.pyc").write_bytes(b"incidental") + shared = directory.parent / "shared" + shared.mkdir() + (shared / "ref.md").write_bytes(b"reference\r\n") + unrelated = root / "unrelated" + unrelated.mkdir() + (unrelated / "run.sh").write_text("not executed\n") + if indexed: + (root / "skill-index.json").write_text(json.dumps({"schema_version": 1, "skills": { + "selected": {"path": "methods/selected", "backend": "common"}}})) + git(repo, "add", ".") + git(repo, "update-index", "--chmod=+x", "library/unrelated/run.sh") + return commit(repo), directory + + +def commit(repo): + git(repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "core.hooksPath=/dev/null", "commit", "-qm", "synthetic fixture") + return git(repo, "rev-parse", "HEAD") + + +class SelectedSnapshotTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + def test_complete_closure_raw_bytes_and_original_guard(self): + tmp_path = self.root + for indexed in [True, False]: + with self.subTest(indexed=indexed), tempfile.TemporaryDirectory() as temporary: + tmp_path = Path(temporary) + repo = tmp_path / "repo" + repo.mkdir() + revision, directory = fixture(repo, indexed=indexed) + result = selected_snapshot(repo, revision, "library", "selected", tmp_path / "out") + relative = directory.relative_to(repo).as_posix() + assert result["entries"][relative + "/resource.bin"]["sha256"] == hashlib.sha256( + b"\x00\xff\r\n").hexdigest() + incidental = tmp_path / "out" / relative / "__pycache__/retained.pyc" + assert incidental.read_bytes() == b"incidental" + bundle = KnowledgeReader(skill_root=result["root"]).read("checkout:selected") + assert bundle.revision == result["bundle_revision"] + assert any(data == b"reference\r\n" for data in bundle.content.values()) + assert all("unrelated" not in path for path in result["entries"]) + if indexed: + projected = result["projected_index"] + assert projected["source_git_blob"] == git( + repo, "rev-parse", revision + ":library/skill-index.json") + assert projected["sha256"] == hashlib.sha256( + (result["root"] / "skill-index.json").read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError, "Executable"): + snapshot(repo, revision, "library", tmp_path / "all") + + def test_reject_executable_anywhere_in_closure(self): + tmp_path = self.root + for target in ["methods/selected/resource.bin", "methods/shared/ref.md", + "methods/selected/__pycache__/retained.pyc"]: + with self.subTest(target=target), tempfile.TemporaryDirectory() as temporary: + tmp_path = Path(temporary) + revision, _ = fixture(tmp_path) + git(tmp_path, "update-index", "--chmod=+x", "library/" + target) + revision = commit(tmp_path) + with self.assertRaisesRegex(ValueError, "executable"): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out") + assert not (tmp_path / "out").exists() + + def test_bad_reference_fails_without_partial_output(self): + tmp_path = self.root + for reference in ["../../../escape", "../shared/missing", "/absolute", + "..\\shared\\ref.md", "../../skill-index.json"]: + with self.subTest(reference=reference), tempfile.TemporaryDirectory() as temporary: + tmp_path = Path(temporary) + _, directory = fixture(tmp_path) + (directory / "metadata.json").write_text(json.dumps({"references": [reference]})) + git(tmp_path, "add", ".") + revision = commit(tmp_path) + with self.assertRaisesRegex(ValueError, ""): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out") + assert not (tmp_path / "out").exists() + + def test_symlink_and_submodule_rejected(self): + tmp_path = self.root + _, directory = fixture(tmp_path) + target = directory / "link" + target.symlink_to("resource.bin") + git(tmp_path, "add", ".") + revision = commit(tmp_path) + with self.assertRaisesRegex(ValueError, "symlink"): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out") + target.unlink() + git(tmp_path, "add", "-u") + git(tmp_path, "update-index", "--add", "--cacheinfo", "160000," + revision + + ",library/methods/selected/submodule") + revision = commit(tmp_path) + with self.assertRaisesRegex(ValueError, "submodule"): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out") + + def test_index_traversal_alias_rejected(self): + tmp_path = self.root + for path in ['methods/../methods/selected', 'methods/selected.']: + with self.subTest(path=path), tempfile.TemporaryDirectory() as temporary: + tmp_path = Path(temporary) + fixture(tmp_path) + (tmp_path / "library/skill-index.json").write_text(json.dumps({"schema_version": 1, + "skills": {"selected": {"path": path, "backend": "common"}}})) + git(tmp_path, "add", ".") + revision = commit(tmp_path) + with self.assertRaisesRegex(ValueError, ""): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out") + + def test_duplicate_index_fields_rejected(self): + tmp_path = self.root + fixture(tmp_path) + (tmp_path / "library/skill-index.json").write_text( + '{"schema_version":1,"skills":{},"skills":{}}') + git(tmp_path, "add", ".") + revision = commit(tmp_path) + with self.assertRaisesRegex(ValueError, "Duplicate"): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out") + + def test_directory_case_alias_rejected(self): + tmp_path = self.root + _, directory = fixture(tmp_path) + (directory / "Assets").mkdir() + (directory / "assets").mkdir() + (directory / "Assets/a").write_text("a") + (directory / "assets/b").write_text("b") + git(tmp_path, "add", ".") + revision = commit(tmp_path) + with self.assertRaisesRegex(ValueError, "Aliased"): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out") + + def test_revision_and_destination_guard(self): + tmp_path = self.root + revision, _ = fixture(tmp_path) + with self.assertRaisesRegex(ValueError, "immutable"): + selected_snapshot(tmp_path, "HEAD", "library", "selected", tmp_path / "out") + with self.assertRaisesRegex(ValueError, "destination"): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path) + with self.assertRaisesRegex(ValueError, "Revision"): + selected_snapshot(tmp_path, revision, "library", "selected", tmp_path / "out", + bundle_revision="sha256:" + "0" * 64) + assert not (tmp_path / "out").exists() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/skill_eval_auto.py b/tools/skill_eval_auto.py index 009ec7bc9..2e38b2b37 100644 --- a/tools/skill_eval_auto.py +++ b/tools/skill_eval_auto.py @@ -13,8 +13,11 @@ from inferrsi.infra.artifacts import write_json from tools import skill_eval_pr as pr -from tools.skill_eval_budget import Binding, BudgetError, PRKey, WriteResult, claim_once, reserve -from tools.skill_eval_budget_git import GitHubGitStore, LedgerAnchor +from tools.skill_eval_budget import (Binding, BudgetError, DailyKey, PRKey, WriteResult, + claim_once, daily_claim_once, daily_reserve, reserve, + shanghai_day) +from tools.skill_eval_budget_git import (DailyLedgerAnchor, GitHubDailyStore, GitHubGitStore, + LedgerAnchor) CONFIG = pr.ROOT / "tools/skill_eval_auto_config.json" @@ -24,9 +27,11 @@ def configuration(path=None, *, now=None): value = json.loads(Path(CONFIG if path is None else path).read_text(encoding="utf-8")) fields = {"schema_version", "repository_id", "label", "operators", "expires_at", "provider", "provider_sha256", "anchors", "review_evidence"} + if isinstance(value, dict) and value.get("schema_version") == 2: + fields.add("daily_budget") if not isinstance(value, dict) or set(value) != fields: raise ValueError("Incomplete trusted automatic-execution configuration") - if (type(value["schema_version"]) is not int or value["schema_version"] != 1 + if (type(value["schema_version"]) is not int or value["schema_version"] not in {1, 2} or type(value["repository_id"]) is not int or value["repository_id"] <= 0 or not isinstance(value["label"], str) or not value["label"] or not isinstance(value["operators"], list) @@ -44,6 +49,8 @@ def configuration(path=None, *, now=None): or set(anchor) != {"ref", "genesis_commit", "checkpoint_commit"}): raise ValueError("Invalid trusted ledger anchor configuration") LedgerAnchor(key=PRKey(value["repository_id"], int(number)), **anchor) + if value["schema_version"] == 2: + daily_anchor(value) provider = value["provider"] required = {"model", "client", "pricing", "spend_limit"} if (not isinstance(provider, dict) or not required <= provider.keys() @@ -72,6 +79,24 @@ def configuration(path=None, *, now=None): return value +def daily_anchor(config): + """v1 remains readable for retained evidence/planning, never paid admission.""" + budget = config.get("daily_budget") + if (config.get("schema_version") != 2 or not isinstance(budget, dict) + or set(budget) != {"currency", "amount", "timezone", "anchor"} + or budget["currency"] != "USD" or budget["amount"] != "1000" + or budget["timezone"] != "Asia/Shanghai" + or not isinstance(budget["anchor"], dict) + or set(budget["anchor"]) != {"ref", "genesis_commit", "checkpoint_commit"}): + raise BudgetError("Audited global daily genesis and USD 1000 configuration required") + return DailyLedgerAnchor(key=DailyKey(config["repository_id"]), **budget["anchor"]) + + +def daily_store(api, config): + anchor = daily_anchor(config) + return GitHubDailyStore(api, anchor), anchor.key + + def ledger_store(api, config, number): key = PRKey(config["repository_id"], number) anchor = config["anchors"].get(str(number)) @@ -130,6 +155,7 @@ def live_label(api, plan, config): def eligible(api, plan, config): if plan.get("execution_blocked"): raise ValueError("Planning-only plan is blocked from paid execution") + daily_anchor(config) if not plan["estimated_calls"] or not plan["comparisons"]: raise ValueError("No meaningful mapped comparisons") if any(c.get("coverage") == "unsupported" for c in plan["coverage"]): @@ -156,8 +182,13 @@ def eligible(api, plan, config): def status(api, plan, config): store, key = ledger_store(api, config, plan["pr"]) ledger = store.read_verified(key).ledger + global_store, global_key = daily_store(api, config) + daily = global_store.read_verified(global_key).ledger + daily.require_active() + day = shanghai_day() actual = ledger.genesis.legacy_charge + sum(e.actual_cost or 0 for e in ledger.entries) - return {"label": config["label"], "spent_or_legacy_usd": str(Decimal(actual) / 1000000), + return {"daily_day": day, "daily_remaining_usd": str(Decimal(daily.remaining_on(day)) / 1000000), + "label": config["label"], "spent_or_legacy_usd": str(Decimal(actual) / 1000000), "reserved_or_unresolved_usd": str(Decimal(ledger.charged - actual) / 1000000), "remaining_usd": str(Decimal(ledger.remaining) / 1000000)} @@ -177,11 +208,21 @@ def controlled_execute(api, output, *, config_path=CONFIG, run_id, attempt, laun amount = Decimal(plan["reservation"]["amount"]) * 1000000 binding = Binding(pr.fingerprint(plan), f"{run_id}.{attempt}", int(amount.to_integral_value(rounding=ROUND_CEILING))) + global_store, global_key = daily_store(api, config) + # Pre-read both: no genesis creation, missing history never means zero spend. snapshot = store.read_verified(key) + global_snapshot = global_store.read_verified(global_key) + admitted_day = shanghai_day() + global_successor = daily_reserve(global_snapshot.ledger, key, binding, admitted_day) + if (global_successor == global_snapshot.ledger + or global_store.compare_and_swap(global_snapshot, global_successor) + is not WriteResult.COMMITTED): + raise BudgetError("Duplicate, conflicted or uncertain global reservation; no paid execution") successor = reserve(snapshot.ledger, binding) if (successor == snapshot.ledger or store.compare_and_swap(snapshot, successor) is not WriteResult.COMMITTED): raise BudgetError("Duplicate, conflicted or uncertain reservation; no paid execution") + daily_claim_once(global_store, key, binding, shanghai_day()) claim_once(store, key, binding) # Not transferable: launch once in this process only. def check(): @@ -194,7 +235,16 @@ def check(): ledger = store.read_verified(key).ledger if not any(e.binding == binding and e.phase == "claimed" for e in ledger.entries): raise ValueError("Live ledger claim missing") + daily = global_store.read_verified(global_key).ledger + daily.require_active() + if not any(e.key == key and e.binding == binding and e.phase == "claimed" + and e.admitted_day == admitted_day for e in daily.entries): + raise BudgetError("Live global claim missing") + # Check AFTER live reads as well: no stale admission-day grant at midnight. + if shanghai_day() != admitted_day: + raise BudgetError("Shanghai admission day ended; no cross-day paid requests") + check() # Both durable claims and the current day must still hold before launch. write_json(output / "approval.json", {"execution_id": pr.execution_id(plan), **authorization}) context = f"{binding.plan_digest}:{binding.run_id}" with authorization_broker(check, plan["reservation"]["requests"], @@ -248,7 +298,7 @@ def report_control(api, plan): "remaining_usd": "unknown"} try: config = configuration() - result["reason"] = "Audited PR ledger missing, unavailable or unsafe" + result["reason"] = "Audited PR/global daily ledger missing, unavailable, quarantined or unsafe" result.update(status(api, plan, config)) result["reason"] = "Plan has no supported pairs or provider/bounds are stale" eligible(api, plan, config) diff --git a/tools/skill_eval_budget.py b/tools/skill_eval_budget.py index c2f3f32f3..34b886420 100644 --- a/tools/skill_eval_budget.py +++ b/tools/skill_eval_budget.py @@ -272,3 +272,251 @@ def unique(pairs): value["revision"], tuple(entries)) except (TypeError, KeyError, ValueError, OverflowError, RecursionError) as exc: raise BudgetError("Missing, corrupt, or unsupported ledger") from exc + + +DAILY_LIMIT_MICROUSD = 1_000_000_000 + + +def _day(value): + from datetime import date + + try: + valid = type(value) is str and date.fromisoformat(value).isoformat() == value + except ValueError: + valid = False + if not valid: + raise BudgetError("Expected canonical calendar day") + return value + + +def shanghai_day(now=None): + """Trusted aware wall clock only; a missing timezone database fails closed.""" + from datetime import datetime, timezone + from zoneinfo import ZoneInfo + + now = datetime.now(timezone.utc) if now is None else now + if not isinstance(now, datetime) or now.tzinfo is None or now.utcoffset() is None: + raise BudgetError("An aware trusted clock is required") + return now.astimezone(ZoneInfo("Asia/Shanghai")).date().isoformat() + + +@dataclass(frozen=True) +class DailyKey: + repository_id: int + + def __post_init__(self): + _amount(self.repository_id, positive=True) + + +@dataclass(frozen=True) +class DailyEntry: + key: PRKey + binding: Binding + admitted_day: str + phase: str = "reserved" + closed_day: str | None = None + actual_cost: int | None = None + + def __post_init__(self): + if type(self.key) is not PRKey: + raise BudgetError("Missing daily reservation PR identity") + Entry(self.binding, self.phase, self.actual_cost) + _day(self.admitted_day) + if self.phase in {"finished", "cancelled"}: + if _day(self.closed_day) < self.admitted_day: + raise BudgetError("Closing day precedes admission") + elif self.closed_day is not None: + raise BudgetError("Outstanding reservation cannot have a closing day") + + +@dataclass(frozen=True) +class DailyLedger: + """Repository-wide history, not one ref per date or per PR. + + Audited genesis must import ALL historical daily charges and ALL outstanding + upper bounds, including legacy PR reservations. Unknown costs block genesis. + Outstanding work counts on every day from admission until trusted settlement. + Settlement retains the FULL bound on every day through closure, inclusive; + it never refunds prior days. A missing settlement safely reduces availability. + """ + + key: DailyKey + evidence: str + entries: tuple[DailyEntry, ...] = () + revision: int = 0 + quarantine: str | None = None + + def __post_init__(self): + if type(self.key) is not DailyKey: + raise BudgetError("Missing daily ledger repository identity") + _text(self.evidence) + _amount(self.revision) + if self.quarantine is not None: + _text(self.quarantine) + if (type(self.entries) is not tuple + or any(type(e) is not DailyEntry or e.key.repository_id != self.key.repository_id + for e in self.entries)): + raise BudgetError("Invalid daily ledger entries") + plans = [(e.key, e.binding.plan_digest) for e in self.entries] + runs = [e.binding.run_id for e in self.entries] + if len(set(plans)) != len(plans) or len(set(runs)) != len(runs): + raise BudgetError("Duplicate global plan or execution attempt") + # Charges can only increase at admissions; checking these boundaries + # proves the cap for the entire timeline, including open-ended entries. + if any(self.charged_on(e.admitted_day) > DAILY_LIMIT_MICROUSD for e in self.entries): + raise BudgetError("Global Shanghai daily budget exhausted") + + def require_active(self): + if self.quarantine is not None: + raise BudgetError("Global budget quarantined; audited reconciliation required") + + def charged_on(self, day): + _day(day) + return sum(e.binding.worst_case for e in self.entries + if e.admitted_day <= day and (e.closed_day is None or day <= e.closed_day)) + + def remaining_on(self, day): + return DAILY_LIMIT_MICROUSD - self.charged_on(day) + + +def daily_reserve(ledger, key, binding, day): + _day(day) + if type(ledger) is not DailyLedger or type(key) is not PRKey or type(binding) is not Binding: + raise BudgetError("Invalid global reservation") + ledger.require_active() + for entry in ledger.entries: + if entry.key == key and entry.binding == binding: + return ledger # Across midnight too: never create another reservation. + if ((entry.key == key and entry.binding.plan_digest == binding.plan_digest) + or entry.binding.run_id == binding.run_id): + raise BudgetError("Global plan or run already bound differently") + if any(day < e.admitted_day or (e.closed_day is not None and day < e.closed_day) + for e in ledger.entries): + raise BudgetError("Daily ledger clock moved backwards") + return replace(ledger, revision=ledger.revision + 1, + entries=ledger.entries + (DailyEntry(key, binding, day),)) + + +def daily_advance(ledger, key, binding, phase, day, *, actual_cost=None): + _day(day) + if type(ledger) is not DailyLedger: + raise BudgetError("Invalid global ledger") + ledger.require_active() + if any(day < e.admitted_day or (e.closed_day is not None and day < e.closed_day) + for e in ledger.entries): + raise BudgetError("Daily ledger clock moved backwards") + for i, entry in enumerate(ledger.entries): + if entry.key != key or entry.binding != binding: + continue + if phase == "claimed" and day != entry.admitted_day: + raise BudgetError("A reservation must be claimed on its admission day") + closed = day if phase in {"finished", "cancelled"} else None + updated = replace(entry, phase=phase, closed_day=closed, actual_cost=actual_cost) + if updated == entry and phase in {"finished", "cancelled"}: + return ledger + allowed = {"reserved": {"claimed", "cancelled"}, "claimed": {"finished"}} + if phase not in allowed.get(entry.phase, set()): + raise BudgetError("Unsafe or duplicate global execution transition") + return replace(ledger, revision=ledger.revision + 1, + entries=ledger.entries[:i] + (updated,) + ledger.entries[i + 1:]) + raise BudgetError("No exact global reservation") + + +@dataclass(frozen=True) +class DailySnapshot: + token: str + ledger: DailyLedger + + def __post_init__(self): + _text(self.token) + if type(self.ledger) is not DailyLedger: + raise BudgetError("Missing global ledger") + + +def daily_claim_once(store, key, binding, day): + snapshot = store.read_verified(DailyKey(key.repository_id)) + if type(snapshot) is not DailySnapshot or snapshot.ledger.key != DailyKey(key.repository_id): + raise BudgetError("Storage returned wrong global ledger") + successor = daily_advance(snapshot.ledger, key, binding, "claimed", day) + if store.compare_and_swap(snapshot, successor) is not WriteResult.COMMITTED: + raise BudgetError("Global claim conflicted or outcome unknown") + return successor + + +def daily_quarantine(ledger, evidence): + """Durably stop ALL global execution on unknown/over-bound exposure. + + This is a one-way transition. There is deliberately no reset/reconciliation + writer. Record only bounded non-secret evidence; do not include credentials + or raw provider errors. Existing quarantine is sufficient and unchanged. + """ + if type(ledger) is not DailyLedger: + raise BudgetError("Invalid global ledger") + _text(evidence) + if ledger.quarantine is not None: + return ledger + return replace(ledger, quarantine=evidence, revision=ledger.revision + 1) + + +def validate_daily_successor(before, after): + if type(before) is not DailyLedger or type(after) is not DailyLedger: + raise BudgetError("Invalid global transition state") + if (before.key != after.key or before.evidence != after.evidence + or after.revision != before.revision + 1): + raise BudgetError("Global ledger reset or identity change forbidden") + if before.quarantine is not None: + raise BudgetError("Quarantine cannot be reset or edited") + if after.quarantine is not None: + if after != daily_quarantine(before, after.quarantine): + raise BudgetError("Quarantine must preserve all existing charges") + return + candidates = [] + if len(after.entries) == len(before.entries) + 1: + new = after.entries[-1] + candidates.append(daily_reserve(before, new.key, new.binding, new.admitted_day)) + elif len(after.entries) == len(before.entries): + for old, new in zip(before.entries, after.entries): + if old != new: + candidates.append(daily_advance(before, new.key, new.binding, new.phase, + new.closed_day or new.admitted_day, + actual_cost=new.actual_cost)) + if after not in candidates: + raise BudgetError("Not a single monotonic global budget transition") + + +def encode_daily(ledger): + if type(ledger) is not DailyLedger: + raise BudgetError("Invalid global ledger") + return json.dumps({"schema_version": 1, "kind": "shanghai_daily", **asdict(ledger)}, + sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def decode_daily(payload): + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise BudgetError("Duplicate ledger JSON field") + result[key] = value + return result + + try: + value = json.loads(payload, object_pairs_hook=unique) + if (type(value) is not dict + or set(value) != {"schema_version", "kind", "key", "evidence", + "entries", "revision", "quarantine"} + or type(value["schema_version"]) is not int or value["schema_version"] != 1 + or value["kind"] != "shanghai_daily" or type(value["entries"]) is not list): + raise BudgetError("Unsupported global ledger schema") + entries = [] + for entry in value["entries"]: + if type(entry) is not dict or set(entry) != { + "key", "binding", "admitted_day", "phase", "closed_day", "actual_cost" + }: + raise BudgetError("Invalid global reservation schema") + entries.append(DailyEntry(**{**entry, "key": PRKey(**entry["key"]), + "binding": Binding(**entry["binding"])})) + return DailyLedger(DailyKey(**value["key"]), value["evidence"], + tuple(entries), value["revision"], value["quarantine"]) + except (TypeError, KeyError, ValueError, OverflowError, RecursionError) as exc: + raise BudgetError("Missing, corrupt, or unsupported global ledger") from exc diff --git a/tools/skill_eval_budget_git.py b/tools/skill_eval_budget_git.py index 2a638defe..4aaeeb37a 100644 --- a/tools/skill_eval_budget_git.py +++ b/tools/skill_eval_budget_git.py @@ -13,12 +13,17 @@ from tools.skill_eval_budget import ( BudgetError, + DailyKey, + DailySnapshot, Ledger, PRKey, Snapshot, WriteResult, decode, + decode_daily, encode, + encode_daily, + validate_daily_successor, validate_successor, ) @@ -63,8 +68,17 @@ class GitHubGitStore: No ref creation, deletion, force update, or settings endpoint is used. """ + anchor_type = LedgerAnchor + snapshot_type = Snapshot + decode_state = staticmethod(decode) + encode_state = staticmethod(encode) + validate_transition = staticmethod(validate_successor) + + def valid_genesis(self, ledger): + return ledger.revision == 0 and not ledger.entries + def __init__(self, api, anchor: LedgerAnchor, *, max_history=10000): - if type(anchor) is not LedgerAnchor or type(max_history) is not int or max_history < 1: + if type(anchor) is not self.anchor_type or type(max_history) is not int or max_history < 1: raise BudgetError("Trusted anchor and bounded history are required") self.api, self.anchor, self.max_history = api, anchor, max_history self._observed = anchor.checkpoint_commit @@ -109,7 +123,7 @@ def _state(self, commit_sha): raw = base64.b64decode("".join(blob["content"].split()), validate=True) if len(raw) > 4_000_000 or blob["size"] != len(raw): raise ValueError("Invalid ledger size") - ledger = decode(raw.decode("utf-8")) + ledger = self.decode_state(raw.decode("utf-8")) if ledger.key != self.anchor.key: raise ValueError("Wrong PR identity") return parents, ledger @@ -131,7 +145,7 @@ def read_verified(self, key: PRKey) -> Snapshot: parents, ledger = self._state(current) states.append(ledger) if current == self.anchor.genesis_commit: - if len(parents) > 1 or ledger.revision != 0 or ledger.entries: + if len(parents) > 1 or not self.valid_genesis(ledger): raise BudgetError("Invalid audited genesis") break if len(parents) != 1: @@ -142,22 +156,22 @@ def read_verified(self, key: PRKey) -> Snapshot: if self.anchor.checkpoint_commit not in seen or self._observed not in seen: raise BudgetError("Ledger rollback or divergent history detected") for before, after in zip(reversed(states[1:]), reversed(states[:-1])): - validate_successor(before, after) + self.validate_transition(before, after) # A racing writer is harmless, but do not label an already stale read current. if self._tip() != tip: raise BudgetError("Ledger changed during verification; retry a fresh read") self._observed = tip - return Snapshot(tip, states[0]) + return self.snapshot_type(tip, states[0]) def compare_and_swap(self, expected: Snapshot, successor: Ledger) -> WriteResult: - validate_successor(expected.ledger, successor) + self.validate_transition(expected.ledger, successor) verified = self.read_verified(self.anchor.key) if verified != expected: return WriteResult.CONFLICT try: tree = self.api.request("POST", "/git/trees", {"tree": [{ "path": "ledger.json", "mode": "100644", "type": "blob", - "content": encode(successor), + "content": self.encode_state(successor), }]}) tree_sha = _sha(tree["sha"]) commit = self.api.request("POST", "/git/commits", { @@ -177,3 +191,40 @@ def compare_and_swap(self, expected: Snapshot, successor: Ledger) -> WriteResult return WriteResult.UNKNOWN self._observed = commit_sha return WriteResult.COMMITTED + + +@dataclass(frozen=True) +class DailyLedgerAnchor: + """One independently pinned repository-wide ledger; never a per-day ref.""" + + key: DailyKey + ref: str + genesis_commit: str + checkpoint_commit: str + + def __post_init__(self): + if type(self.key) is not DailyKey: + raise BudgetError("Invalid global ledger identity") + if self.ref != f"refs/heads/skill-eval-budget/{self.key.repository_id}/daily": + raise BudgetError("Global budget requires its single repository-wide ref") + _sha(self.genesis_commit) + _sha(self.checkpoint_commit) + + +class GitHubDailyStore(GitHubGitStore): + """Reuse the same nonce-bearing Git CAS and full lineage verification. + + Genesis is separately audited, including imported historical and outstanding + entries. This adapter never creates or resets it. Existing PR ledgers remain + independently mandatory; commit daily admission first, then PR admission, + then both one-shot claims before launching. Partial failures burn capacity. + """ + + anchor_type = DailyLedgerAnchor + snapshot_type = DailySnapshot + decode_state = staticmethod(decode_daily) + encode_state = staticmethod(encode_daily) + validate_transition = staticmethod(validate_daily_successor) + + def valid_genesis(self, ledger): + return ledger.revision == 0 diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index e7fd3df51..e07b10f67 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -11,6 +11,7 @@ from dataclasses import replace from pathlib import Path +from inferrsi.harness import skill_eval_diagnostic as diagnostic from inferrsi.harness.skill_evaluation import ( CodexClient, export_bundle, @@ -24,6 +25,7 @@ from inferrsi.model.anthropic_agent import AnthropicAgentClient, validate_schema, validate_spec from inferrsi.model.request_guard import SOURCE as GUARD_SOURCE from tools.skill_eval_pilot import PROMPT, SCHEMA, observed_tokens +from tools.skill_eval_snapshot import selected_snapshot from tools.skill_eval_verifier import verify ROOT = Path(__file__).resolve().parents[1] @@ -31,6 +33,7 @@ GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "src/inferrsi/harness/skill_evaluation.py", + "src/inferrsi/harness/skill_eval_diagnostic.py", "tools/skill_eval_snapshot.py", "src/inferrsi/infra/executors/local.py", "src/inferrsi/infra/processes.py", "src/inferrsi/infra/artifacts.py") EVIDENCE = ("execution.json", "agent/stdout.log", "agent/stderr.log", @@ -171,8 +174,17 @@ def validate_default_audit(task: dict) -> None: def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> None: if set(config) - {"mode", "skill", "skill_root", "cases", "schema", "grader", - "snapshots", "model", "limits", "client", "pricing", "spend_limit"}: + "snapshots", "model", "limits", "client", "pricing", "spend_limit", + "adapter"}: raise ValueError("Unknown configuration field") + adapter = config.get("adapter", "audit") + if adapter not in ("audit", diagnostic.ADAPTER): + raise ValueError("Unknown experiment adapter") + if adapter == diagnostic.ADAPTER: + if config["mode"] != "update": + raise ValueError("Diagnostic comparisons require paired old/new bundles (update mode)") + if "grader" in config or "schema" in config: + raise ValueError("Diagnostic adapter owns its frozen schema and verifier") if config["mode"] not in ("addition", "update"): raise ValueError("mode must be addition or update") if not re.fullmatch(r"[a-z0-9][a-z0-9._-]*", config["skill"]): @@ -206,8 +218,11 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if not tasks: raise ValueError("cases must be nonempty with unique IDs") for task in tasks: - validate_input(task, CASE_INPUT) - validate_audit_values(task) + if adapter == diagnostic.ADAPTER: + diagnostic.validate_task(task) + else: + validate_input(task, CASE_INPUT) + validate_audit_values(task) missing = {"id", "split", "request"} - task.keys() if missing: raise ValueError("Missing required case fields: " + ", ".join(sorted(missing))) @@ -215,7 +230,7 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> raise ValueError("case IDs must be safe path components") if task["split"] not in ("development", "holdout", "discovery"): raise ValueError("Unknown case split") - if default_grader: + if default_grader and adapter == "audit": validate_default_audit(task) if len({task["id"] for task in tasks}) != len(tasks): raise ValueError("cases must be nonempty with unique IDs") @@ -289,13 +304,17 @@ def require_git_version(version: str) -> None: def prepare(config_path: Path, repo: Path, output: Path) -> str: config = read_json(config_path) tasks = read_json(config_path.parent / config["cases"]) - grader = config_path.parent / config["grader"] if "grader" in config else ( - ROOT / "tools/skill_eval_verifier.py" + is_diagnostic = config.get("adapter") == diagnostic.ADAPTER + grader = Path(diagnostic.__file__) if is_diagnostic else ( + config_path.parent / config["grader"] if "grader" in config else ( + ROOT / "tools/skill_eval_verifier.py" + ) ) validate(config, tasks, default_grader=( grader.read_bytes() == (ROOT / "tools/skill_eval_verifier.py").read_bytes())) require_git_version(subprocess.check_output(("git", "--version"), cwd=repo, text=True)) - schema = read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA + schema = diagnostic.SCHEMA if is_diagnostic else ( + read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA) if "client" in config: validate_schema(schema) output.mkdir(parents=True, exist_ok=False) @@ -308,9 +327,17 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: with tempfile.TemporaryDirectory(prefix="se-") as temporary: for role, short in zip(ROLES, ("b", "c")): selected = config["snapshots"][role] - root = snapshot(repo, selected["git_revision"], config["skill_root"], - Path(temporary) / short, - allow_empty=role == "baseline" and config["mode"] == "addition") + if is_diagnostic: + closure = selected_snapshot( + repo, selected["git_revision"], config["skill_root"], config["skill"], + Path(temporary) / short, bundle_revision=selected.get("bundle_revision")) + root = closure["root"] + write_json(output / "plan/snapshots" / f"{role}.json", + {key: value for key, value in closure.items() if key != "root"}) + else: + root = snapshot(repo, selected["git_revision"], config["skill_root"], + Path(temporary) / short, + allow_empty=role == "baseline" and config["mode"] == "addition") bundle = None if role == "candidate" or config["mode"] == "update": reader = KnowledgeReader(skill_root=root) @@ -339,13 +366,15 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: "exam": {"cases": files["plan/tasks.json"], "schema": files["plan/schema.json"], "grader": files["validator/verify.py"]}, "same_bundle": before is not None and before["revision"] == after["revision"], - "prompt": PROMPT, "files": files, + "prompt": diagnostic.PROMPT if is_diagnostic else PROMPT, "files": files, "controllers": {name: sha256(ROOT / name) for name in CONTROLLERS}, "retry_policy": "none", "cost": None, "isolation": "Fresh sessions and workspaces; selected bundle only. " "CodexClient disables global skills, plugins, apps, memories and web. " "Controlled exposure is not OS read confinement.", } + if is_diagnostic: + plan["adapter"] = diagnostic.ADAPTER if "client" in config: client = AnthropicAgentClient(config["model"]["name"], config["client"], config["limits"]["timeout_seconds"], @@ -728,7 +757,11 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: retained answer. A custom grader has no in-tree authority to re-derive from; that exam is reported as unverified rather than silently trusted. """ - if plan["exam"]["grader"] != sha256(ROOT / "tools/skill_eval_verifier.py"): + trusted_verifier = { + sha256(ROOT / "tools/skill_eval_verifier.py"): verify, + sha256(Path(diagnostic.__file__)): diagnostic.verify, + }.get(plan["exam"]["grader"]) + if trusted_verifier is None: for trial in trials: if trial["verdict"] in ("PASS", "FAIL"): # No in-tree authority can re-derive this grade, and the retained @@ -743,7 +776,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: for trial in trials: if trial["verdict"] not in ("PASS", "FAIL"): continue - checks = verify(tasks[trial["task_id"]], json.loads(trial["output"])) + checks = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) if checks != trial["checks"] or checks["passed"] != (trial["verdict"] == "PASS"): raise ValueError("Untrusted grading disagrees with the trusted grader") trial["grader_trust"] = "reverified" diff --git a/tools/skill_eval_pr.py b/tools/skill_eval_pr.py index 67b595e6e..b5ac1fbeb 100644 --- a/tools/skill_eval_pr.py +++ b/tools/skill_eval_pr.py @@ -18,7 +18,6 @@ from inferrsi.model.anthropic_agent import KIND, money, reservation from tools import skill_eval_compare as compare from tools.check_knowledge_model_policy import GitHubHTTPError -from tools.skill_eval_verifier import verify ROOT = Path(__file__).resolve().parents[1] MAPPING = ROOT / "tools/skill_eval_pr_cases.json" @@ -110,6 +109,10 @@ def prepare(repo, mapping_path, identity, output, *, provider=None, planning_blo "snapshots": {"baseline": {"git_revision": identity["base"]}, "candidate": {"git_revision": identity["head"]}}, "model": mapping["model"], "limits": mapping["limits"]} + if "adapter" in item: + # Trusted mapping chooses an explicit typed contract; compare validates it. + # Do not infer a diagnostic adapter from arbitrary PR file contents. + config["adapter"] = item["adapter"] if provider is not None: config.update({key: provider[key] for key in ("client", "pricing", "spend_limit")}) config_path = output / (item["skill"] + ".json") @@ -324,12 +327,11 @@ def reports(output): if not isinstance(status, dict) or not isinstance(status.get("execution_state"), str): raise ValueError("Invalid retained execution status shape") tasks = {t["id"]: t for t in compare.read_json(directory / "plan/tasks.json")} - for trial in [p[r] for p in result["pairs"] for r in compare.ROLES] + result["discovery"]: - if trial["verdict"] in ("PASS", "FAIL"): - # Recheck the supported grader using trusted code, not artifact verify.py. - checks = verify(tasks[trial["task_id"]], json.loads(trial["output"])) - if checks != trial["checks"] or checks["passed"] != (trial["verdict"] == "PASS"): - raise ValueError("Untrusted grading disagrees with the trusted grader") + # Recheck the exact built-in adapter with trusted code, not artifact verify.py. + compare.verify_retained_grader( + compare.load_plan(directory), + [p[r] for p in result["pairs"] for r in compare.ROLES] + result["discovery"], + tasks) results.append(result) return results @@ -870,11 +872,13 @@ def main(): reason = "Trusted automatic configuration or pricing is expired or invalid" else: try: - from tools.skill_eval_auto import eligible, ledger_store + from tools.skill_eval_auto import daily_store, eligible, ledger_store eligible(api, plan, config) live_label(api, plan, config) store, key = ledger_store(api, config, pr) store.read_verified(key) + global_store, global_key = daily_store(api, config) + global_store.read_verified(global_key).ledger.require_active() candidate = True reason = "Eligible; durable reservation required before execution" except (ValueError, KeyError, TypeError): diff --git a/tools/skill_eval_quarantine.py b/tools/skill_eval_quarantine.py new file mode 100644 index 000000000..6695bbfca --- /dev/null +++ b/tools/skill_eval_quarantine.py @@ -0,0 +1,89 @@ +"""Explicit trusted incident stop; never admission, billing ingestion or unquarantine.""" + +import argparse +import json +import os +import re + +from tools.skill_eval_auto import CONFIG, daily_store +from tools.skill_eval_budget import BudgetError, DailySnapshot, daily_quarantine +from tools.skill_eval_pr import GitHub + + +def evidence_reference(value): + # Public incident references only; never accept arbitrary raw billing/error text, + # credential URLs, query strings or fragments containing private payloads. + if (not isinstance(value, str) or len(value) > 512 or not re.fullmatch( + r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/" + r"(?:issues|pull)/[1-9][0-9]*" + r"(?:#(?:issuecomment|discussion_r|pullrequestreview)-[1-9][0-9]*)?", + value)): + raise BudgetError("A public GitHub incident evidence reference is required") + return value + + +def persist_quarantine(store, key, evidence, *, attempts=3): + """Only verified durable stop is success, including after a lost CAS receipt. + + Reread/retry is safe for this monotonic stop, NEVER for an execution grant. + Any existing quarantine satisfies the stop; preserve its original evidence. + A bounded failure is NOT success: caller must keep paid work stopped and + escalate reconciliation, then retry this operation when storage is usable. + Already issued/in-flight provider calls cannot be recalled by this operation. + """ + evidence_reference(evidence) + if type(attempts) is not int or not 1 <= attempts <= 10: + raise BudgetError("Invalid bounded quarantine retry count") + for _ in range(attempts): + try: + snapshot = store.read_verified(key) + if type(snapshot) is not DailySnapshot or snapshot.ledger.key != key: + raise BudgetError("Wrong global quarantine history") + if snapshot.ledger.quarantine is not None: + return snapshot + successor = daily_quarantine(snapshot.ledger, evidence) + # All receipts (including explicit COMMITTED) require fresh verified + # readback. A competing admission can win; recompute against its tip. + store.compare_and_swap(snapshot, successor) + except (OSError, ValueError, KeyError, TypeError): + # Never expose provider material, tokens, or backend exception text. + continue + # Last write may have committed despite lost response; final read is read-only. + try: + snapshot = store.read_verified(key) + if (type(snapshot) is DailySnapshot and snapshot.ledger.key == key + and snapshot.ledger.quarantine is not None): + return snapshot + except (OSError, ValueError, KeyError, TypeError): + pass + raise BudgetError("Global quarantine NOT verified; keep paid work stopped and reconcile") + + +def quarantine(api, evidence, *, config_path=CONFIG): + """Trusted root anchor only; expired provider pricing must not prevent a stop.""" + evidence_reference(evidence) + with open(config_path, encoding="utf-8") as handle: + config = json.load(handle) + store, key = daily_store(api, config) + return persist_quarantine(store, key, evidence) + + +def main(): + parser = argparse.ArgumentParser(description="Persist global skill-eval incident quarantine") + parser.add_argument("--repository", required=True, help="Trusted owner/repository") + parser.add_argument("--evidence", required=True, help="Public GitHub incident reference") + args = parser.parse_args() + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", args.repository): + parser.error("Invalid repository identity") + try: + evidence_reference(args.evidence) + # Reuse the existing authenticated client; no provider key or model call. + api = GitHub(args.repository, os.environ["GITHUB_TOKEN"]) + snapshot = quarantine(api, args.evidence) + except (OSError, ValueError, KeyError, TypeError): + parser.exit(1, "Global quarantine NOT verified; keep paid work stopped and reconcile.\n") + print(json.dumps({"state": "quarantine_verified", "revision": snapshot.ledger.revision})) + + +if __name__ == "__main__": + main() diff --git a/tools/skill_eval_snapshot.py b/tools/skill_eval_snapshot.py new file mode 100644 index 000000000..ff85b7835 --- /dev/null +++ b/tools/skill_eval_snapshot.py @@ -0,0 +1,159 @@ +"""Opt-in raw-Git selected checkout closure; not the whole-library audit snapshot. + +Closure follows KnowledgeReader: every selected-directory resource and each +metadata-declared reference. Markdown navigation links are not dependencies. +The index is explicitly projected, never represented as an original Git blob. +""" + +import hashlib +import json +import posixpath +import re +import subprocess +import tempfile +import unicodedata +from pathlib import Path, PurePosixPath + +from inferrsi.catalog import BACKENDS +from inferrsi.knowledge_operations import KnowledgeReader, safe_path +from inferrsi.skill_catalog import _skill_name, _unique_fields, skill_metadata + +GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") + + +def _path(value: str) -> str: + safe_path(value) + if unicodedata.normalize("NFC", value) != value: + raise ValueError("Noncanonical Unicode snapshot path") + # Reject filesystem aliases even when preparation happens on Linux. + for part in value.split("/"): + if part.endswith((".", " ")) or part.split(".")[0].upper() in { + "CON", "PRN", "AUX", "NUL", *(f"COM{i}" for i in range(1, 10)), + *(f"LPT{i}" for i in range(1, 10)), + }: + raise ValueError(f"Aliased snapshot path: {value}") + return value + + +def selected_snapshot(repo: Path, revision: str, skill_root: str, skill: str, + destination: Path, *, bundle_revision: str | None = None) -> dict: + """Materialize a selected closure into a new destination, returning provenance. + + No checkout/filter execution, chmod, provider call, or network fetch is used. + All selected Git entries must be ordinary non-executable blobs, including + files the reader considers incidental. Failure publishes no destination. + """ + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Selected snapshot requires a full immutable Git commit SHA") + _path(skill_root) + _skill_name(skill) + if destination.exists() or destination.is_symlink(): + raise ValueError("Selected snapshot destination must not exist") + version = subprocess.check_output(["git", "--version"], cwd=repo, text=True) + match = re.match(r"git version (\d+)\.(\d+)", version) + if match is None or tuple(map(int, match.groups())) < (2, 45): + raise ValueError("Selected snapshot requires Git >= 2.45 for --no-lazy-fetch") + if subprocess.check_output([*GIT_READ, "cat-file", "-t", revision], + cwd=repo).strip() != b"commit": + raise ValueError("Selected snapshot revision must identify a Git commit") + inventory = {} + aliases = {} + for entry in subprocess.check_output( + [*GIT_READ, "ls-tree", "-rz", revision, "--", skill_root], cwd=repo, + ).split(b"\0"): + if not entry: + continue + header, name = entry.split(b"\t", 1) + mode, kind, digest = header.decode("ascii").split() + path = _path(name.decode("utf-8")) + if not path.startswith(skill_root + "/"): + raise ValueError("Snapshot entry escapes skill root") + parts = path.split("/") + for end in range(1, len(parts) + 1): + component = "/".join(parts[:end]) + key = component.casefold() + if key in aliases and aliases[key] != component: + raise ValueError("Aliased snapshot entries") + aliases[key] = component + inventory[path] = (mode, kind, digest) + + provenance = {} + + def read(path): + if path not in inventory: + raise ValueError(f"Missing selected snapshot resource: {path}") + mode, kind, digest = inventory[path] + if mode != "100644" or kind != "blob": + raise ValueError(f"Selected executable, symlink or submodule is unsupported: {path}") + data = subprocess.check_output([*GIT_READ, "cat-file", "blob", digest], cwd=repo) + provenance[path] = {"mode": mode, "git_blob": digest, + "sha256": hashlib.sha256(data).hexdigest()} + return data + + index_path = skill_root + "/skill-index.json" + projection = None + if index_path in inventory: + original = read(index_path) + index = json.loads(original, object_pairs_hook=_unique_fields) + if (not isinstance(index, dict) or type(index.get("schema_version")) is not int + or index["schema_version"] != 1 or not isinstance(index.get("skills"), dict)): + raise ValueError("Invalid skill index") + paths = set() + for name, item in index["skills"].items(): + _skill_name(name) + if (not isinstance(item, dict) or not isinstance(item.get("path"), str) + or item.get("backend") not in (*BACKENDS, "common")): + raise ValueError("Invalid skill index entry") + path = _path(item["path"]) + if PurePosixPath(path).name != name or path.casefold() in paths: + raise ValueError("Duplicate or aliased skill index path") + paths.add(path.casefold()) + if skill not in index["skills"]: + raise ValueError(f"Unknown skill: {skill}") + relative = index["skills"][skill]["path"] + projection = json.dumps({**index, "skills": {skill: index["skills"][skill]}}, + sort_keys=True).encode() + else: + relative = skill + prefix = skill_root + "/" + relative + selected = {path for path in inventory if path.startswith(prefix + "/")} + if prefix + "/SKILL.md" not in selected: + raise ValueError(f"Missing selected SKILL.md: {skill}") + # Stage only in our own fresh directory; never follow caller-created children. + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="selected-", dir=destination.parent) as temporary: + staging = Path(temporary) / "snapshot" + root = staging / skill_root + + def write(path, data): + target = staging / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + + for path in sorted(selected): + write(path, read(path)) + if projection is not None: + write(index_path, projection) + metadata = skill_metadata(root, skill) + for reference in metadata.get("references", []): + if not reference or "\\" in reference or ":" in reference or reference.startswith("/"): + raise ValueError("Reference must use a relative POSIX path") + path = posixpath.normpath(prefix + "/" + reference) + _path(path) + if not path.startswith(skill_root + "/"): + raise ValueError("Reference escapes skills root") + if projection is not None and path == index_path: + raise ValueError("Declared reference conflicts with projected index") + # Missing leaf behind a symlink/submodule is rejected, never traversed. + write(path, read(path)) + bundle = KnowledgeReader(skill_root=root).read("checkout:" + skill, + revision=bundle_revision) + staging.rename(destination) + return {"root": destination / skill_root, "git_revision": revision, + "skill": skill, "bundle_revision": bundle.revision, + "entries": dict(sorted(provenance.items())), + "projected_index": None if projection is None else { + "path": index_path, "source_git_blob": provenance[index_path]["git_blob"], + "sha256": hashlib.sha256(projection).hexdigest(), + "transformation": "existing skill index restricted to selected identity"}, + "closure_semantics": "selected directory plus metadata.references; navigation excluded"} From 9a523f5e169d69d095f6352b5d1cdaa034a64191 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:23:20 +0900 Subject: [PATCH 2/2] style: format daily admission imports and messages Agent-assisted-by: OpenAI Codex Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> --- tools/skill_eval_auto.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/skill_eval_auto.py b/tools/skill_eval_auto.py index 2e38b2b37..9abcfa7f1 100644 --- a/tools/skill_eval_auto.py +++ b/tools/skill_eval_auto.py @@ -13,11 +13,24 @@ from inferrsi.infra.artifacts import write_json from tools import skill_eval_pr as pr -from tools.skill_eval_budget import (Binding, BudgetError, DailyKey, PRKey, WriteResult, - claim_once, daily_claim_once, daily_reserve, reserve, - shanghai_day) -from tools.skill_eval_budget_git import (DailyLedgerAnchor, GitHubDailyStore, GitHubGitStore, - LedgerAnchor) +from tools.skill_eval_budget import ( + Binding, + BudgetError, + DailyKey, + PRKey, + WriteResult, + claim_once, + daily_claim_once, + daily_reserve, + reserve, + shanghai_day, +) +from tools.skill_eval_budget_git import ( + DailyLedgerAnchor, + GitHubDailyStore, + GitHubGitStore, + LedgerAnchor, +) CONFIG = pr.ROOT / "tools/skill_eval_auto_config.json" @@ -187,7 +200,8 @@ def status(api, plan, config): daily.require_active() day = shanghai_day() actual = ledger.genesis.legacy_charge + sum(e.actual_cost or 0 for e in ledger.entries) - return {"daily_day": day, "daily_remaining_usd": str(Decimal(daily.remaining_on(day)) / 1000000), + return {"daily_day": day, + "daily_remaining_usd": str(Decimal(daily.remaining_on(day)) / 1000000), "label": config["label"], "spent_or_legacy_usd": str(Decimal(actual) / 1000000), "reserved_or_unresolved_usd": str(Decimal(ledger.charged - actual) / 1000000), "remaining_usd": str(Decimal(ledger.remaining) / 1000000)} @@ -217,7 +231,8 @@ def controlled_execute(api, output, *, config_path=CONFIG, run_id, attempt, laun if (global_successor == global_snapshot.ledger or global_store.compare_and_swap(global_snapshot, global_successor) is not WriteResult.COMMITTED): - raise BudgetError("Duplicate, conflicted or uncertain global reservation; no paid execution") + raise BudgetError( + "Duplicate, conflicted or uncertain global reservation; no paid execution") successor = reserve(snapshot.ledger, binding) if (successor == snapshot.ledger or store.compare_and_swap(snapshot, successor) is not WriteResult.COMMITTED): @@ -298,7 +313,8 @@ def report_control(api, plan): "remaining_usd": "unknown"} try: config = configuration() - result["reason"] = "Audited PR/global daily ledger missing, unavailable, quarantined or unsafe" + result["reason"] = ( + "Audited PR/global daily ledger missing, unavailable, quarantined or unsafe") result.update(status(api, plan, config)) result["reason"] = "Plan has no supported pairs or provider/bounds are stale" eligible(api, plan, config)