From ec24b85bcf73476a3b1c826405151eee7ac2a69b Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:11:48 +0900 Subject: [PATCH 1/2] feat: add label-gated skill evaluation with cumulative PR budget Keep ledger authority in the trusted controller and gate every provider request. Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> Agent-assisted-by: OpenAI Codex --- .github/workflows/skill-eval-pr.yml | 31 +-- docs/skill-eval-budget-core.md | 167 ++++++++++++++ docs/skill-eval-pr.md | 174 ++++++++------- src/inferrsi/model/anthropic_agent.py | 27 ++- src/inferrsi/model/request_guard.py | 143 ++++++++++++ tests/test_request_guard.py | 141 ++++++++++++ tests/test_skill_eval_auto.py | 270 +++++++++++++++++++++++ tests/test_skill_eval_budget.py | 242 +++++++++++++++++++++ tests/test_skill_eval_budget_git.py | 302 ++++++++++++++++++++++++++ tests/test_skill_eval_pr.py | 24 +- tests/test_skill_eval_pr_summary.py | 4 +- tests/test_yibuapi_runtime.py | 15 +- tools/skill_eval_auto.py | 199 +++++++++++++++++ tools/skill_eval_budget.py | 274 +++++++++++++++++++++++ tools/skill_eval_budget_git.py | 179 +++++++++++++++ tools/skill_eval_compare.py | 11 +- tools/skill_eval_pr.py | 70 ++++-- 17 files changed, 2139 insertions(+), 134 deletions(-) create mode 100644 docs/skill-eval-budget-core.md create mode 100644 src/inferrsi/model/request_guard.py create mode 100644 tests/test_request_guard.py create mode 100644 tests/test_skill_eval_auto.py create mode 100644 tests/test_skill_eval_budget.py create mode 100644 tests/test_skill_eval_budget_git.py create mode 100644 tools/skill_eval_auto.py create mode 100644 tools/skill_eval_budget.py create mode 100644 tools/skill_eval_budget_git.py diff --git a/.github/workflows/skill-eval-pr.yml b/.github/workflows/skill-eval-pr.yml index e77f32a9..eefd7c3d 100644 --- a/.github/workflows/skill-eval-pr.yml +++ b/.github/workflows/skill-eval-pr.yml @@ -7,9 +7,11 @@ run-name: >- on: pull_request_target: branches: [main] - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled, closed] paths: - src/inferrsi/data/skill_library/** + - src/inferrsi/data/knowledge_packs/** + - src/inferrsi/data/repo_packs/**/knowledge/** - tools/skill_eval_* - docs/examples/skill-comparison/** - .github/workflows/skill-eval-pr.yml @@ -58,7 +60,8 @@ jobs: plan: outputs: client_kind: ${{ steps.prepare.outputs.client_kind }} - if: github.ref == 'refs/heads/main' + auto_candidate: ${{ steps.prepare.outputs.auto_candidate }} + if: github.ref == 'refs/heads/main' && github.event.action != 'closed' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -79,11 +82,6 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: python -m tools.skill_eval_pr prepare --output prepared - - name: Validate explicit operator authorization and reserve execution - if: github.event_name == 'workflow_dispatch' && inputs.action == 'execute' - env: - GITHUB_TOKEN: ${{ github.token }} - run: python -m tools.skill_eval_pr authorize --output prepared - uses: actions/upload-artifact@v4 with: name: skill-eval-plan @@ -92,12 +90,12 @@ jobs: retention-days: 30 execute: - if: github.event_name == 'workflow_dispatch' && inputs.action == 'execute' + if: github.event_name == 'pull_request_target' && needs.plan.outputs.auto_candidate == 'true' needs: plan runs-on: ubuntu-latest timeout-minutes: 20 permissions: - contents: read + contents: write actions: read pull-requests: read steps: @@ -113,21 +111,11 @@ jobs: with: name: skill-eval-plan path: prepared - - name: Existing Codex client preflight - if: needs.plan.outputs.client_kind == 'codex' - run: codex --version - - name: Recheck live revision and authorization before execution + - name: Trusted cumulative-budget controller (never executes PR code) env: GITHUB_TOKEN: ${{ github.token }} - run: python -m tools.skill_eval_pr authorize --output prepared - - name: Execute existing Codex comparison without publication credentials - if: needs.plan.outputs.client_kind == 'codex' - run: python -m tools.skill_eval_pr execute --output prepared - - name: Execute configured Yibuapi comparison without publication credentials - if: needs.plan.outputs.client_kind == 'anthropic-messages-agent-v1' - env: SKILL_EVAL_YIBUAPI_KEY: ${{ secrets.SKILL_EVAL_YIBUAPI_KEY }} - run: python -m tools.skill_eval_pr execute --output prepared + run: python -m tools.skill_eval_pr control --output prepared - name: Package retained results even after interruption if: always() run: python -m tools.skill_eval_pr pack --output prepared @@ -163,6 +151,7 @@ jobs: run: python -m tools.skill_eval_pr prepare --output prepared - name: Update the single revision-bound summary from retained data env: + SKILL_EVAL_AUTO_SOURCE: ${{ github.event_name == 'pull_request_target' && needs.plan.outputs.auto_candidate == 'true' }} GITHUB_TOKEN: ${{ github.token }} run: python -m tools.skill_eval_pr publish --output prepared - uses: actions/upload-artifact@v4 diff --git a/docs/skill-eval-budget-core.md b/docs/skill-eval-budget-core.md new file mode 100644 index 00000000..27ba40ea --- /dev/null +++ b/docs/skill-eval-budget-core.md @@ -0,0 +1,167 @@ +# Cumulative PR budget and Git ledger + +Refs [#215](https://github.com/InferMatrixLab/InferRSI/issues/215). +`tools/skill_eval_budget.py` implements pure accounting transitions. +`tools/skill_eval_budget_git.py` implements the GitHub Git-data adapter using the +existing authenticated `GitHub.request` client and injected `GITHUB_TOKEN`. +The adapter never loads credentials or executes PR code. Tests use mocks only. + +## Accounting + +- Key: immutable repository ID plus PR number; never branch name, repository + name, head revision, workflow run or label epoch. Renames, pushes, retries and + label removal/reapplication cannot create a fresh PR allowance. +- The requested ceiling is USD 200 per PR, represented as 200,000,000 integer USD + microdollars. Trusted pricing calculations must round worst-case costs upward. + Floats, booleans, negative costs and zero-cost reservations are rejected. +- A trusted frozen-plan digest covers all revisions, mapping, exam, provider, + pricing, retries, request/token limits and other cost-relevant inputs. The core + receives its independently computed worst-case total; it does not derive cost + or prove that the plan producer is trustworthy. +- Reserve the entire plan before execution. Charge = audited legacy obligation + plus **every reservation**, including cancelled, failed, interrupted, completed + and ambiguous executions. Actual cost is evidence only; there are no refunds. +- An exact plan/run/cost binding is reserve-idempotent. Reusing a plan with a + different run, or a run with a different plan/cost, fails closed. This first + version deliberately does not support rerunning an identical plan. A distinct + authorized attempt must have a distinct frozen identity and a fresh reservation. + Run identity includes workflow run attempt, not just workflow run ID. + +```text +verified genesis → reserved → claimed → finished + └───→ cancelled +``` + +Claim is non-idempotent: at most one process can obtain a successful claim CAS. +No takeover, lease expiry, automatic retry or reset exists. A process crash after +claim burns capacity even when no request happened. `claim_once` returns only on +an explicit, durable CAS success for that invocation. Its return is **not** a +replayable execution credential. The trusted caller must execute at most once, +respect all frozen bounds (including provider retries), and not expose a result +that another worker can reuse to execute again. An actual-cost overrun is rejected +and requires external quarantine/reconciliation before further paid activity; +preventing overruns depends on the future bounded executor. + +## Genesis is an audited migration, never a missing-file default + +`Genesis(evidence, legacy_charge)` requires an explicit audit reference and an +explicit integer charge. Zero is acceptable only when the authorized audit proves +no earlier paid obligations. Include settled legacy spend and conservative upper +bounds for all outstanding/ambiguous legacy runs; quiesce or reconcile older +writers before migration. Unknown obligations block genesis. If the obligations +exceed the ceiling, initialization fails and the PR is ineligible. + +Constructing this value is not authorization or cryptographic proof. The separately authorized +trusted initializer must verify the audit and atomically create only for a key +that has **never** had a ledger. There is intentionally no genesis initializer in the core or Git adapter. Missing/corrupt existing state, HTTP 404, expired artifacts, empty +workflow history and label reapplication are never permission to initialize. + +## Atomic storage contract + +`AtomicStore.read_verified(key)` returns a `Snapshot(history_token, ledger)` only +from authenticated, durable, current history. `decode` validates schema and local +invariants, **not provenance, rollback resistance or history completeness**. +The backend retains an independently configured genesis/checkpoint anchor that +survives ledger ref deletion. A ledger's own revision cannot detect its reset. +See the precise rollback trust boundary below. +Missing, corrupt, untrusted or discontinuous history must raise and block work. + +`compare_and_swap(snapshot, successor)` must atomically append exactly one +successor against the current immutable history token, using +`validate_successor` as well as key/lineage checks. All writers use the same CAS +boundary. A matching body already stored by another request is **not** this +caller's successful commit. On conflict, reread verified state and recompute; +never overwrite a newer charge. On unknown write outcome, do not execute. + +Reservation writes can reconcile by rereading the exact binding and charge. +Claim writes cannot reconcile into a new execution grant: an observed `claimed` +entry may already have executed. Duplicate/ambiguous claims stay blocked. If a +verified read still shows `reserved`, a new CAS can compete safely only against +that current token; a delayed earlier write must conflict with the same boundary. +Terminal evidence retries are idempotent only when identical; no terminal update +changes charged capacity. + +## Git-data adapter and trusted bootstrap + +Each PR uses exactly `refs/heads/skill-eval-budget//`. +A trusted `LedgerAnchor` supplies immutable repository/PR identity, expected +`genesis_commit`, and independently retained `checkpoint_commit`. These values +must come from reviewed root configuration, **never PR files, artifacts, labels, +comments, caller-selected refs or ledger contents**. The API repository ID is +checked on every verified read; repository renaming does not refresh capacity. + +Before activation an authorized operator must quiesce previous paid writers, +audit settled **and outstanding/ambiguous** legacy obligations, and create a +genesis commit with exactly `ledger.json` holding revision-zero state and the +explicit `Genesis(evidence, legacy_charge)`. The genesis may be parentless or have +exactly one explicitly audited existing-main parent when the authorized Git API +requires a parent. Its tree must still contain **only** `ledger.json`; do not +inherit the main tree. Pin the complete genesis commit SHA and checkpoint in +trusted root configuration. That immutable SHA binds the genesis parent as well +as its state. Parent ancestry before genesis is provenance, not ledger history: +verification stops at the pinned audited genesis and never interprets main commits +as budget transitions. Merge genesis commits are rejected. All commits after +genesis still require exactly one parent and one valid accounting transition. An empty ledger is valid only after an explicit no-prior-spend +audit; unknown obligations block bootstrap. Bootstrap/ref creation is deliberately +not automated by this patch and no missing-ledger fallback exists. If the PR had +an earlier ledger, recover/reconcile that history rather than creating new zero +capacity. Unknown/reset history requires operator investigation, not retry-reset. + +Reads fetch the exact ref, every single-parent commit and exact ledger tree/blob +back to pinned genesis. They check all accounting transitions and require both +the configured checkpoint and the last tip observed by that store instance to +be ancestors. Missing refs/objects, merges, extra tree entries, corruption, +discontinuous transitions, divergent roots, rollback behind the checkpoint, +identity changes, racing reads and excessive history all block execution. +History verification is bounded to 10,000 commits and 4 MB per ledger; reaching +bounds fails closed, never truncates accounting or resets capacity. + +Writes create an immutable tree and a single-parent child of the verified tip, +then PATCH the existing ref with `force: false`. The +[GitHub reference API](https://docs.github.com/en/rest/git/refs#update-a-reference) +requires a fast-forward when force is false. Under append-only writers, two +children of the same tip cannot both become the next tip. A random nonce makes +even identical state transitions different commits, preventing two identical +claims from receiving successful idempotent updates for a shared commit SHA. +Stale snapshots return conflict; failed/uncertain writes return unknown, including +permission/rules rejection. No automatic PATCH retry or claim reconciliation +confers execution permission. Unreachable trees/commits after a crash do not +change accounting; a committed reservation/claim remains permanently charged. +No branch creation, force update, deletion or repository-settings API is used. + +### Trust boundary: not protection against malicious administrators + +Git refs are mutable by sufficiently privileged principals. The security model +requires trusted root configuration/code and repository administrators, plus no +out-of-band force writes/deletion/recreation of the ledger ref. Use independently +administered append-only restrictions where available; this implementation does +not change settings or bypass rules. A static genesis pin detects a replaced +root but **cannot detect rollback to a valid later ancestor across new processes**. +A trusted checkpoint detects rollback before that checkpoint, and process memory +detects rollback behind a tip already seen in that process. Neither is an external +monotonic witness. The tests explicitly demonstrate this limitation. + +Absolute rollback resistance against malicious repository administrators would +require separately authorized, independently durable monotonic storage or an +independently administered enforcement boundary. Do not claim this Git branch +provides it. Retain and advance reviewed checkpoints as audit evidence; never +lower or replace a checkpoint to make a failed verification pass. This document +does not authorize any settings changes, migration, or remote writes. + +## Verification and integration boundary + +`pytest tests/test_skill_eval_budget.py tests/test_skill_eval_budget_git.py` +covers cumulative limits, permanent reservations, legacy charges, duplicate +plan/run bindings, malformed state, reservation/claim races, unique claim commit +identities, crashes/lost responses, rules rejection, history corruption/deletion, +checkpoint rollback detection and its explicit administrator limitation. The +mock Git API enforces non-force fast-forward semantics. CPU tests do not prove +provider billing or live GitHub rules/permission compatibility. + +The adapter does not create a plan, grant label authority, run a provider or +reconcile actual billing. The trusted controller must reserve and claim before +executing, revalidate admission, enforce all worst-case plan bounds including +retries, and keep credentials away from untrusted PR code. A returned claim is +process-local permission for one execution, not a reusable workflow output. +Live activation still requires the audited genesis/root configuration and +successful permitted writes; absent prerequisites fail closed. diff --git a/docs/skill-eval-pr.md b/docs/skill-eval-pr.md index 6c9b5e74..18180d5e 100644 --- a/docs/skill-eval-pr.md +++ b/docs/skill-eval-pr.md @@ -1,13 +1,13 @@ # Skill PR planning and feedback -The report-only [Skill PR workflow](../.github/workflows/skill-eval-pr.yml) connects +The advisory [Skill PR workflow](../.github/workflows/skill-eval-pr.yml) connects skill changes to the existing [comparison command](skill-eval-comparison.md). It does not add an execution engine or a required efficacy gate. Refs #101; parent #88. ```mermaid flowchart LR A["PR Git objects"] --> B["Trusted mapping → frozen plan"] - B --> C["Authorized operator: versions + hash + budget"] + B --> C["Trusted label actor + exact plan + cumulative PR reservation"] C --> D["Existing comparison client; no publication credentials"] D --> E["Retained results: 30 days"] E --> F["Trusted publisher → exact-plan advisory + explicit gaps"] @@ -63,10 +63,9 @@ also rejects `action=plan` with its authorize, execute or pack CLI modes. Supplying execution fields or a `source_run` to this action does not authorize execution or import earlier results. -Only `action=execute` schedules authorization and execution. Its five approval -fields remain mandatory in the controller, although optional in the shared -dispatch form. `action=publish` still requires a validated original execution -run; a planning-only run is not a supported source for retained results. +Legacy `action=execute` is now explicitly blocked. Automatic paid execution uses +the trusted opt-in label controller described below. `action=publish` requires a +validated original source run and never schedules model execution. A new trusted implementation or changed PR inputs creates a new plan identity; old hashes and approvals cannot authorize it. A plan dispatch has a non-execution @@ -79,50 +78,25 @@ automatic PR planning can also exercise that publisher. Retain the safe HTTP diagnostic or successful summary receipt. Neither entry repairs a permission denial. This verifies publication only, not evaluation. -## Explicit execution authorization - -Planning grants no execution budget. An operator with repository `write`, -`maintain` or `admin` permission dispatches **Skill PR evaluation** on **main**: - -| Dispatch input | Required approval | -| --- | --- | -| `action=execute`, `pr` | The PR being evaluated | -| `plan_sha256` | The exact PR plan hash displayed in its summary | -| `version` | The full trusted implementation SHA from that summary | -| `base`, `head` | Both full compared revisions | -| `budget` | The complete displayed Budget JSON, including limits per comparison and total estimated calls | - -The authenticated dispatch actor's current collaborator permission is queried; -knowledge of a hash or a PR comment is insufficient. The explicit inputs must -match the newly reconstructed plan exactly. Authorization and current revisions -are rechecked in the execution job immediately before starting the existing -comparison command. No shell evaluates user-provided inputs. - -The selected model is **`gpt-6.1-sol`, effort `xhigh`**, requested by the owner. -Each selected comparison schedules **2 task calls**: one absent/old and one new arm, -one case, one repeat. Limits are 180 seconds per trial, 600 seconds per comparison, -and an observed-token stop threshold of 20,000. This stops new trials; it is not -a provider token or monetary hard cap. One trial can cross the threshold, and -unreported usage/cost remain unknown. Selecting both mapped owners schedules four -calls. Mapping more skills increases the total; -the exact total is shown in the plan and requires renewed authorization. - -### Execution environment prerequisite - -Execution reuses the configured comparison client and cleanup path. The client -and separately authorized model access must already be available in that job's -environment; the Codex path checks `codex --version` and never substitutes fixed -outputs. The optional provider configuration has its own readiness checks and -reservation contract. A historical successful run does not establish current -credential availability, permission, provider readiness or remaining budget. -Do not dispatch paid work without current exact-plan authorization. This guide -and an issue/label proposal do not activate paid execution or provision secrets. - -The execute job has read-only repository permissions, no publication token in -its model step, and checkout uses `persist-credentials: false`. The privileged -publisher is a separate fresh job. Model access must not also grant PR publishing. -Only the publish job requests `pull-requests: write`, alongside its existing -`issues: write`; planning and execution retain read-only repository permissions. +## Execution authorization and environment + +Planning grants no execution budget. The trusted deployment configuration, +latest authorized label-add event and current actor permission are required; +PR text and dispatch inputs cannot supply that authority. Automatic execution +uses the existing frozen provider plans and enforces their complete worst-case +reservation against USD 200 cumulatively per PR. The non-provider default mapping +remains useful for offline planning, but cannot execute as a paid PR fallback. + +No provider credential, label or deployment configuration is created by this +change. A historical successful run does not establish current credentials, +permission, provider readiness or remaining budget. See the deployment checklist +below before enabling the switch. + +The controller alone has owner-approved `contents: write` for the dedicated Git +budget ledger. Its existing model executor runs in a sanitized child process with +no GitHub/Actions write tokens. Checkout uses `persist-credentials: false`. +The publisher remains a separate fresh job and alone requests PR/issue comment +write permissions. Planning and publication have read-only repository contents. In the controlled fork [baseline](https://github.com/Dong1017/OmniRSI/actions/runs/40455073055), the same Actions job could create/update an ordinary Issue comment but received permission HTTP403 for a PR conversation comment with PullRequests read. @@ -141,26 +115,17 @@ duplicate attempts. ## Duplicate prevention and interrupted execution -Actions records the authorization dispatch as -`skill-eval execute PR plan `. Under the shared concurrency -group, only the earliest matching execution run may proceed. Every status is -considered, including queued, failed and cancelled runs. Attempts other than -`run_attempt=1` cannot execute. An exclusive local claim file additionally prevents -two callers sharing the same retained directory from entering the comparison. - -The Actions run is the durable execution tombstone, not the expiring artifact. -An interrupted or failed attempt is consumed even if it might have stopped -before any paid work. There is no automatic paid retry/resume. Do not delete -execution workflow runs; inaccessible/incomplete run history fails closed. -Authorization requires a stable integer `total_count`, exactly that many unique -run IDs and no overlapping pages. GitHub's filtered workflow-run searches have a -[1,000-result limit](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow). -Reaching that limit is ambiguous, so a reported total of 1,000 or more blocks -execution immediately. Missing/changing totals or an early short/empty page -also block execution; a short page alone never proves complete history. -Deleting this coordination history removes duplicate-prevention evidence and -requires human reconciliation before any further execution of the same plan. -Workflow reruns may republish results but must not rerun evaluation. +The durable Git ledger, not expiring artifacts or Actions titles, binds each +frozen plan to one workflow run/attempt. All revisions of one PR share the same +USD 200 ceiling. Reservation and claim are separate compare-and-swap transitions; +an ambiguous write, duplicate plan, exhausted allowance or missing/corrupt history +blocks execution. Claims are deliberately non-idempotent. Interrupted requests +retain their full conservative reservations, including when no cost receipt was +produced. No takeover, automatic paid retry, refund or reset is provided. + +An exclusive local claim file also prevents duplicate execution in one retained +directory. Attempts other than `run_attempt=1` cannot execute. Publication-only +retry makes no new model calls and consumes no new reservation. ## Publication and retention @@ -264,12 +229,12 @@ content or another boundary. The automated report does not invent that decision or silently count the gap as passed. Content/source review, authorship admission, and merge authority remain independent. -Knowledge-only changes under `src/inferrsi/data/repo_packs/**` or -`src/inferrsi/data/knowledge_packs/**` are outside the current automatic path -filter. Use the existing **plan-only dispatch** for an explicit scope report; -unmapped paths remain uncovered and select no paid work. Absence of a triggered -workflow is not an evaluation pass. This slice does not expand automatic paths -or introduce an unrelated audit grader for knowledge content. +Knowledge-only changes under `src/inferrsi/data/repo_packs/**/knowledge/**` and +`src/inferrsi/data/knowledge_packs/**` now trigger bounded automatic planning. +Unmapped paths remain uncovered, select no paid work, and publish insufficient +evidence rather than a pass. Other repository-pack paths can use the existing +**plan-only dispatch** for an explicit scope report. No unrelated audit grader +is introduced for knowledge content. ### Relevant #179 integrity reconciliation @@ -288,3 +253,60 @@ The PR path's trusted reconstruction narrows that boundary; it does not create cryptographic provenance. Review guidance is advisory evidence interpretation, not execution attestation or skill-efficacy certification. Unrelated harness lifecycle/Windows contention work remains outside this slice and under #179. + +## Label-gated cumulative paid execution (deployment prerequisite) + +The trusted controller extends the existing prepare/execute/pack/publish pipeline; +it is not another runner. It uses only the existing Yibuapi Messages agent and +frozen comparisons. Automatic execution is opt-in, advisory and never a merge +approval. The bounded skill/knowledge path filters remain; label addition, +removal, closing and revision events are included. A removal or closure stops +subsequent requests through a live GitHub check, independently of whether a +path-filtered notification starts another workflow. Already in-flight calls +cannot be revoked. Closed events never start preparation or paid work. + +**Not activated by this change.** An owner must separately review and deploy +`tools/skill_eval_auto_config.json` in trusted main. No working config, label, +credential or ledger genesis is created by this implementation. The config is +strictly shaped as: + +- `schema_version: 1`, immutable `repository_id`, dedicated `label`, and explicit + owner-authorized GitHub logins in `operators`; +- finite Unix `expires_at` and `review_evidence` identifying the owner review; +- complete `provider` (existing provider-config schema) and its canonical JSON + `provider_sha256`. Only Yibuapi `claude-opus-5-5`, USD 8 input / 40 output / 0.4 + cache-read per million are accepted. Cache-write, request fees, effective group, + full-context input bound, output/request limits, and current rate-evidence + comment must **also** be explicitly reviewed. No unknown rate defaults to zero; +- `anchors`, keyed by PR number string: each entry contains `ref`, + `genesis_commit`, and `checkpoint_commit`, referencing the separately audited + per-PR Git ledger described in [the budget contract](skill-eval-budget-core.md). + Unknown historical spending is a blocker, never an empty zero-cost ledger. + +Every eligible plan reserves its entire enforceable worst case against the same +repository-ID + PR-number USD 200 ceiling, then wins a one-shot durable claim +before any provider request. Duplicate events, attempts, label re-additions and +changed heads cannot replenish the allowance. Reservations are deliberately never +refunded automatically: a successful, interrupted or uncertain run retains its +full bound. The report separates known/legacy spending, conservative unresolved +reservations and remaining capacity. Unknown accounting is explicitly blocked. + +The latest applicable label-add action must belong to an allowlisted operator +who **currently** retains repository write/maintain/admin permission. Merely +having the label, being the PR author, or supplying dispatch text does not grant +authority. Each provider request rechecks live label event identity, open/non-draft +state, exact head/base, operator permission, configuration expiry and durable +claim. A revision produces a fresh frozen binding against the original PR ledger. + +Only the trusted controller job has `contents: write`. Its executor child gets an +allowlisted environment, a short-lived local request-gate capability and the +provider key, not the GitHub/Actions write tokens. PR content remains data, not +executed code. The separate publisher never makes model calls. Publication-only +retry remains supported via the original source run and exact current plan; +legacy paid `workflow_dispatch execute` is explicitly blocked instead of bypassing +the cumulative ledger. Existing credential-free injected-client tests still work. + +Live acceptance remains pending until the owner-reviewed provider configuration, +explicit label/operator policy, current pricing evidence and audited historical +spend/genesis anchors exist, followed by an authorized real run and publication. +Offline controller tests do not establish provider charges or skill efficacy. diff --git a/src/inferrsi/model/anthropic_agent.py b/src/inferrsi/model/anthropic_agent.py index 08246cdc..9e71eed2 100644 --- a/src/inferrsi/model/anthropic_agent.py +++ b/src/inferrsi/model/anthropic_agent.py @@ -6,6 +6,7 @@ import math import os import re +import runpy import sys import time import urllib.request @@ -21,6 +22,11 @@ # Full context bound from https://platform.claude.com/docs/en/models/opus-5-5/overview MODEL_CONTEXT = {"claude-opus-5-5": 1000000} KEY_ENV = "SKILL_EVAL_YIBUAPI_KEY" +# The isolated (-I) worker must load its adjacent, frozen trusted guard, not an +# installed package or a module from the model workspace/PYTHONPATH. +_guard = runpy.run_path(str(Path(__file__).with_name("request_guard.py"))) +authorize_request = _guard["authorize_request"] +gate_environment = _guard["gate_environment"] def canonical(value): @@ -169,6 +175,7 @@ def execution_environment(environment, *, with_key=True): or any(c.isspace() for c in key)): raise ValueError("Provider credential is missing or malformed") safe[KEY_ENV] = key + safe.update({k: environment[k] for k in _guard["ENV_KEYS"] if k in environment}) return safe @@ -179,11 +186,14 @@ class AnthropicAgentClient: timeout_seconds: float pricing: dict | None = None spend_limit: dict | None = None + require_authorization: bool = False def frozen_identity(self): validate_spec({"name": self.model, "effort": "not_applicable"}, self.spec) value = {"settings": json.loads(canonical(self.spec)), "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "guard_source_sha256": hashlib.sha256( + Path(__file__).with_name("request_guard.py").read_bytes()).hexdigest(), "runtime_state": "unconfigured", "request_policy": {"tool_choice": "auto", "thinking": "model_default", "sampling": "model_default", "retries": 0}} @@ -198,6 +208,8 @@ def require_live_readiness(self, trials=1, *, now=None): return reservation(self.model, self.spec, self.pricing, self.spend_limit, trials, now=now) def environment(self): + if self.require_authorization: + gate_environment(os.environ) return execution_environment(os.environ) def verifier_environment(self): @@ -210,7 +222,8 @@ def command(self, workspace, final, schema, prompt): return (sys.executable, "-I", str(Path(__file__).resolve()), "--workspace", str(workspace), "--final", str(final), "--schema", str(schema), "--settings", canonical(settings), "--prompt", prompt, - "--timeout", str(self.timeout_seconds)) + "--timeout", str(self.timeout_seconds), + *(("--require-authorization",) if self.require_authorization else ())) @staticmethod def trace_metrics(path): @@ -234,7 +247,7 @@ def unique_object(pairs): return result -def send_message(payload, spec, key, timeout): +def send_message(payload, spec, key, timeout, *, environment=None, require_authorization=False): """One non-streaming request. Callers must supply approved runtime credentials.""" validate_spec({"name": payload.get("model"), "effort": "not_applicable"}, spec) if not isinstance(key, str) or not key or not key.isascii() or any(c.isspace() for c in key): @@ -248,6 +261,12 @@ def send_message(payload, spec, key, timeout): "x-api-key": key, "anthropic-version": spec["api_version"], "Content-Type": "application/json", "Accept": "application/json", "User-Agent": "inferrsi-anthropic-agent/1"}) + runtime = os.environ if environment is None else environment + if require_authorization or any(k in runtime for k in _guard["ENV_KEYS"]): + try: + authorize_request(raw, environment=runtime, timeout=timeout) + except Exception: + raise AgentError("request_authorization_denied") from None try: opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) with opener.open(request, timeout=timeout) as response: @@ -452,6 +471,7 @@ def main(argv=None, *, environment=None, transport=None, emit=None): for name in ("workspace", "final", "schema", "settings", "prompt"): parser.add_argument("--" + name, required=True) parser.add_argument("--timeout", type=float, required=True) + parser.add_argument("--require-authorization", action="store_true") args = parser.parse_args(argv) emit = emit or (lambda event: print(canonical(event), flush=True)) try: @@ -466,6 +486,9 @@ def main(argv=None, *, environment=None, transport=None, emit=None): sender = transport or send_message def send(payload, spec, timeout): + if transport is None: + return sender(payload, spec, key, timeout, environment=runtime, + require_authorization=args.require_authorization) return sender(payload, spec, key, timeout) schema = json.loads(Path(args.schema).read_text(encoding="utf-8")) diff --git a/src/inferrsi/model/request_guard.py b/src/inferrsi/model/request_guard.py new file mode 100644 index 00000000..f5137acb --- /dev/null +++ b/src/inferrsi/model/request_guard.py @@ -0,0 +1,143 @@ +"""Ephemeral per-request gate owned by the trusted evaluation controller. + +This is a trusted-client boundary, not a sandbox against arbitrary child code. +The child has the provider key and a bounded IPC capability, never GitHub authority. +No grant permits retries; ambiguous/lost replies consume their slot. Authorization +can change after a check or during an in-flight request, which cannot be recalled. +""" + +import hashlib +import hmac +import http.client +import json +import os +import re +import secrets +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +SOURCE = "src/inferrsi/model/request_guard.py" +ENV_KEYS = ("SKILL_EVAL_GATE_PORT", "SKILL_EVAL_GATE_TOKEN", "SKILL_EVAL_GATE_CONTEXT") + + +class AuthorizationError(RuntimeError): + """Fixed text only: never propagate controller failures or credentials.""" + + +def gate_environment(environment): + values = {key: environment.get(key, "") for key in ENV_KEYS} + port, token, context = (values[key] for key in ENV_KEYS) + if (not isinstance(port, str) or not re.fullmatch(r"[0-9]{1,5}", port) + or not 0 < int(port) < 65536 + or not isinstance(token, str) or not re.fullmatch(r"[0-9a-f]{64}", token) + or not isinstance(context, str) or not re.fullmatch(r"[0-9a-f]{64}", context)): + raise AuthorizationError("request_authorization_unavailable") + return values + + +def authorize_request(payload: bytes, *, environment=None, timeout=10): + """Consume one fresh grant immediately before a single real provider call.""" + env = gate_environment(os.environ if environment is None else environment) + identity = secrets.token_hex(32) + digest = hashlib.sha256(payload).hexdigest() + context = env[ENV_KEYS[2]] + body = json.dumps({"context": context, "request_id": identity, + "payload_sha256": digest}, separators=(",", ":")).encode() + connection = http.client.HTTPConnection("127.0.0.1", int(env[ENV_KEYS[0]]), + timeout=min(timeout, 10)) + try: + connection.request("POST", "/authorize", body, { + "Authorization": "Bearer " + env[ENV_KEYS[1]], "Content-Type": "application/json"}) + response = connection.getresponse() + raw = response.read(1025) + if response.status != 200 or len(raw) > 1024: + raise ValueError("denied") + expected = {"context": context, "request_id": identity, "payload_sha256": digest} + if json.loads(raw) != expected: + raise ValueError("invalid grant") + except Exception: + raise AuthorizationError("request_authorization_denied") from None + finally: + connection.close() + + +@contextmanager +def authorization_broker(check_callback, max_requests, *, context): + """Yield only an ephemeral loopback capability; callback stays in this process. + + ``context`` binds a frozen plan digest and reservation/run identity. Callback + must re-read live authorization and raise (or return False) to deny. Failure + permanently closes this broker to new calls, even if authorization returns. + """ + if (type(max_requests) is not int or not 0 < max_requests <= 1000000 + or not isinstance(context, str) or not context or len(context) > 1024 + or not callable(check_callback)): + raise ValueError("Invalid request authorization scope") + token = secrets.token_hex(32) + binding = hashlib.sha256(context.encode()).hexdigest() + lock = threading.Lock() + seen = set() + stopped = False + + class Handler(BaseHTTPRequestHandler): + def setup(self): + super().setup() + self.connection.settimeout(5) + + def log_message(self, *args): + pass # Never retain bearer tokens, request material, or exception text. + + def do_POST(self): + nonlocal stopped + supplied = self.headers.get("Authorization", "") + if (self.path != "/authorize" or len(supplied) > 100 or not supplied.isascii() + or not hmac.compare_digest(supplied, "Bearer " + token)): + self.send_error(403) + return + try: + length = self.headers.get("Content-Length", "") + if not re.fullmatch(r"[0-9]{1,4}", length) or not 0 < int(length) <= 1024: + raise ValueError("invalid length") + if self.headers.get("Transfer-Encoding"): + raise ValueError("invalid encoding") + data = json.loads(self.rfile.read(int(length))) + if (not isinstance(data, dict) + or set(data) != {"context", "request_id", "payload_sha256"} + or data["context"] != binding + or any(not isinstance(data[k], str) + or not re.fullmatch(r"[0-9a-f]{64}", data[k]) + for k in ("request_id", "payload_sha256"))): + raise ValueError("invalid identity") + with lock: + if stopped or len(seen) >= max_requests or data["request_id"] in seen: + raise ValueError("denied") + # Burn identity before checking: failures/replays never replenish slots. + seen.add(data["request_id"]) + try: + if check_callback() is False: + raise ValueError("revoked") + except Exception: + stopped = True + raise ValueError("denied") from None + body = json.dumps(data, separators=(",", ":")).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except Exception: + self.send_error(403) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield dict(zip(ENV_KEYS, (str(server.server_port), token, binding))) + finally: + with lock: + stopped = True + server.shutdown() + server.server_close() + thread.join() diff --git a/tests/test_request_guard.py b/tests/test_request_guard.py new file mode 100644 index 00000000..6f1df430 --- /dev/null +++ b/tests/test_request_guard.py @@ -0,0 +1,141 @@ +"""Credential-free checks: only loopback traffic and injected provider transports.""" + +import concurrent.futures +import hashlib +import http.client +import json + +import pytest + +from inferrsi.model import anthropic_agent as agent +from inferrsi.model import request_guard as guard + + +def request(env, *, token=None, context=None, identity="a" * 64): + connection = http.client.HTTPConnection("127.0.0.1", int(env[guard.ENV_KEYS[0]]), timeout=2) + body = {"context": context or env[guard.ENV_KEYS[2]], "request_id": identity, + "payload_sha256": hashlib.sha256(b"payload").hexdigest()} + try: + connection.request("POST", "/authorize", json.dumps(body), { + "Authorization": "Bearer " + (token or env[guard.ENV_KEYS[1]])}) + response = connection.getresponse() + response.read() + return response.status + finally: + connection.close() + + +def test_each_request_checks_live_authorization_and_revocation_latches(): + calls = [] + + def check(): + calls.append(1) + return len(calls) == 1 + + with guard.authorization_broker(check, 5, context="plan:run") as env: + guard.authorize_request(b"first", environment=env) + with pytest.raises(guard.AuthorizationError): + guard.authorize_request(b"second", environment=env) + with pytest.raises(guard.AuthorizationError): + guard.authorize_request(b"third", environment=env) + assert len(calls) == 2 + + +def test_failed_callback_fails_closed_without_disclosing_details(): + def check(): + raise RuntimeError("private GitHub credential failure detail") + + with guard.authorization_broker(check, 2, context="plan:run") as env: + with pytest.raises(guard.AuthorizationError, match="^request_authorization_denied$"): + guard.authorize_request(b"request", environment=env) + + +def test_unauthenticated_wrong_context_and_replays_do_not_consume_extra_grants(): + calls = [] + with guard.authorization_broker(lambda: calls.append(1), 2, context="plan:run") as env: + assert request(env, token="b" * 64) == 403 + assert request(env, context="c" * 64) == 403 + assert calls == [] + assert request(env) == 200 + assert request(env) == 403 + assert request(env, identity="d" * 64) == 200 + assert request(env, identity="e" * 64) == 403 + assert calls == [1, 1] + + +def test_concurrent_requests_cannot_exceed_total_bound(): + calls = [] + with guard.authorization_broker(lambda: calls.append(1), 3, context="plan:run") as env: + def send(index): + return request(env, identity=f"{index:064x}") + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool: + statuses = list(pool.map(send, range(6))) + assert statuses.count(200) == 3 and len(calls) == 3 + + +def test_broker_loss_missing_and_partial_configuration_fail_closed(): + with guard.authorization_broker(lambda: None, 1, context="plan:run") as env: + pass + for value in ({}, {guard.ENV_KEYS[0]: "1234"}, env): + with pytest.raises(guard.AuthorizationError): + guard.authorize_request(b"request", environment=value) + + +def test_capability_cannot_be_reused_for_other_plan_or_run(): + with guard.authorization_broker(lambda: None, 1, context="plan:run-one") as first: + with guard.authorization_broker(lambda: None, 1, context="plan:run-two") as second: + mixed = {**second, guard.ENV_KEYS[1]: first[guard.ENV_KEYS[1]]} + with pytest.raises(guard.AuthorizationError): + guard.authorize_request(b"request", environment=mixed) + guard.authorize_request(b"request", environment=second) + + +def test_required_provider_guard_stops_before_network(monkeypatch): + spec = {"kind": agent.KIND, "provider": "yibuapi", "endpoint": agent.ENDPOINT, + "api_version": "2023-06-01", "max_requests": 2, "max_output_tokens": 256, + "max_request_bytes": 8192, "max_read_bytes": 1024, "max_tool_calls": 8} + monkeypatch.setattr(agent.urllib.request, "build_opener", + lambda *args: pytest.fail("Provider must not be contacted")) + with pytest.raises(agent.AgentError, match="request_authorization_denied"): + agent.send_message({"model": "claude-opus-5-5"}, spec, "offline-key", 2, + environment={}, require_authorization=True) + + +def test_runtime_environment_passes_only_gate_and_provider_not_github(): + with guard.authorization_broker(lambda: None, 1, context="plan:run") as env: + runtime = agent.execution_environment({**env, agent.KEY_ENV: "offline-key", + "GH_TOKEN": "not-a-real-token", + "GITHUB_TOKEN": "not-a-real-token"}) + assert all(runtime[key] == env[key] for key in guard.ENV_KEYS) + assert "GH_TOKEN" not in runtime and "GITHUB_TOKEN" not in runtime + verifier = agent.execution_environment(runtime, with_key=False) + assert all(key not in verifier for key in guard.ENV_KEYS) + assert agent.KEY_ENV not in verifier + + +def test_real_transport_hook_checks_each_call_and_stops_after_revocation(monkeypatch): + import io + + spec = {"kind": agent.KIND, "provider": "yibuapi", "endpoint": agent.ENDPOINT, + "api_version": "2023-06-01", "max_requests": 3, "max_output_tokens": 256, + "max_request_bytes": 8192, "max_read_bytes": 1024, "max_tool_calls": 8} + checks, sends = [], [] + + def check(): + checks.append(1) + return len(checks) < 3 + + class OfflineProvider: + def open(self, request, timeout): + sends.append(request) + return io.BytesIO(b'{"type":"message"}') + + monkeypatch.setattr(agent.urllib.request, "build_opener", lambda *args: OfflineProvider()) + with guard.authorization_broker(check, 5, context="plan:run") as env: + for _ in range(2): + agent.send_message({"model": "claude-opus-5-5"}, spec, "offline-key", 2, + environment=env, require_authorization=True) + with pytest.raises(agent.AgentError, match="request_authorization_denied"): + agent.send_message({"model": "claude-opus-5-5"}, spec, "offline-key", 2, + environment=env, require_authorization=True) + assert len(checks) == 3 and len(sends) == 2 diff --git a/tests/test_skill_eval_auto.py b/tests/test_skill_eval_auto.py new file mode 100644 index 00000000..fca2a2e3 --- /dev/null +++ b/tests/test_skill_eval_auto.py @@ -0,0 +1,270 @@ +"""Credential-free label/controller acceptance controls.""" +import copy +import json +from unittest.mock import Mock + +import pytest +import yaml + +from inferrsi.model import anthropic_agent as agent +from tools import skill_eval_auto as auto +from tools import skill_eval_pr as pr +from tools.skill_eval_budget import BudgetError + + +@pytest.fixture +def config(): + provider = { + "model": {"name": "claude-opus-5-5", "effort": "not_applicable"}, + "client": {"kind": agent.KIND, "provider": "yibuapi", "endpoint": agent.ENDPOINT, + "api_version": "2023-06-01", "max_requests": 2, "max_output_tokens": 100, + "max_request_bytes": 8192, "max_read_bytes": 1024, "max_tool_calls": 8}, + "pricing": {"currency": "USD", "unit": "per_million_tokens", + "model": "claude-opus-5-5", "group": "offline-reviewed-fixture", + "rates": {"input": "8", "output": "40", "cache_read": "0.4", + "cache_write": "8"}, + "request_fee": "0", "max_input_tokens": 1000000, + "rate_evidence": "https://github.com/example/repo/issues/1#issuecomment-1", + "expires_at": 4102444800}, + "spend_limit": {"currency": "USD", "amount": "200"}} + return {"schema_version": 1, "repository_id": 123, "label": "skill-eval:execute", + "operators": ["operator"], "expires_at": 4102444800, "provider": provider, + "provider_sha256": pr.fingerprint(provider), "review_evidence": "offline fixture", + "anchors": {}} + + +@pytest.fixture +def plan(config): + result = {"pr": 7, "repository": "example/repo", "base": "a" * 40, "head": "b" * 40, + "version": "a" * 40, + "estimated_calls": 2, "comparisons": [{"plan_sha256": "c" * 64, "calls": 2}], + "coverage": [{"coverage": "covered"}], "provider_config": config["provider"]} + result["reservation"] = pr.provider_reservation(result) + return result + + +class API: + def __init__(self, plan, config): + self.current = {"state": "open", "base": {"sha": plan["base"], "ref": "main", + "repo": {"id": 123}}, + "head": {"sha": plan["head"]}, "labels": [{"name": config["label"]}]} + self.events = [{"id": 1, "event": "labeled", "label": {"name": config["label"]}, + "actor": {"login": "operator"}, "created_at": "2026-01-01T00:00:00Z"}] + self.permission = "write" + self.calls = [] + + def request(self, method, path): + assert method == "GET" + self.calls.append(path) + if not path: + return {"id": 123} + if path.startswith("/pulls/"): + return copy.deepcopy(self.current) + if path.startswith("/issues/"): + return copy.deepcopy(self.events) + if path.startswith("/collaborators/"): + return {"permission": self.permission} + raise AssertionError(path) + + +def test_current_label_actor_and_permission(plan, config): + api = API(plan, config) + assert auto.live_label(api, plan, config) == {"actor": "operator", "label_event_id": 1} + api.permission = "read" + with pytest.raises(ValueError, match="permission"): + auto.live_label(api, plan, config) + + +@pytest.mark.parametrize("change", ["closed", "draft", "head", "base", "removed", + "untrusted_actor", "readded", "no_history", "unordered", + "target_branch", "target_repo"]) +def test_revocation_and_incomplete_authorization_block(plan, config, change): + api = API(plan, config) + if change == "target_branch": + api.current["base"]["ref"] = "other" + elif change == "target_repo": + api.current["base"]["repo"]["id"] = 456 + elif change == "closed": + api.current["state"] = "closed" + elif change == "draft": + api.current["draft"] = True + elif change in {"head", "base"}: + api.current[change]["sha"] = "d" * 40 + elif change == "removed": + api.current["labels"] = [] + elif change == "untrusted_actor": + api.events[0]["actor"]["login"] = "untrusted" + elif change == "readded": + api.events += [{**api.events[0], "id": 2, "event": "unlabeled"}, + {**api.events[0], "id": 3, "actor": {"login": "untrusted"}}] + elif change == "no_history": + api.events = [] + elif change == "unordered": + api.events *= 2 + with pytest.raises(ValueError): + auto.live_label(api, plan, config) + + +def test_incomplete_full_page_history_blocks(plan, config): + api = API(plan, config) + api.events = [{**api.events[0], "id": n} for n in range(1, 101)] + with pytest.raises(ValueError, match="Ambiguous"): + auto.live_label(api, plan, config) + + +def test_trusted_configuration_validates_identity_rates_expiry(tmp_path, config): + path = tmp_path / "config.json" + path.write_text(json.dumps(config)) + assert auto.configuration(path, now=1) == config + for change in ("expired", "hash", "rate", "cache_missing", "group_missing"): + altered = copy.deepcopy(config) + if change == "expired": + altered["expires_at"] = 1 + elif change == "hash": + altered["provider_sha256"] = "f" * 64 + elif change == "rate": + altered["provider"]["pricing"]["rates"]["input"] = "1" + elif change == "cache_missing": + del altered["provider"]["pricing"]["rates"]["cache_write"] + else: + del altered["provider"]["pricing"]["group"] + if change != "hash": + altered["provider_sha256"] = pr.fingerprint(altered["provider"]) + path.write_text(json.dumps(altered)) + with pytest.raises((ValueError, KeyError)): + auto.configuration(path, now=2) + + +def test_missing_legacy_spend_audit_cannot_bootstrap(plan, config): + with pytest.raises(BudgetError, match="genesis"): + auto.ledger_store(Mock(), config, plan["pr"]) + + +@pytest.mark.parametrize("change", ["none", "unsupported", "provider", "stale_cost", "old_policy"]) +def test_eligibility_fails_closed(plan, config, change): + if change == "old_policy": + plan["version"] = "e" * 40 + elif change == "none": + plan["estimated_calls"] = 0 + elif change == "unsupported": + plan["coverage"][0]["coverage"] = "unsupported" + elif change == "provider": + plan["provider_config"] = {} + else: + plan["reservation"]["amount"] = "0" + with pytest.raises(ValueError): + auto.eligible(plan, config) + + +def test_workflow_is_opt_in_and_controller_is_only_budget_writer(): + workflow = yaml.safe_load((pr.ROOT / pr.WORKFLOW).read_text()) + trigger = workflow.get("on", workflow.get(True))["pull_request_target"] + assert {"labeled", "unlabeled", "closed", "synchronize"} <= set(trigger["types"]) + assert trigger["paths"] and "**" not in trigger["paths"] + job = workflow["jobs"]["execute"] + assert "auto_candidate == 'true'" in job["if"] + assert job["permissions"]["contents"] == "write" + control = [s for s in job["steps"] if "control --output" in s.get("run", "")] + assert len(control) == 1 + assert not any(" execute --output" in s.get("run", "") for s in job["steps"]) + assert workflow["jobs"]["publish"]["permissions"]["contents"] == "read" + + +def test_claim_once_launch_scrubs_tokens_and_rechecks_every_request( + plan, config, tmp_path, monkeypatch): + import contextlib + import sys + import types + + from tools.skill_eval_budget import Genesis, Ledger, PRKey, Snapshot, WriteResult + + class Store: + def __init__(self): + self.ledger = Ledger(PRKey(123, 7), Genesis("audited offline history", 0)) + + def read_verified(self, key): + assert key == self.ledger.key + return Snapshot(str(self.ledger.revision), self.ledger) + + def compare_and_swap(self, before, after): + assert before.ledger == self.ledger + self.ledger = after + return WriteResult.COMMITTED + + store, api = Store(), API(plan, config) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(config)) + (tmp_path / "pr-plan.json").write_text(json.dumps(plan)) + monkeypatch.setattr(auto, "ledger_store", lambda *args: (store, store.ledger.key)) + monkeypatch.setenv("GITHUB_TOKEN", "must-not-reach-worker") + monkeypatch.setenv("ACTIONS_RUNTIME_TOKEN", "must-not-reach-worker") + monkeypatch.setenv("SKILL_EVAL_YIBUAPI_KEY", "offline-only") + checks = [] + + @contextlib.contextmanager + def broker(check, max_requests, *, context): + assert max_requests == plan["reservation"]["requests"] + assert context == pr.fingerprint(plan) + ":42.1" + checks.append(check) + yield {"SKILL_EVAL_GUARD_TEST": "offline-only"} + + monkeypatch.setitem(sys.modules, "inferrsi.model.request_guard", + types.SimpleNamespace(authorization_broker=broker)) + + def launch(command, *, env, cwd, check): + assert "GITHUB_TOKEN" not in env and "ACTIONS_RUNTIME_TOKEN" not in env + assert env["SKILL_EVAL_YIBUAPI_KEY"] == "offline-only" + assert store.ledger.entries[0].phase == "claimed" + checks[0]() # First provider call succeeds. + api.current["labels"] = [] + with pytest.raises(ValueError, match="absent"): + checks[0]() # Next provider call cannot proceed. + + auto.controlled_execute(api, tmp_path, config_path=config_path, + run_id=42, attempt=1, launch=launch) + charged = store.ledger.charged + api.current["labels"] = [{"name": config["label"]}] + with pytest.raises(BudgetError, match="Duplicate"): + auto.controlled_execute(api, tmp_path, config_path=config_path, + run_id=42, attempt=1, launch=Mock(side_effect=AssertionError)) + with pytest.raises(BudgetError): + auto.controlled_execute(api, tmp_path, config_path=config_path, + run_id=43, attempt=1, launch=Mock(side_effect=AssertionError)) + assert store.ledger.charged == charged + plan["head"] = "d" * 40 + api.current["head"]["sha"] = plan["head"] + (tmp_path / "pr-plan.json").write_text(json.dumps(plan)) + with pytest.raises(ValueError, match="reruns"): + auto.controlled_execute(api, tmp_path, config_path=config_path, + run_id=43, attempt=2, launch=Mock(side_effect=AssertionError)) + assert store.ledger.charged == charged # Head changes never reset the PR ledger. + + +def test_configured_unmapped_pr_prepares_without_paid_reservation(tmp_path, config): + from tests.fixtures.skill_eval_pr_support import setup + + repo, mapping, identity = setup(tmp_path, "addition", ids=("one",)) + mapping_value = json.loads(mapping.read_text()) + mapping_value["skills"] = [] + mapping.write_text(json.dumps(mapping_value)) + output = tmp_path / "unmapped" + plan = pr.prepare(repo, mapping, identity, output, provider=config["provider"]) + assert plan["estimated_calls"] == 0 and plan["comparisons"] == [] + assert "reservation" not in plan and pr.budget(plan)["reservation"] is None + report = pr.summary(plan, [], "https://example.invalid/evidence", 100, 1) + assert "obtain missing evidence" in report + with pytest.raises(ValueError): + auto.eligible(plan, config) + + +def test_missing_provider_key_does_not_reserve(plan, config, tmp_path, monkeypatch): + monkeypatch.delenv("SKILL_EVAL_YIBUAPI_KEY", raising=False) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(config)) + (tmp_path / "pr-plan.json").write_text(json.dumps(plan)) + ledger = Mock(side_effect=AssertionError("Must not reserve or claim")) + monkeypatch.setattr(auto, "ledger_store", ledger) + with pytest.raises(ValueError, match="credential is unavailable"): + auto.controlled_execute(API(plan, config), tmp_path, config_path=config_path, + run_id=42, attempt=1, launch=Mock(side_effect=AssertionError)) + ledger.assert_not_called() diff --git a/tests/test_skill_eval_budget.py b/tests/test_skill_eval_budget.py new file mode 100644 index 00000000..8563934f --- /dev/null +++ b/tests/test_skill_eval_budget.py @@ -0,0 +1,242 @@ +"""Credential-free tests: the in-memory CAS is NOT a production backend.""" + +import json +from dataclasses import replace + +import pytest + +from tools import skill_eval_budget as budget + +KEY = budget.PRKey(123, 7) + + +def genesis(legacy=0): + return budget.Ledger(KEY, budget.Genesis("audited-genesis-fixture", legacy)) + + +def binding(number=1, amount=50_000_000): + return budget.Binding(f"{number:064x}", f"workflow-{number}:attempt-1", amount) + + +class MemoryStore: + def __init__(self, ledger): + self.snapshot = budget.Snapshot("lineage-0", ledger) + self.outcome = budget.WriteResult.COMMITTED + self.write_on_unknown = False + + def read_verified(self, key): + if self.snapshot is None or self.snapshot.ledger.key != key: + raise budget.BudgetError("Missing existing history") + return self.snapshot + + def compare_and_swap(self, expected, successor): + if self.snapshot != expected: + return budget.WriteResult.CONFLICT + budget.validate_successor(expected.ledger, successor) + if self.outcome is budget.WriteResult.COMMITTED or self.write_on_unknown: + self.snapshot = budget.Snapshot(f"lineage-{successor.revision}", successor) + return self.outcome + + +def test_budget_cumulative_across_revisions_retries_and_label_epochs(): + ledger = genesis(20_000_000) + for number in range(1, 4): + plan = binding(number, 60_000_000) + ledger = budget.reserve(ledger, plan) + ledger = budget.advance(ledger, plan, "claimed") + ledger = budget.advance(ledger, plan, "finished", actual_cost=1) + assert ledger.charged == 200_000_000 + assert ledger.remaining == 0 + with pytest.raises(budget.BudgetError): + budget.reserve(ledger, binding(4, 1)) + assert budget.decode(budget.encode(ledger)) == ledger + + +def test_no_refunds_on_cancel_or_actual_cost(): + plan = binding(amount=200_000_000) + ledger = budget.reserve(genesis(), plan) + cancelled = budget.advance(ledger, plan, "cancelled") + assert cancelled.remaining == 0 + assert budget.advance(cancelled, plan, "cancelled") == cancelled + with pytest.raises(budget.BudgetError): + budget.advance(cancelled, plan, "claimed") + assert ledger.entries[0].phase == "reserved" # pure transition + + +def test_reserve_exact_idempotency_and_conflicting_bindings(): + plan = binding() + ledger = budget.reserve(genesis(), plan) + assert budget.reserve(ledger, plan) is ledger + for changed in (replace(plan, run_id="workflow-1:attempt-2"), + replace(plan, worst_case=1), replace(plan, plan_digest="a" * 64)): + with pytest.raises(budget.BudgetError): + budget.reserve(ledger, changed) + + +def test_claim_once_and_duplicate_execution_rejected(): + plan = binding() + store = MemoryStore(budget.reserve(genesis(), plan)) + claimed = budget.claim_once(store, KEY, plan) + assert claimed.entries[0].phase == "claimed" + with pytest.raises(budget.BudgetError): + budget.claim_once(store, KEY, plan) + finished = budget.advance(claimed, plan, "finished", actual_cost=100) + assert budget.advance(finished, plan, "finished", actual_cost=100) is finished + with pytest.raises(budget.BudgetError): + budget.advance(finished, plan, "finished", actual_cost=101) + + +@pytest.mark.parametrize("outcome,applied", [ + (budget.WriteResult.UNKNOWN, True), (budget.WriteResult.UNKNOWN, False), + (budget.WriteResult.CONFLICT, False), ("committed", False), (None, False), +]) +def test_uncertain_or_conflicted_claim_never_grants_execution(outcome, applied): + plan = binding() + store = MemoryStore(budget.reserve(genesis(), plan)) + store.outcome, store.write_on_unknown = outcome, applied + with pytest.raises(budget.BudgetError, match="blocked"): + budget.claim_once(store, KEY, plan) + if applied: + store.outcome = budget.WriteResult.COMMITTED + with pytest.raises(budget.BudgetError, match="duplicate"): + budget.claim_once(store, KEY, plan) + + +def test_lost_update_cannot_overwrite_or_overspend(): + store = MemoryStore(genesis()) + first = store.read_verified(KEY) + stale = store.read_verified(KEY) + a = budget.reserve(first.ledger, binding(1, 150_000_000)) + b = budget.reserve(stale.ledger, binding(2, 150_000_000)) + assert store.compare_and_swap(first, a) is budget.WriteResult.COMMITTED + assert store.compare_and_swap(stale, b) is budget.WriteResult.CONFLICT + with pytest.raises(budget.BudgetError, match="exhausted"): + budget.reserve(store.read_verified(KEY).ledger, binding(2, 150_000_000)) + + +def test_concurrent_claims_only_one_wins(): + plan = binding() + store = MemoryStore(budget.reserve(genesis(), plan)) + first = store.read_verified(KEY) + second = store.read_verified(KEY) + claim = budget.advance(first.ledger, plan, "claimed") + assert store.compare_and_swap(first, claim) is budget.WriteResult.COMMITTED + assert store.compare_and_swap(second, claim) is budget.WriteResult.CONFLICT + + +def test_missing_history_does_not_create_zero_budget(): + store = MemoryStore(genesis()) + store.snapshot = None + with pytest.raises(budget.BudgetError, match="Missing"): + budget.claim_once(store, KEY, binding()) + with pytest.raises(budget.BudgetError): + budget.Genesis("", 0) + with pytest.raises(budget.BudgetError): + budget.Genesis("unreconciled", None) + assert genesis(200_000_000).remaining == 0 + + +def test_wrong_pr_and_reset_rejected(): + old = budget.reserve(genesis(), binding()) + for reset in (genesis(), replace(old, key=budget.PRKey(124, 7)), + replace(old, genesis=budget.Genesis("replacement-audit", 0))): + with pytest.raises(budget.BudgetError): + budget.validate_successor(old, reset) + store = MemoryStore(old) + store.read_verified = lambda key: store.snapshot + with pytest.raises(budget.BudgetError, match="wrong PR"): + budget.claim_once(store, budget.PRKey(123, 8), binding()) + + +@pytest.mark.parametrize("amount", [-1, True, 1.2, "1", None]) +def test_non_integer_or_negative_money_rejected(amount): + with pytest.raises(budget.BudgetError): + budget.Genesis("audit", amount) + with pytest.raises(budget.BudgetError): + binding(amount=amount) + + +def test_zero_reservation_overrun_and_unreserved_execution_rejected(): + with pytest.raises(budget.BudgetError): + binding(amount=0) + plan = binding() + with pytest.raises(budget.BudgetError): + budget.advance(genesis(), plan, "claimed") + ledger = budget.advance(budget.reserve(genesis(), plan), plan, "claimed") + for phase, cost in (("cancelled", None), ("finished", plan.worst_case + 1), + ("finished", None), ("finished", -1)): + with pytest.raises(budget.BudgetError): + budget.advance(ledger, plan, phase, actual_cost=cost) + + +@pytest.mark.parametrize("payload", ["", "null", "{}", "[]", '{"x":1,"x":2}', + '{"schema_version":NaN}', '{"revision":Infinity}']) +def test_corrupt_missing_ledger_rejected(payload): + with pytest.raises(budget.BudgetError): + budget.decode(payload) + + +@pytest.mark.parametrize("mutation", [ + lambda v: v.update(extra=1), lambda v: v.update(schema_version=True), + lambda v: v.update(revision=0), lambda v: v["key"].update(repository_id=True), + lambda v: v["genesis"].update(legacy_charge=-1), + lambda v: v["entries"].append(v["entries"][0]), + lambda v: v["entries"][0].update(phase="unknown"), + lambda v: v["entries"][0].update(actual_cost=0), + lambda v: v["entries"][0]["binding"].update(worst_case=1.5), +]) +def test_schema_corruption_rejected(mutation): + value = json.loads(budget.encode(budget.reserve(genesis(), binding()))) + mutation(value) + with pytest.raises(budget.BudgetError): + budget.decode(json.dumps(value)) + + +def test_storage_successor_validation_allows_only_one_transition(): + old = genesis() + reserved = budget.reserve(old, binding()) + claimed = budget.advance(reserved, binding(), "claimed") + finished = budget.advance(claimed, binding(), "finished", actual_cost=5) + for before, after in ((old, reserved), (reserved, claimed), (claimed, finished)): + budget.validate_successor(before, after) + for before, after in ((old, claimed), (reserved, reserved), (finished, claimed)): + with pytest.raises(budget.BudgetError): + budget.validate_successor(before, after) + + +def test_unknown_reservation_write_reconciles_without_new_charge(): + store = MemoryStore(genesis()) + initial = store.read_verified(KEY) + plan = binding() + successor = budget.reserve(initial.ledger, plan) + store.outcome = budget.WriteResult.UNKNOWN + store.write_on_unknown = True + assert store.compare_and_swap(initial, successor) is budget.WriteResult.UNKNOWN + observed = store.read_verified(KEY) + assert budget.reserve(observed.ledger, plan) is observed.ledger + assert observed.ledger.charged == plan.worst_case + + +def test_storage_validator_rejects_reservation_rewrite_and_removal(): + first, second = binding(1), binding(2) + old = budget.reserve(budget.reserve(genesis(), first), second) + changed = replace(first, worst_case=1) + rewritten = budget.advance( + budget.reserve(budget.reserve(genesis(), changed), second), changed, "claimed") + with pytest.raises(budget.BudgetError): + budget.validate_successor(old, rewritten) + removed = budget.advance(budget.advance(budget.reserve(genesis(), first), first, + "claimed"), first, "finished", actual_cost=1) + with pytest.raises(budget.BudgetError): + budget.validate_successor(old, removed) + + +def test_storage_transport_exception_never_grants_execution(): + store = MemoryStore(budget.reserve(genesis(), binding())) + + def unknown(*args): + raise OSError("Synthetic lost response") + + store.compare_and_swap = unknown + with pytest.raises(OSError): + budget.claim_once(store, KEY, binding()) diff --git a/tests/test_skill_eval_budget_git.py b/tests/test_skill_eval_budget_git.py new file mode 100644 index 00000000..0e7c0485 --- /dev/null +++ b/tests/test_skill_eval_budget_git.py @@ -0,0 +1,302 @@ +"""Credential-free Git-data API model; no remote writes/provider requests.""" + +import base64 +import copy +import hashlib +import json +from dataclasses import replace + +import pytest + +from tools import skill_eval_budget as budget +from tools.skill_eval_budget_git import GitHubGitStore, LedgerAnchor + +KEY = budget.PRKey(123, 7) +REF = "refs/heads/skill-eval-budget/123/7" + + +def binding(number=1, amount=50_000_000): + return budget.Binding(f"{number:064x}", f"run-{number}:attempt-1", amount) + + +class GitAPI: + """Immutable objects plus non-force fast-forward reference updates.""" + + def __init__(self): + self.objects, self.calls = {}, [] + self.tip = None + self.before_patch = None + self.lose_response = False + self.reject_write = False + self.repository_id = KEY.repository_id + ledger = budget.Ledger(KEY, budget.Genesis("audited legacy fixture", 0)) + self.genesis = self.append(ledger, parents=[]) + self.anchor = LedgerAnchor(KEY, REF, self.genesis, self.genesis) + + def put(self, value): + sha = hashlib.sha1(json.dumps(value, sort_keys=True).encode()).hexdigest() + self.objects[sha] = {**value, "sha": sha} + return sha + + def tree(self, content): + raw = content.encode() + blob = self.put({"encoding": "base64", "size": len(raw), + "content": base64.b64encode(raw).decode()}) + return self.put({"truncated": False, "tree": [{"path": "ledger.json", + "mode": "100644", "type": "blob", "sha": blob}]}) + + def append(self, ledger, parents=None): + tree = self.tree(budget.encode(ledger)) + parents = [self.tip] if parents is None else parents + self.tip = self.put({"tree": {"sha": tree}, + "parents": [{"sha": p} for p in parents]}) + return self.tip + + def ref(self): + if self.tip is None: + raise OSError("404") + return {"ref": REF, "object": {"type": "commit", "sha": self.tip}} + + def request(self, method, path, data=None): + self.calls.append((method, path, copy.deepcopy(data))) + if method == "GET": + if path == "": + return {"id": self.repository_id} + if path.startswith("/git/ref/"): + return self.ref() + return copy.deepcopy(self.objects[path.rsplit("/", 1)[1]]) + if self.reject_write: + raise OSError("rules reject writes") + if path == "/git/trees": + assert set(data) == {"tree"} + return {"sha": self.tree(data["tree"][0]["content"])} + if path == "/git/commits": + return {"sha": self.put({"tree": {"sha": data["tree"]}, + "parents": [{"sha": p} for p in data["parents"]], + "message": data["message"]})} + assert method == "PATCH" and path == "/git/refs/heads/skill-eval-budget/123/7" + assert data["force"] is False + if self.before_patch: + callback, self.before_patch = self.before_patch, None + callback() + if self.tip is None: + raise OSError("404") + parents = self.objects[data["sha"]]["parents"] + if self.tip != data["sha"] and parents != [{"sha": self.tip}]: + raise OSError("422 non fast-forward") + self.tip = data["sha"] + if self.lose_response: + raise TimeoutError("lost response after commit") + return self.ref() + + +def test_real_adapter_read_reserve_claim_and_permanent_charge(): + api = GitAPI() + store = GitHubGitStore(api, api.anchor) + snap = store.read_verified(KEY) + reserved = budget.reserve(snap.ledger, binding()) + assert store.compare_and_swap(snap, reserved) is budget.WriteResult.COMMITTED + claimed = budget.claim_once(store, KEY, binding()) + assert claimed.charged == 50_000_000 + with pytest.raises(budget.BudgetError): + budget.claim_once(store, KEY, binding()) + snap = store.read_verified(KEY) + done = budget.advance(snap.ledger, binding(), "finished", actual_cost=1) + assert store.compare_and_swap(snap, done) is budget.WriteResult.COMMITTED + assert store.read_verified(KEY).ledger.remaining == 150_000_000 + + +def test_racing_identical_claim_has_only_one_grant_and_unique_commit(): + api = GitAPI() + a, b = GitHubGitStore(api, api.anchor), GitHubGitStore(api, api.anchor) + snap = a.read_verified(KEY) + a.compare_and_swap(snap, budget.reserve(snap.ledger, binding())) + grants = [] + api.before_patch = lambda: grants.append(budget.claim_once(b, KEY, binding())) + with pytest.raises(budget.BudgetError): + budget.claim_once(a, KEY, binding()) + assert len(grants) == 1 + messages = [data["message"] for method, path, data in api.calls + if path == "/git/commits"] + assert len(messages) == len(set(messages)) + + +def test_racing_reservations_cannot_overwrite_or_exceed_cap(): + api = GitAPI() + a, b = GitHubGitStore(api, api.anchor), GitHubGitStore(api, api.anchor) + snap = a.read_verified(KEY) + one = budget.reserve(snap.ledger, binding(1, 150_000_000)) + two = budget.reserve(snap.ledger, binding(2, 150_000_000)) + api.before_patch = lambda: b.compare_and_swap(snap, two) + assert a.compare_and_swap(snap, one) is budget.WriteResult.UNKNOWN + latest = a.read_verified(KEY) + assert latest.ledger == two + assert a.compare_and_swap(snap, one) is budget.WriteResult.CONFLICT + with pytest.raises(budget.BudgetError): + budget.reserve(latest.ledger, binding(1, 150_000_000)) + + +@pytest.mark.parametrize("phase", ["reserved", "claimed"]) +def test_unknown_write_is_never_execution_permission(phase): + api = GitAPI() + store = GitHubGitStore(api, api.anchor) + snap = store.read_verified(KEY) + reserved = budget.reserve(snap.ledger, binding()) + if phase == "claimed": + store.compare_and_swap(snap, reserved) + api.lose_response = True + with pytest.raises(budget.BudgetError): + budget.claim_once(store, KEY, binding()) + api.lose_response = False + with pytest.raises(budget.BudgetError): + budget.claim_once(GitHubGitStore(api, api.anchor), KEY, binding()) + else: + api.lose_response = True + assert store.compare_and_swap(snap, reserved) is budget.WriteResult.UNKNOWN + assert store.read_verified(KEY).ledger.charged == 50_000_000 + + +def test_missing_ledger_never_recreated_and_rule_rejection_blocks(): + api = GitAPI() + store = GitHubGitStore(api, api.anchor) + snap = store.read_verified(KEY) + api.reject_write = True + assert store.compare_and_swap(snap, budget.reserve(snap.ledger, binding())) is ( + budget.WriteResult.UNKNOWN) + api.tip = None + with pytest.raises(budget.BudgetError): + store.read_verified(KEY) + assert not any(path == "/git/refs" for _, path, _ in api.calls) + + +@pytest.mark.parametrize("damage", ["reset", "merge", "key", "tree", "blob", "missing", + "repository", "truncated", "genesis"]) +def test_corruption_and_discontinuous_history_fail_closed(damage): + api = GitAPI() + store = GitHubGitStore(api, api.anchor) + initial = store.read_verified(KEY).ledger + api.append(budget.reserve(initial, binding())) + if damage == "reset": + api.append(initial) + elif damage == "merge": + api.objects[api.tip]["parents"].append({"sha": api.genesis}) + elif damage == "key": + api.append(replace(initial, key=budget.PRKey(124, 7))) + elif damage == "repository": + api.repository_id = 124 + elif damage == "genesis": + api.anchor = replace(api.anchor, genesis_commit="f" * 40) + store = GitHubGitStore(api, api.anchor) + else: + tree = api.objects[api.objects[api.tip]["tree"]["sha"]] + if damage == "tree": + tree["tree"].append(dict(tree["tree"][0])) + elif damage == "truncated": + tree["truncated"] = True + elif damage == "blob": + api.objects[tree["tree"][0]["sha"]]["content"] = "invalid%" + else: + del api.objects[api.genesis] + with pytest.raises(budget.BudgetError): + store.read_verified(KEY) + + +def test_checkpoint_and_observed_tip_detect_rollback(): + api = GitAPI() + store = GitHubGitStore(api, api.anchor) + snap = store.read_verified(KEY) + store.compare_and_swap(snap, budget.reserve(snap.ledger, binding())) + anchor = replace(api.anchor, checkpoint_commit=api.tip) + api.tip = api.genesis + for reader in (store, GitHubGitStore(api, anchor)): + with pytest.raises(budget.BudgetError, match="rollback"): + reader.read_verified(KEY) + # Explicit limit: a new reader pinned ONLY to genesis cannot detect an + # administrator's rollback after genesis. Do not claim otherwise. + assert GitHubGitStore(api, api.anchor).read_verified(KEY).ledger.revision == 0 + + +def test_history_bound_and_anchor_validation(): + api = GitAPI() + initial = GitHubGitStore(api, api.anchor).read_verified(KEY).ledger + api.append(budget.reserve(initial, binding())) + with pytest.raises(budget.BudgetError, match="bound"): + GitHubGitStore(api, api.anchor, max_history=1).read_verified(KEY) + with pytest.raises(budget.BudgetError): + replace(api.anchor, ref="refs/heads/main") + with pytest.raises(budget.BudgetError): + replace(api.anchor, genesis_commit="not-a-sha") + + +@pytest.mark.parametrize("failure", ["before_commit", "before_patch", "deleted", "bad_response"]) +def test_crashes_and_uncertain_responses_cannot_grant_claim(failure): + api = GitAPI() + store = GitHubGitStore(api, api.anchor) + snap = store.read_verified(KEY) + store.compare_and_swap(snap, budget.reserve(snap.ledger, binding())) + request = api.request + + def fail(method, path, data=None): + if failure == "before_commit" and method == "POST" and path == "/git/commits": + raise TimeoutError("commit creation unknown") + if failure == "before_patch" and method == "PATCH": + raise TimeoutError("patch not sent") + if failure == "deleted" and method == "PATCH": + api.tip = None + result = request(method, path, data) + if failure == "bad_response" and method == "PATCH": + return {} + return result + + api.request = fail + with pytest.raises(budget.BudgetError): + budget.claim_once(store, KEY, binding()) + api.request = request + if failure == "deleted": + with pytest.raises(budget.BudgetError): + GitHubGitStore(api, api.anchor).read_verified(KEY) + else: + latest = GitHubGitStore(api, api.anchor).read_verified(KEY).ledger + assert latest.charged == 50_000_000 + if failure == "bad_response": + with pytest.raises(budget.BudgetError): + budget.claim_once(store, KEY, binding()) + else: + # Only an independently successful new claim can permit execution; + # no caller received a grant for the failed attempt. + assert latest.entries[0].phase == "reserved" + + +def test_caller_forged_snapshot_and_identity_do_not_write(): + api = GitAPI() + store = GitHubGitStore(api, api.anchor) + snap = store.read_verified(KEY) + fake = budget.Snapshot("f" * 40, snap.ledger) + assert store.compare_and_swap(fake, budget.reserve(fake.ledger, binding())) is ( + budget.WriteResult.CONFLICT) + with pytest.raises(budget.BudgetError): + store.read_verified(budget.PRKey(123, 8)) + assert all(method == "GET" for method, _, _ in api.calls) + + +@pytest.mark.parametrize("parent_count", [1, 2]) +def test_audited_genesis_can_have_one_main_parent_but_not_merge(parent_count): + api = GitAPI() + # Deliberately absent from the ledger object model: main ancestry is not + # ledger history and must not be read or interpreted as budget transitions. + main_parents = [f"{i + 1:040x}" for i in range(parent_count)] + genesis = budget.Ledger(KEY, budget.Genesis("audited main-parent migration", 10)) + api.genesis = api.append(genesis, parents=main_parents) + api.anchor = replace(api.anchor, genesis_commit=api.genesis, + checkpoint_commit=api.genesis) + store = GitHubGitStore(api, api.anchor) + if parent_count > 1: + with pytest.raises(budget.BudgetError, match="genesis"): + store.read_verified(KEY) + return + snap = store.read_verified(KEY) + assert snap.ledger == genesis + reserved = budget.reserve(snap.ledger, binding()) + assert store.compare_and_swap(snap, reserved) is budget.WriteResult.COMMITTED + assert store.read_verified(KEY).ledger.charged == 50_000_010 + assert not any(path == "/git/commits/" + main_parents[0] for _, path, _ in api.calls) diff --git a/tests/test_skill_eval_pr.py b/tests/test_skill_eval_pr.py index 4b52c7ee..c32d7726 100644 --- a/tests/test_skill_eval_pr.py +++ b/tests/test_skill_eval_pr.py @@ -345,7 +345,9 @@ def test_file_only_plan_artifact_round_trip_can_execute(tmp_path, mode): @pytest.mark.parametrize("mode", ["addition", "update"]) -def test_retained_results_publish_once_retry_without_model_and_reject_stale_jobs(tmp_path, mode): +@pytest.mark.parametrize("event_name", ["workflow_dispatch", "pull_request_target"]) +def test_retained_results_publish_once_retry_without_model_and_reject_stale_jobs( + tmp_path, mode, event_name): repo, mapping, identity = setup(tmp_path, mode, token_stop_new_trials=200) output = tmp_path / "execution" plan = integration.prepare(repo, mapping, identity, output) @@ -357,6 +359,9 @@ def test_retained_results_publish_once_retry_without_model_and_reject_stale_jobs publisher = tmp_path / "publisher" integration.prepare(repo, mapping, identity, publisher) api = SimulatedGitHub(plan, data) + if event_name == "pull_request_target": + api.run.update(event=event_name, + display_title=f"skill-eval plan or publish PR {plan['pr']}") api.fail_response = True with pytest.raises(ConnectionError): integration.publish_retained(api, publisher, 10, 11, 1) @@ -470,7 +475,7 @@ def test_workflow_separates_execution_and_publication_credentials(): readers = {"contents": "read", "actions": "read", "pull-requests": "read"} assert workflow["permissions"] == {} assert workflow["jobs"]["plan"]["permissions"] == readers - assert executor["permissions"] == readers + assert executor["permissions"] == {**readers, "contents": "write"} # PR conversation writes belong only to the separate trusted publisher. assert workflow["jobs"]["publish"]["permissions"] == { **readers, "pull-requests": "write", "issues": "write"} @@ -478,8 +483,8 @@ def test_workflow_separates_execution_and_publication_credentials(): checkout = next(step for step in job["steps"] if step.get("uses") == "actions/checkout@v4") assert checkout["with"]["persist-credentials"] is False assert checkout["with"]["ref"] == "${{ github.workflow_sha }}" - step = next(s for s in executor["steps"] if "execute --output" in s.get("run", "")) - assert "env" not in step + step = next(s for s in executor["steps"] if "control --output" in s.get("run", "")) + assert set(step["env"]) == {"GITHUB_TOKEN", "SKILL_EVAL_YIBUAPI_KEY"} def test_download_redirect_drops_the_github_token(): @@ -654,9 +659,10 @@ def test_plan_dispatch_workflow_keeps_execution_guards(): assert all(inputs[key]["required"] is False for key in ("plan_sha256", "version", "base", "head", "budget")) guard = "github.event_name == 'workflow_dispatch' && inputs.action == 'execute'" - assert workflow["jobs"]["plan"]["if"] == "github.ref == 'refs/heads/main'" - assert workflow["jobs"]["execute"]["if"] == guard - authorize = next(step for step in workflow["jobs"]["plan"]["steps"] - if "authorize --output" in step.get("run", "")) - assert authorize["if"] == guard + assert workflow["jobs"]["plan"]["if"] == ( + "github.ref == 'refs/heads/main' && github.event.action != 'closed'") + assert workflow["jobs"]["execute"]["if"] == ( + "github.event_name == 'pull_request_target' && needs.plan.outputs.auto_candidate == 'true'") + assert not any("authorize --output" in step.get("run", "") + for step in workflow["jobs"]["plan"]["steps"]) assert workflow["run-name"].startswith("${{ " + guard) diff --git a/tests/test_skill_eval_pr_summary.py b/tests/test_skill_eval_pr_summary.py index 6b73a8a5..a9cf07c7 100644 --- a/tests/test_skill_eval_pr_summary.py +++ b/tests/test_skill_eval_pr_summary.py @@ -47,8 +47,8 @@ def assert_boundaries(text): assert text.startswith(integration.MARKER + "\n") assert text.count("
") == text.count("
") == 2 assert "
512: + raise BudgetError("Expected a bounded nonempty identifier") + + +def _amount(value, *, positive=False): + if type(value) is not int or value < int(positive): + raise BudgetError("Expected nonnegative integer USD microdollars") + + +@dataclass(frozen=True) +class PRKey: + repository_id: int + pr_number: int + + def __post_init__(self): + _amount(self.repository_id, positive=True) + _amount(self.pr_number, positive=True) + + +@dataclass(frozen=True) +class Genesis: + """Trusted operator audit, NOT a proof inferred from an absent ledger. + + legacy_charge includes settled spend and upper bounds for ALL outstanding + work. Unknown legacy spend is not representable and must block creation. + evidence identifies the audited no-prior-ledger/legacy reconciliation record. + """ + + evidence: str + legacy_charge: int + + def __post_init__(self): + _text(self.evidence) + _amount(self.legacy_charge) + + +@dataclass(frozen=True) +class Binding: + """Immutable plan digest + execution attempt identity + worst-case cost. + + plan_digest must cover revisions, mapping, exam, provider/pricing and limits. + run_id must include the workflow run attempt, not only its run number. + """ + + plan_digest: str + run_id: str + worst_case: int + + def __post_init__(self): + if (type(self.plan_digest) is not str or len(self.plan_digest) != 64 + or any(c not in "0123456789abcdef" for c in self.plan_digest)): + raise BudgetError("Expected SHA-256 plan digest") + _text(self.run_id) + _amount(self.worst_case, positive=True) + + +@dataclass(frozen=True) +class Entry: + binding: Binding + phase: str = "reserved" + actual_cost: int | None = None + + def __post_init__(self): + if type(self.binding) is not Binding or self.phase not in { + "reserved", "claimed", "finished", "cancelled" + }: + raise BudgetError("Invalid reservation") + if self.phase == "finished": + _amount(self.actual_cost) + if self.actual_cost > self.binding.worst_case: + raise BudgetError("Actual cost exceeds reservation; quarantine externally") + elif self.actual_cost is not None: + raise BudgetError("Only finished reservations have actual cost") + + +@dataclass(frozen=True) +class Ledger: + key: PRKey + genesis: Genesis + revision: int = 0 + entries: tuple[Entry, ...] = () + + def __post_init__(self): + if type(self.key) is not PRKey or type(self.genesis) is not Genesis: + raise BudgetError("Missing ledger identity or genesis audit") + _amount(self.revision) + if type(self.entries) is not tuple or any(type(e) is not Entry for e in self.entries): + raise BudgetError("Invalid ledger entries") + expected_revision = sum({"reserved": 1, "claimed": 2, "finished": 3, + "cancelled": 2}[e.phase] for e in self.entries) + if self.revision != expected_revision: + raise BudgetError("Inconsistent transition revision") + plans = [e.binding.plan_digest for e in self.entries] + runs = [e.binding.run_id for e in self.entries] + if len(set(plans)) != len(plans) or len(set(runs)) != len(runs): + raise BudgetError("Duplicate plan or run binding") + if self.charged > PR_LIMIT_MICROUSD: + raise BudgetError("PR cumulative budget exhausted") + + @property + def charged(self): + return self.genesis.legacy_charge + sum(e.binding.worst_case for e in self.entries) + + @property + def remaining(self): + return PR_LIMIT_MICROUSD - self.charged + + +def reserve(ledger: Ledger, binding: Binding) -> Ledger: + """Idempotent for the exact binding only; never authorize execution here.""" + if type(ledger) is not Ledger or type(binding) is not Binding: + raise BudgetError("Invalid ledger or binding") + for entry in ledger.entries: + if entry.binding == binding: + return ledger + if (entry.binding.plan_digest == binding.plan_digest + or entry.binding.run_id == binding.run_id): + raise BudgetError("Plan or run already bound differently") + return replace(ledger, revision=ledger.revision + 1, + entries=ledger.entries + (Entry(binding),)) + + +def advance(ledger: Ledger, binding: Binding, phase: str, *, actual_cost=None) -> Ledger: + """Claim is deliberately NOT idempotent: a second claimant must not execute. + + Finished/cancelled writes may be retried with identical evidence. No phase + frees capacity; a crash in claimed is permanently ambiguous, not retryable. + """ + if type(ledger) is not Ledger or type(binding) is not Binding: + raise BudgetError("Invalid ledger or binding") + for i, entry in enumerate(ledger.entries): + if entry.binding != binding: + continue + updated = Entry(binding, phase, actual_cost) + if updated == entry and phase in {"finished", "cancelled"}: + return ledger + allowed = {"reserved": {"claimed", "cancelled"}, "claimed": {"finished"}} + if phase not in allowed.get(entry.phase, set()): + raise BudgetError("Unsafe or duplicate execution transition") + return replace(ledger, revision=ledger.revision + 1, + entries=ledger.entries[:i] + (updated,) + ledger.entries[i + 1:]) + raise BudgetError("No exact reservation for this plan/run") + + +@dataclass(frozen=True) +class Snapshot: + """Verified history token and immutable state, never a cached/missing read.""" + + token: str + ledger: Ledger + + def __post_init__(self): + _text(self.token) + if type(self.ledger) is not Ledger: + raise BudgetError("Missing or corrupt existing ledger") + + +class WriteResult(Enum): + COMMITTED = "committed" + CONFLICT = "conflict" + UNKNOWN = "unknown" + + +class AtomicStore(Protocol): + """Durable storage contract; see skill_eval_budget_git for the GitHub adapter. + + read_verified MUST reject absent/corrupt/untrusted/reset history. A durable + independent lineage anchor must survive ledger deletion/rollback within the + documented backend trust boundary. Creation + MUST be separately audited and atomically create-if-never-created, not an + upsert. Genesis cannot be inferred from an HTTP 404 or empty history. + + compare_and_swap atomically appends exactly one successor only if the current + immutable history token equals expected.token; it validates key, genesis, + revision, monotonic entries/charges and allowed transitions. It returns + COMMITTED only for this invocation's durable successful write, never merely + because matching content is already present (especially for claim writes). + Unknown write outcome must not be reported as CONFLICT or COMMITTED. + """ + + def read_verified(self, key: PRKey) -> Snapshot: ... + + def compare_and_swap(self, expected: Snapshot, successor: Ledger) -> WriteResult: ... + + +def claim_once(store: AtomicStore, key: PRKey, binding: Binding) -> Ledger: + """Return only after this caller wins the durable one-shot execution claim. + + A returned ledger is NOT a transferable/replayable execution token. The + trusted caller must execute at most once in this process, enforce the frozen + plan's bounds, and never retry paid requests outside those bounds. A crash + after commit, including before return, burns the reservation. No takeover. + """ + snapshot = store.read_verified(key) + if type(snapshot) is not Snapshot or snapshot.ledger.key != key: + raise BudgetError("Storage returned the wrong PR ledger") + successor = advance(snapshot.ledger, binding, "claimed") + result = store.compare_and_swap(snapshot, successor) + if result is not WriteResult.COMMITTED: + raise BudgetError("Execution blocked: claim write conflicted or outcome unknown") + return successor + + +def validate_successor(before: Ledger, after: Ledger) -> None: + """Storage adapters must enforce this in addition to atomic history CAS.""" + if type(before) is not Ledger or type(after) is not Ledger: + raise BudgetError("Invalid transition state") + if (before.key != after.key or before.genesis != after.genesis + or after.revision != before.revision + 1): + raise BudgetError("Reset or identity change forbidden") + candidates = [] + if len(after.entries) == len(before.entries) + 1: + candidates.append(reserve(before, after.entries[-1].binding)) + elif len(after.entries) == len(before.entries): + for old, new in zip(before.entries, after.entries): + if old != new: + candidates.append(advance(before, new.binding, new.phase, + actual_cost=new.actual_cost)) + if after not in candidates: + raise BudgetError("Not a single monotonic budget transition") + + +def encode(ledger: Ledger) -> str: + if type(ledger) is not Ledger: + raise BudgetError("Invalid ledger") + return json.dumps({"schema_version": 1, **asdict(ledger)}, + sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def decode(payload: str) -> Ledger: + """Strict schema/number/duplicate validation, not history authentication.""" + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise BudgetError("Duplicate ledger JSON field") + result[key] = value + return result + + try: + value = json.loads(payload, object_pairs_hook=unique) + if (type(value) is not dict + or set(value) != {"schema_version", "key", "genesis", "revision", "entries"} + or type(value["schema_version"]) is not int or value["schema_version"] != 1 + or type(value["entries"]) is not list): + raise BudgetError("Unsupported ledger schema") + entries = [] + for entry in value["entries"]: + if type(entry) is not dict or set(entry) != {"binding", "phase", "actual_cost"}: + raise BudgetError("Invalid reservation schema") + entries.append(Entry(Binding(**entry["binding"]), entry["phase"], entry["actual_cost"])) + return Ledger(PRKey(**value["key"]), Genesis(**value["genesis"]), + value["revision"], tuple(entries)) + except (TypeError, KeyError, ValueError, OverflowError, RecursionError) as exc: + raise BudgetError("Missing, corrupt, or unsupported ledger") from exc diff --git a/tools/skill_eval_budget_git.py b/tools/skill_eval_budget_git.py new file mode 100644 index 00000000..2a638def --- /dev/null +++ b/tools/skill_eval_budget_git.py @@ -0,0 +1,179 @@ +"""Fail-closed GitHub Git-data ledger adapter; never initializes a ledger. + +The caller supplies the existing authenticated GitHub API client and trusted +root configuration, never values from a PR checkout/artifact. See the budget +contract for the administrator/rollback trust boundary. +""" + +import base64 +import binascii +import re +import uuid +from dataclasses import dataclass + +from tools.skill_eval_budget import ( + BudgetError, + Ledger, + PRKey, + Snapshot, + WriteResult, + decode, + encode, + validate_successor, +) + + +def _sha(value): + if type(value) is not str or not re.fullmatch(r"[0-9a-f]{40}", value): + raise BudgetError("Invalid Git object identity") + return value + + +@dataclass(frozen=True) +class LedgerAnchor: + """Reviewed root configuration, pinned independently of the mutable ref. + + Genesis is audited and contains only ledger.json. It may have no parent or + one audited existing-main parent; that ancestry is outside ledger history. + Checkpoint is an independently retained known tip (genesis at first setup). + Refreshing checkpoints strengthens rollback detection, never refunds spend. + """ + + key: PRKey + ref: str + genesis_commit: str + checkpoint_commit: str + + def __post_init__(self): + if type(self.key) is not PRKey: + raise BudgetError("Invalid immutable PR identity") + expected_ref = f"refs/heads/skill-eval-budget/{self.key.repository_id}/{self.key.pr_number}" + if self.ref != expected_ref: + raise BudgetError("Ledger must use its dedicated immutable PR ref") + _sha(self.genesis_commit) + _sha(self.checkpoint_commit) + + +class GitHubGitStore: + """Single-parent, unique-commit, non-force fast-forward CAS. + + A failed/uncertain PATCH is UNKNOWN, never an execution grant. Each commit + has a random operation nonce so simultaneous identical claims cannot share + a commit SHA and both receive successful idempotent ref-update responses. + No ref creation, deletion, force update, or settings endpoint is used. + """ + + def __init__(self, api, anchor: LedgerAnchor, *, max_history=10000): + if type(anchor) is not LedgerAnchor or type(max_history) is not int or max_history < 1: + raise BudgetError("Trusted anchor and bounded history are required") + self.api, self.anchor, self.max_history = api, anchor, max_history + self._observed = anchor.checkpoint_commit + + def _get(self, path): + try: + value = self.api.request("GET", path) + if type(value) is not dict: + raise ValueError("Not an object") + return value + except Exception: + raise BudgetError("Ledger read failed; uncertain history blocks work") from None + + def _tip(self): + value = self._get("/git/ref/" + self.anchor.ref.removeprefix("refs/")) + try: + if value["ref"] != self.anchor.ref or value["object"]["type"] != "commit": + raise ValueError("Invalid ref") + return _sha(value["object"]["sha"]) + except (KeyError, TypeError, ValueError): + raise BudgetError("Invalid ledger ref") from None + + def _state(self, commit_sha): + try: + commit = self._get("/git/commits/" + commit_sha) + if commit["sha"] != commit_sha or type(commit["parents"]) is not list: + raise ValueError("Invalid commit") + parents = [_sha(p["sha"]) for p in commit["parents"]] + tree_sha = _sha(commit["tree"]["sha"]) + tree = self._get("/git/trees/" + tree_sha) + if tree["sha"] != tree_sha or tree.get("truncated") is not False: + raise ValueError("Incomplete tree") + entries = tree["tree"] + if (type(entries) is not list or len(entries) != 1 + or entries[0]["path"] != "ledger.json" + or entries[0]["mode"] != "100644" or entries[0]["type"] != "blob"): + raise ValueError("Invalid ledger tree") + blob_sha = _sha(entries[0]["sha"]) + blob = self._get("/git/blobs/" + blob_sha) + if blob["sha"] != blob_sha or blob["encoding"] != "base64": + raise ValueError("Invalid ledger blob") + raw = base64.b64decode("".join(blob["content"].split()), validate=True) + if len(raw) > 4_000_000 or blob["size"] != len(raw): + raise ValueError("Invalid ledger size") + ledger = decode(raw.decode("utf-8")) + if ledger.key != self.anchor.key: + raise ValueError("Wrong PR identity") + return parents, ledger + except (KeyError, TypeError, ValueError, UnicodeError, binascii.Error): + raise BudgetError("Corrupt or incomplete ledger history") from None + + def read_verified(self, key: PRKey) -> Snapshot: + if key != self.anchor.key: + raise BudgetError("Wrong PR key for trusted anchor") + repository_id = self._get("").get("id") + if type(repository_id) is not int or repository_id != key.repository_id: + raise BudgetError("Repository identity changed") + tip = current = self._tip() + seen, states = set(), [] + for _ in range(self.max_history): + if current in seen: + raise BudgetError("Cyclic ledger history") + seen.add(current) + parents, ledger = self._state(current) + states.append(ledger) + if current == self.anchor.genesis_commit: + if len(parents) > 1 or ledger.revision != 0 or ledger.entries: + raise BudgetError("Invalid audited genesis") + break + if len(parents) != 1: + raise BudgetError("Ledger must be a complete single-parent history") + current = parents[0] + else: + raise BudgetError("Ledger history exceeds verification bound") + if self.anchor.checkpoint_commit not in seen or self._observed not in seen: + raise BudgetError("Ledger rollback or divergent history detected") + for before, after in zip(reversed(states[1:]), reversed(states[:-1])): + validate_successor(before, after) + # A racing writer is harmless, but do not label an already stale read current. + if self._tip() != tip: + raise BudgetError("Ledger changed during verification; retry a fresh read") + self._observed = tip + return Snapshot(tip, states[0]) + + def compare_and_swap(self, expected: Snapshot, successor: Ledger) -> WriteResult: + validate_successor(expected.ledger, successor) + verified = self.read_verified(self.anchor.key) + if verified != expected: + return WriteResult.CONFLICT + try: + tree = self.api.request("POST", "/git/trees", {"tree": [{ + "path": "ledger.json", "mode": "100644", "type": "blob", + "content": encode(successor), + }]}) + tree_sha = _sha(tree["sha"]) + commit = self.api.request("POST", "/git/commits", { + "message": "budget append " + uuid.uuid4().hex, + "tree": tree_sha, "parents": [expected.token], + }) + commit_sha = _sha(commit["sha"]) + updated = self.api.request( + "PATCH", "/git/refs/" + self.anchor.ref.removeprefix("refs/"), + {"sha": commit_sha, "force": False}) + if (updated["ref"] != self.anchor.ref or updated["object"]["type"] != "commit" + or updated["object"]["sha"] != commit_sha): + return WriteResult.UNKNOWN + except Exception: + # Includes rule/permission rejection and uncertain or lost responses. + # Never reconcile a claim by treating an observed commit as our grant. + return WriteResult.UNKNOWN + self._observed = commit_sha + return WriteResult.COMMITTED diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 751f1ca7..e7fd3df5 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -8,6 +8,7 @@ import sys import tempfile import time +from dataclasses import replace from pathlib import Path from inferrsi.harness.skill_evaluation import ( @@ -21,13 +22,14 @@ from inferrsi.knowledge_operations import KnowledgeReader, safe_path from inferrsi.model.anthropic_agent import SOURCE as AGENT_SOURCE from inferrsi.model.anthropic_agent import AnthropicAgentClient, validate_schema, validate_spec +from inferrsi.model.request_guard import SOURCE as GUARD_SOURCE from tools.skill_eval_pilot import PROMPT, SCHEMA, observed_tokens from tools.skill_eval_verifier import verify ROOT = Path(__file__).resolve().parents[1] ROLES = ("baseline", "candidate") GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") -CONTROLLERS = ("tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", +CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "src/inferrsi/harness/skill_evaluation.py", "src/inferrsi/infra/executors/local.py", "src/inferrsi/infra/processes.py", "src/inferrsi/infra/artifacts.py") @@ -482,14 +484,19 @@ def provider_client(plan): def execute(output: Path, authorization: str, *, client=None, campaign_deadline: float | None = None, - stop_on_unknown_usage: bool = True) -> dict: + stop_on_unknown_usage: bool = True, + require_request_authorization: bool = False) -> dict: if authorization != sha256(output / "plan/freeze.json"): raise ValueError("Execution requires authorization for this exact plan SHA-256") plan = load_plan(output) if any((output / "results").iterdir()): raise ValueError("Execution requires fresh results; prepare a new output directory") + if require_request_authorization and "client" not in plan: + raise ValueError("PR execution requires the guarded bounded provider") if "client" in plan: configured = provider_client(plan) + if require_request_authorization: + configured = replace(configured, require_authorization=True) configured.require_live_readiness(len(plan["schedule"])) if client is not None: raise ValueError("A paid provider plan cannot substitute an injected client") diff --git a/tools/skill_eval_pr.py b/tools/skill_eval_pr.py index d331ee5b..9c2562a2 100644 --- a/tools/skill_eval_pr.py +++ b/tools/skill_eval_pr.py @@ -136,7 +136,8 @@ def prepare(repo, mapping_path, identity, output, *, provider=None): "retention_days": RETENTION_DAYS} if provider is not None: plan["provider_config"] = provider - plan["reservation"] = provider_reservation(plan, now=0) + if plan["estimated_calls"]: + plan["reservation"] = provider_reservation(plan, now=0) write_json(output / "pr-plan.json", plan) return plan @@ -160,7 +161,7 @@ def budget(plan): value = {"estimated_calls": plan["estimated_calls"], "limits_per_comparison": plan["limits"], "comparisons": sum("plan_sha256" in c for c in plan["comparisons"])} if "provider_config" in plan: - value.update(reservation=plan["reservation"], + value.update(reservation=plan.get("reservation"), provider_config_sha256=fingerprint(plan["provider_config"])) return value @@ -225,7 +226,8 @@ def execute(output, approval, *, client=None): # Artifact downloads omit empty directories required by the executor. (output / item["skill"] / "results").mkdir(exist_ok=True) result = compare.execute(output / item["skill"], item["plan_sha256"], client=client, - stop_on_unknown_usage="provider_config" in plan) + stop_on_unknown_usage="provider_config" in plan, + require_request_authorization=client is None) if "provider_config" in plan and result["status"]["execution_state"] != "completed": break @@ -387,12 +389,16 @@ def review_guidance(plan, results, *, evidence_state): "evidence_state": evidence_state, "plan_sha256": fingerprint(plan)} -def summary(plan, results, evidence_url, expires_at, now, *, evidence_state=None): +def summary(plan, results, evidence_url, expires_at, now, *, evidence_state=None, + execution_control=None): expired = now >= expires_at evidence_state = ("expired" if expired else evidence_state or ("available" if results else "planned / unexecuted")) guidance = review_guidance(plan, results, evidence_state=evidence_state) lines = [MARKER, "## Skill evaluation"] + if execution_control is not None: + lines += ["", "Automatic execution / cumulative PR budget: `" + + canonical(execution_control) + "`", ""] lines += ["### Advisory: " + guidance["disposition"], "Retained evidence availability: **" + evidence_state + "**. " "No approval or merge authorization is issued."] @@ -425,8 +431,8 @@ def summary(plan, results, evidence_url, expires_at, now, *, evidence_state=None f"**{plan['estimated_calls']}**.", f"Uncovered/unsupported changes: **{len(gaps)}**. Gaps never count as passes.", "", "Applicable only while these revisions and inputs match the current PR.", - "Execution requires exact plan, versions and budget approval; " - "not a hard provider token or monetary cap. Cost remains unknown.", + "Paid execution requires trusted label authorization and cumulative PR reservation. " + "Unknown accounting blocks new paid calls; planning is not execution.", "Pipeline evidence only; offline fixed outputs do not establish skill efficacy.", f"[Evidence and logs]({evidence_url}) · Retention: {RETENTION_DAYS} days.", f"Outcomes are historical; verifiable only before Unix time `{int(expires_at)}` " @@ -697,8 +703,11 @@ def artifact_zip(api, run_id, name, now): def publish_retained(api, output, source_run, writer_run, now): plan = compare.read_json(output / "pr-plan.json") run = api.request("GET", f"/actions/runs/{source_run}") - if (run["path"] != WORKFLOW or run["event"] != "workflow_dispatch" - or run["display_title"] != run_title(plan["pr"], fingerprint(plan)) + trusted_source = (run["event"] == "workflow_dispatch" + and run["display_title"] == run_title(plan["pr"], fingerprint(plan))) + trusted_source |= (run["event"] == "pull_request_target" + and run["display_title"] == f"skill-eval plan or publish PR {plan['pr']}") + if (run["path"] != WORKFLOW or not trusted_source or run["head_sha"] != plan["version"] or run["status"] != "completed" and source_run != writer_run): raise ValueError("Untrusted or mismatched execution workflow") @@ -716,13 +725,16 @@ def publish_retained(api, output, source_run, writer_run, now): # Never expose artifact-controlled exception text or reuse old counts. evidence_state = "invalid" url = f"https://github.com/{plan['repository']}/actions/runs/{source_run}" + from tools.skill_eval_auto import report_control return api.publish(plan, summary(plan, results, url, expires, now, - evidence_state=evidence_state), writer_run) + evidence_state=evidence_state, + execution_control=report_control(api, plan)), writer_run) def main(): parser = argparse.ArgumentParser(description="Revision-bound skill PR integration") - parser.add_argument("mode", choices=("prepare", "authorize", "execute", "pack", "publish")) + parser.add_argument("mode", choices=("prepare", "authorize", "execute", "pack", + "publish", "control")) parser.add_argument("--output", required=True, type=Path) args = parser.parse_args() output = args.output.resolve() @@ -733,6 +745,14 @@ def main(): pr = int(inputs.get("pr") or event["pull_request"]["number"]) repository = os.environ["GITHUB_REPOSITORY"] run_id = int(os.environ["GITHUB_RUN_ID"]) + if inputs.get("action") == "execute": + raise ValueError("Legacy paid dispatch is blocked; use the audited label controller") + if args.mode == "control": + from tools.skill_eval_auto import controlled_execute + api = GitHub(repository, os.environ["GITHUB_TOKEN"]) + controlled_execute(api, output, run_id=run_id, + attempt=int(os.environ["GITHUB_RUN_ATTEMPT"])) + return if args.mode == "execute": execute(output, compare.read_json(output / "approval.json")) return @@ -746,11 +766,28 @@ def main(): "https://github.com/" + repository + ".git", current["base"], f"refs/pull/{pr}/head"], cwd=ROOT, check=True) version = git(ROOT, "rev-parse", "HEAD").decode().strip() + from tools.skill_eval_auto import CONFIG, configuration, live_label + config = configuration(now=0) if CONFIG.exists() else None + provider = config["provider"] if config else (json.loads(inputs["provider_config"]) + if inputs.get("provider_config") else None) plan = prepare(ROOT, MAPPING, {"repository": repository, "pr": pr, "version": version, **current}, output, - provider=json.loads(inputs["provider_config"]) - if inputs.get("provider_config") else None) + provider=provider) + candidate, reason = False, "Trusted provider configuration and audited PR genesis required" + if config: + try: + from tools.skill_eval_auto import eligible, ledger_store + configuration() + eligible(plan, config) + live_label(api, plan, config) + store, key = ledger_store(api, config, pr) + store.read_verified(key) + candidate, reason = True, "Eligible; durable reservation required before execution" + except (ValueError, KeyError, TypeError): + reason = "Blocked: label/operator, suitability, provider or audited ledger" + write_json(output / "control-status.json", {"eligible": candidate, "reason": reason}) with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as handle: + handle.write(f"auto_candidate={str(candidate).lower()}\n") handle.write(f"pr={pr}\nplan_sha256={fingerprint(plan)}\n") handle.write("client_kind=" + (KIND if "provider_config" in plan else "codex") + "\n") return @@ -766,15 +803,18 @@ def main(): claim(plan, runs, run_id, int(os.environ["GITHUB_RUN_ATTEMPT"])) write_json(output / "approval.json", approval) return - if inputs.get("action") in ("execute", "publish"): - if inputs["action"] == "publish" and not inputs.get("source_run"): + if (inputs.get("action") in ("execute", "publish") + or os.environ.get("SKILL_EVAL_AUTO_SOURCE") == "true"): + if inputs.get("action") == "publish" and not inputs.get("source_run"): raise ValueError("Publish-only retry requires the original execution run ID") receipt = publish_retained(api, output, int(inputs.get("source_run") or run_id), run_id, time.time()) else: now = time.time() url = f"https://github.com/{repository}/actions/runs/{run_id}" - body = summary(plan, [], url, now + RETENTION_DAYS * 86400, now) + from tools.skill_eval_auto import report_control + body = summary(plan, [], url, now + RETENTION_DAYS * 86400, now, + execution_control=report_control(api, plan)) receipt = api.publish(plan, body, run_id) print(receipt["html_url"]) From 0c8103de1dc065eb8dcb6a3c6f9826bbe1558e76 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:29:26 +0900 Subject: [PATCH 2/2] fix: bind automatic evidence to its executed plan and pin bounded rollout Reject stale source plans before importing results; retain audited legacy charges. Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> Agent-assisted-by: OpenAI Codex --- docs/skill-eval-deployment-215.md | 78 ++++++++++++++++++++ docs/skill-eval-pr.md | 19 +++-- tests/fixtures/skill_eval_pr_support.py | 13 +++- tests/test_skill_eval_deployment.py | 97 +++++++++++++++++++++++++ tests/test_skill_eval_pr.py | 96 ++++++++++++++++++++++++ tools/skill_eval_auto_config.json | 63 ++++++++++++++++ tools/skill_eval_pr.py | 59 ++++++++++++++- 7 files changed, 414 insertions(+), 11 deletions(-) create mode 100644 docs/skill-eval-deployment-215.md create mode 100644 tests/test_skill_eval_deployment.py create mode 100644 tools/skill_eval_auto_config.json diff --git a/docs/skill-eval-deployment-215.md b/docs/skill-eval-deployment-215.md new file mode 100644 index 00000000..fa11a538 --- /dev/null +++ b/docs/skill-eval-deployment-215.md @@ -0,0 +1,78 @@ +# Issue 215: bounded PR159/160 deployment record + +This is an owner-authorized, issue-scoped deployment configuration, not a portable +provider default or a claim that live execution passed. Governing issue: +[#215](https://github.com/InferMatrixLab/InferRSI/issues/215). +Only the two audited PR anchors in `tools/skill_eval_auto_config.json` are enabled; +other PRs have no usable ledger anchor. The sole authorized label operator is the +public repository owner `Dong1017`, using `skill-eval:execute`. Current repository +write/maintain/admin permission, open non-draft PR state, current main/head, +matching plan/configuration and a durable claim remain mandatory runtime checks. + +## Audit and price provenance + +[Reviewed evidence](https://github.com/InferMatrixLab/InferRSI/issues/215#issuecomment-6467531341) +contains the retained-workflow historical-spend reconciliation and the fresh +public price observation. Its machine-readable companion SHA256 is +`0cf72dd857d642ab9f6e810c100bb8d661cbb4c28568455dc49eca6c58cea501`. +The audited carry-forward is conservatively rounded upward: PR159 **$0.68**; +PR160 **$63.53**, including the entire failed-run reservation. It is not an exact +invoice or a universal claim about deleted/outside executions. Each genesis has +only `ledger.json`, revision zero and no entries; genesis/checkpoint identities +are pinned in the deployment file. Never initialize a replacement or reset spend. + +Observed 2026-10-28 10:53:48 UTC at [public pricing](https://yibuapi.com/pricing), +with [public API](https://yibuapi.com/api/pricing) pricing version +`a42d372ccf0b5dd13ecf71203521f9d2`: exact `claude-opus-5-5`, default group, +USD per million input **8**, output **40**, cache-read **0.4**, cache-write **10**. +The account's default group is separately owner-confirmed, not inferred from the +logged-out public page. `request_fee=0` is the reviewed token-tariff inference +(`quota_type=0`, `model_price=0`), not a contractual no-surcharge guarantee. + +Both price and authorization expire **2026-10-29 10:53:48 UTC** (1793271228), +24 hours after observation. This is an explicit conservative **operational +freshness deadline**, not a vendor quote-validity promise. Expiry blocks new +requests; refresh requires new reviewed evidence and configuration, not silently +extending the timestamp. No provider/model/Codex fallback is authorized. + +## Hard cost bound + +The existing Messages client uses its known 1,000,000-token context ceiling; +reservation charges every input token at the largest input/cache rate ($10/M). +Six requests per trial, 4,096 output tokens per request and a complete two-trial +pair give: + +- Raw request bound: `(1,000,000 × 10 + 4,096 × 40) / 1,000,000 = $10.16384`. +- Upward cent rounding: **$10.17/request**, **12 requests**, **$122.04/pair**. +- Cumulative $200/PR leaves **$199.32 / $136.47** before this round, and + **$77.28 / $14.43** after reserving one pair each. +- Only these two PRs can reserve. Their combined remaining capacity is + **$335.79**, strictly below this round's $400 total ceiling; no separate global + ledger or additional authorized PR is implied. One pair each reserves $244.08. +- Reservations never refund on success, cancellation, uncertainty or head change. + A second full pair on either PR is rejected even with a different plan/run. + +Request bytes (131072), read bytes (32768), tool calls (12), and the soft +`token_stop_new_trials=200000` stop match successful PR160 +[run 40499987048](https://github.com/InferMatrixLab/InferRSI/actions/runs/40499987048), +plan artifact 12731459808, plan SHA256 +`fc8b6978e2f176e5c1999fcd0fab34d649096ebf1226d87196d4677c36bba0ab`. +The soft stop is not a spending guarantee; the independent hard bound above is. +The historical expired price record is not reused. + +The deployment retains the existing `SKILL_EVAL_YIBUAPI_KEY` runtime injection; +no secret value is read or stored here. Tests use offline mocks, not provider +requests. Config presence changes real plan preparation, so synthetic-repository +CLI tests explicitly isolate their own absent configuration rather than bypassing +production guards. + +## Activation prerequisites and stopping + +Deployment is not complete merely because this file exists. The trusted main +controller, all blocking review fixes, authenticated Git-data non-force CAS, +audited anchor readback and exact-head checks must pass before a label is added +or any paid request is launched. This local preparation performs none of those +external mutations, does not dispatch a run, and does not claim token/storage +permissions have been live-tested. Keep existing protection/settings unchanged. +Remove the label to revoke authorization before the next request; never delete +or roll back a ledger to recover capacity. Retain all ambiguous reservations. diff --git a/docs/skill-eval-pr.md b/docs/skill-eval-pr.md index 18180d5e..f6938949 100644 --- a/docs/skill-eval-pr.md +++ b/docs/skill-eval-pr.md @@ -87,10 +87,11 @@ uses the existing frozen provider plans and enforces their complete worst-case reservation against USD 200 cumulatively per PR. The non-provider default mapping remains useful for offline planning, but cannot execute as a paid PR fallback. -No provider credential, label or deployment configuration is created by this -change. A historical successful run does not establish current credentials, -permission, provider readiness or remaining budget. See the deployment checklist -below before enabling the switch. +The bounded [issue 215 deployment record](skill-eval-deployment-215.md) supplies +the reviewed configuration for PR159/160 only. It creates no provider credential +or execution label. A historical successful run does not establish current +credentials, permission, provider readiness or remaining budget. See the +deployment checklist below before enabling the switch. The controller alone has owner-approved `contents: write` for the dedicated Git budget ledger. Its existing model executor runs in a sanitized child process with @@ -137,7 +138,15 @@ and updates the same marked bot comment instead of creating another summary. The publisher reconstructs trusted inputs from Git and the maintained mapping. It requires the source run's workflow path, event, trusted revision and exact -plan identity. It downloads only that run's uniquely named result artifact. +plan identity. Automatic runs have only a PR-number title, so before downloading +results the publisher reads that source run's unique `skill-eval-plan` artifact +through the authenticated Actions API and requires its original `pr-plan.json` +fingerprint to equal the complete freshly reconstructed plan. This binds both +initial publication and publish-only retries to the actually planned PR head, +even when main and skill bytes stayed unchanged. Missing, expired, ambiguous or +invalid source plans cannot substantiate results; artifact files are never +executed. Dispatch sources retain their exact plan-hash title requirement. +It then downloads only that run's uniquely named result artifact. ZIP members must be bounded, unique, regular files at explicitly scheduled result paths; traversal, symlinks, executable scripts and replacement plans or Markdown are rejected before extraction. GitHub authorization is stripped on diff --git a/tests/fixtures/skill_eval_pr_support.py b/tests/fixtures/skill_eval_pr_support.py index 0578973d..dcab140e 100644 --- a/tests/fixtures/skill_eval_pr_support.py +++ b/tests/fixtures/skill_eval_pr_support.py @@ -65,19 +65,28 @@ def __init__(self, plan, data=b"", run_id=10): self.artifact = {"id": 90, "name": "skill-eval-results", "expired": False, "expires_at": "2030-01-01T00:00:00Z", "size_in_bytes": len(data)} self.data = envelope(data) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("pr-plan.json", integration.canonical(plan)) + self.plan_data = buffer.getvalue() + self.plan_artifact = {**self.artifact, "id": 91, "name": "skill-eval-plan", + "size_in_bytes": len(self.plan_data)} + self.reads = [] def request(self, method, path, data=None, *, binary=False): if method == "GET": + self.reads.append(path) if path.startswith("/pulls/"): return {"state": "open", **{key: {"sha": value} for key, value in self.current.items()}} if path.startswith("/actions/runs/") and "/artifacts" not in path: return self.run if path.startswith("/actions/runs/") and "/artifacts" in path: - return {"artifacts": [self.artifact]} + return {"artifacts": [self.artifact, *([self.plan_artifact] + if self.plan_artifact is not None else [])]} if path.startswith("/actions/artifacts/"): assert binary - return self.data + return self.plan_data if path == "/actions/artifacts/91/zip" else self.data if path.startswith("/issues/") and "/comments?" in path: if self.change_during_read: self.current["head"] = "f" * 40 diff --git a/tests/test_skill_eval_deployment.py b/tests/test_skill_eval_deployment.py new file mode 100644 index 00000000..0e080c38 --- /dev/null +++ b/tests/test_skill_eval_deployment.py @@ -0,0 +1,97 @@ +"""Offline regression checks for the issue-scoped reviewed deployment configuration.""" + +import json +from decimal import Decimal +from unittest.mock import Mock + +import pytest + +from inferrsi.model import anthropic_agent as agent +from tools import skill_eval_auto as auto +from tools import skill_eval_pr as pr +from tools.skill_eval_budget import Binding, BudgetError, Genesis, Ledger, reserve + +# Freeze observation time: an expired deployment must not make CI depend on wall time. +OBSERVED = 1793184828 + + +@pytest.fixture +def deployed(): + return auto.configuration(now=OBSERVED) + + +def test_deployment_identity_and_expiry_are_closed(deployed): + assert deployed["repository_id"] == 1409534281 + assert deployed["operators"] == ["Dong1017"] + assert deployed["label"] == "skill-eval:execute" + expected = {"159": "727d31e55f496c53fbf02cf4f624008487de27cb", + "160": "1fdbd8adc35e2df7439c3cd74780f869d2d6e668"} + assert set(deployed["anchors"]) == set(expected) + for number, sha in expected.items(): + assert deployed["anchors"][number] == { + "ref": f"refs/heads/skill-eval-budget/1409534281/{number}", + "genesis_commit": sha, "checkpoint_commit": sha} + assert deployed["expires_at"] == OBSERVED + 86400 + assert deployed["provider"]["pricing"]["expires_at"] == deployed["expires_at"] + assert deployed["provider_sha256"] == pr.fingerprint(deployed["provider"]) + with pytest.raises(ValueError, match="expired"): + auto.configuration(now=deployed["expires_at"]) + with pytest.raises(BudgetError, match="genesis"): + auto.ledger_store(Mock(), deployed, 161) + + +def test_deployment_uses_existing_client_and_no_secret_config(deployed): + provider = deployed["provider"] + agent.validate_spec(provider["model"], provider["client"]) + assert provider["model"] == {"name": "claude-opus-5-5", "effort": "not_applicable"} + assert provider["client"]["endpoint"] == agent.ENDPOINT + assert provider["client"]["max_requests"] == 6 + assert provider["client"]["max_output_tokens"] == 4096 + assert provider["client"]["max_request_bytes"] == 131072 + assert provider["client"]["max_read_bytes"] == 32768 + assert provider["client"]["max_tool_calls"] == 12 + assert provider["limits"] == {"token_stop_new_trials": 200000} + assert provider["pricing"]["group"] == "default" + assert provider["pricing"]["max_input_tokens"] == agent.MODEL_CONTEXT["claude-opus-5-5"] + assert provider["pricing"]["rates"] == { + "input": "8", "output": "40", "cache_read": "0.4", "cache_write": "10"} + assert provider["pricing"]["request_fee"] == "0" + assert "key" not in json.dumps(deployed).lower() + + +def test_pair_reservation_fits_both_audited_residuals(deployed): + value = pr.provider_reservation({"provider_config": deployed["provider"], + "estimated_calls": 2, + "comparisons": [{"plan_sha256": "a" * 64, "calls": 2}]}, + now=OBSERVED) + assert value["per_request"] == "10.17" # ceil cents of 10 + 4096 * 40 / 1M + assert value["requests"] == 12 and value["amount"] == "122.04" + total_remaining = 0 + for number, legacy, remaining in [(159, 680000, 199320000), (160, 63530000, 136470000)]: + store, key = auto.ledger_store(Mock(), deployed, number) + assert store.anchor.genesis_commit == store.anchor.checkpoint_commit + ledger = Ledger(key, Genesis(deployed["review_evidence"], legacy)) + assert ledger.remaining == remaining + total_remaining += remaining + binding = Binding("a" * 64, "offline.1", int(Decimal(value["amount"]) * 1000000)) + charged = reserve(ledger, binding) + assert charged.remaining == remaining - 122040000 + # Even a new plan cannot reset the cumulative PR allowance. + with pytest.raises(BudgetError, match="exhausted"): + reserve(charged, Binding("b" * 64, "offline.2", binding.worst_case)) + assert total_remaining == 335790000 < 400000000 + + +@pytest.mark.parametrize("mode", ["addition", "update"]) +def test_deployment_provider_prepares_a_complete_pair_without_execution(tmp_path, deployed, mode): + from tests.fixtures.skill_eval_pr_support import setup + + repo, mapping, identity = setup(tmp_path, mode, ids=("one",)) + plan = pr.prepare(repo, mapping, identity, tmp_path / "planned", + provider=deployed["provider"]) + assert plan["estimated_calls"] == 2 + assert plan["reservation"]["amount"] == "122.04" + assert plan["limits"]["token_stop_new_trials"] == 200000 + assert plan["provider_config"] == deployed["provider"] + assert plan["comparisons"][0]["mode"] == mode + assert not list((tmp_path / "planned/selected/results").iterdir()) diff --git a/tests/test_skill_eval_pr.py b/tests/test_skill_eval_pr.py index c32d7726..d76bce4f 100644 --- a/tests/test_skill_eval_pr.py +++ b/tests/test_skill_eval_pr.py @@ -514,7 +514,12 @@ def dispatch_event(monkeypatch, root, output, inputs, mode, run_id=11): def test_plan_dispatch_prepare_publish_has_no_execution_side_effects( tmp_path, monkeypatch, mode, extra_inputs, ): + from tools import skill_eval_auto as auto + repo, mapping, identity = setup(tmp_path, mode, ids=("one",)) + # This synthetic repository has no deployed controller configuration. + # Do not let the real checkout's opt-in configuration change its plan. + monkeypatch.setattr(auto, "CONFIG", repo / "tools/skill_eval_auto_config.json") plan = integration.prepare(repo, mapping, identity, tmp_path / "expected") api = SimulatedGitHub(plan) request = Mock(wraps=api.request) @@ -666,3 +671,94 @@ def test_plan_dispatch_workflow_keeps_execution_guards(): assert not any("authorize --output" in step.get("run", "") for step in workflow["jobs"]["plan"]["steps"]) assert workflow["run-name"].startswith("${{ " + guard) + + +@pytest.mark.parametrize("prior_publication", [False, True]) +def test_automatic_source_plan_rejects_new_head_with_same_trusted_main( + tmp_path, prior_publication): + repo, mapping, identity = setup(tmp_path, ids=("one",)) + identity["version"] = identity["base"] # Current trusted main stays fixed across the PR push. + execution = tmp_path / "execution" + old = integration.prepare(repo, mapping, identity, execution) + client = FixedOutputClient(tmp_path) + integration.execute(execution, approval(old), client=client) + api = SimulatedGitHub(old, integration.result_archive(execution).read_bytes()) + api.run.update(event="pull_request_target", display_title="skill-eval plan or publish PR 7") + if prior_publication: + integration.publish_retained(api, execution, 10, 11, 1) + assert '"unchanged_correct":1' in api.comments[0]["body"] + # PR push after execution (or after the first publish), with unchanged main. + # Even unchanged skill bytes do not establish that this new revision was tested. + (repo / "unrelated.txt").write_text("new PR revision") + git(repo, "add", "unrelated.txt") + git(repo, "commit", "--quiet", "-m", "Offline PR push after execution") + current = {**identity, "head": git(repo, "rev-parse", "HEAD")} + publisher = tmp_path / "publisher" + new = integration.prepare(repo, mapping, current, publisher) + assert old["version"] == new["version"] and old["head"] != new["head"] + api.current["head"] = new["head"] + api.reads.clear() + calls = len(client.calls) + integration.publish_retained(api, publisher, 10, 12, 1) + body = api.comments[0]["body"] + assert "artifacts are invalid" in body and "obtain missing evidence" in body + assert '"unchanged_correct":1' not in body + assert "/actions/artifacts/90/zip" not in api.reads + assert not list((publisher / "selected/results").iterdir()) + assert len(client.calls) == calls + + +@pytest.mark.parametrize("damage", ["missing", "expired", "malformed", "duplicate", + "symlink", "oversized", "ambiguous", "rerun_mismatch", + "unsafe_path", "member_oversize", "plan_only"]) +def test_automatic_missing_or_invalid_source_plan_never_imports_results(tmp_path, damage): + repo, mapping, identity = setup(tmp_path, ids=("one",)) + output = tmp_path / "execution" + plan = integration.prepare(repo, mapping, identity, output) + api = SimulatedGitHub(plan) + api.run.update(event="pull_request_target", display_title="skill-eval plan or publish PR 7") + if damage == "missing": + api.plan_artifact = None + elif damage == "expired": + api.plan_artifact["expired"] = True + elif damage == "oversized": + api.plan_artifact["size_in_bytes"] = integration.MAX_ARCHIVE + 1 + elif damage == "plan_only": + original_pages = api.pages + api.pages = lambda path, *a, **kw: ([api.plan_artifact] if "/artifacts" in path + else original_pages(path, *a, **kw)) + elif damage == "ambiguous": + original_pages = api.pages + api.pages = lambda path, *a, **kw: (original_pages(path, *a, **kw) + + ([api.plan_artifact] if "/artifacts" in path else [])) + else: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + entry = zipfile.ZipInfo("pr-plan.json") + if damage == "symlink": + entry.external_attr = (stat.S_IFLNK | 0o777) << 16 + value = {**plan, "head": "f" * 40} if damage == "rerun_mismatch" else plan + archive.writestr(entry, "not JSON" if damage == "malformed" + else integration.canonical(value)) + if damage == "unsafe_path": + archive.writestr("../untrusted", "must not extract") + if damage == "member_oversize": + archive.writestr("oversized", b"x" * (integration.MAX_ARCHIVE + 1)) + if damage == "duplicate": + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr("pr-plan.json", integration.canonical(plan)) + api.plan_data = buffer.getvalue() + if damage == "rerun_mismatch": + api.run["run_attempt"] = 2 + integration.publish_retained(api, output, 10, 11, 1) + assert "/actions/artifacts/90/zip" not in api.reads + assert '"unchanged_correct":1' not in api.comments[0]["body"] + assert "obtain missing evidence" in api.comments[0]["body"] + + +def test_source_plan_artifact_is_immutable_across_workflow_reruns(): + workflow = yaml.safe_load((integration.ROOT / integration.WORKFLOW).read_text()) + uploads = [step["with"] for step in workflow["jobs"]["plan"]["steps"] + if step.get("uses") == "actions/upload-artifact@v4"] + assert len(uploads) == 1 and uploads[0]["name"] == "skill-eval-plan" + assert uploads[0].get("overwrite", False) is False diff --git a/tools/skill_eval_auto_config.json b/tools/skill_eval_auto_config.json new file mode 100644 index 00000000..451640ed --- /dev/null +++ b/tools/skill_eval_auto_config.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "repository_id": 1409534281, + "label": "skill-eval:execute", + "operators": [ + "Dong1017" + ], + "expires_at": 1793271228, + "provider": { + "model": { + "name": "claude-opus-5-5", + "effort": "not_applicable" + }, + "client": { + "kind": "anthropic-messages-agent-v1", + "provider": "yibuapi", + "endpoint": "https://yibuapi.com/v1/messages", + "api_version": "2023-06-01", + "max_requests": 6, + "max_output_tokens": 4096, + "max_request_bytes": 131072, + "max_read_bytes": 32768, + "max_tool_calls": 12 + }, + "pricing": { + "currency": "USD", + "unit": "per_million_tokens", + "model": "claude-opus-5-5", + "group": "default", + "rates": { + "input": "8", + "output": "40", + "cache_read": "0.4", + "cache_write": "10" + }, + "request_fee": "0", + "max_input_tokens": 1000000, + "rate_evidence": "https://github.com/InferMatrixLab/InferRSI/issues/215#issuecomment-6467531341", + "expires_at": 1793271228 + }, + "spend_limit": { + "currency": "USD", + "amount": "200" + }, + "limits": { + "token_stop_new_trials": 200000 + } + }, + "provider_sha256": "253cf88a7ff04311fd1dae897cb68a8c986f136e837dd00dbe5f8872bd094b5f", + "anchors": { + "159": { + "ref": "refs/heads/skill-eval-budget/1409534281/159", + "genesis_commit": "727d31e55f496c53fbf02cf4f624008487de27cb", + "checkpoint_commit": "727d31e55f496c53fbf02cf4f624008487de27cb" + }, + "160": { + "ref": "refs/heads/skill-eval-budget/1409534281/160", + "genesis_commit": "1fdbd8adc35e2df7439c3cd74780f869d2d6e668", + "checkpoint_commit": "1fdbd8adc35e2df7439c3cd74780f869d2d6e668" + } + }, + "review_evidence": "https://github.com/InferMatrixLab/InferRSI/issues/215#issuecomment-6467531341" +} diff --git a/tools/skill_eval_pr.py b/tools/skill_eval_pr.py index 9c2562a2..2bbb41d7 100644 --- a/tools/skill_eval_pr.py +++ b/tools/skill_eval_pr.py @@ -675,7 +675,7 @@ class MissingEvidence(ValueError): """A known absent artifact, not an API access or integrity failure.""" -def artifact_zip(api, run_id, name, now): +def artifact_download(api, run_id, name, now): artifacts = api.pages(f"/actions/runs/{run_id}/artifacts", "artifacts") selected = [a for a in artifacts if a["name"] == name] if not selected: @@ -691,6 +691,15 @@ def artifact_zip(api, run_id, name, now): if artifact["expired"] or now >= expires: return None, min(expires, now) raw = api.request("GET", f"/actions/artifacts/{artifact['id']}/zip", binary=True) + if len(raw) > MAX_ARCHIVE: + raise ValueError("Oversized artifact download") + return raw, expires + + +def artifact_zip(api, run_id, name, now): + raw, expires = artifact_download(api, run_id, name, now) + if raw is None: + return None, expires with zipfile.ZipFile(io.BytesIO(raw)) as zipped: if zipped.namelist() != ["results.zip"]: raise ValueError("Unexpected artifact envelope") @@ -700,6 +709,41 @@ def artifact_zip(api, run_id, name, now): return zipped.read(member), expires +def source_plan_expiry(api, run_id, plan, now): + """Bind automatic evidence to the trusted planner artifact from this source run. + + The run's authenticated workflow identity is checked by the caller. Neither a + result's self-reported head nor the PR-only automatic run title is provenance. + Read data only; never extract or execute any file from the planning archive. + """ + raw, expires = artifact_download(api, run_id, "skill-eval-plan", now) + if raw is None: + return expires + with zipfile.ZipFile(io.BytesIO(raw)) as zipped: + members = zipped.infolist() + names = [m.orig_filename for m in members] + if (len(members) > 10000 or len(names) != len(set(names)) + or sum(m.file_size for m in members) > MAX_ARCHIVE + or names.count("pr-plan.json") != 1): + raise ValueError("Invalid source planning archive") + for item, name in zip(members, names): + path = PurePosixPath(name) + mode = item.external_attr >> 16 + if (path.is_absolute() or ".." in path.parts or "\\" in name + or stat.S_ISLNK(mode) + or stat.S_IFMT(mode) not in (0, stat.S_IFREG, stat.S_IFDIR)): + raise ValueError("Unsafe source planning archive member") + member = members[names.index("pr-plan.json")] + mode = member.external_attr >> 16 + if (member.file_size > 4_000_000 or stat.S_ISLNK(mode) + or stat.S_IFMT(mode) not in (0, stat.S_IFREG)): + raise ValueError("Invalid source plan member") + original = json.loads(zipped.read(member)) + if fingerprint(original) != fingerprint(plan): + raise ValueError("Source execution plan does not match current PR plan") + return expires + + def publish_retained(api, output, source_run, writer_run, now): plan = compare.read_json(output / "pr-plan.json") run = api.request("GET", f"/actions/runs/{source_run}") @@ -707,15 +751,22 @@ def publish_retained(api, output, source_run, writer_run, now): and run["display_title"] == run_title(plan["pr"], fingerprint(plan))) trusted_source |= (run["event"] == "pull_request_target" and run["display_title"] == f"skill-eval plan or publish PR {plan['pr']}") - if (run["path"] != WORKFLOW or not trusted_source + if (run["id"] != source_run or run["path"] != WORKFLOW or not trusted_source or run["head_sha"] != plan["version"] or run["status"] != "completed" and source_run != writer_run): raise ValueError("Untrusted or mismatched execution workflow") evidence_state = "available" + data, expires = None, now + RETENTION_DAYS * 86400 try: - data, expires = artifact_zip(api, source_run, "skill-eval-results", now) + if run["event"] == "pull_request_target": + expires = source_plan_expiry(api, source_run, plan, now) + if now < expires: + data, result_expiry = artifact_zip(api, source_run, "skill-eval-results", now) + expires = min(expires, result_expiry) except MissingEvidence: - data, expires, evidence_state = None, now + RETENTION_DAYS * 86400, "missing" + evidence_state = "missing" + except (ValueError, KeyError, TypeError, zipfile.BadZipFile): + evidence_state = "invalid" results = [] if data is not None: try: