feat: plan and report skill PR evaluations with explicit authorization

.github/workflows/skill-eval-pr.yml

@@ -0,0 +1,164 @@
+name: Skill PR evaluation
+run-name: >-
+  ${{ github.event_name == 'workflow_dispatch' && inputs.action == 'execute'
+      && format('skill-eval execute PR {0} plan {1}', inputs.pr, inputs.plan_sha256)
+      || format('skill-eval plan or publish PR {0}', inputs.pr || github.event.pull_request.number) }}
+
+on:
+  pull_request_target:
+    branches: [main]
+    types: [opened, synchronize, reopened]
+    paths:
+      - src/inferrsi/data/skill_library/**
+      - tools/skill_eval_*
+      - docs/examples/skill-comparison/**
+      - .github/workflows/skill-eval-pr.yml
+  workflow_dispatch:
+    inputs:
+      action:
+        type: choice
+        options: [execute, publish]
+        required: true
+        description: Execute consumes authorization once; publish reuses retained results
+      pr:
+        required: true
+        description: PR number
+      plan_sha256:
+        required: true
+        description: Exact PR plan SHA-256 shown by automatic planning
+      version:
+        required: true
+        description: Exact trusted implementation SHA shown in the plan
+      base:
+        required: true
+        description: Exact compared base SHA
+      head:
+        required: true
+        description: Exact compared head SHA
+      budget:
+        required: true
+        description: Explicit approval of the complete Budget JSON shown in the plan
+      source_run:
+        description: Execution workflow run ID, required for publish-only retries
+
+permissions: {}
+
+# Every writer and execution for a PR is serialized; never cancel paid work.
+concurrency:
+  group: skill-eval-pr-${{ inputs.pr || github.event.pull_request.number }}
+  cancel-in-progress: false
+
+env:
+  PYTHONPATH: src
+
+jobs:
+  plan:
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      pull-requests: read
+      actions: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install '.[skills]'
+      - name: Prepare from trusted code and raw PR Git objects
+        id: prepare
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python -m tools.skill_eval_pr prepare --output prepared
+      - name: Validate explicit operator authorization and reserve execution
+        if: github.event_name == 'workflow_dispatch' && inputs.action == 'execute'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python -m tools.skill_eval_pr authorize --output prepared
+      - uses: actions/upload-artifact@v4
+        with:
+          name: skill-eval-plan
+          path: prepared
+          include-hidden-files: true
+          retention-days: 30
+
+  execute:
+    if: github.event_name == 'workflow_dispatch' && inputs.action == 'execute'
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      actions: read
+      pull-requests: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install '.[skills]'
+      - uses: actions/download-artifact@v4
+        with:
+          name: skill-eval-plan
+          path: prepared
+      - name: Existing client preflight (requires separately provisioned model access)
+        run: codex --version
+      - name: Recheck live revision and authorization before execution
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python -m tools.skill_eval_pr authorize --output prepared
+      - name: Execute existing comparison without publication credentials
+        run: python -m tools.skill_eval_pr execute --output prepared
+      - name: Package retained results even after interruption
+        if: always()
+        run: python -m tools.skill_eval_pr pack --output prepared
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: skill-eval-results
+          path: prepared/results.zip
+          retention-days: 30
+
+  publish:
+    if: always() && needs.plan.result == 'success'
+    needs: [plan, execute]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      actions: read
+      pull-requests: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install '.[skills]'
+      - name: Rebuild trusted inputs; never execute artifact code
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python -m tools.skill_eval_pr prepare --output prepared
+      - name: Update the single revision-bound summary from retained data
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python -m tools.skill_eval_pr publish --output prepared
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: skill-eval-report
+          path: |
+            prepared/pr-plan.json
+            prepared/*/report.json
+            prepared/*/report.md
+          retention-days: 30

docs/examples/skill-eval-pr/summary-example.md

@@ -0,0 +1,38 @@
+# Offline skill PR summary example
+
+Fixed-output pipeline control, not a model measurement or a skill-efficacy claim.
+Revision/hash values below are placeholders for the temporary Git fixture.
+
+PR plan: `<canonical-pr-plan-sha256>`
+
+Base: `<fixture-base-sha>` · Head: `<fixture-head-sha>`
+
+Comparison: **old/new** (`addition` uses **absent/new**)
+
+Client: **FixedOutputClient**, model label `offline-fixture`, effort `test`
+
+| Pair outcome | Count |
+| --- | ---: |
+| fix | 1 |
+| regression | 1 |
+| unchanged_correct | 1 |
+| unchanged_incorrect | 1 |
+| inconclusive | 1 |
+| missing_pair | 1 |
+
+One unrelated source change remains **uncovered** and never counts as a pass.
+The deliberately incorrect candidate is graded as a regression; malformed JSON
+is inconclusive. The final pair is missing after the observed-token stop.
+
+Budget: 12 scheduled calls, one repeat, 60 seconds/trial, 900 seconds/comparison,
+observed-token stop threshold 200. The fixed client records **10 calls**; no real
+model calls occur. Monetary cost is unknown. Publication retry updates the same
+single simulated comment with **zero additional evaluation calls**.
+
+The [offline integration control](../../../tests/test_skill_eval_pr.py) verifies
+these outcomes, including addition/update, lost publication response, stale
+heads and retained evidence validation. The
+[demo command](../../../tests/fixtures/skill_eval_pr_demo.py) saves the complete
+plan, results, regenerated reports and simulated summary for local inspection.
+Simulated GitHub links are not live evidence. Real Actions evidence is requested
+for 30 days; after expiry the comment alone cannot substantiate its outcomes.

docs/operations.md

@@ -29,6 +29,7 @@ Effect classes used below:
 | Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.model.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution |
 | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring |
 | Compare skill additions or updates | [Configured skill comparison](skill-eval-comparison.md), `python -m tools.skill_eval_compare` | Read-only preparation / experiment-executing / report-generating | Fixed Git snapshots and one shared exam; explicit plan authorization for execution; JSON + Markdown regeneration makes no model calls |
+| Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials |
 | Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority |
 | Summarize campaign gains | `tools/campaign_summary_chart.py`; `report`/`run` figure | Figure-generating | Renders recorded values into one gain figure plus a matching table; validates the specification and computes nothing new |
 

docs/skill-eval-comparison.md

@@ -5,6 +5,9 @@ generates JSON and Markdown reports. It reuses the existing reader,
 `CodexClient`, `run_trial`, verifier and error handling. The original
 [pilot](skill-evaluation.md) remains available with its original interfaces.
 
+The [PR integration](skill-eval-pr.md) adds automatic planning, explicit operator
+authorization and retained-result publication around these interfaces.
+
 ```mermaid
 flowchart LR
     A["New: small JSON config"] --> B["New: fixed Git snapshots"]