fix(ci): fail loudly on attestation lookup failure; decide the codecov pair (LAB-2528)

.github/workflows/attestation-check.yml

@@ -16,16 +16,64 @@ jobs:
       - name: Get latest release tag
         id: release
         run: |
-          TAG=$(gh release list --repo ${{ github.repository }} --limit 1 --json tagName --jq '.[0].tagName' 2>/dev/null || echo "")
-          if [ -z "$TAG" ]; then
-            echo "No releases found, skipping"
+          # Three outcomes, three exit paths. The old `2>/dev/null || echo ""`
+          # collapsed the first into the third, so a week with no attestation
+          # verification at all exited 0 and read as healthy:
+          #   1. lookup failed -> hard fail; we know nothing either way.
+          #   2. releases exist but none is GitHub's "latest" -> hard fail. On a repo
+          #      that publishes releases that is a tamper or misconfiguration signal,
+          #      and skipping it is the exact shape LAB-984 fell into.
+          #   3. zero releases at all -> skip; nothing has been published yet.
+          #
+          # Two lookups, each answering exactly one question, because one call
+          # cannot answer both without guessing a page size. `gh release list`
+          # is a paged view over ALL releases (drafts included) — it answers
+          # "does this repo publish anything at all?", and --limit 1 is enough
+          # for that. `gh release view` with no tag resolves /releases/latest
+          # server-side: the same newest-non-draft-non-prerelease release the
+          # `isLatest` flag marks, but with no page window it can fall outside
+          # of. An earlier form filtered `isLatest` out of a --limit N listing,
+          # which meant picking an N and hard-failing a perfectly healthy repo
+          # once the latest release aged past it. A tripwire that cries wolf is
+          # the same trust bug as one that stays silent, so the window is gone
+          # rather than widened.
+          if ! RELEASES=$(gh release list --repo "$REPO" --limit 1 --json tagName); then
+            echo "::error::gh release list failed for $REPO — cannot determine whether any release exists; refusing to report a green skip"
+            exit 1
+          fi
+          # Validate the payload in its own statement before anything branches on it.
+          # The `type=="array"` guard makes every non-array shape (null, an object,
+          # an error envelope) return null, which `jq -e` reports as a non-zero exit
+          # alongside parse errors and no-output — so empty or truncated stdout from
+          # a `gh` that still exited 0 lands on the annotated hard failure instead of
+          # a raw jq trace. Doing this inline as `[ "$(jq ...)" -ne 0 ]` would hide
+          # jq's exit code in a command substitution, suppress `set -e` inside the
+          # `if`, and — when `[` itself errored on non-numeric input — evaluate FALSE
+          # straight into the skip path: the same fail-open shape as the
+          # `|| echo ""` this commit removes.
+          if ! COUNT=$(jq -e 'if type == "array" then length else null end' <<<"$RELEASES"); then
+            echo "::error::gh release list returned no parseable release array for $REPO — refusing to report a green skip"
+            exit 1
+          fi
+          if [ "$COUNT" -eq 0 ]; then
+            echo "$REPO has no published releases — nothing to attest, skipping"
             echo "skip=true" >> "$GITHUB_OUTPUT"
+          # Releases exist, so a latest MUST resolve. Both ways this can fail are
+          # incidents, not idle states: an API failure means we know nothing, and
+          # "releases exist but none is latest" (all drafts/prereleases) on a repo
+          # that publishes to PyPI is tamper or misconfiguration. Skipping either
+          # is the exact shape LAB-984 fell into, so both go red.
+          elif ! TAG=$(gh release view --repo "$REPO" --json tagName --jq '.tagName'); then
+            echo "::error::$REPO has release(s) but no resolvable latest release — refusing to skip a release that may need attesting"
+            exit 1
           else
+            echo "Latest release: $TAG"
             echo "tag=$TAG" >> "$GITHUB_OUTPUT"
             echo "skip=false" >> "$GITHUB_OUTPUT"
           fi
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
 
       - name: Set up Python
         if: steps.release.outputs.skip != 'true'
@@ -34,13 +82,19 @@ jobs:
           python-version: '3.12'
 
       - name: Verify attestations
+        id: verify
         if: steps.release.outputs.skip != 'true'
         run: |
           # `gh attestation` has no `list` subcommand — the old query errored in
           # ~8s every run (that was the real cause of the weekly red, not missing
           # attestations). Verify the actual published artifact instead.
-          VER="${{ steps.release.outputs.tag }}"
-          VER="${VER#v}"
+          #
+          # TAG/REPO arrive via env, never template-interpolated into this body: a
+          # git tag may legally contain `$(...)` or backticks, and whoever can name
+          # a tag is exactly the adversary this tripwire exists to catch —
+          # interpolating it would hand them code execution in a job holding
+          # GH_TOKEN and issues: write.
+          VER="${TAG#v}"
           echo "Verifying attestations for cachekit ${VER}"
           mkdir -p attest-check
           if ! pip download "cachekit==${VER}" --no-deps --only-binary :all: -d attest-check; then
@@ -55,21 +109,34 @@ jobs:
           fi
           for f in "${files[@]}"; do
             echo "Verifying attestation for $f"
-            gh attestation verify "$f" --repo ${{ github.repository }} || {
+            gh attestation verify "$f" --repo "$REPO" || {
               echo "::error::Attestation verification failed for $f"
               exit 1
             }
           done
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          TAG: ${{ steps.release.outputs.tag }}
 
+      # Gated on the verify step's own outcome, not a bare `failure()`. Previously a
+      # failed *lookup* landed here and filed a public "Attestation verification
+      # failed for " issue — empty tag, blaming attestations for an API outage that
+      # never reached the verify step; a setup-python or PyPI-lag failure did the
+      # same with a tag attached. A tripwire that cries wolf is the same trust bug
+      # as one that stays silent, so the title says "health check" (the step covers
+      # both the PyPI download and the attestation check) and the body sends the
+      # reader to the log to find out which. For any earlier failure the red run is
+      # the signal and no issue is filed.
       - name: Open issue on failure
-        if: failure()
+        if: failure() && steps.verify.outcome == 'failure'
         run: |
           gh issue create \
-            --repo ${{ github.repository }} \
-            --title "Attestation verification failed for ${{ steps.release.outputs.tag }}" \
-            --body "Weekly attestation health check failed. Verify that the release workflow produced valid attestations." \
+            --repo "$REPO" \
+            --title "Attestation health check failed for $TAG" \
+            --body "The weekly attestation health check failed for ${TAG}. Check the run log to see whether the wheel download or the attestation verification failed, then verify that the release workflow produced valid attestations." \
             --label "bug"
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          TAG: ${{ steps.release.outputs.tag }}

.github/workflows/ci.yml

@@ -153,24 +153,78 @@ jobs:
         REDIS_URL: redis://localhost:6379  # autouse redis-isolation fixture uses external Redis when set (else spawns a binary the runner lacks)
       run: uv run pytest tests/performance/ -m "performance and slow" -q
 
+    # The two uploads below are deliberately treated differently, not defaulted
+    # (LAB-2528 finding 3). coverage.xml is the ONLY input to the
+    # project/patch statuses codecov.yml declares, and every flag there sets
+    # `carryforward: true` — so a silently-dropped upload does not remove the
+    # status, it answers "is this PR's new code 80% covered?" with a previous
+    # run's numbers.
+    #
+    # Scoped to same-repo events rather than a bare `true`, because of what the
+    # action actually does on a fork (read at the pinned SHA, not assumed): its
+    # `Get OIDC token` step is guarded `CC_USE_OIDC == 'true' && CC_FORK != 'true'`,
+    # so on a fork it never attempts OIDC, `CC_TOKEN` stays empty, and the upload
+    # proceeds TOKENLESS — Codecov's rate-limited path. A bare `true` would let a
+    # 429 nobody controls redden an outside contribution, which on a repo with no
+    # branch protection trains maintainers to merge over red CI: the opposite of
+    # what this ticket is hardening. The flag therefore applies exactly where OIDC
+    # really authenticates.
+    #
+    # ACCEPTED RESIDUAL RISK, stated because the scoping creates it: on a fork PR a
+    # dropped tokenless upload is still silent, and carryforward then answers the
+    # patch question with an earlier commit's numbers — a stale green on exactly the
+    # least-trusted contributions. The wrapper's CLI signature check is likewise
+    # unenforced there (fail_ci_if_error is its switch; see the pin note below).
+    # Accepted because a fork PR cannot reach the self-hosted `cachekit` runner
+    # without a maintainer approving the run, no fork PR has run here to date, and
+    # a tampered binary reaches same-repo runs first, where it fails closed. The real
+    # fix is to stop depending on an external upload for the floor (a local
+    # `--cov-fail-under` on the PR pytest invocation); that is a coverage-policy
+    # change, tracked separately rather than smuggled into this diff.
+    #
+    # Do not re-pin below v7.0.0 (fb8b3582): releases published before Codecov's
+    # June 2026 keybase migration fetch the CLI signing key from a deleted account,
+    # so `gpg --verify` can never pass. The wrapper stops there only when
+    # fail_ci_if_error is true; otherwise it prints "CLI integrity verified" and runs
+    # the unverified binary. Accepted flip side: a keybase.io outage now fails
+    # same-repo CI closed, not open.
     - name: Upload coverage to Codecov
       if: ${{ !cancelled() }}
-      uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
       with:
         files: ./coverage.xml
         use_oidc: true
-        fail_ci_if_error: false
+        fail_ci_if_error: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
         flags: ${{ github.event_name == 'push' && 'full' || 'critical' }}-python-${{ matrix.python-version }}
 
+    # fail_ci_if_error is true here too, for a different reason: it is also the
+    # wrapper's signature-enforcement switch (see the pin note above), so false
+    # would let an unverified binary run. The "don't redden CI" intent lives in
+    # continue-on-error instead: junit.xml feeds Codecov Test Analytics (flaky-test
+    # history) only and nothing gates on it, so a Codecov-side outage marks this
+    # step failed-and-continued and the job stays green.
+    # That downgrade is visibility only: the wrapper exits before it chmods or runs
+    # the binary, so a bad signature here still runs no downloaded code. Nor does it hide
+    # an integrity failure on same-repo events: the coverage step above verifies the
+    # same CLI, key and checksum hard-fail first, so a signature that cannot pass
+    # reddens the job there before this step runs.
+    #
+    # `handle_no_reports_found` is left at its default (false) on both uploads on
+    # purpose: it would also swallow "the report was never written", which is the
+    # silent-degradation this ticket exists to remove. On coverage.xml a green job
+    # that uploaded nothing is a trust bug; on junit.xml continue-on-error already
+    # accepts a green job, and false there keeps the failed step visible as an
+    # annotation instead of hiding it.
     - name: Upload test results to Codecov
       if: ${{ !cancelled() }}
-      uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5
+      continue-on-error: true
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
       with:
         files: ./junit.xml
         report_type: test_results
         use_oidc: true
         flags: ${{ github.event_name == 'push' && 'full' || 'critical' }}-python-${{ matrix.python-version }}
-        fail_ci_if_error: false
+        fail_ci_if_error: true
 
   # Version sync + doc tests (push to main only)
   post-merge:
@@ -196,7 +250,7 @@ jobs:
     - name: Scan Python dependencies for CVEs
       run: |
         # No suppressions: every prior CVE is resolved at source on the py3.10+
-        # resolution. urllib3>=2.7.0 and pip>=26.1.2 are pinned via
+        # resolution. urllib3>=2.7.0 and pip>=26.2 are pinned via
         # [tool.uv] constraint-dependencies; pygments/pyarrow advisories cleared
         # by their py3.10+ fix versions. Keep this list IDENTICAL to
         # security-fast.yml's pip-audit so the two cannot drift.

.github/workflows/security-fast.yml

@@ -91,7 +91,7 @@ jobs:
     - name: Run pip-audit
       run: |
         # No suppressions: every prior CVE is resolved at source on the py3.10+
-        # resolution. urllib3>=2.7.0 and pip>=26.1.2 are pinned via
+        # resolution. urllib3>=2.7.0 and pip>=26.2 are pinned via
         # [tool.uv] constraint-dependencies; pygments/pyarrow advisories cleared
         # by their py3.10+ fix versions. Keep this list IDENTICAL to ci.yml's
         # post-merge pip-audit so the two cannot drift.

pyproject.toml

@@ -249,8 +249,9 @@ constraint-dependencies = [
     "werkzeug>=3.1.4",
     # pip is a dev-only transitive dep (pip-audit -> pip-api -> pip). 26.1.2 fixes
     # PYSEC-2026-196 (entry-point path traversal), GHSA-58qw-9mgm-455v (tar/zip
-    # confusion) and GHSA-jp4c-xjxw-mgf9 (self-update import ordering).
-    "pip>=26.1.2",
+    # confusion) and GHSA-jp4c-xjxw-mgf9 (self-update import ordering); 26.2 fixes
+    # PYSEC-2026-3721 (doubly-encoded index URLs install to arbitrary paths).
+    "pip>=26.2",
     # h2 is a transitive dep (httpx[http2] -> h2). 4.4.1 fixes
     # GHSA-6hr6-w5qg-qmwg (duplicate Host headers forwarded on HTTP/2 ->
     # HTTP/1.1 downgrade — request smuggling primitive).