0x73746F66 · Nov 13, 2026 · Nov 13, 2026
diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock
@@ -33,6 +33,7 @@ workflows:
         - 'actions/checkout@v7'
         - 'sigstore/cosign-installer@756fe2d284c7292c4e9b62c8a28f1e5629c59532'
     '.github/workflows/slsa-provenance.yml':
+        - 'actions/checkout@v7'
         - 'actions/download-artifact@v8'
         - 'actions/upload-artifact@v7'
         - 'sigstore/cosign-installer@v4.1.2'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -211,6 +211,9 @@ jobs:
       newest: ${{ steps.newest.outputs.newest }}
       major: ${{ steps.newest.outputs.major }}
       module: ${{ steps.module.outputs.module }}
+      # The release's numeric id: later uploads go to it rather than looking
+      # the release up by tag (scripts/release-asset-upload.sh has why).
+      release_id: ${{ steps.publish.outputs.release_id }}
     steps:
       - uses: actions/checkout@v7
         with:
@@ -311,20 +314,44 @@ jobs:
           path: bin/checksums.txt
           retention-days: 1
 
+      # Whether the release exists is asked over GraphQL, not `gh release
+      # view`: gh reports a REST lookup refused for a spent quota as "release
+      # not found", which would send a re-run to create a release that exists.
       - name: Create or update Release
+        id: publish
         run: |
+          set -euo pipefail
           VERSION=${{ needs.version.outputs.version }}
           if ! git tag -l "$VERSION" | grep -q "$VERSION"; then
             git tag -a "$VERSION" -m "Release $VERSION"
           fi
-          if gh release view "$VERSION" > /dev/null 2>&1; then
-            gh release upload "$VERSION" bin/* --clobber
+          release_id() {
+            # shellcheck disable=SC2016 # $owner, $name and $tag are GraphQL variables
+            gh api graphql \
+              -f query='query($owner:String!,$name:String!,$tag:String!){repository(owner:$owner,name:$name){release(tagName:$tag){databaseId}}}' \
+              -f owner="${GITHUB_REPOSITORY%/*}" -f name="${GITHUB_REPOSITORY#*/}" -f tag="$VERSION" \
+              --jq '.data.repository.release.databaseId // empty'
+          }
+          ID=$(release_id)
+          if [ -n "$ID" ]; then
+            bash scripts/release-asset-upload.sh "$GITHUB_REPOSITORY" "$VERSION" "$ID" bin/*
           else
             gh release create "$VERSION" bin/* \
               --title "Release $VERSION" \
               --latest=${{ steps.newest.outputs.newest }} \
               --generate-notes
+            for attempt in 1 2 3 4 5 6; do
+              ID=$(release_id) || ID=''
+              [ -n "$ID" ] && break
+              sleep $(( attempt * 10 ))
+            done
+          fi
+          if [ -z "$ID" ]; then
+            echo "::error::GraphQL has no release for $VERSION after creating it"
+            exit 1
           fi
+          echo "release $VERSION is id $ID"
+          echo "release_id=$ID" >> "$GITHUB_OUTPUT"
         env:
           GH_TOKEN: ${{ secrets.PACKAGES_TOKEN }}
 
@@ -443,6 +470,7 @@ jobs:
     with:
       subjects_artifact: release-subjects
       version: ${{ needs.release.outputs.version }}
+      release_id: ${{ needs.release.outputs.release_id }}
     secrets:
       release_token: ${{ secrets.PACKAGES_TOKEN }}
 
@@ -661,13 +689,19 @@ jobs:
   # one.
   publish-tea:
     needs: [release, update-packages, provenance]
-    # A provenance failure publishes the release without the bundle, with a
-    # warning below, rather than leaving the transparency log silent about it.
-    # The other two needs still have to succeed, as they did before.
-    # update-packages is skipped, not failed, for a release older than the newest.
-    if: ${{ !cancelled() && needs.release.result == 'success' && (needs.update-packages.result == 'success' || needs.update-packages.result == 'skipped') }}
+    # A TEA release cannot be unpublished, so it waits for the provenance
+    # bundle it attaches: a failed provenance job is re-run, and this job with
+    # it. update-packages may be skipped only because the release is older
+    # than the newest; skipped because its canary never ran (which needs
+    # provenance) it is a failure upstream, and publishing then put a release
+    # in the transparency log that Homebrew, Scoop and the flake never got.
+    if: >-
+      ${{ !cancelled() && needs.release.result == 'success' && needs.provenance.result == 'success'
+      && (needs.update-packages.result == 'success'
+      || (needs.update-packages.result == 'skipped' && needs.release.outputs.newest != 'true')) }}
     runs-on: [self-hosted, Linux, X64, vulnetix-local]
-    timeout-minutes: 20
+    # Room for the signature upload to wait out a spent REST quota.
+    timeout-minutes: 90
     # id-token lets this job get an OIDC token from GitHub, which is the
     # identity public Fulcio issues a short-lived certificate against. That is
     # what makes the signing below keyless: there is no key to hold or rotate.
@@ -691,22 +725,17 @@ jobs:
           aws s3 cp "s3://$STAGE_BUCKET/$STAGE_PREFIX/cross-binaries.tgz" - | tar -xzf - -C bin
 
       # The bundle for every binary in the release, signed by the provenance
-      # job. Not fatal: `tea release` drops a file it cannot read and says so.
+      # job, which this job needs to have succeeded.
       - name: Download the provenance bundle
-        id: provenance
-        continue-on-error: true
         uses: actions/download-artifact@v8
         with:
           name: slsa-provenance
           path: provenance
 
-      - name: Report whether the provenance was produced
+      - name: Check the provenance bundle
         run: |
-          if [ -s provenance/provenance.sigstore.json ]; then
-            echo "provenance: $(wc -c < provenance/provenance.sigstore.json) bytes"
-          else
-            echo "::warning::no provenance bundle was produced; publishing the release without it"
-          fi
+          test -s provenance/provenance.sigstore.json
+          echo "provenance: $(wc -c < provenance/provenance.sigstore.json) bytes"
 
       - name: Install the binary this release built
         run: |
@@ -785,12 +814,13 @@ jobs:
             --certificate-oidc-issuer https://token.actions.githubusercontent.com \
             vulnetix.cdx.json
 
-          gh release upload "${{ needs.release.outputs.version }}" \
-            vulnetix.cdx.json.sig vulnetix.cdx.json.pem \
-            vulnetix.cdx.json.intoto.jsonl --clobber
+          # By release id: `gh release upload <tag>` reports a spent REST
+          # quota as "release not found" (scripts/release-asset-upload.sh).
+          bash scripts/release-asset-upload.sh "$GITHUB_REPOSITORY" \
+            "${{ needs.release.outputs.version }}" "${{ needs.release.outputs.release_id }}" \
+            vulnetix.cdx.json.sig vulnetix.cdx.json.pem vulnetix.cdx.json.intoto.jsonl
         env:
-          # The token that created the release: GITHUB_TOKEN gets "release
-          # not found" here, as in slsa-provenance.yml.
+          # The token that created the release.
           GH_TOKEN: ${{ secrets.PACKAGES_TOKEN }}
 
       - name: Publish product, release, artifacts and distributions

diff --git a/.github/workflows/slsa-provenance.yml b/.github/workflows/slsa-provenance.yml
@@ -34,10 +34,19 @@ on:
         description: 'release tag the bundle is uploaded to (e.g. v1.2.3)'
         type: string
         required: true
+      release_id:
+        description: 'numeric id of that release, when the caller has it; otherwise it is looked up by tag over GraphQL'
+        type: string
+        required: false
+        default: ''
     secrets:
-      # The token that created the release. The job's own GITHUB_TOKEN gets
-      # "release not found" from the release API on the self-hosted runners,
-      # so no bundle was ever attached; the creating token can see it.
+      # The token that created the release (other callers keep GITHUB_TOKEN).
+      # "release not found" from `gh release upload` was never about which
+      # token could see the release: gh (since cli/cli#13723) reports a REST
+      # rate-limit answer on its tag lookup as "release not found" whenever
+      # the GraphQL draft lookup beside it finds no draft. The upload below
+      # therefore never resolves the release by tag over REST, and it tells a
+      # spent quota apart from a missing release.
       release_token:
         required: false
 
@@ -48,7 +57,10 @@ jobs:
   provenance:
     name: Sign and verify provenance
     runs-on: [self-hosted, Linux, X64, vulnetix-local]
-    timeout-minutes: 15
+    # Signing takes about a minute. The rest is room for the upload to wait
+    # out a spent REST quota (up to an hour, see the upload step) rather than
+    # fail the release.
+    timeout-minutes: 80
     steps:
       - uses: actions/download-artifact@v8
         with:
@@ -58,6 +70,7 @@ jobs:
       - uses: sigstore/cosign-installer@v4.1.2
 
       - name: Build the statement
+        id: statement
         env:
           VERSION: ${{ inputs.version }}
         run: |
@@ -78,7 +91,14 @@ jobs:
             "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sigstore" | jq -r .value)"
           payload="$(printf '%s' "$token" | cut -d. -f2 | tr '_-' '/+')"
           while [ $(( ${#payload} % 4 )) -ne 0 ]; do payload="${payload}="; done
-          signer="$(printf '%s' "$payload" | base64 -d | jq -r .job_workflow_ref)"
+          claims="$(printf '%s' "$payload" | base64 -d)"
+          signer="$(jq -r .job_workflow_ref <<<"$claims")"
+          # The commit of this workflow, for the upload script below.
+          workflow_sha="$(jq -r '.job_workflow_sha // empty' <<<"$claims")"
+          if ! [[ $workflow_sha =~ ^[0-9a-f]{40}$ ]]; then
+            echo "::error::the OIDC token names no job_workflow_sha"; exit 1
+          fi
+          echo "workflow_sha=$workflow_sha" >> "$GITHUB_OUTPUT"
           case "$signer" in
             Vulnetix/cli/.github/workflows/slsa-provenance.yml@*) ;;
             *) echo "::error::unexpected signing workflow: $signer"; exit 1 ;;
@@ -148,15 +168,36 @@ jobs:
 
           mv candidate.sigstore.json provenance.sigstore.json
 
-      - name: Upload to the release
-        env:
-          GH_TOKEN: ${{ secrets.release_token || github.token }}
-          VERSION: ${{ inputs.version }}
-        run: gh release upload "$VERSION" provenance.sigstore.json --clobber --repo "$GITHUB_REPOSITORY"
-
+      # The in-run hand-off goes first: it does not depend on the REST quota,
+      # so a release upload that fails no longer takes the bundle with it.
       - uses: actions/upload-artifact@v7
         with:
           name: slsa-provenance
           path: provenance.sigstore.json
           # Only the caller's publish-tea job reads this, in this same run.
           retention-days: 1
+          # A re-run of this job replaces the bundle an earlier attempt left.
+          overwrite: true
+
+      # The upload script, from this workflow's own commit (the OIDC claim the
+      # statement step read), so a caller in another repository runs the code
+      # this workflow is signed with.
+      - uses: actions/checkout@v7
+        with:
+          repository: Vulnetix/cli
+          ref: ${{ steps.statement.outputs.workflow_sha }}
+          path: .slsa-provenance
+          sparse-checkout: scripts/release-asset-upload.sh
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+
+      # By release id, never by tag over REST: gh reports a spent REST quota on
+      # its tag lookup as "release not found" (the script's header has why),
+      # which failed v4.24.8, v4.24.32 and v4.24.34 until a re-run. A spent
+      # quota now waits for its reset rather than failing the release.
+      - name: Upload to the release
+        env:
+          GH_TOKEN: ${{ secrets.release_token || github.token }}
+          VERSION: ${{ inputs.version }}
+          RELEASE_ID: ${{ inputs.release_id }}
+        run: bash .slsa-provenance/scripts/release-asset-upload.sh "$GITHUB_REPOSITORY" "$VERSION" "$RELEASE_ID" provenance.sigstore.json
diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md
@@ -44,9 +44,23 @@ provenance of the build workflow on this organisation's runners (the
 `runner_environment` field says so); it is not an isolated builder, so it is not
 SLSA Build L3.
 
-A provenance failure does not stop the TEA publish: the release is published
-without the bundle and the job warns. A public TEA product cannot be
-unpublished, so preview a release first. Do not re-dispatch an existing tag for
+A public TEA product cannot be unpublished, so `publish-tea` waits for the
+bundle: it runs only when `provenance` succeeded (and, for the newest release,
+when `update-packages` did, which needs the canary, which needs provenance).
+A failed `provenance` job is re-run (`gh run rerun <run-id> --failed`), and
+everything after it runs again.
+
+Release assets are uploaded by release id (`scripts/release-asset-upload.sh`),
+never with `gh release upload <tag>`. The uploads use `PACKAGES_TOKEN`, a
+personal token whose 5,000 REST requests an hour are shared with everything
+else that uses it. When they are spent, gh's lookup of a release by tag prints
+`release not found` for a published release (it prefers that answer when its
+REST lookup and its GraphQL draft lookup both fail, cli/cli#13723), which
+failed v4.24.8, v4.24.32 and v4.24.34. The script resolves the id over
+GraphQL, which has its own quota, waits for a spent REST quota to reset (up to
+65 minutes) and honours `Retry-After`, and every error names its HTTP status.
+
+Preview a release before publishing it. Do not re-dispatch an existing tag for
 that: it rebuilds the binaries and replaces the release assets. Run the CLI
 locally against the release's own checksums instead; `--dry-run` publishes
 nothing:

diff --git a/scripts/release-asset-upload.sh b/scripts/release-asset-upload.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# Upload files to a GitHub release, replacing an asset of the same name.
+#
+#   release-asset-upload.sh <owner/repo> <tag> <release-id or ""> <file>...
+#
+# Environment: GH_TOKEN (contents: write on the repository). QUOTA_WAIT_MAX
+# (seconds, default 3900) bounds the total time spent waiting on rate limits.
+#
+# Why not `gh release upload <tag>`: gh looks the release up by tag over REST
+# and as a draft over GraphQL at once, and since cli/cli#13723 it reports
+# "release not found" when the REST lookup is refused for a spent quota and
+# the GraphQL lookup finds no draft, which is every published release. The
+# release workflows upload with a personal token whose 5,000 REST requests an
+# hour everything else using it shares, so v4.24.8, v4.24.32 and v4.24.34
+# failed that way and passed on a re-run once the quota had reset.
+#
+# So the release is resolved by id: the caller's, or GraphQL's (its own
+# quota, which REST traffic does not spend). Every REST answer is classified:
+# a spent quota waits for its reset, a secondary limit waits for Retry-After,
+# a 404, 5xx or connection failure is retried a few times, anything else
+# fails at once, and every failure names the HTTP status it got.
+set -euo pipefail
+
+if [ "$#" -lt 4 ]; then
+  echo "usage: $0 <owner/repo> <tag> <release-id or \"\"> <file>..." >&2
+  exit 2
+fi
+repo=$1 tag=$2 release_id=$3
+shift 3
+: "${GH_TOKEN:?GH_TOKEN is required}"
+quota_wait_max=${QUOTA_WAIT_MAX:-3900}
+
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+api_status=''
+waited=0
+
+header() {
+  tr -d '\r' < "$tmp/head" | awk -v h="$1" \
+    'index(tolower($0), h ":") == 1 { sub(/^[^:]*:[ \t]*/, ""); v = $0 } END { print v }'
+}
+
+# api METHOD URL [curl args...]: returns 0 on a 2xx, the body in $tmp/body.
+api() {
+  local method=$1 url=$2 attempt wait remaining reset retry
+  shift 2
+  for attempt in 1 2 3 4 5 6 7 8; do
+    api_status=$(curl -sS -o "$tmp/body" -D "$tmp/head" -w '%{http_code}' \
+      -X "$method" -H "Authorization: Bearer $GH_TOKEN" \
+      -H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' \
+      "$@" "$url") || api_status=000
+    case "$api_status" in 2??) return 0 ;; esac
+    remaining=$(header x-ratelimit-remaining)
+    reset=$(header x-ratelimit-reset)
+    retry=$(header retry-after)
+    if [[ $api_status =~ ^(403|429)$ && $remaining == 0 && -n $reset ]]; then
+      wait=$((reset - $(date +%s) + 5))
+      if ((wait < 5)); then wait=5; fi
+      echo "::warning::$method $url: HTTP $api_status, the token's REST quota is spent until $(date -u -d "@$reset" +%H:%M:%SZ); waiting ${wait}s"
+    elif [[ $api_status =~ ^(403|429)$ ]] && { [ -n "$retry" ] || grep -qi 'secondary rate limit' "$tmp/body"; }; then
+      wait=${retry:-60}
+      echo "::warning::$method $url: HTTP $api_status, secondary rate limit; waiting ${wait}s"
+    elif [[ $api_status =~ ^(404|5..|000)$ ]] && ((attempt < 4)); then
+      wait=$((attempt * 10))
+      echo "$method $url: HTTP $api_status, retrying in ${wait}s"
+    elif [ "$api_status" = 422 ]; then
+      # The caller decides: on an upload it is an asset already there.
+      echo "$method $url: HTTP 422: $(head -c 300 "$tmp/body")"
+      return 1
+    else
+      echo "::error::$method $url: HTTP $api_status: $(head -c 500 "$tmp/body")"
+      return 1
+    fi
+    if ((waited + wait > quota_wait_max)); then
+      echo "::error::$method $url: still HTTP $api_status after waiting ${waited}s; re-run the job once the quota resets"
+      return 1
+    fi
+    sleep "$wait"
+    waited=$((waited + wait))
+  done
+  echo "::error::$method $url: HTTP $api_status after $attempt attempts"
+  return 1
+}
+
+if [ -z "$release_id" ]; then
+  for attempt in 1 2 3 4 5 6; do
+    # shellcheck disable=SC2016 # $owner, $name and $tag are GraphQL variables
+    release_id=$(gh api graphql \
+      -f query='query($owner:String!,$name:String!,$tag:String!){repository(owner:$owner,name:$name){release(tagName:$tag){databaseId}}}' \
+      -f owner="${repo%/*}" -f name="${repo#*/}" -f tag="$tag" \
+      --jq '.data.repository.release.databaseId // empty') || release_id=''
+    [ -n "$release_id" ] && break
+    sleep $((attempt * 10))
+  done
+  if [ -z "$release_id" ]; then
+    echo "::error::GraphQL has no release for $tag in $repo"
+    exit 1
+  fi
+fi
+echo "release $tag in $repo is id $release_id"
+
+for file in "$@"; do
+  name=$(basename "$file")
+  upload="https://uploads.github.com/repos/$repo/releases/$release_id/assets?name=$name"
+  if ! api POST "$upload" -H 'Content-Type: application/octet-stream' --data-binary "@$file"; then
+    # 422 is an asset of that name already there (a re-run): replace it.
+    if [ "$api_status" != 422 ]; then exit 1; fi
+    echo "replacing the $name already on $tag"
+    api GET "https://api.github.com/repos/$repo/releases/$release_id/assets?per_page=100"
+    asset=$(jq -r --arg n "$name" '.[] | select(.name == $n) | .id' "$tmp/body")
+    if [ -z "$asset" ]; then
+      echo "::error::upload of $name refused with HTTP 422 and the release has no asset of that name to replace"
+      exit 1
+    fi
+    api DELETE "https://api.github.com/repos/$repo/releases/assets/$asset"
+    api POST "$upload" -H 'Content-Type: application/octet-stream' --data-binary "@$file"
+  fi
+  echo "uploaded $name to $tag"
+done