diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 256ccf06b7..509f093cc7 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -33,6 +33,7 @@ workflows: - 'actions/checkout@v7' - 'sigstore/cosign-installer@756fe2d284c7292c4e9b62c8a28f1e5629c59532' '.github/workflows/slsa-provenance.yml': + - 'actions/checkout@v7' - 'actions/download-artifact@v8' - 'actions/upload-artifact@v7' - 'sigstore/cosign-installer@v4.1.2' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cee262712..e14fe991c1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -211,6 +211,9 @@ jobs: newest: ${{ steps.newest.outputs.newest }} major: ${{ steps.newest.outputs.major }} module: ${{ steps.module.outputs.module }} + # The release's numeric id: later uploads go to it rather than looking + # the release up by tag (scripts/release-asset-upload.sh has why). + release_id: ${{ steps.publish.outputs.release_id }} steps: - uses: actions/checkout@v7 with: @@ -311,20 +314,44 @@ jobs: path: bin/checksums.txt retention-days: 1 + # Whether the release exists is asked over GraphQL, not `gh release + # view`: gh reports a REST lookup refused for a spent quota as "release + # not found", which would send a re-run to create a release that exists. - name: Create or update Release + id: publish run: | + set -euo pipefail VERSION=${{ needs.version.outputs.version }} if ! git tag -l "$VERSION" | grep -q "$VERSION"; then git tag -a "$VERSION" -m "Release $VERSION" fi - if gh release view "$VERSION" > /dev/null 2>&1; then - gh release upload "$VERSION" bin/* --clobber + release_id() { + # shellcheck disable=SC2016 # $owner, $name and $tag are GraphQL variables + gh api graphql \ + -f query='query($owner:String!,$name:String!,$tag:String!){repository(owner:$owner,name:$name){release(tagName:$tag){databaseId}}}' \ + -f owner="${GITHUB_REPOSITORY%/*}" -f name="${GITHUB_REPOSITORY#*/}" -f tag="$VERSION" \ + --jq '.data.repository.release.databaseId // empty' + } + ID=$(release_id) + if [ -n "$ID" ]; then + bash scripts/release-asset-upload.sh "$GITHUB_REPOSITORY" "$VERSION" "$ID" bin/* else gh release create "$VERSION" bin/* \ --title "Release $VERSION" \ --latest=${{ steps.newest.outputs.newest }} \ --generate-notes + for attempt in 1 2 3 4 5 6; do + ID=$(release_id) || ID='' + [ -n "$ID" ] && break + sleep $(( attempt * 10 )) + done + fi + if [ -z "$ID" ]; then + echo "::error::GraphQL has no release for $VERSION after creating it" + exit 1 fi + echo "release $VERSION is id $ID" + echo "release_id=$ID" >> "$GITHUB_OUTPUT" env: GH_TOKEN: ${{ secrets.PACKAGES_TOKEN }} @@ -443,6 +470,7 @@ jobs: with: subjects_artifact: release-subjects version: ${{ needs.release.outputs.version }} + release_id: ${{ needs.release.outputs.release_id }} secrets: release_token: ${{ secrets.PACKAGES_TOKEN }} @@ -661,13 +689,19 @@ jobs: # one. publish-tea: needs: [release, update-packages, provenance] - # A provenance failure publishes the release without the bundle, with a - # warning below, rather than leaving the transparency log silent about it. - # The other two needs still have to succeed, as they did before. - # update-packages is skipped, not failed, for a release older than the newest. - if: ${{ !cancelled() && needs.release.result == 'success' && (needs.update-packages.result == 'success' || needs.update-packages.result == 'skipped') }} + # A TEA release cannot be unpublished, so it waits for the provenance + # bundle it attaches: a failed provenance job is re-run, and this job with + # it. update-packages may be skipped only because the release is older + # than the newest; skipped because its canary never ran (which needs + # provenance) it is a failure upstream, and publishing then put a release + # in the transparency log that Homebrew, Scoop and the flake never got. + if: >- + ${{ !cancelled() && needs.release.result == 'success' && needs.provenance.result == 'success' + && (needs.update-packages.result == 'success' + || (needs.update-packages.result == 'skipped' && needs.release.outputs.newest != 'true')) }} runs-on: [self-hosted, Linux, X64, vulnetix-local] - timeout-minutes: 20 + # Room for the signature upload to wait out a spent REST quota. + timeout-minutes: 90 # id-token lets this job get an OIDC token from GitHub, which is the # identity public Fulcio issues a short-lived certificate against. That is # what makes the signing below keyless: there is no key to hold or rotate. @@ -691,22 +725,17 @@ jobs: aws s3 cp "s3://$STAGE_BUCKET/$STAGE_PREFIX/cross-binaries.tgz" - | tar -xzf - -C bin # The bundle for every binary in the release, signed by the provenance - # job. Not fatal: `tea release` drops a file it cannot read and says so. + # job, which this job needs to have succeeded. - name: Download the provenance bundle - id: provenance - continue-on-error: true uses: actions/download-artifact@v8 with: name: slsa-provenance path: provenance - - name: Report whether the provenance was produced + - name: Check the provenance bundle run: | - if [ -s provenance/provenance.sigstore.json ]; then - echo "provenance: $(wc -c < provenance/provenance.sigstore.json) bytes" - else - echo "::warning::no provenance bundle was produced; publishing the release without it" - fi + test -s provenance/provenance.sigstore.json + echo "provenance: $(wc -c < provenance/provenance.sigstore.json) bytes" - name: Install the binary this release built run: | @@ -785,12 +814,13 @@ jobs: --certificate-oidc-issuer https://token.actions.githubusercontent.com \ vulnetix.cdx.json - gh release upload "${{ needs.release.outputs.version }}" \ - vulnetix.cdx.json.sig vulnetix.cdx.json.pem \ - vulnetix.cdx.json.intoto.jsonl --clobber + # By release id: `gh release upload ` reports a spent REST + # quota as "release not found" (scripts/release-asset-upload.sh). + bash scripts/release-asset-upload.sh "$GITHUB_REPOSITORY" \ + "${{ needs.release.outputs.version }}" "${{ needs.release.outputs.release_id }}" \ + vulnetix.cdx.json.sig vulnetix.cdx.json.pem vulnetix.cdx.json.intoto.jsonl env: - # The token that created the release: GITHUB_TOKEN gets "release - # not found" here, as in slsa-provenance.yml. + # The token that created the release. GH_TOKEN: ${{ secrets.PACKAGES_TOKEN }} - name: Publish product, release, artifacts and distributions diff --git a/.github/workflows/slsa-provenance.yml b/.github/workflows/slsa-provenance.yml index 1f3fa77bab..e33ec03497 100644 --- a/.github/workflows/slsa-provenance.yml +++ b/.github/workflows/slsa-provenance.yml @@ -34,10 +34,19 @@ on: description: 'release tag the bundle is uploaded to (e.g. v1.2.3)' type: string required: true + release_id: + description: 'numeric id of that release, when the caller has it; otherwise it is looked up by tag over GraphQL' + type: string + required: false + default: '' secrets: - # The token that created the release. The job's own GITHUB_TOKEN gets - # "release not found" from the release API on the self-hosted runners, - # so no bundle was ever attached; the creating token can see it. + # The token that created the release (other callers keep GITHUB_TOKEN). + # "release not found" from `gh release upload` was never about which + # token could see the release: gh (since cli/cli#13723) reports a REST + # rate-limit answer on its tag lookup as "release not found" whenever + # the GraphQL draft lookup beside it finds no draft. The upload below + # therefore never resolves the release by tag over REST, and it tells a + # spent quota apart from a missing release. release_token: required: false @@ -48,7 +57,10 @@ jobs: provenance: name: Sign and verify provenance runs-on: [self-hosted, Linux, X64, vulnetix-local] - timeout-minutes: 15 + # Signing takes about a minute. The rest is room for the upload to wait + # out a spent REST quota (up to an hour, see the upload step) rather than + # fail the release. + timeout-minutes: 80 steps: - uses: actions/download-artifact@v8 with: @@ -58,6 +70,7 @@ jobs: - uses: sigstore/cosign-installer@v4.1.2 - name: Build the statement + id: statement env: VERSION: ${{ inputs.version }} run: | @@ -78,7 +91,14 @@ jobs: "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sigstore" | jq -r .value)" payload="$(printf '%s' "$token" | cut -d. -f2 | tr '_-' '/+')" while [ $(( ${#payload} % 4 )) -ne 0 ]; do payload="${payload}="; done - signer="$(printf '%s' "$payload" | base64 -d | jq -r .job_workflow_ref)" + claims="$(printf '%s' "$payload" | base64 -d)" + signer="$(jq -r .job_workflow_ref <<<"$claims")" + # The commit of this workflow, for the upload script below. + workflow_sha="$(jq -r '.job_workflow_sha // empty' <<<"$claims")" + if ! [[ $workflow_sha =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::the OIDC token names no job_workflow_sha"; exit 1 + fi + echo "workflow_sha=$workflow_sha" >> "$GITHUB_OUTPUT" case "$signer" in Vulnetix/cli/.github/workflows/slsa-provenance.yml@*) ;; *) echo "::error::unexpected signing workflow: $signer"; exit 1 ;; @@ -148,15 +168,36 @@ jobs: mv candidate.sigstore.json provenance.sigstore.json - - name: Upload to the release - env: - GH_TOKEN: ${{ secrets.release_token || github.token }} - VERSION: ${{ inputs.version }} - run: gh release upload "$VERSION" provenance.sigstore.json --clobber --repo "$GITHUB_REPOSITORY" - + # The in-run hand-off goes first: it does not depend on the REST quota, + # so a release upload that fails no longer takes the bundle with it. - uses: actions/upload-artifact@v7 with: name: slsa-provenance path: provenance.sigstore.json # Only the caller's publish-tea job reads this, in this same run. retention-days: 1 + # A re-run of this job replaces the bundle an earlier attempt left. + overwrite: true + + # The upload script, from this workflow's own commit (the OIDC claim the + # statement step read), so a caller in another repository runs the code + # this workflow is signed with. + - uses: actions/checkout@v7 + with: + repository: Vulnetix/cli + ref: ${{ steps.statement.outputs.workflow_sha }} + path: .slsa-provenance + sparse-checkout: scripts/release-asset-upload.sh + sparse-checkout-cone-mode: false + persist-credentials: false + + # By release id, never by tag over REST: gh reports a spent REST quota on + # its tag lookup as "release not found" (the script's header has why), + # which failed v4.24.8, v4.24.32 and v4.24.34 until a re-run. A spent + # quota now waits for its reset rather than failing the release. + - name: Upload to the release + env: + GH_TOKEN: ${{ secrets.release_token || github.token }} + VERSION: ${{ inputs.version }} + RELEASE_ID: ${{ inputs.release_id }} + run: bash .slsa-provenance/scripts/release-asset-upload.sh "$GITHUB_REPOSITORY" "$VERSION" "$RELEASE_ID" provenance.sigstore.json diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index dca16ae5c1..ffbcb70e4b 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -44,9 +44,23 @@ provenance of the build workflow on this organisation's runners (the `runner_environment` field says so); it is not an isolated builder, so it is not SLSA Build L3. -A provenance failure does not stop the TEA publish: the release is published -without the bundle and the job warns. A public TEA product cannot be -unpublished, so preview a release first. Do not re-dispatch an existing tag for +A public TEA product cannot be unpublished, so `publish-tea` waits for the +bundle: it runs only when `provenance` succeeded (and, for the newest release, +when `update-packages` did, which needs the canary, which needs provenance). +A failed `provenance` job is re-run (`gh run rerun --failed`), and +everything after it runs again. + +Release assets are uploaded by release id (`scripts/release-asset-upload.sh`), +never with `gh release upload `. The uploads use `PACKAGES_TOKEN`, a +personal token whose 5,000 REST requests an hour are shared with everything +else that uses it. When they are spent, gh's lookup of a release by tag prints +`release not found` for a published release (it prefers that answer when its +REST lookup and its GraphQL draft lookup both fail, cli/cli#13723), which +failed v4.24.8, v4.24.32 and v4.24.34. The script resolves the id over +GraphQL, which has its own quota, waits for a spent REST quota to reset (up to +65 minutes) and honours `Retry-After`, and every error names its HTTP status. + +Preview a release before publishing it. Do not re-dispatch an existing tag for that: it rebuilds the binaries and replaces the release assets. Run the CLI locally against the release's own checksums instead; `--dry-run` publishes nothing: diff --git a/scripts/release-asset-upload.sh b/scripts/release-asset-upload.sh new file mode 100755 index 0000000000..50fd70b4cc --- /dev/null +++ b/scripts/release-asset-upload.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Upload files to a GitHub release, replacing an asset of the same name. +# +# release-asset-upload.sh ... +# +# Environment: GH_TOKEN (contents: write on the repository). QUOTA_WAIT_MAX +# (seconds, default 3900) bounds the total time spent waiting on rate limits. +# +# Why not `gh release upload `: gh looks the release up by tag over REST +# and as a draft over GraphQL at once, and since cli/cli#13723 it reports +# "release not found" when the REST lookup is refused for a spent quota and +# the GraphQL lookup finds no draft, which is every published release. The +# release workflows upload with a personal token whose 5,000 REST requests an +# hour everything else using it shares, so v4.24.8, v4.24.32 and v4.24.34 +# failed that way and passed on a re-run once the quota had reset. +# +# So the release is resolved by id: the caller's, or GraphQL's (its own +# quota, which REST traffic does not spend). Every REST answer is classified: +# a spent quota waits for its reset, a secondary limit waits for Retry-After, +# a 404, 5xx or connection failure is retried a few times, anything else +# fails at once, and every failure names the HTTP status it got. +set -euo pipefail + +if [ "$#" -lt 4 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi +repo=$1 tag=$2 release_id=$3 +shift 3 +: "${GH_TOKEN:?GH_TOKEN is required}" +quota_wait_max=${QUOTA_WAIT_MAX:-3900} + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +api_status='' +waited=0 + +header() { + tr -d '\r' < "$tmp/head" | awk -v h="$1" \ + 'index(tolower($0), h ":") == 1 { sub(/^[^:]*:[ \t]*/, ""); v = $0 } END { print v }' +} + +# api METHOD URL [curl args...]: returns 0 on a 2xx, the body in $tmp/body. +api() { + local method=$1 url=$2 attempt wait remaining reset retry + shift 2 + for attempt in 1 2 3 4 5 6 7 8; do + api_status=$(curl -sS -o "$tmp/body" -D "$tmp/head" -w '%{http_code}' \ + -X "$method" -H "Authorization: Bearer $GH_TOKEN" \ + -H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' \ + "$@" "$url") || api_status=000 + case "$api_status" in 2??) return 0 ;; esac + remaining=$(header x-ratelimit-remaining) + reset=$(header x-ratelimit-reset) + retry=$(header retry-after) + if [[ $api_status =~ ^(403|429)$ && $remaining == 0 && -n $reset ]]; then + wait=$((reset - $(date +%s) + 5)) + if ((wait < 5)); then wait=5; fi + echo "::warning::$method $url: HTTP $api_status, the token's REST quota is spent until $(date -u -d "@$reset" +%H:%M:%SZ); waiting ${wait}s" + elif [[ $api_status =~ ^(403|429)$ ]] && { [ -n "$retry" ] || grep -qi 'secondary rate limit' "$tmp/body"; }; then + wait=${retry:-60} + echo "::warning::$method $url: HTTP $api_status, secondary rate limit; waiting ${wait}s" + elif [[ $api_status =~ ^(404|5..|000)$ ]] && ((attempt < 4)); then + wait=$((attempt * 10)) + echo "$method $url: HTTP $api_status, retrying in ${wait}s" + elif [ "$api_status" = 422 ]; then + # The caller decides: on an upload it is an asset already there. + echo "$method $url: HTTP 422: $(head -c 300 "$tmp/body")" + return 1 + else + echo "::error::$method $url: HTTP $api_status: $(head -c 500 "$tmp/body")" + return 1 + fi + if ((waited + wait > quota_wait_max)); then + echo "::error::$method $url: still HTTP $api_status after waiting ${waited}s; re-run the job once the quota resets" + return 1 + fi + sleep "$wait" + waited=$((waited + wait)) + done + echo "::error::$method $url: HTTP $api_status after $attempt attempts" + return 1 +} + +if [ -z "$release_id" ]; then + for attempt in 1 2 3 4 5 6; do + # shellcheck disable=SC2016 # $owner, $name and $tag are GraphQL variables + release_id=$(gh api graphql \ + -f query='query($owner:String!,$name:String!,$tag:String!){repository(owner:$owner,name:$name){release(tagName:$tag){databaseId}}}' \ + -f owner="${repo%/*}" -f name="${repo#*/}" -f tag="$tag" \ + --jq '.data.repository.release.databaseId // empty') || release_id='' + [ -n "$release_id" ] && break + sleep $((attempt * 10)) + done + if [ -z "$release_id" ]; then + echo "::error::GraphQL has no release for $tag in $repo" + exit 1 + fi +fi +echo "release $tag in $repo is id $release_id" + +for file in "$@"; do + name=$(basename "$file") + upload="https://uploads.github.com/repos/$repo/releases/$release_id/assets?name=$name" + if ! api POST "$upload" -H 'Content-Type: application/octet-stream' --data-binary "@$file"; then + # 422 is an asset of that name already there (a re-run): replace it. + if [ "$api_status" != 422 ]; then exit 1; fi + echo "replacing the $name already on $tag" + api GET "https://api.github.com/repos/$repo/releases/$release_id/assets?per_page=100" + asset=$(jq -r --arg n "$name" '.[] | select(.name == $n) | .id' "$tmp/body") + if [ -z "$asset" ]; then + echo "::error::upload of $name refused with HTTP 422 and the release has no asset of that name to replace" + exit 1 + fi + api DELETE "https://api.github.com/repos/$repo/releases/assets/$asset" + api POST "$upload" -H 'Content-Type: application/octet-stream' --data-binary "@$file" + fi + echo "uploaded $name to $tag" +done