From acecb578bbfe26021bc37f2426f9bcbb40356b38 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:24:32 +1000 Subject: [PATCH] fix(release): stop compiling cryo and Go in the release path Releases were exceeding goreleaser's 1h timeout and failing. The release Dockerfile compiled two things per architecture - cryo (Rust: polars + alloy) and the Go binary - and the arm64 leg did both under QEMU. Neither architecture finished in time. The buildx GHA cache that was meant to absorb this never worked, so every release started from a cold cache. It was broken two ways over: the goreleaser container is never passed ACTIONS_RUNTIME_TOKEN, and the default docker driver it falls back to inside that container cannot import or export type=gha at all. The flags are removed rather than left misleading. Both compiles now happen ahead of the release: - Dockerfile.cryo + .github/workflows/cryo-image.yml build the pinned cryo as ethpandaops/cryo:, each architecture on a native runner, joined into a manifest. It only runs when the pin actually moves, and reads the pin back out of the root Dockerfile so there is a single source of truth. - The release Dockerfile now compiles nothing. It takes cryo from that image and the server binary from goreleaser, which already cross-compiles both architectures natively with CGO_ENABLED=0. Also fixes ldflags that pointed at cmd.Release and cmd.GitCommit, which do not exist - the symbols live in internal/version. -X against a missing symbol fails silently, so released archives have been reporting the version as "dev". This mattered here because the images previously got their version via make build-binary and now inherit goreleaser's ldflags. Note the release Dockerfile now expects a prebuilt binary in its context, so it is no longer buildable standalone with `docker build .`. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/cryo-image.yml | 144 +++++++++++++++++++++++++++++++ .goreleaser.yaml | 38 ++------ Dockerfile | 54 ++++-------- Dockerfile.cryo | 50 +++++++++++ 4 files changed, 221 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/cryo-image.yml create mode 100644 Dockerfile.cryo diff --git a/.github/workflows/cryo-image.yml b/.github/workflows/cryo-image.yml new file mode 100644 index 0000000..29982b5 --- /dev/null +++ b/.github/workflows/cryo-image.yml @@ -0,0 +1,144 @@ +name: cryo image + +# Publishes the pinned cryo binary as a multi-arch image so the release path +# never has to compile Rust. cryo pulls in polars and alloy, which is minutes +# natively and hours under emulation - so each architecture is built on a native +# runner and the results are joined into a manifest. +# +# The pin lives in the root Dockerfile (ARG CRYO_SHA) and is read back out here, +# so there is one place to bump it. Bump it, merge, wait for this workflow, then +# tag the release. + +on: + push: + branches: + - master + paths: + - 'Dockerfile' + - 'Dockerfile.cryo' + - '.github/workflows/cryo-image.yml' + workflow_dispatch: + +env: + IMAGE_NAME: ethpandaops/cryo + +jobs: + pin: + runs-on: ubuntu-24.04 + outputs: + cryo_sha: ${{ steps.pin.outputs.cryo_sha }} + exists: ${{ steps.exists.outputs.exists }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Read cryo pin from Dockerfile + id: pin + run: | + CRYO_SHA=$(grep -oE '^ARG CRYO_SHA=[0-9a-f]{40}' Dockerfile | cut -d= -f2) + + if [[ -z "$CRYO_SHA" ]]; then + echo "::error::could not read 'ARG CRYO_SHA=<40 hex chars>' from Dockerfile" + exit 1 + fi + + echo "cryo_sha=$CRYO_SHA" >> "$GITHUB_OUTPUT" + echo "cryo pin: $CRYO_SHA" + + - name: Login to DockerHub + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check whether this pin is already published + id: exists + run: | + IMAGE="${{ env.IMAGE_NAME }}:${{ steps.pin.outputs.cryo_sha }}" + + # Skip the (expensive) rebuild when the pin has not moved. Manual runs + # always rebuild, so there is an escape hatch if a push went wrong. + if [[ "${{ github.event_name }}" != "workflow_dispatch" ]] && \ + docker manifest inspect "$IMAGE" > /dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "$IMAGE already published, skipping build" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "$IMAGE needs building" + fi + + build: + needs: pin + if: needs.pin.outputs.exists == 'false' + strategy: + fail-fast: false + matrix: + include: + # Native runners on both sides - no QEMU. These are GitHub-hosted + # because this workflow runs rarely and the two legs are parallel, so + # wall clock is bound by the slower (arm) leg either way. + - arch: amd64 + platform: linux/amd64 + runs-on: ubuntu-24.04 + - arch: arm64 + platform: linux/arm64 + runs-on: ubuntu-24.04-arm + runs-on: ${{ matrix.runs-on }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Login to DockerHub + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push ${{ matrix.arch }} + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: Dockerfile.cryo + platforms: ${{ matrix.platform }} + push: true + provenance: false + build-args: | + CRYO_SHA=${{ needs.pin.outputs.cryo_sha }} + tags: ${{ env.IMAGE_NAME }}:${{ needs.pin.outputs.cryo_sha }}-${{ matrix.arch }} + cache-from: type=gha,scope=cryo-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=cryo-${{ matrix.arch }} + + - name: Smoke test + run: | + # Native runner, so the freshly built image actually runs here. + docker run --rm \ + "${{ env.IMAGE_NAME }}:${{ needs.pin.outputs.cryo_sha }}-${{ matrix.arch }}" \ + --version + + manifest: + needs: + - pin + - build + runs-on: ubuntu-24.04 + steps: + - name: Login to DockerHub + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Create multi-arch manifest + run: | + IMAGE="${{ env.IMAGE_NAME }}" + SHA="${{ needs.pin.outputs.cryo_sha }}" + + docker buildx imagetools create \ + --tag "$IMAGE:$SHA" \ + --tag "$IMAGE:latest" \ + "$IMAGE:$SHA-amd64" \ + "$IMAGE:$SHA-arm64" + + docker buildx imagetools inspect "$IMAGE:$SHA" diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 296ceab..1f8786e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -17,8 +17,8 @@ builds: - amd64 ldflags: - -s -w - - -X github.com/ethpandaops/execution-processor/cmd.Release={{.Tag}}-{{.ShortCommit}} - - -X github.com/ethpandaops/execution-processor/cmd.GitCommit={{.ShortCommit}} + - -X github.com/ethpandaops/execution-processor/internal/version.Release={{.Tag}}-{{.ShortCommit}} + - -X github.com/ethpandaops/execution-processor/internal/version.GitCommit={{.ShortCommit}} mod_timestamp: "{{ .CommitTimestamp }}" - id: linux-arm64 @@ -32,8 +32,8 @@ builds: - arm64 ldflags: - -s -w - - -X github.com/ethpandaops/execution-processor/cmd.Release={{.Tag}}-{{.ShortCommit}} - - -X github.com/ethpandaops/execution-processor/cmd.GitCommit={{.ShortCommit}} + - -X github.com/ethpandaops/execution-processor/internal/version.Release={{.Tag}}-{{.ShortCommit}} + - -X github.com/ethpandaops/execution-processor/internal/version.GitCommit={{.ShortCommit}} mod_timestamp: "{{ .CommitTimestamp }}" checksum: name_template: 'checksums.txt' @@ -48,55 +48,33 @@ changelog: dockers: - use: buildx + ids: + - linux-amd64 goos: linux goarch: amd64 dockerfile: Dockerfile - extra_files: - - go.mod - - go.sum - - cmd - - pkg - - internal - - example_config.yaml - - Makefile - - main.go image_templates: - "ethpandaops/{{ .ProjectName }}:{{ .Version }}-amd64" - "ethpandaops/{{ .ProjectName }}:{{ if .Env.RELEASE_SUFFIX }}{{ .Env.RELEASE_SUFFIX }}-{{ end }}latest-amd64" build_flag_templates: - "--platform=linux/amd64" - "--provenance=false" - - "--cache-from=type=gha,scope=cryo-amd64" - - "--cache-to=type=gha,mode=max,ignore-error=true,scope=cryo-amd64" - - "--build-arg=VERSION={{.Tag}}" - - "--build-arg=GIT_COMMIT={{.ShortCommit}}" - "--label=org.opencontainers.image.created={{.Date}}" - "--label=org.opencontainers.image.title={{.ProjectName}}" - "--label=org.opencontainers.image.revision={{.FullCommit}}" - "--label=org.opencontainers.image.version={{.Version}}" - use: buildx + ids: + - linux-arm64 goos: linux goarch: arm64 dockerfile: Dockerfile - extra_files: - - go.mod - - go.sum - - cmd - - pkg - - internal - - example_config.yaml - - Makefile - - main.go image_templates: - "ethpandaops/{{ .ProjectName }}:{{ .Version }}-arm64v8" - "ethpandaops/{{ .ProjectName }}:{{ if .Env.RELEASE_SUFFIX }}{{ .Env.RELEASE_SUFFIX }}-{{ end }}latest-arm64v8" build_flag_templates: - "--platform=linux/arm64/v8" - "--provenance=false" - - "--cache-from=type=gha,scope=cryo-arm64" - - "--cache-to=type=gha,mode=max,ignore-error=true,scope=cryo-arm64" - - "--build-arg=VERSION={{.Tag}}" - - "--build-arg=GIT_COMMIT={{.ShortCommit}}" - "--label=org.opencontainers.image.created={{.Date}}" - "--label=org.opencontainers.image.title={{.ProjectName}}" - "--label=org.opencontainers.image.revision={{.FullCommit}}" diff --git a/Dockerfile b/Dockerfile index 108ec25..c93dff5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,39 +1,21 @@ -# cryo ships no release binaries at all, so it is built from a pinned commit. -# The build must be static: cryo links glibc and OpenSSL 3 dynamically by -# default, and neither exists on the alpine runtime. +# Assembles the release image. Deliberately compiles nothing. # -# Pin the Rust version as well as the commit. cryo itself has been stable for -# ~18 months, but rustc has not: 1.97 fails to compile the ethnum dependency -# with "cannot transmute between types of different sizes", against both the -# musl and gnu targets. -FROM rust:1.83-alpine AS cryo-builder - +# Both inputs are produced ahead of this build: +# - cryo comes from a prebuilt multi-arch image (see Dockerfile.cryo and +# .github/workflows/cryo-image.yml), so the Rust toolchain never runs here. +# - the server binary is cross-compiled natively by goreleaser with +# CGO_ENABLED=0 and placed in the build context. +# +# That keeps the arm64 image free of emulated compilation, which is what +# previously pushed releases past goreleaser's timeout. +# +# Bumping cryo: change CRYO_SHA here, then let .github/workflows/cryo-image.yml +# publish the matching image before tagging a release. This ARG is the single +# source of truth - the workflow reads the pin back out of this file. ARG CRYO_SHA=559b65455d7ef6b03e8e9e96a0e50fd4fe8a9c86 +ARG CRYO_IMAGE=ethpandaops/cryo -RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static pkgconfig perl make git - -ENV OPENSSL_STATIC=1 OPENSSL_DIR=/usr - -RUN git clone https://github.com/paradigmxyz/cryo /src && \ - cd /src && \ - git checkout "${CRYO_SHA}" && \ - cargo build --release -p cryo_cli && \ - strip target/release/cryo - -FROM golang:1.25.4-alpine AS builder - -RUN apk add --no-cache make git ca-certificates - -WORKDIR /build - -COPY . . - -# Build arguments for version info -ARG VERSION=dev -ARG GIT_COMMIT=dev - -# All code is pre-generated by the workflow, just build the binary -RUN VERSION=${VERSION} GIT_COMMIT=${GIT_COMMIT} make build-binary +FROM ${CRYO_IMAGE}:${CRYO_SHA} AS cryo FROM alpine:latest @@ -43,8 +25,10 @@ RUN apk --no-cache add ca-certificates && \ WORKDIR /app -COPY --from=cryo-builder /src/target/release/cryo /usr/local/bin/cryo -COPY --from=builder /build/bin/execution-processor . +COPY --from=cryo /usr/local/bin/cryo /usr/local/bin/cryo + +# Produced by the matching goreleaser build id, not compiled here. +COPY server /app/execution-processor RUN chown -R appuser:appuser /app diff --git a/Dockerfile.cryo b/Dockerfile.cryo new file mode 100644 index 0000000..b71b5e9 --- /dev/null +++ b/Dockerfile.cryo @@ -0,0 +1,50 @@ +# Builds the pinned cryo binary as a standalone, publishable image. +# +# This exists so that the Rust toolchain never runs in the release path. cryo +# pulls in polars and alloy, which is tens of minutes of compilation even on a +# fast native runner - and hours under QEMU. Building it here means the +# application Dockerfile only has to COPY --from an already-built image, and +# cryo is only recompiled when CRYO_SHA is bumped. +# +# Built natively per-architecture by .github/workflows/cryo-image.yml, then +# joined into a multi-arch manifest. Do not build this with --platform for a +# foreign architecture; that reintroduces the emulation this is avoiding. +# +# cryo ships no release binaries at all, so it is built from a pinned commit. +# The build must be static: cryo links glibc and OpenSSL 3 dynamically by +# default, and neither exists on the alpine runtime. +# +# Pin the Rust version as well as the commit. cryo itself has been stable for +# ~18 months, but rustc has not: 1.97 fails to compile the ethnum dependency +# with "cannot transmute between types of different sizes", against both the +# musl and gnu targets. +ARG RUST_VERSION=1.83 + +FROM rust:${RUST_VERSION}-alpine AS builder + +ARG CRYO_SHA + +RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static pkgconfig perl make git + +ENV OPENSSL_STATIC=1 OPENSSL_DIR=/usr + +RUN test -n "${CRYO_SHA}" || (echo "CRYO_SHA build-arg is required" >&2 && exit 1) + +RUN git clone https://github.com/paradigmxyz/cryo /src && \ + cd /src && \ + git checkout "${CRYO_SHA}" && \ + cargo build --release -p cryo_cli && \ + strip target/release/cryo + +FROM alpine:latest + +ARG CRYO_SHA + +LABEL org.opencontainers.image.title="cryo" \ + org.opencontainers.image.source="https://github.com/paradigmxyz/cryo" \ + org.opencontainers.image.revision="${CRYO_SHA}" \ + org.opencontainers.image.description="Statically linked cryo CLI, built from a pinned commit for execution-processor" + +COPY --from=builder /src/target/release/cryo /usr/local/bin/cryo + +ENTRYPOINT ["/usr/local/bin/cryo"]