diff --git a/Dockerfile b/Dockerfile index 4c9a399..877b77f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,13 @@ FROM ubuntu:24.04 # Prevent interactive prompts during package installation ENV DEBIAN_FRONTEND=noninteractive +# Populated by BuildKit with "amd64" or "arm64". Must be declared WITHOUT a +# default: a default shadows the value the builder injects, which would silently +# fetch the wrong architecture's binaries. Steps below fall back to +# `dpkg --print-architecture` (same amd64/arm64 vocabulary) when it is unset, +# so non-BuildKit builds still resolve the host architecture correctly. +ARG TARGETARCH + # ============================================================================ # Base system dependencies (GitHub Actions Runner) # ============================================================================ @@ -55,19 +62,74 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash # ============================================================================ # Install Pkl (Apple's configuration language - used by canvas) # ============================================================================ -RUN curl -L -o /usr/local/bin/pkl https://github.com/apple/pkl/releases/download/0.30.1/pkl-linux-amd64 && \ +RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ + case "$ARCH" in \ + amd64) PKL_ARCH=amd64 ;; \ + arm64) PKL_ARCH=aarch64 ;; \ + *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;; \ + esac && \ + curl -fL -o /usr/local/bin/pkl "https://github.com/apple/pkl/releases/download/0.30.1/pkl-linux-${PKL_ARCH}" && \ chmod +x /usr/local/bin/pkl # ============================================================================ # Install uv (fast Python package manager) and maturin (Rust-Python build tool) # ============================================================================ -RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ - # Add uv to PATH - . $HOME/.local/bin/env && \ - # Install maturin globally via uv - uv tool install maturin +# Installed into shared, world-readable locations rather than under /root, which +# is mode 0700: a tool symlinked out of /root is unusable by the unprivileged +# runner user that actually executes jobs. +ENV UV_TOOL_DIR=/opt/uv/tools + +RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh && \ + # Install maturin globally, with its launcher on the shared PATH + UV_TOOL_BIN_DIR=/usr/local/bin uv tool install maturin && \ + chmod -R a+rX /opt/uv + +# ============================================================================ +# Web UI and Tauri desktop dependencies (hbf) +# ============================================================================ +# Deliberately placed AFTER the espup layer. Docker invalidates every layer +# below an edited one, and rebuilding the Xtensa toolchain costs many minutes, +# so anything added later must stay later. +# +# `cargo build -p hbf-gui` links against webkit2gtk-4.1 and fails at +# pkg-config time without the -dev package; librsvg2 and appindicator3 are +# Tauri's SVG and tray-icon dependencies. This mirrors the apt list hbf CI +# installs per job, minus what the firmware layers above already provide +# (libudev-dev, pkg-config, libssl-dev). +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libwebkit2gtk-4.1-dev libayatana-appindicator3-dev \ + librsvg2-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists/* -ENV PATH="/root/.local/bin:${PATH}" +# Node is needed even though bun is the package manager, because bun does not +# replace it as a script *interpreter*. hbf's `ts_export` test execs +# `ui/node_modules/.bin/prettier` directly from Rust; that file is a .cjs script +# whose shebang is `#!/usr/bin/env node`, so without node the exec fails with +# status 127 and the drift check reports "bindings would drift". `bun run lint` +# and `bun run check` are unaffected because `bun run` interprets the JS itself +# and never consults the shebang -- which is exactly why this gap is invisible +# until something shells out to a .bin entry. +# +# npm comes along for `npx`, which the same test falls back to when the +# project-local binary is absent. GitHub-hosted runners preinstall both, which is +# why this only surfaced on the fleet. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + nodejs npm && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + node --version && npx --version + +# bun builds the SvelteKit bundle that `tauri::generate_context!()` embeds at +# COMPILE time, so it is a build dependency of hbf-gui rather than a test-only +# tool. Pinned to the version hbf CI's `oven-sh/setup-bun` requests so lockfile +# resolution is identical on both. BUN_INSTALL places the binary on the shared +# PATH instead of under /root, which is mode 0700 and therefore invisible to the +# unprivileged runner user -- the same trap the uv block above documents. +ENV BUN_INSTALL=/usr/local +RUN curl -fsSL https://bun.sh/install | bash -s "bun-v1.3.14" && \ + chmod a+rx /usr/local/bin/bun && \ + bun --version # ============================================================================ # Create runner directory and download GitHub Actions Runner @@ -75,13 +137,19 @@ ENV PATH="/root/.local/bin:${PATH}" RUN mkdir -p /actions-runner WORKDIR /actions-runner -RUN LATEST_TAG=$(curl -s https://api.github.com/repos/actions/runner/releases/latest | jq -r .tag_name) && \ +RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ + case "$ARCH" in \ + amd64) RUNNER_ARCH=x64 ;; \ + arm64) RUNNER_ARCH=arm64 ;; \ + *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;; \ + esac && \ + LATEST_TAG=$(curl -s https://api.github.com/repos/actions/runner/releases/latest | jq -r .tag_name) && \ RUNNER_VERSION=${LATEST_TAG#v} && \ - echo "Downloading Runner Version: ${RUNNER_VERSION}" && \ - curl -L -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \ - "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" && \ - tar xzf actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \ - rm actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz + echo "Downloading Runner Version: ${RUNNER_VERSION} (${RUNNER_ARCH})" && \ + curl -fL -o runner.tar.gz \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" && \ + tar xzf runner.tar.gz && \ + rm runner.tar.gz # ============================================================================ # Setup SSH for private repository access (submodules) @@ -105,9 +173,12 @@ RUN useradd -m runner && \ cp -r /root/.rustup/* /home/runner/.rustup/ 2>/dev/null || true && \ # Copy export-esp.sh to runner home cp /root/export-esp.sh /home/runner/export-esp.sh 2>/dev/null || true && \ - # Copy uv and tools to runner user - mkdir -p /home/runner/.local && \ - cp -r /root/.local/* /home/runner/.local/ 2>/dev/null || true && \ + # uv and its tools (maturin) live in /usr/local/bin and /opt/uv, which are + # already on the shared PATH and readable by this user — nothing to copy. + # Pre-create the sccache directory so its named volume is seeded with runner + # ownership. A volume mounted over a path that does not exist in the image is + # created root-owned, which the unprivileged runner cannot write to. + mkdir -p /home/runner/.cache/sccache && \ # Copy SSH config to runner user mkdir -p /home/runner/.ssh && \ cp /root/.ssh/known_hosts /home/runner/.ssh/ && \ diff --git a/README.md b/README.md index ad8a82d..318003d 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,23 @@ This runner includes all tools required for the firmware CI pipeline: - **SSH** - Pre-configured with GitHub's host keys for private submodule access - Standard build essentials (`build-essential`, `pkg-config`, `libssl-dev`) +## Architectures + +The image builds for both `linux/amd64` and `linux/arm64` (e.g. Apple Silicon via +OrbStack/Docker Desktop). Architecture-specific downloads (GitHub Actions runner, +Pkl) are selected from BuildKit's `TARGETARCH`; the Rust, ESP (`espup`) and Python +toolchains resolve their own host architecture. + +Docker Compose and `docker build` produce a native image by default. To build +explicitly for one architecture: + +```bash +docker buildx build --platform linux/arm64 -t github-runner . +``` + +> Note: if `TARGETARCH` is unset (a build without BuildKit), the Dockerfile falls +> back to `dpkg --print-architecture`, i.e. the base image's own architecture. + ## Usage ### Environment Variables @@ -40,6 +57,62 @@ This runner includes all tools required for the firmware CI pipeline: | `RUNNER_TOKEN` | One of `GITHUB_PAT` / `RUNNER_TOKEN` | Static runner registration token from GitHub. Expires ~1 hour after creation, so restarts after that will fail unless refreshed. Ignored if `GITHUB_PAT` is set. | | `RUNNER_NAME` | No | Base name for the runner (default: `runner`) | | `RUNNER_LABELS` | No | Comma-separated labels for the runner | +| `RUNNER_CPUS` | No | CPUs per replica; also caps `CARGO_BUILD_JOBS` (default: `2`) | +| `RUNNER_MEMORY` | No | Memory per replica (default: `6g`) | + +### Parallel Jobs + +A GitHub Actions runner executes **one job at a time** — there is no concurrency +setting inside the runner. Total parallelism is therefore just `RUNNER_COUNT`. + +Eight replicas (`runner-1` .. `runner-8`) are declared explicitly in +`docker-compose.yml`, at 2 CPUs and 6 GB each, sized for a 16-core / 64 GB host. +`CARGO_BUILD_JOBS` is pinned to `RUNNER_CPUS` — without that, cargo sizes its +thread pool from the *host* core count and every replica would spawn ~16 +threads, oversubscribing the machine. + +**Memory, not CPU, is what limits the replica count.** 8 x 6 GB = 48 GB of the +~58 GB the OrbStack VM exposes. Adding replicas without lowering `RUNNER_MEMORY` +will overcommit and get builds OOM-killed. + +A single CI run only reaches 5 concurrent jobs (four checks in parallel, then +three builds behind `needs`). The reason more replicas still help is that +`concurrency` in `firmware_ci.yml` is keyed per *branch*, so several runs +execute at once and jobs queue globally. + +To run fewer runners, name the services; to run bigger ones, raise the limits: + +```bash +docker compose up -d --build runner-1 runner-2 runner-3 +RUNNER_CPUS=4 RUNNER_MEMORY=10g docker compose up -d --build +``` + +> Replicas are separate services rather than `deploy.replicas` because a scaled +> service shares one set of volumes, and sccache cannot safely share a cache +> directory between concurrent server processes (see below). + +### Caching + +Two caches survive container recreation, both **per replica**: + +- **`cargo-registry-N`** — the crate download cache. +- **`sccache-N`** — the compiler cache. `setup-rust-dual` in the firmware repo + points sccache at `$HOME/.cache/sccache`. + +Neither may be shared between replicas. sccache keeps its LRU index in memory +per server process, so containers sharing one directory evict against each +other. The registry was shared in an earlier revision and broke CI: unpacked +sources under `registry/src` disappear mid-compile when another container's +cargo garbage-collects the global cache, producing +`could not execute process ... No such file or directory`. The cost of not +sharing is N copies of the same crate downloads, which is the right trade. + +The runner's `_work` directory is deliberately **not** persisted. The firmware +workflow checks out with `clean: false` to reuse `target/`, but a stale +submodule `target/` surviving `git submodule deinit` is what produced +`could not parse/generate dep info ... No such file or directory` build +failures. sccache is content-hashed and immune to that staleness, so it is the +right layer to persist; `_work` is not. ### Running with Docker Compose @@ -48,6 +121,10 @@ This runner includes all tools required for the firmware CI pipeline: export URL=https://github.com/jkuracing export GITHUB_PAT= -# Start the runner -docker compose up -d +# Start the runners +docker compose up -d --build ``` + +> Always pass `--build`. Plain `docker compose up -d` only builds when the image +> is missing, so it will happily keep running a stale image after the Dockerfile +> or `entrypoint.sh` changes. diff --git a/docker-compose.yml b/docker-compose.yml index 235563e..0bcc36a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,149 @@ +# Replicas are declared explicitly rather than via `deploy.replicas` because +# every replica of a scaled service shares one set of volumes, and NOTHING here +# is safe to share between concurrently building runners: +# +# - sccache keeps its LRU index in memory per server process, so several +# containers on one cache directory evict against each other. +# - The cargo registry was shared here initially and caused real CI failures: +# error: could not compile `crc32fast` (lib) +# Caused by: could not execute process `.../bin/rustc --crate-name +# crc32fast .../registry/src/index.crates.io-*/crc32fast-1.5.0/src/lib.rs` +# Caused by: No such file or directory (os error 2) +# Unpacked sources under registry/src vanish mid-compile when another +# container's cargo garbage-collects the global cache, so the spawn fails on +# a working directory that no longer exists. Cargo's package-cache lock does +# not protect a build for its whole duration across separate containers. +# +# Each runner therefore gets its own registry AND sccache volume, which is only +# expressible as its own service. The cost is N copies of the crate downloads. +# +# `docker compose up -d --build` starts all 12. To run fewer, name them: +# docker compose up -d --build runner-1 runner-2 runner-3 +# +# `docker compose build` MUST be run with no service argument. Each service +# declares its own `build: .`, so compose tags a separate image per service, and +# `docker compose build runner-1` silently leaves the others on the old image -- +# they still start and register, so nothing looks wrong until a job needs a tool +# only the rebuilt image has. +x-runner: &runner + build: . + # `on-failure:5` used to be the policy here and it cost the fleet ten days of + # downtime: a bug in entrypoint.sh's config cleanup made every replica exit 1 + # on restart, the five retries were spent in seconds, and Docker then left all + # containers dead with no surviving process to notice. A runner fleet + # should heal rather than latch off, and the entrypoint mints a fresh + # registration token per start, so a genuinely broken image loops visibly in + # the logs instead of failing silently. + restart: unless-stopped + stop_grace_period: 5m + environment: + URL: ${URL} + GITHUB_PAT: ${GITHUB_PAT} + # Defaulted to empty: entrypoint.sh prefers GITHUB_PAT and only falls back + # to a static token, but an unset variable makes `docker compose` print a + # warning per service per invocation -- noise that buries the real errors + # underneath. + RUNNER_TOKEN: ${RUNNER_TOKEN:-} + RUNNER_NAME: ${RUNNER_NAME:-} + # A runner is offered a job only when its label set is a SUPERSET of the + # job's `runs-on`. Both labels therefore go on every replica: splitting them + # across replicas would leave some containers ineligible for hbf jobs and + # idle whenever hbf work is all that is queued. firmware_ci.yml asks for + # [fw-builder]; hbf asks for [hbf-builder]; every replica can serve either. + RUNNER_LABELS: ${RUNNER_LABELS:-fw-builder,hbf-builder} + # Match cargo's internal parallelism to this replica's CPU allotment. + # cargo defaults to one codegen unit per *host* core, so without this each + # replica would spawn ~16 threads and every replica would oversubscribe the + # machine. The cpus limit below only throttles the result; capping the thread + # count is what actually avoids the thrashing. + CARGO_BUILD_JOBS: ${RUNNER_CPUS:-2} + deploy: + resources: + limits: + # 12 replicas at 2 CPU / 4 GB on a 16-core / 64 GB host. + # + # This block previously read "Memory is the binding constraint, not CPU" + # at 8 x 6 GB. Measured under a full load of firmware and hbf jobs, that + # was wrong on both counts: peak usage across all replicas was 756 MiB + # against the 6 GiB limit -- an 8x overshoot -- and only 4 of 8 + # containers were computing at all (~195% CPU each), the rest sitting + # near idle on network and setup. Roughly half the host's cores were + # unused while jobs queued. + # + # The real constraint was SLOTS. An hbf run measured 16.8 minutes of job + # time inside an 11.8 minute span -- an average concurrency of 1.4 -- + # because firmware held 7 of the 8 slots. So: more, smaller replicas. + # + # Total memory is unchanged at 48 GB of the ~58 GB the VM exposes. CPU is + # deliberately oversubscribed 1.5:1 (12 x 2 = 24 on 16 cores), which the + # observed idle time justifies. + # + # DISK is now the limiting factor, not memory: each replica keeps its own + # target/ for both repositories on one 200 GB volume, which was 47% full + # at 8 replicas. Watch `docker system df` before going wider. + cpus: ${RUNNER_CPUS:-2} + memory: ${RUNNER_MEMORY:-4g} + services: - github-runner: - build: . - restart: on-failure:5 - stop_grace_period: 5m - environment: - URL: ${URL} - GITHUB_PAT: ${GITHUB_PAT} - RUNNER_TOKEN: ${RUNNER_TOKEN} - RUNNER_NAME: ${RUNNER_NAME} - RUNNER_LABELS: ${RUNNER_LABELS} - deploy: - replicas: ${RUNNER_COUNT:-1} + runner-1: + <<: *runner + volumes: [cargo-registry-1:/home/runner/.cargo/registry, sccache-1:/home/runner/.cache/sccache] + runner-2: + <<: *runner + volumes: [cargo-registry-2:/home/runner/.cargo/registry, sccache-2:/home/runner/.cache/sccache] + runner-3: + <<: *runner + volumes: [cargo-registry-3:/home/runner/.cargo/registry, sccache-3:/home/runner/.cache/sccache] + runner-4: + <<: *runner + volumes: [cargo-registry-4:/home/runner/.cargo/registry, sccache-4:/home/runner/.cache/sccache] + runner-5: + <<: *runner + volumes: [cargo-registry-5:/home/runner/.cargo/registry, sccache-5:/home/runner/.cache/sccache] + runner-6: + <<: *runner + volumes: [cargo-registry-6:/home/runner/.cargo/registry, sccache-6:/home/runner/.cache/sccache] + runner-7: + <<: *runner + volumes: [cargo-registry-7:/home/runner/.cargo/registry, sccache-7:/home/runner/.cache/sccache] + runner-8: + <<: *runner + volumes: [cargo-registry-8:/home/runner/.cargo/registry, sccache-8:/home/runner/.cache/sccache] + runner-9: + <<: *runner + volumes: [cargo-registry-9:/home/runner/.cargo/registry, sccache-9:/home/runner/.cache/sccache] + runner-10: + <<: *runner + volumes: [cargo-registry-10:/home/runner/.cargo/registry, sccache-10:/home/runner/.cache/sccache] + runner-11: + <<: *runner + volumes: [cargo-registry-11:/home/runner/.cargo/registry, sccache-11:/home/runner/.cache/sccache] + runner-12: + <<: *runner + volumes: [cargo-registry-12:/home/runner/.cargo/registry, sccache-12:/home/runner/.cache/sccache] volumes: - runner-data: + cargo-registry-1: + cargo-registry-2: + cargo-registry-3: + cargo-registry-4: + cargo-registry-5: + cargo-registry-6: + cargo-registry-7: + cargo-registry-8: + cargo-registry-9: + cargo-registry-10: + cargo-registry-11: + cargo-registry-12: + sccache-1: + sccache-2: + sccache-3: + sccache-4: + sccache-5: + sccache-6: + sccache-7: + sccache-8: + sccache-9: + sccache-10: + sccache-11: + sccache-12: diff --git a/entrypoint.sh b/entrypoint.sh index b74e239..88c35b2 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -18,6 +18,16 @@ FULL_RUNNER_NAME="${RUNNER_NAME}-${HOSTNAME}" echo "Fixing permissions for /actions-runner..." chown -R runner:runner /actions-runner +# These are named volumes. Docker seeds them from the image with the right +# ownership, but a volume created before the directory existed in the image (or +# by another image) comes back root-owned and silently breaks every build. +for vol_dir in /home/runner/.cargo/registry /home/runner/.cache/sccache; do + if [[ -d "$vol_dir" ]] && [[ "$(stat -c %U "$vol_dir")" != "runner" ]]; then + echo "Fixing permissions for ${vol_dir}..." + chown -R runner:runner "$vol_dir" + fi +done + # Fetches a short-lived token ($1: "registration-token" or "remove-token") from the # GitHub API, using GITHUB_PAT. Prints the token on stdout, returns non-zero on failure. fetch_runner_token() { @@ -71,8 +81,24 @@ if [[ -f /home/runner/export-esp.sh ]]; then fi echo "Removing any existing runner configuration..." -# Clean up previous runs (crucial for ephemeral runners) -rm -f .runner .credentials .credentials_rsaparams +# Clean up previous runs (crucial for ephemeral runners). +# +# `.runner_migrated` MUST be in this list. The runner self-updates in place, and +# a post-update runner drops that marker beside its config. `config.sh` treats +# the marker ALONE as proof the runner is already configured -- verified by +# creating only `.runner_migrated` and passing a deliberately bogus token: it +# fails with "Cannot configure the runner because it is already configured" +# without even attempting to authenticate. +# +# Because the old list stopped at `.credentials_rsaparams`, every replica that +# had auto-updated crash-looped on its next restart until `restart: +# on-failure:5` exhausted its retries, which silently took the entire fleet +# offline about ten days after it was last rebuilt. Deleting the marker is +# correct rather than merely expedient: this entrypoint always reconfigures from +# a freshly minted registration token, so there is no migrated state worth +# preserving across a restart. +rm -f .runner .credentials .credentials_rsaparams \ + .runner_migrated .credentials_migrated echo "Configuring GitHub Actions Runner as ${FULL_RUNNER_NAME}..." echo "URL: $URL"