Overhaul the runner fleet: arm64, 8 replicas, per-replica caches, and hbf support

Dockerfile

@@ -3,6 +3,13 @@ FROM ubuntu:24.04
 # Prevent interactive prompts during package installation
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Populated by BuildKit with "amd64" or "arm64". Must be declared WITHOUT a
+# default: a default shadows the value the builder injects, which would silently
+# fetch the wrong architecture's binaries. Steps below fall back to
+# `dpkg --print-architecture` (same amd64/arm64 vocabulary) when it is unset,
+# so non-BuildKit builds still resolve the host architecture correctly.
+ARG TARGETARCH
+
 # ============================================================================
 # Base system dependencies (GitHub Actions Runner)
 # ============================================================================
@@ -55,33 +62,94 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash
 # ============================================================================
 # Install Pkl (Apple's configuration language - used by canvas)
 # ============================================================================
-RUN curl -L -o /usr/local/bin/pkl https://github.com/apple/pkl/releases/download/0.30.1/pkl-linux-amd64 && \
+RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \
+    case "$ARCH" in \
+        amd64) PKL_ARCH=amd64 ;; \
+        arm64) PKL_ARCH=aarch64 ;; \
+        *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;; \
+    esac && \
+    curl -fL -o /usr/local/bin/pkl "https://github.com/apple/pkl/releases/download/0.30.1/pkl-linux-${PKL_ARCH}" && \
     chmod +x /usr/local/bin/pkl
 
 # ============================================================================
 # Install uv (fast Python package manager) and maturin (Rust-Python build tool)
 # ============================================================================
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \
-    # Add uv to PATH
-    . $HOME/.local/bin/env && \
-    # Install maturin globally via uv
-    uv tool install maturin
+# Installed into shared, world-readable locations rather than under /root, which
+# is mode 0700: a tool symlinked out of /root is unusable by the unprivileged
+# runner user that actually executes jobs.
+ENV UV_TOOL_DIR=/opt/uv/tools
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh && \
+    # Install maturin globally, with its launcher on the shared PATH
+    UV_TOOL_BIN_DIR=/usr/local/bin uv tool install maturin && \
+    chmod -R a+rX /opt/uv
+
+# ============================================================================
+# Web UI and Tauri desktop dependencies (hbf)
+# ============================================================================
+# Deliberately placed AFTER the espup layer. Docker invalidates every layer
+# below an edited one, and rebuilding the Xtensa toolchain costs many minutes,
+# so anything added later must stay later.
+#
+# `cargo build -p hbf-gui` links against webkit2gtk-4.1 and fails at
+# pkg-config time without the -dev package; librsvg2 and appindicator3 are
+# Tauri's SVG and tray-icon dependencies. This mirrors the apt list hbf CI
+# installs per job, minus what the firmware layers above already provide
+# (libudev-dev, pkg-config, libssl-dev).
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        libwebkit2gtk-4.1-dev libayatana-appindicator3-dev \
+        librsvg2-dev && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
-ENV PATH="/root/.local/bin:${PATH}"
+# Node is needed even though bun is the package manager, because bun does not
+# replace it as a script *interpreter*. hbf's `ts_export` test execs
+# `ui/node_modules/.bin/prettier` directly from Rust; that file is a .cjs script
+# whose shebang is `#!/usr/bin/env node`, so without node the exec fails with
+# status 127 and the drift check reports "bindings would drift". `bun run lint`
+# and `bun run check` are unaffected because `bun run` interprets the JS itself
+# and never consults the shebang -- which is exactly why this gap is invisible
+# until something shells out to a .bin entry.
+#
+# npm comes along for `npx`, which the same test falls back to when the
+# project-local binary is absent. GitHub-hosted runners preinstall both, which is
+# why this only surfaced on the fleet.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        nodejs npm && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    node --version && npx --version
+
+# bun builds the SvelteKit bundle that `tauri::generate_context!()` embeds at
+# COMPILE time, so it is a build dependency of hbf-gui rather than a test-only
+# tool. Pinned to the version hbf CI's `oven-sh/setup-bun` requests so lockfile
+# resolution is identical on both. BUN_INSTALL places the binary on the shared
+# PATH instead of under /root, which is mode 0700 and therefore invisible to the
+# unprivileged runner user -- the same trap the uv block above documents.
+ENV BUN_INSTALL=/usr/local
+RUN curl -fsSL https://bun.sh/install | bash -s "bun-v1.3.14" && \
+    chmod a+rx /usr/local/bin/bun && \
+    bun --version
 
 # ============================================================================
 # Create runner directory and download GitHub Actions Runner
 # ============================================================================
 RUN mkdir -p /actions-runner
 WORKDIR /actions-runner
 
-RUN LATEST_TAG=$(curl -s https://api.github.com/repos/actions/runner/releases/latest | jq -r .tag_name) && \
+RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \
+    case "$ARCH" in \
+        amd64) RUNNER_ARCH=x64 ;; \
+        arm64) RUNNER_ARCH=arm64 ;; \
+        *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;; \
+    esac && \
+    LATEST_TAG=$(curl -s https://api.github.com/repos/actions/runner/releases/latest | jq -r .tag_name) && \
     RUNNER_VERSION=${LATEST_TAG#v} && \
-    echo "Downloading Runner Version: ${RUNNER_VERSION}" && \
-    curl -L -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
-        "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" && \
-    tar xzf actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
-    rm actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz
+    echo "Downloading Runner Version: ${RUNNER_VERSION} (${RUNNER_ARCH})" && \
+    curl -fL -o runner.tar.gz \
+        "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" && \
+    tar xzf runner.tar.gz && \
+    rm runner.tar.gz
 
 # ============================================================================
 # Setup SSH for private repository access (submodules)
@@ -105,9 +173,12 @@ RUN useradd -m runner && \
     cp -r /root/.rustup/* /home/runner/.rustup/ 2>/dev/null || true && \
     # Copy export-esp.sh to runner home
     cp /root/export-esp.sh /home/runner/export-esp.sh 2>/dev/null || true && \
-    # Copy uv and tools to runner user
-    mkdir -p /home/runner/.local && \
-    cp -r /root/.local/* /home/runner/.local/ 2>/dev/null || true && \
+    # uv and its tools (maturin) live in /usr/local/bin and /opt/uv, which are
+    # already on the shared PATH and readable by this user — nothing to copy.
+    # Pre-create the sccache directory so its named volume is seeded with runner
+    # ownership. A volume mounted over a path that does not exist in the image is
+    # created root-owned, which the unprivileged runner cannot write to.
+    mkdir -p /home/runner/.cache/sccache && \
     # Copy SSH config to runner user
     mkdir -p /home/runner/.ssh && \
     cp /root/.ssh/known_hosts /home/runner/.ssh/ && \

README.md

@@ -29,6 +29,23 @@ This runner includes all tools required for the firmware CI pipeline:
 - **SSH** - Pre-configured with GitHub's host keys for private submodule access
 - Standard build essentials (`build-essential`, `pkg-config`, `libssl-dev`)
 
+## Architectures
+
+The image builds for both `linux/amd64` and `linux/arm64` (e.g. Apple Silicon via
+OrbStack/Docker Desktop). Architecture-specific downloads (GitHub Actions runner,
+Pkl) are selected from BuildKit's `TARGETARCH`; the Rust, ESP (`espup`) and Python
+toolchains resolve their own host architecture.
+
+Docker Compose and `docker build` produce a native image by default. To build
+explicitly for one architecture:
+
+```bash
+docker buildx build --platform linux/arm64 -t github-runner .
+```
+
+> Note: if `TARGETARCH` is unset (a build without BuildKit), the Dockerfile falls
+> back to `dpkg --print-architecture`, i.e. the base image's own architecture.
+
 ## Usage
 
 ### Environment Variables
@@ -40,6 +57,62 @@ This runner includes all tools required for the firmware CI pipeline:
 | `RUNNER_TOKEN` | One of `GITHUB_PAT` / `RUNNER_TOKEN` | Static runner registration token from GitHub. Expires ~1 hour after creation, so restarts after that will fail unless refreshed. Ignored if `GITHUB_PAT` is set. |
 | `RUNNER_NAME` | No | Base name for the runner (default: `runner`) |
 | `RUNNER_LABELS` | No | Comma-separated labels for the runner |
+| `RUNNER_CPUS` | No | CPUs per replica; also caps `CARGO_BUILD_JOBS` (default: `2`) |
+| `RUNNER_MEMORY` | No | Memory per replica (default: `6g`) |
+
+### Parallel Jobs
+
+A GitHub Actions runner executes **one job at a time** — there is no concurrency
+setting inside the runner. Total parallelism is therefore just `RUNNER_COUNT`.
+
+Eight replicas (`runner-1` .. `runner-8`) are declared explicitly in
+`docker-compose.yml`, at 2 CPUs and 6 GB each, sized for a 16-core / 64 GB host.
+`CARGO_BUILD_JOBS` is pinned to `RUNNER_CPUS` — without that, cargo sizes its
+thread pool from the *host* core count and every replica would spawn ~16
+threads, oversubscribing the machine.
+
+**Memory, not CPU, is what limits the replica count.** 8 x 6 GB = 48 GB of the
+~58 GB the OrbStack VM exposes. Adding replicas without lowering `RUNNER_MEMORY`
+will overcommit and get builds OOM-killed.
+
+A single CI run only reaches 5 concurrent jobs (four checks in parallel, then
+three builds behind `needs`). The reason more replicas still help is that
+`concurrency` in `firmware_ci.yml` is keyed per *branch*, so several runs
+execute at once and jobs queue globally.
+
+To run fewer runners, name the services; to run bigger ones, raise the limits:
+
+```bash
+docker compose up -d --build runner-1 runner-2 runner-3
+RUNNER_CPUS=4 RUNNER_MEMORY=10g docker compose up -d --build
+```
+
+> Replicas are separate services rather than `deploy.replicas` because a scaled
+> service shares one set of volumes, and sccache cannot safely share a cache
+> directory between concurrent server processes (see below).
+
+### Caching
+
+Two caches survive container recreation, both **per replica**:
+
+- **`cargo-registry-N`** — the crate download cache.
+- **`sccache-N`** — the compiler cache. `setup-rust-dual` in the firmware repo
+  points sccache at `$HOME/.cache/sccache`.
+
+Neither may be shared between replicas. sccache keeps its LRU index in memory
+per server process, so containers sharing one directory evict against each
+other. The registry was shared in an earlier revision and broke CI: unpacked
+sources under `registry/src` disappear mid-compile when another container's
+cargo garbage-collects the global cache, producing
+`could not execute process ... No such file or directory`. The cost of not
+sharing is N copies of the same crate downloads, which is the right trade.
+
+The runner's `_work` directory is deliberately **not** persisted. The firmware
+workflow checks out with `clean: false` to reuse `target/`, but a stale
+submodule `target/` surviving `git submodule deinit` is what produced
+`could not parse/generate dep info ... No such file or directory` build
+failures. sccache is content-hashed and immune to that staleness, so it is the
+right layer to persist; `_work` is not.
 
 ### Running with Docker Compose
 
@@ -48,6 +121,10 @@ This runner includes all tools required for the firmware CI pipeline:
 export URL=https://github.com/jkuracing
 export GITHUB_PAT=<your-pat>
 
-# Start the runner
-docker compose up -d
+# Start the runners
+docker compose up -d --build
 ```
+
+> Always pass `--build`. Plain `docker compose up -d` only builds when the image
+> is missing, so it will happily keep running a stale image after the Dockerfile
+> or `entrypoint.sh` changes.