diff --git a/.github/ciod/prefetch.sh b/.github/ciod/prefetch.sh new file mode 100755 index 0000000..dfded75 --- /dev/null +++ b/.github/ciod/prefetch.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Prefetches the pinned Redis image of the store tests (tests/redis-image.sh) as an OCI image +# archive that `docker load` accepts: $CI_CACHE_DIR/web-proxy/tools/redis-.tar, with +# a .complete marker. Prints "redis image: prefetched PATH" when it was already there, otherwise +# "redis image: installed now PATH"; exits non-zero on any failure. +# +# The nightly CI image build runs it with no argument from a fresh clone of main, as a user that +# may write only $CI_CACHE_DIR/web-proxy, with HOME=$CI_CACHE_DIR/web-proxy and no Docker. The +# rust-test part of .github/scripts/ci-run.sh runs it as the fallback for images built without it. +# +# No Docker, so it uses the registry API v2 with an anonymous token: Docker Hub first, and +# public.ecr.aws (the same digests) only if Docker Hub fails; one attempt each, no retry. Every +# body is checked against its sha256 digest before use, so the bytes are the pinned image whichever +# registry sent them. The archive is built in a temporary folder and renamed into place. +# REGISTRY_URL, MIRROR_URL and AUTH_URL (the Docker Hub token service) override the endpoints. +set -euo pipefail +: "${CI_CACHE_DIR:?is not set}" +# shellcheck source=../../tests/redis-image.sh +. "$(dirname "$0")/../../tests/redis-image.sh" +registry=${REGISTRY_URL:-https://registry-1.docker.io} +mirror=${MIRROR_URL:-https://public.ecr.aws} +auth=${AUTH_URL:-https://auth.docker.io/token} +tools="$CI_CACHE_DIR/web-proxy/tools" +archive="$tools/$REDIS_ARCHIVE_NAME" +index_types=application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json +manifest_types=application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json + +if [ -f "$archive.complete" ] && [ -f "$archive" ]; then + echo "redis image: prefetched $archive" + exit 0 +fi + +# get URL TOKEN ACCEPT DIGEST OUT: one GET, then the body's sha256 must be DIGEST. +get() { + curl -fsSL --connect-timeout 10 --max-time 120 -H "Authorization: Bearer $2" -H "Accept: $3" \ + -o "$5" "$1" || return 1 + if [ "sha256:$(sha256sum "$5" | cut -d ' ' -f 1)" != "$4" ]; then + echo "redis image: digest mismatch for $4 from $1" >&2 + return 1 + fi +} + +# download BASE REPO TOKEN_URL DIR: the linux/amd64 image from one registry into DIR as an OCI +# image layout. Called in a condition, so every step checks its own status. +download() { + local base=$1 repo=$2 dir=$4 token manifest blob blobs list + token=$(curl -fsS --connect-timeout 10 --max-time 30 "$3" | jq -er .token) || return 1 + blobs="$dir/blobs/sha256" + mkdir -p "$blobs" || return 1 + get "$base/v2/$repo/manifests/$REDIS_INDEX_DIGEST" "$token" "$index_types" "$REDIS_INDEX_DIGEST" \ + "$dir/image-index.json" || return 1 + manifest=$(jq -er '[.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64") + | .digest][0] // empty' "$dir/image-index.json") || return 1 + get "$base/v2/$repo/manifests/$manifest" "$token" "$manifest_types" "$manifest" \ + "$blobs/${manifest#sha256:}" || return 1 + list=$(jq -er '.config.digest, .layers[].digest' "$blobs/${manifest#sha256:}") || return 1 + for blob in $list; do + get "$base/v2/$repo/blobs/$blob" "$token" '*/*' "$blob" "$blobs/${blob#sha256:}" || return 1 + done + # The layout's own index names the one manifest; docker load reads the image name from it. + jq -n --arg digest "$manifest" --argjson size "$(wc -c < "$blobs/${manifest#sha256:}")" \ + --arg type "$(jq -r '.mediaType // "application/vnd.oci.image.manifest.v1+json"' "$blobs/${manifest#sha256:}")" \ + --arg name "docker.io/$REDIS_LOCAL_IMAGE" --arg tag "$REDIS_TAG" \ + '{schemaVersion: 2, mediaType: "application/vnd.oci.image.index.v1+json", + manifests: [{mediaType: $type, digest: $digest, size: $size, + platform: {os: "linux", architecture: "amd64"}, + annotations: {"io.containerd.image.name": $name, "org.opencontainers.image.ref.name": $tag}}]}' \ + > "$dir/index.json" || return 1 + rm -f "$dir/image-index.json" + echo '{"imageLayoutVersion":"1.0.0"}' > "$dir/oci-layout" +} + +mkdir -p "$tools" +work=$(mktemp -d "$tools/.redis.XXXXXX") +trap 'rm -rf "$work"' EXIT +hub_token="$auth?service=registry.docker.io&scope=repository:$REDIS_REPO_HUB:pull" +mirror_token="$mirror/token/?scope=repository:$REDIS_REPO_MIRROR:pull" +if ! download "$registry" "$REDIS_REPO_HUB" "$hub_token" "$work/layout"; then + echo "redis image: $registry failed; trying $mirror" >&2 + rm -rf "$work/layout" + if ! download "$mirror" "$REDIS_REPO_MIRROR" "$mirror_token" "$work/layout"; then + echo "redis image: download failed from $registry and $mirror" >&2 + exit 1 + fi +fi +tar -cf "$work/image.tar" -C "$work/layout" oci-layout index.json blobs +rm -f "$archive.complete" +mv -f "$work/image.tar" "$archive" +touch "$archive.complete" +echo "redis image: installed now $archive" diff --git a/.github/scripts/ci-run.sh b/.github/scripts/ci-run.sh index 0f45427..20a4028 100755 --- a/.github/scripts/ci-run.sh +++ b/.github/scripts/ci-run.sh @@ -9,6 +9,7 @@ # EVENT the GitHub event name # ACTIONLINT the verified actionlint binary # CI_EVIDENCE, RUNNER_TEMP evidence and scratch directories +# CI_CACHE_DIR the runner's cache (rust-test: the prefetched Redis image) # A part that does not run is recorded as skipped with the rule that skipped it. set -euo pipefail cd "$(git rev-parse --show-toplevel)" @@ -42,6 +43,20 @@ docs() { fi } +# The store tests' Redis never comes from a registry here: the server image prefetched it +# (.github/ciod/prefetch.sh; on an image built without the hook, the hook downloads it now, once), +# docker loads the archive and tests/allowance-redis.sh runs it with --pull never and no network. +rust_test() { + local state + # shellcheck source=tests/redis-image.sh + . tests/redis-image.sh + state=$(bash .github/ciod/prefetch.sh) + echo "$state" | tee "$CI_EVIDENCE/redis-image.txt" + docker load -i "$CI_CACHE_DIR/web-proxy/tools/$REDIS_ARCHIVE_NAME" + REDIS_IMAGE=$REDIS_LOCAL_IMAGE REDIS_PULL=never sh tests/allowance-redis.sh \ + cargo nextest run --locked --profile "$NEXTEST_PROFILE" --target-dir target/test --run-ignored all +} + node_test() { local files=(tests/*.test.mjs) if [ "$MODE" = full ]; then @@ -60,7 +75,7 @@ part actionlint "$ACTIONLINT" if [ "$SCOPE_RUST" = true ]; then part fmt cargo fmt --check part clippy cargo clippy --all-targets --locked --target-dir target/clippy -- -D warnings - part rust-test cargo nextest run --locked --profile "$NEXTEST_PROFILE" --target-dir target/test + part rust-test rust_test else rule=$rust_rule if [ "$SCOPE_NODE" != true ]; then rule=$docs_rule; fi @@ -90,4 +105,14 @@ else part_skip npm-audit "$package_rule" fi -parts_wait +status=0 +parts_wait || status=$? +# rust-test's Redis image line goes to the job summary; a download on this run is a warning. +if [ -s "$CI_EVIDENCE/redis-image.txt" ]; then + state=$(cat "$CI_EVIDENCE/redis-image.txt") + printf '\n%s\n' "$state" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + if [[ $state == "redis image: installed now"* ]]; then + echo "::warning::$state: this server image was built without .github/ciod/prefetch.sh" + fi +fi +exit "$status" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bc2da6..431580d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,13 @@ # One job, tier1, on one fresh CI-SelfHosted server per run ([self-hosted, ci-8cpu]; a dispatch # may pick ci-16cpu or ci-32cpu). Its "Checks and tests in parallel" step runs every part at the # same time through .github/scripts/ci-run.sh (parts runner: .github/scripts/ci-parts.sh): policy, -# docs, actionlint, fmt, clippy, rust-test (cargo nextest), npm-ci, then node-test, then vendor, +# docs, actionlint, fmt, clippy, rust-test (cargo nextest, with the real-Redis store tests on a +# private Redis from tests/allowance-redis.sh), npm-ci, then node-test, then vendor, # and npm-audit. The job summary holds the parts table; a failed part's log is printed in a group. +# rust-test's Redis image is never pulled from a registry: the server image prefetched it into +# CI_CACHE_DIR (.github/ciod/prefetch.sh, verified by digest), docker loads it and Redis runs with +# --pull never. On a server image built without the hook, the hook downloads it once with a warning; +# the summary says "redis image: prefetched" or "redis image: installed now". # # Triggers and modes: # - pull_request (ready, not draft): fast mode. Rust tests named slow_* and tests/slow/*.test.mjs @@ -22,8 +27,7 @@ # The server's job-started guard refuses forks, pull requests by anyone but the owner and every # event not listed above, before any step runs. # Not run here: the Docker stack suites, browser tests, live smoke tests, the credential scan -# (tests/secret-scan.py) and the 11 real-Redis store tests, which need REDIS_TEST_URL. Run them -# locally as ENGINEERING.md "Testing and validation" describes. +# (tests/secret-scan.py). Run them locally as ENGINEERING.md "Testing and validation" describes. name: ci on: diff --git a/ENGINEERING.md b/ENGINEERING.md index 876a4ee..adb6a19 100644 --- a/ENGINEERING.md +++ b/ENGINEERING.md @@ -589,7 +589,7 @@ The gateway binds sessions to their assigned region and rejects a region mismatc | `actionlint` | actionlint 1.7.12 (pinned, checksum-verified download), all workflows | always | | `fmt` | `cargo fmt --check` | Rust inputs changed | | `clippy` | `cargo clippy --all-targets --locked --target-dir target/clippy -- -D warnings` | Rust inputs changed | -| `rust-test` | `cargo nextest run --locked --profile ci` (or `full`) `--target-dir target/test` | Rust inputs changed | +| `rust-test` | the Redis image (`.github/ciod/prefetch.sh`, then `docker load`), then `sh tests/allowance-redis.sh cargo nextest run --locked --profile ci` (or `full`) `--target-dir target/test --run-ignored all` | Rust inputs changed | | `npm-ci` | `npm ci --ignore-scripts --no-audit --no-fund --prefer-offline` | not documentation only | | `node-test` | `node --test --test-timeout=90000 tests/*.test.mjs` (plus `tests/slow/*.test.mjs` in full mode), after `npm-ci` | not documentation only | | `vendor` | `node scripts/vendor.mjs`, after `node-test` (it rewrites `web/runtime/vendor/`, which the tests read) | not documentation only | @@ -615,7 +615,9 @@ Merging: pull requests merge only through the merge queue, as squash commits. Th Server size and times: `ci-8cpu` is the smallest size whose measured runs all keep the parts step under 90 seconds and the job under 4 minutes. Measured with a warm cache (three manual runs per size, November 2026): on `ci-8cpu` the parts step takes 22–24 seconds and the job 33–34 seconds, with sccache at 100% and about $0.002 per run; `ci-16cpu` takes 13–15 seconds and `ci-32cpu` 9–11 seconds for the parts step, at 1.3 to 2.4 times the cost. Getting a fresh server takes about 90–115 seconds before the job starts, whatever the size; the slowest test takes under 3 seconds. -CI does not run the Docker stack tests (`integration.mjs`, `limits.mjs`, `load.mjs`, `failover.mjs`, `configuration.mjs`, `allowance-redis.sh` and the other stack scripts), the browser tests, the credential scan or the live smoke tests: they need Docker, a running stack with Chrome, or private credentials. The 11 real-Redis store tests are skipped because they need an isolated `REDIS_TEST_URL`. Run them locally as described below. The CI server refuses pull requests from forks or from anyone but the repository owner, before any step runs. +CI runs the 11 real-Redis store tests (`src/store_tests.rs`) in `rust-test`: `tests/allowance-redis.sh` starts Redis on the server's rootless Docker with `--pull never`, `--network none`, a Unix socket in a temporary folder and no published port, runs nextest with `REDIS_TEST_URL` set and removes the container afterwards, also when tests fail. The image is the `redis:7.4-alpine` index digest pinned in `tests/redis-image.sh`, and a test run never reaches a registry for it. The nightly CI server image build runs the repository's hook `.github/ciod/prefetch.sh`, which downloads the linux/amd64 image through the registry API (Docker Hub, or `public.ecr.aws` with the same digests when Docker Hub fails; one attempt each), checks every part against its sha256 digest and stores it as an OCI archive in the runner's cache. `rust-test` runs the same hook first: on a prefetched image it does nothing, and on a server image built without the hook it downloads the image once, with a warning. It then loads the archive with `docker load`. The job summary says `redis image: prefetched` or `redis image: installed now`. A failed download, a digest mismatch, a failed load or a Redis that does not start fails the part; nothing is retried. Mark a Rust test `#[ignore]` only when it needs `REDIS_TEST_URL`; CI runs every ignored test (`--run-ignored all`), so `#[ignore]` never hides a test from CI. + +CI does not run the Docker stack tests (`integration.mjs`, `limits.mjs`, `load.mjs`, `failover.mjs`, `configuration.mjs` and the other stack scripts), the browser tests, the credential scan or the live smoke tests: they need a running stack with Chrome, or private credentials. Run them locally as described below. The CI server refuses pull requests from forks or from anyone but the repository owner, before any step runs. Local equivalents of the CI commands: @@ -624,7 +626,7 @@ bash .github/scripts/repo-policy.sh python3 scripts/render-readme.py && git diff --exit-code -- docs/readme.html docs/engineering.html cargo fmt --check cargo clippy --all-targets --locked -- -D warnings -cargo nextest run --locked --profile ci # or: cargo test --locked +sh tests/allowance-redis.sh cargo nextest run --locked --profile ci --run-ignored all # Linux Docker npm ci --ignore-scripts node --test tests/*.test.mjs node scripts/vendor.mjs diff --git a/docs/engineering.html b/docs/engineering.html index f73b05a..19ce4fc 100644 --- a/docs/engineering.html +++ b/docs/engineering.html @@ -243,7 +243,7 @@

HAProxy, regions, and recovery

Testing and validation

Continuous integration

.github/workflows/ci.yml is the merge gate. It runs on the shared self-hosted CI pool (CI-SelfHosted servers; GitHub-hosted runners are never used) as one job, tier1, on one fresh server per run: [self-hosted, ci-8cpu], or the size a manual run picks. Its step "Checks and tests in parallel" starts every part at the same time through .github/scripts/ci-run.sh (parts runner: .github/scripts/ci-parts.sh). Each part has its own log; the job summary shows the parts table (result and seconds per part), and the log of every failed part is printed in its own group. A part that is killed fails the run as "no result". Each run checks the pull request's own head commit.

-
PartCommandRuns when
policybash .github/scripts/repo-policy.shalways
docspython3 scripts/render-readme.py, then git diff --exit-code -- docs/readme.html docs/engineering.html, in its own worktreealways
actionlintactionlint 1.7.12 (pinned, checksum-verified download), all workflowsalways
fmtcargo fmt --checkRust inputs changed
clippycargo clippy --all-targets --locked --target-dir target/clippy -- -D warningsRust inputs changed
rust-testcargo nextest run --locked --profile ci (or full) --target-dir target/testRust inputs changed
npm-cinpm ci --ignore-scripts --no-audit --no-fund --prefer-offlinenot documentation only
node-testnode --test --test-timeout=90000 tests/*.test.mjs (plus tests/slow/*.test.mjs in full mode), after npm-cinot documentation only
vendornode scripts/vendor.mjs, after node-test (it rewrites web/runtime/vendor/, which the tests read)not documentation only
npm-auditnpm audit --audit-level=high --package-lock-onlypull requests that change package files or .github/, and manual runs
+
PartCommandRuns when
policybash .github/scripts/repo-policy.shalways
docspython3 scripts/render-readme.py, then git diff --exit-code -- docs/readme.html docs/engineering.html, in its own worktreealways
actionlintactionlint 1.7.12 (pinned, checksum-verified download), all workflowsalways
fmtcargo fmt --checkRust inputs changed
clippycargo clippy --all-targets --locked --target-dir target/clippy -- -D warningsRust inputs changed
rust-testthe Redis image (.github/ciod/prefetch.sh, then docker load), then sh tests/allowance-redis.sh cargo nextest run --locked --profile ci (or full) --target-dir target/test --run-ignored allRust inputs changed
npm-cinpm ci --ignore-scripts --no-audit --no-fund --prefer-offlinenot documentation only
node-testnode --test --test-timeout=90000 tests/*.test.mjs (plus tests/slow/*.test.mjs in full mode), after npm-cinot documentation only
vendornode scripts/vendor.mjs, after node-test (it rewrites web/runtime/vendor/, which the tests read)not documentation only
npm-auditnpm audit --audit-level=high --package-lock-onlypull requests that change package files or .github/, and manual runs

Before the parts, the job checks that rust-toolchain.toml (Rust 1.94.1 with rustfmt and clippy) names the version of the Dockerfile's FROM rust:<version>- build stage, so change both together, and that Node is major 24, as in the Dockerfile's web stage. The CI server image pre-installs the toolchain rust-toolchain.toml names. Rust builds share the pool's sccache; build folders are set with --target-dir, never with a CARGO_* variable, which sccache would put into its cache key.

Triggers and modes:

@@ -252,13 +252,14 @@

Continuous integration

Slow tests and the 90-second rule: pull requests leave out Rust tests whose name starts with slow_ and JavaScript test files under tests/slow/; the merge queue, the 6-hour run and manual runs include them. In every mode a test that runs 90 seconds fails the run (nextest slow-timeout in .config/nextest.toml, node --test-timeout and the "CI time" step, .github/scripts/ci-timing.py), and a test over 60 seconds gets a warning to mark it slow. Tests are never retried (retries = 0); a flaky test gets its own issue. The "CI time" step adds the test counts and the slowest tests to the job summary, and the run keeps them with the JUnit files as the ci-timing artifact for 90 days.

Merging: pull requests merge only through the merge queue, as squash commits. The required checks are tier1, maestro-policy, maestro/review and maestro/qa, enforced for administrators too.

Server size and times: ci-8cpu is the smallest size whose measured runs all keep the parts step under 90 seconds and the job under 4 minutes. Measured with a warm cache (three manual runs per size, November 2026): on ci-8cpu the parts step takes 22–24 seconds and the job 33–34 seconds, with sccache at 100% and about $0.002 per run; ci-16cpu takes 13–15 seconds and ci-32cpu 9–11 seconds for the parts step, at 1.3 to 2.4 times the cost. Getting a fresh server takes about 90–115 seconds before the job starts, whatever the size; the slowest test takes under 3 seconds.

-

CI does not run the Docker stack tests (integration.mjs, limits.mjs, load.mjs, failover.mjs, configuration.mjs, allowance-redis.sh and the other stack scripts), the browser tests, the credential scan or the live smoke tests: they need Docker, a running stack with Chrome, or private credentials. The 11 real-Redis store tests are skipped because they need an isolated REDIS_TEST_URL. Run them locally as described below. The CI server refuses pull requests from forks or from anyone but the repository owner, before any step runs.

+

CI runs the 11 real-Redis store tests (src/store_tests.rs) in rust-test: tests/allowance-redis.sh starts Redis on the server's rootless Docker with --pull never, --network none, a Unix socket in a temporary folder and no published port, runs nextest with REDIS_TEST_URL set and removes the container afterwards, also when tests fail. The image is the redis:7.4-alpine index digest pinned in tests/redis-image.sh, and a test run never reaches a registry for it. The nightly CI server image build runs the repository's hook .github/ciod/prefetch.sh, which downloads the linux/amd64 image through the registry API (Docker Hub, or public.ecr.aws with the same digests when Docker Hub fails; one attempt each), checks every part against its sha256 digest and stores it as an OCI archive in the runner's cache. rust-test runs the same hook first: on a prefetched image it does nothing, and on a server image built without the hook it downloads the image once, with a warning. It then loads the archive with docker load. The job summary says redis image: prefetched or redis image: installed now. A failed download, a digest mismatch, a failed load or a Redis that does not start fails the part; nothing is retried. Mark a Rust test #[ignore] only when it needs REDIS_TEST_URL; CI runs every ignored test (--run-ignored all), so #[ignore] never hides a test from CI.

+

CI does not run the Docker stack tests (integration.mjs, limits.mjs, load.mjs, failover.mjs, configuration.mjs and the other stack scripts), the browser tests, the credential scan or the live smoke tests: they need a running stack with Chrome, or private credentials. Run them locally as described below. The CI server refuses pull requests from forks or from anyone but the repository owner, before any step runs.

Local equivalents of the CI commands:

bash .github/scripts/repo-policy.sh
 python3 scripts/render-readme.py && git diff --exit-code -- docs/readme.html docs/engineering.html
 cargo fmt --check
 cargo clippy --all-targets --locked -- -D warnings
-cargo nextest run --locked --profile ci   # or: cargo test --locked
+sh tests/allowance-redis.sh cargo nextest run --locked --profile ci --run-ignored all   # Linux Docker
 npm ci --ignore-scripts
 node --test tests/*.test.mjs
 node scripts/vendor.mjs
diff --git a/tests/allowance-redis.sh b/tests/allowance-redis.sh
index f062f3f..f65dada 100644
--- a/tests/allowance-redis.sh
+++ b/tests/allowance-redis.sh
@@ -1,6 +1,11 @@
 #!/bin/sh
 # Real Redis store tests without a TCP listener, published ports or live traffic.
+# REDIS_IMAGE (default: the pin in tests/redis-image.sh) and REDIS_PULL (docker run --pull, default
+# missing) let CI run the image it loaded from the server's cache with --pull never.
 set -eu
+. "$(dirname "$0")/redis-image.sh"
+image=${REDIS_IMAGE:-redis:$REDIS_TAG@$REDIS_INDEX_DIGEST}
+pull=${REDIS_PULL:-missing}
 fixture_dir=$(mktemp -d "${TMPDIR:-/tmp}/allowance-redis.XXXXXX")
 container=""
 cleanup() {
@@ -8,9 +13,13 @@ cleanup() {
   rm -rf "$fixture_dir"
 }
 trap cleanup EXIT HUP INT TERM
-chmod 777 "$fixture_dir"
-container=$(docker run -d --rm --network none -v "$fixture_dir:/data" \
-  redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 \
+# The image hands /data to its redis user, under rootless Docker another host user. Mounting a
+# subfolder keeps $fixture_dir ours, so cleanup can still remove it from a sticky /tmp.
+socket_dir="$fixture_dir/redis"
+mkdir "$socket_dir"
+chmod 777 "$socket_dir"
+echo "redis fixture: docker run --pull $pull --network none $image" >&2
+container=$(docker run -d --rm --pull "$pull" --network none -v "$socket_dir:/data" "$image" \
   redis-server --port 0 --save '' --appendonly no --protected-mode yes \
   --unixsocket /data/redis.sock --unixsocketperm 777)
 attempt=0
@@ -22,4 +31,4 @@ done
 if [ "$#" -eq 0 ]; then
   set -- cargo test --locked store::tests -- --include-ignored
 fi
-REDIS_TEST_URL="redis+unix://$fixture_dir/redis.sock" "$@"
+REDIS_TEST_URL="redis+unix://$socket_dir/redis.sock" "$@"
diff --git a/tests/ci-run.test.mjs b/tests/ci-run.test.mjs
index 9b89bce..1f531ae 100644
--- a/tests/ci-run.test.mjs
+++ b/tests/ci-run.test.mjs
@@ -7,18 +7,27 @@ import {dirname, join} from 'node:path';
 import {fileURLToPath} from 'node:url';
 
 const repo = fileURLToPath(new URL('..', import.meta.url));
-const scripts = ['.github/scripts/ci-run.sh', '.github/scripts/ci-parts.sh'];
+const scripts = ['.github/scripts/ci-run.sh', '.github/scripts/ci-parts.sh', 'tests/allowance-redis.sh',
+  'tests/redis-image.sh'];
 
 // Every tool ci-run.sh calls is this stub: it logs its command line and fails (exit 7) when the line
-// starts with $STUB_FAIL.
+// starts with $STUB_FAIL. A line run with REDIS_TEST_URL set is also logged to $STUB_LOG.redis, and
+// `docker run` prints a container name.
 const stub = `#!/bin/sh
 line="$(basename "$0") $*"
 echo "$line" >> "$STUB_LOG"
 if [ -n "$STUB_FAIL" ]; then
   case "$line" in "$STUB_FAIL"*) echo "stub failed: $line"; exit 7 ;; esac
 fi
+if [ -n "$REDIS_TEST_URL" ]; then echo "$line" >> "$STUB_LOG.redis"; fi
+if [ "$1" = run ]; then echo redis-fixture; fi
 `;
 
+// The ciod hook: the stub, then the line the real hook prints ($PREFETCH_STATE: prefetched by default).
+const prefetch = `${stub}echo "redis image: \${PREFETCH_STATE:-prefetched} $CI_CACHE_DIR/web-proxy/tools/redis.tar"
+`;
+const archive = 'web-proxy/tools/redis-858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499.tar';
+
 // A renderer that leaves the docs alone, or makes them stale when $STALE is set.
 const renderer = `import os, pathlib
 if os.environ.get("STALE"):
@@ -28,6 +37,7 @@ if os.environ.get("STALE"):
 const docsRule = 'skipped: only documentation changed (.github/scripts/changed-scope.sh)';
 const rustRule = 'skipped: no Rust input changed (.github/scripts/changed-scope.sh)';
 const packageRule = 'skipped: package files did not change (.github/scripts/changed-scope.sh)';
+const nextest = profile => `cargo nextest run --locked --profile ${profile} --target-dir target/test --run-ignored all`;
 
 // Runs ci-run.sh in a temporary committed repository whose tools are stubs.
 function run(t, vars) {
@@ -37,6 +47,7 @@ function run(t, vars) {
   const bin = join(dir, 'bin');
   const files = {
     '.github/scripts/repo-policy.sh': 'echo "$0" >> "$STUB_LOG"\n',
+    '.github/ciod/prefetch.sh': prefetch,
     'scripts/render-readme.py': renderer,
     'docs/readme.html': 'readme\n',
     'docs/engineering.html': 'engineering\n',
@@ -52,18 +63,20 @@ function run(t, vars) {
     writeFileSync(join(work, path), content);
   }
   mkdirSync(bin);
-  for (const tool of ['cargo', 'npm', 'node', 'actionlint']) {
+  for (const tool of ['cargo', 'npm', 'node', 'actionlint', 'docker']) {
     writeFileSync(join(bin, tool), stub);
     chmodSync(join(bin, tool), 0o755);
   }
   const env = {
     ...process.env, GIT_CEILING_DIRECTORIES: dir, PATH: `${bin}:${process.env.PATH}`,
     STUB_LOG: join(dir, 'calls.log'), STUB_FAIL: '', ACTIONLINT: join(bin, 'actionlint'),
-    CI_EVIDENCE: join(dir, 'ev'), RUNNER_TEMP: join(dir, 'tmp'), GITHUB_STEP_SUMMARY: join(dir, 'summary.md'),
+    CI_EVIDENCE: join(dir, 'ev'), CI_CACHE_DIR: join(dir, 'cache'), RUNNER_TEMP: join(dir, 'tmp'),
+    GITHUB_STEP_SUMMARY: join(dir, 'summary.md'),
     SCOPE_RUST: 'true', SCOPE_NODE: 'true', SCOPE_AUDIT: 'true', MODE: 'full', NEXTEST_PROFILE: 'full',
     EVENT: 'merge_group', ...vars,
   };
   delete env.STALE;
+  delete env.REDIS_TEST_URL;
   if (vars.STALE) env.STALE = vars.STALE;
   const git = args => assert.equal(spawnSync('git', ['-c', 'user.name=ci', '-c', 'user.email=ci@example.invalid', ...args],
     {cwd: work, env}).status, 0, `git ${args.join(' ')}`);
@@ -72,11 +85,13 @@ function run(t, vars) {
   git(['commit', '-qm', 'fixture']);
   mkdirSync(env.RUNNER_TEMP);
   const result = spawnSync('bash', [join(work, scripts[0])], {cwd: work, env, encoding: 'utf8', timeout: 60_000});
-  const calls = existsSync(env.STUB_LOG) ? readFileSync(env.STUB_LOG, 'utf8').trim().split('\n').sort() : [];
+  const lines = path => existsSync(path) ? readFileSync(path, 'utf8').trim().split('\n') : [];
+  const calls = lines(env.STUB_LOG).sort();
+  const withRedis = lines(`${env.STUB_LOG}.redis`);
   const summary = existsSync(env.GITHUB_STEP_SUMMARY) ? readFileSync(env.GITHUB_STEP_SUMMARY, 'utf8') : '';
   const parts = Object.fromEntries(summary.split('\n').filter(line => /^\| [a-z]/.test(line) && !line.startsWith('| Part'))
     .map(line => line.split(' | ').slice(0, 2).map(cell => cell.replace(/^\| /, ''))));
-  return {...result, calls, parts};
+  return {...result, calls, withRedis, parts, summary, cache: env.CI_CACHE_DIR};
 }
 
 test('the merge queue runs every part in full mode except npm-audit', t => {
@@ -86,7 +101,7 @@ test('the merge queue runs every part in full mode except npm-audit', t => {
     'node-test', 'vendor', 'npm-audit']);
   for (const name of Object.keys(r.parts).slice(0, -1)) assert.equal(r.parts[name], 'passed', name);
   assert.match(r.parts['npm-audit'], /^skipped: dependency audits run on PRs that change package files, on dispatch/);
-  assert.ok(r.calls.includes('cargo nextest run --locked --profile full --target-dir target/test'), r.calls.join('\n'));
+  assert.ok(r.calls.includes(nextest('full')), r.calls.join('\n'));
   assert.ok(r.calls.includes('cargo clippy --all-targets --locked --target-dir target/clippy -- -D warnings'));
   assert.ok(r.calls.includes('npm ci --ignore-scripts --no-audit --no-fund --prefer-offline'));
   assert.ok(r.calls.includes('node scripts/vendor.mjs'));
@@ -100,7 +115,7 @@ test('a pull request in fast mode leaves out the slow test files', t => {
   const r = run(t, {EVENT: 'pull_request', MODE: 'fast', NEXTEST_PROFILE: 'ci'});
   assert.equal(r.status, 0, r.stdout + r.stderr);
   assert.match(r.calls.find(call => call.startsWith('node --test')), / tests\/a\.test\.mjs$/);
-  assert.ok(r.calls.includes('cargo nextest run --locked --profile ci --target-dir target/test'));
+  assert.ok(r.calls.includes(nextest('ci')));
   assert.ok(r.calls.includes('npm audit --audit-level=high --package-lock-only'));
   assert.equal(r.parts['npm-audit'], 'passed');
 });
@@ -140,6 +155,55 @@ test('a failed npm ci fails the run and the parts after it do not run', t => {
   assert.match(r.stdout, /::group::npm-ci failed\nstub failed: npm ci/);
 });
 
+test('rust-test runs every test, ignored ones included, against the prefetched Redis it removes', t => {
+  const r = run(t, {});
+  assert.equal(r.status, 0, r.stdout + r.stderr);
+  assert.ok(r.calls.some(call => call.trim() === 'prefetch.sh'), r.calls.join('\n'));
+  assert.ok(r.calls.includes(`docker load -i ${join(r.cache, archive)}`), r.calls.join('\n'));
+  const start = r.calls.find(call => call.startsWith('docker run '));
+  assert.match(start, /^docker run -d --rm --pull never --network none -v \S+\/redis:\/data web-proxy-ci\/redis:7\.4-alpine redis-server /);
+  assert.doesNotMatch(start, / -p | --publish /);
+  assert.deepEqual(r.withRedis, [nextest('full')]);
+  assert.ok(r.calls.includes('docker rm -f redis-fixture'), r.calls.join('\n'));
+  assert.match(r.summary, /^redis image: prefetched \S+$/m);
+  assert.doesNotMatch(r.stdout, /::warning::/);
+});
+
+test('a Redis image downloaded on this run is a warning and is named in the summary', t => {
+  const r = run(t, {PREFETCH_STATE: 'installed now'});
+  assert.equal(r.status, 0, r.stdout + r.stderr);
+  assert.match(r.summary, /^redis image: installed now \S+$/m);
+  assert.match(r.stdout, /^::warning::redis image: installed now \S+: this server image was built without/m);
+});
+
+test('a failed Redis image download fails rust-test before Docker or any test runs', t => {
+  const r = run(t, {STUB_FAIL: 'prefetch.sh'});
+  assert.equal(r.status, 1, r.stdout + r.stderr);
+  assert.equal(r.parts['rust-test'], 'failed (exit 7)');
+  assert.ok(!r.calls.some(call => /^(docker|cargo nextest) /.test(call)), r.calls.join('\n'));
+});
+
+test('a failed docker load fails rust-test before any test runs', t => {
+  const r = run(t, {STUB_FAIL: 'docker load'});
+  assert.equal(r.status, 1, r.stdout + r.stderr);
+  assert.equal(r.parts['rust-test'], 'failed (exit 7)');
+  assert.ok(!r.calls.some(call => /^(docker run|cargo nextest) /.test(call)), r.calls.join('\n'));
+});
+
+test('a failing store test fails rust-test and the Redis container is still removed', t => {
+  const r = run(t, {STUB_FAIL: 'cargo nextest'});
+  assert.equal(r.status, 1, r.stdout + r.stderr);
+  assert.equal(r.parts['rust-test'], 'failed (exit 7)');
+  assert.ok(r.calls.includes('docker rm -f redis-fixture'), r.calls.join('\n'));
+});
+
+test('a Redis that does not start fails rust-test before any test runs', t => {
+  const r = run(t, {STUB_FAIL: 'docker run'});
+  assert.equal(r.status, 1, r.stdout + r.stderr);
+  assert.equal(r.parts['rust-test'], 'failed (exit 7)');
+  assert.ok(!r.calls.some(call => call.startsWith('cargo nextest')), r.calls.join('\n'));
+});
+
 test('stale documentation fails the docs part', t => {
   const r = run(t, {STALE: '1'});
   assert.equal(r.status, 1, r.stdout + r.stderr);
diff --git a/tests/ciod-prefetch.test.mjs b/tests/ciod-prefetch.test.mjs
new file mode 100644
index 0000000..f4adfeb
--- /dev/null
+++ b/tests/ciod-prefetch.test.mjs
@@ -0,0 +1,166 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import {spawn, spawnSync} from 'node:child_process';
+import {createHash} from 'node:crypto';
+import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
+import {createServer} from 'node:http';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+// .github/ciod/prefetch.sh against fake registries: local HTTP servers that serve the token, the
+// image index, the manifest, the config and a layer, all content-addressed like the real ones.
+const repo = fileURLToPath(new URL('..', import.meta.url));
+const sha = body => `sha256:${createHash('sha256').update(body).digest('hex')}`;
+const blob = (mediaType, body) => ({mediaType, digest: sha(body), size: body.length});
+
+const layer = Buffer.from('a layer of the fake Redis image');
+const config = Buffer.from(JSON.stringify({architecture: 'amd64', os: 'linux', rootfs: {type: 'layers'}}));
+const manifest = Buffer.from(JSON.stringify({
+  schemaVersion: 2, mediaType: 'application/vnd.oci.image.manifest.v1+json',
+  config: blob('application/vnd.oci.image.config.v1+json', config),
+  layers: [blob('application/vnd.oci.image.layer.v1.tar+gzip', layer)],
+}));
+const other = Buffer.from('the arm64 manifest, never fetched');
+const index = Buffer.from(JSON.stringify({
+  schemaVersion: 2, mediaType: 'application/vnd.oci.image.index.v1+json',
+  manifests: [
+    {...blob('application/vnd.oci.image.manifest.v1+json', other), platform: {os: 'linux', architecture: 'arm64'}},
+    {...blob('application/vnd.oci.image.manifest.v1+json', manifest), platform: {os: 'linux', architecture: 'amd64'}},
+    {...blob('application/vnd.oci.image.manifest.v1+json', other), platform: {os: 'unknown', architecture: 'unknown'}},
+  ],
+}));
+const bodies = new Map([index, manifest, config, layer].map(body => [sha(body), body]));
+
+// A registry serving REPO. mode: ok, down (503 for everything) or corrupt (the layer's bytes differ).
+async function registry(t, repoName, mode) {
+  const token = `token-for-${repoName}`;
+  const server = createServer((req, res) => {
+    server.requests += 1;
+    const url = new URL(req.url, 'http://registry');
+    if (mode === 'down') return res.writeHead(503).end();
+    if (url.pathname === '/token' || url.pathname === '/token/') {
+      if (url.searchParams.get('scope') !== `repository:${repoName}:pull`) return res.writeHead(400).end();
+      return res.end(JSON.stringify({token}));
+    }
+    const match = url.pathname.match(/^\/v2\/(.+)\/(manifests|blobs)\/(sha256:[0-9a-f]{64})$/);
+    if (!match || match[1] !== repoName || req.headers.authorization !== `Bearer ${token}`) {
+      return res.writeHead(match ? 401 : 404).end();
+    }
+    let body = bodies.get(match[3]);
+    if (!body) return res.writeHead(404).end();
+    if (mode === 'corrupt' && body === layer) body = Buffer.from('tampered layer bytes');
+    res.end(body);
+  });
+  server.requests = 0;
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  t.after(() => server.close());
+  return {server, url: `http://127.0.0.1:${server.address().port}`};
+}
+
+// The hook in a temporary tree whose tests/redis-image.sh pins the fake index.
+function fixture(t) {
+  const dir = mkdtempSync(join(tmpdir(), 'ciod-prefetch-'));
+  t.after(() => rmSync(dir, {recursive: true, force: true}));
+  mkdirSync(join(dir, 'repo/.github/ciod'), {recursive: true});
+  mkdirSync(join(dir, 'repo/tests'));
+  copyFileSync(join(repo, '.github/ciod/prefetch.sh'), join(dir, 'repo/.github/ciod/prefetch.sh'));
+  const pin = readFileSync(join(repo, 'tests/redis-image.sh'), 'utf8')
+    .replace(/^REDIS_INDEX_DIGEST=\S+$/m, `REDIS_INDEX_DIGEST=${sha(index)}`);
+  assert.match(pin, new RegExp(sha(index)));
+  writeFileSync(join(dir, 'repo/tests/redis-image.sh'), pin);
+  const cache = join(dir, 'cache');
+  const tools = join(cache, 'web-proxy/tools');
+  return {dir, cache, tools, archive: join(tools, `redis-${sha(index).slice(7)}.tar`)};
+}
+
+function hook(f, hub, mirror) {
+  const env = {...process.env, CI_CACHE_DIR: f.cache, HOME: join(f.cache, 'web-proxy'),
+    REGISTRY_URL: hub.url, MIRROR_URL: mirror.url, AUTH_URL: `${hub.url}/token`};
+  return new Promise(resolve => {
+    const child = spawn('bash', [join(f.dir, 'repo/.github/ciod/prefetch.sh')], {env});
+    let stdout = '', stderr = '';
+    child.stdout.on('data', data => { stdout += data; });
+    child.stderr.on('data', data => { stderr += data; });
+    child.on('close', status => resolve({status, stdout, stderr}));
+  });
+}
+
+const tarFile = (archive, name) => spawnSync('tar', ['-xOf', archive, name], {encoding: 'utf8'}).stdout;
+
+test('the first run installs a verified OCI archive; the second finds it and makes no request', async t => {
+  const f = fixture(t);
+  const hub = await registry(t, 'library/redis', 'ok');
+  const mirror = await registry(t, 'docker/library/redis', 'ok');
+  const first = await hook(f, hub, mirror);
+  assert.equal(first.status, 0, first.stderr);
+  assert.equal(first.stdout, `redis image: installed now ${f.archive}\n`);
+  assert.ok(existsSync(`${f.archive}.complete`));
+  assert.equal(mirror.server.requests, 0);
+  const names = spawnSync('tar', ['-tf', f.archive], {encoding: 'utf8'}).stdout.trim().split('\n').sort();
+  const blobs = [manifest, config, layer].map(body => `blobs/sha256/${sha(body).slice(7)}`);
+  assert.deepEqual(names.filter(name => !name.endsWith('/')), [...blobs, 'index.json', 'oci-layout'].sort());
+  for (const body of [manifest, config, layer]) {
+    assert.equal(tarFile(f.archive, `blobs/sha256/${sha(body).slice(7)}`), body.toString());
+  }
+  const layout = JSON.parse(tarFile(f.archive, 'index.json'));
+  assert.equal(layout.manifests.length, 1);
+  assert.equal(layout.manifests[0].digest, sha(manifest));
+  assert.equal(layout.manifests[0].size, manifest.length);
+  assert.equal(layout.manifests[0].annotations['io.containerd.image.name'], 'docker.io/web-proxy-ci/redis:7.4-alpine');
+  assert.deepEqual(JSON.parse(tarFile(f.archive, 'oci-layout')), {imageLayoutVersion: '1.0.0'});
+
+  const before = hub.server.requests;
+  const second = await hook(f, hub, mirror);
+  assert.equal(second.status, 0, second.stderr);
+  assert.equal(second.stdout, `redis image: prefetched ${f.archive}\n`);
+  assert.equal(hub.server.requests, before);
+  assert.equal(mirror.server.requests, 0);
+});
+
+test('with Docker Hub down the mirror is used', async t => {
+  const f = fixture(t);
+  const hub = await registry(t, 'library/redis', 'down');
+  const mirror = await registry(t, 'docker/library/redis', 'ok');
+  const r = await hook(f, hub, mirror);
+  assert.equal(r.status, 0, r.stderr);
+  assert.equal(r.stdout, `redis image: installed now ${f.archive}\n`);
+  assert.match(r.stderr, /redis image: http:\S+ failed; trying http:/);
+  assert.equal(hub.server.requests, 1);
+  assert.ok(mirror.server.requests > 0);
+});
+
+test('a digest mismatch fails, and with the mirror down nothing is left behind', async t => {
+  const f = fixture(t);
+  const hub = await registry(t, 'library/redis', 'corrupt');
+  const mirror = await registry(t, 'docker/library/redis', 'down');
+  const r = await hook(f, hub, mirror);
+  assert.notEqual(r.status, 0);
+  assert.match(r.stderr, new RegExp(`redis image: digest mismatch for ${sha(layer)} from http:`));
+  assert.match(r.stderr, /redis image: download failed from http:\S+ and http:/);
+  assert.equal(r.stdout, '');
+  assert.deepEqual(readdirSync(f.tools), []);
+});
+
+test('a digest mismatch on Docker Hub falls back to the mirror', async t => {
+  const f = fixture(t);
+  const hub = await registry(t, 'library/redis', 'corrupt');
+  const mirror = await registry(t, 'docker/library/redis', 'ok');
+  const r = await hook(f, hub, mirror);
+  assert.equal(r.status, 0, r.stderr);
+  assert.match(r.stderr, /digest mismatch/);
+  assert.equal(tarFile(f.archive, `blobs/sha256/${sha(layer).slice(7)}`), layer.toString());
+});
+
+test('with both registries down it exits non-zero and leaves no partial file', async t => {
+  const f = fixture(t);
+  const hub = await registry(t, 'library/redis', 'down');
+  const mirror = await registry(t, 'docker/library/redis', 'down');
+  const r = await hook(f, hub, mirror);
+  assert.notEqual(r.status, 0);
+  assert.match(r.stderr, /redis image: download failed from/);
+  assert.equal(r.stdout, '');
+  assert.deepEqual(readdirSync(f.tools), []);
+  assert.equal(hub.server.requests, 1);
+  assert.equal(mirror.server.requests, 1);
+});
diff --git a/tests/redis-image.sh b/tests/redis-image.sh
new file mode 100644
index 0000000..6b693a5
--- /dev/null
+++ b/tests/redis-image.sh
@@ -0,0 +1,13 @@
+# The Redis image of the store tests: the one pin, sourced by tests/allowance-redis.sh, the ciod hook
+# .github/ciod/prefetch.sh and .github/scripts/ci-run.sh. Change the digest only here.
+# The index digest names every platform; the hook verifies each body against it and keeps the
+# linux/amd64 manifest (sha256:ca0acbb137c1...).
+# shellcheck shell=sh disable=SC2034
+REDIS_INDEX_DIGEST=sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499
+REDIS_TAG=7.4-alpine
+REDIS_REPO_HUB=library/redis
+REDIS_REPO_MIRROR=docker/library/redis
+# The archive the hook writes under $CI_CACHE_DIR/web-proxy/tools, and the local name it loads as
+# (never pushed anywhere).
+REDIS_ARCHIVE_NAME="redis-${REDIS_INDEX_DIGEST#sha256:}.tar"
+REDIS_LOCAL_IMAGE="web-proxy-ci/redis:$REDIS_TAG"