Run the 11 real-Redis store tests in CI

.github/ciod/prefetch.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Prefetches the pinned Redis image of the store tests (tests/redis-image.sh) as an OCI image
+# archive that `docker load` accepts: $CI_CACHE_DIR/web-proxy/tools/redis-<index digest>.tar, with
+# a .complete marker. Prints "redis image: prefetched PATH" when it was already there, otherwise
+# "redis image: installed now PATH"; exits non-zero on any failure.
+#
+# The nightly CI image build runs it with no argument from a fresh clone of main, as a user that
+# may write only $CI_CACHE_DIR/web-proxy, with HOME=$CI_CACHE_DIR/web-proxy and no Docker. The
+# rust-test part of .github/scripts/ci-run.sh runs it as the fallback for images built without it.
+#
+# No Docker, so it uses the registry API v2 with an anonymous token: Docker Hub first, and
+# public.ecr.aws (the same digests) only if Docker Hub fails; one attempt each, no retry. Every
+# body is checked against its sha256 digest before use, so the bytes are the pinned image whichever
+# registry sent them. The archive is built in a temporary folder and renamed into place.
+# REGISTRY_URL, MIRROR_URL and AUTH_URL (the Docker Hub token service) override the endpoints.
+set -euo pipefail
+: "${CI_CACHE_DIR:?is not set}"
+# shellcheck source=../../tests/redis-image.sh
+. "$(dirname "$0")/../../tests/redis-image.sh"
+registry=${REGISTRY_URL:-https://registry-1.docker.io}
+mirror=${MIRROR_URL:-https://public.ecr.aws}
+auth=${AUTH_URL:-https://auth.docker.io/token}
+tools="$CI_CACHE_DIR/web-proxy/tools"
+archive="$tools/$REDIS_ARCHIVE_NAME"
+index_types=application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json
+manifest_types=application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json
+
+if [ -f "$archive.complete" ] && [ -f "$archive" ]; then
+  echo "redis image: prefetched $archive"
+  exit 0
+fi
+
+# get URL TOKEN ACCEPT DIGEST OUT: one GET, then the body's sha256 must be DIGEST.
+get() {
+  curl -fsSL --connect-timeout 10 --max-time 120 -H "Authorization: Bearer $2" -H "Accept: $3" \
+    -o "$5" "$1" || return 1
+  if [ "sha256:$(sha256sum "$5" | cut -d ' ' -f 1)" != "$4" ]; then
+    echo "redis image: digest mismatch for $4 from $1" >&2
+    return 1
+  fi
+}
+
+# download BASE REPO TOKEN_URL DIR: the linux/amd64 image from one registry into DIR as an OCI
+# image layout. Called in a condition, so every step checks its own status.
+download() {
+  local base=$1 repo=$2 dir=$4 token manifest blob blobs list
+  token=$(curl -fsS --connect-timeout 10 --max-time 30 "$3" | jq -er .token) || return 1
+  blobs="$dir/blobs/sha256"
+  mkdir -p "$blobs" || return 1
+  get "$base/v2/$repo/manifests/$REDIS_INDEX_DIGEST" "$token" "$index_types" "$REDIS_INDEX_DIGEST" \
+    "$dir/image-index.json" || return 1
+  manifest=$(jq -er '[.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64")
+    | .digest][0] // empty' "$dir/image-index.json") || return 1
+  get "$base/v2/$repo/manifests/$manifest" "$token" "$manifest_types" "$manifest" \
+    "$blobs/${manifest#sha256:}" || return 1
+  list=$(jq -er '.config.digest, .layers[].digest' "$blobs/${manifest#sha256:}") || return 1
+  for blob in $list; do
+    get "$base/v2/$repo/blobs/$blob" "$token" '*/*' "$blob" "$blobs/${blob#sha256:}" || return 1
+  done
+  # The layout's own index names the one manifest; docker load reads the image name from it.
+  jq -n --arg digest "$manifest" --argjson size "$(wc -c < "$blobs/${manifest#sha256:}")" \
+    --arg type "$(jq -r '.mediaType // "application/vnd.oci.image.manifest.v1+json"' "$blobs/${manifest#sha256:}")" \
+    --arg name "docker.io/$REDIS_LOCAL_IMAGE" --arg tag "$REDIS_TAG" \
+    '{schemaVersion: 2, mediaType: "application/vnd.oci.image.index.v1+json",
+      manifests: [{mediaType: $type, digest: $digest, size: $size,
+        platform: {os: "linux", architecture: "amd64"},
+        annotations: {"io.containerd.image.name": $name, "org.opencontainers.image.ref.name": $tag}}]}' \
+    > "$dir/index.json" || return 1
+  rm -f "$dir/image-index.json"
+  echo '{"imageLayoutVersion":"1.0.0"}' > "$dir/oci-layout"
+}
+
+mkdir -p "$tools"
+work=$(mktemp -d "$tools/.redis.XXXXXX")
+trap 'rm -rf "$work"' EXIT
+hub_token="$auth?service=registry.docker.io&scope=repository:$REDIS_REPO_HUB:pull"
+mirror_token="$mirror/token/?scope=repository:$REDIS_REPO_MIRROR:pull"
+if ! download "$registry" "$REDIS_REPO_HUB" "$hub_token" "$work/layout"; then
+  echo "redis image: $registry failed; trying $mirror" >&2
+  rm -rf "$work/layout"
+  if ! download "$mirror" "$REDIS_REPO_MIRROR" "$mirror_token" "$work/layout"; then
+    echo "redis image: download failed from $registry and $mirror" >&2
+    exit 1
+  fi
+fi
+tar -cf "$work/image.tar" -C "$work/layout" oci-layout index.json blobs
+rm -f "$archive.complete"
+mv -f "$work/image.tar" "$archive"
+touch "$archive.complete"
+echo "redis image: installed now $archive"

.github/scripts/ci-run.sh

@@ -9,6 +9,7 @@
 #   EVENT                                the GitHub event name
 #   ACTIONLINT                           the verified actionlint binary
 #   CI_EVIDENCE, RUNNER_TEMP             evidence and scratch directories
+#   CI_CACHE_DIR                         the runner's cache (rust-test: the prefetched Redis image)
 # A part that does not run is recorded as skipped with the rule that skipped it.
 set -euo pipefail
 cd "$(git rev-parse --show-toplevel)"
@@ -42,6 +43,20 @@ docs() {
   fi
 }
 
+# The store tests' Redis never comes from a registry here: the server image prefetched it
+# (.github/ciod/prefetch.sh; on an image built without the hook, the hook downloads it now, once),
+# docker loads the archive and tests/allowance-redis.sh runs it with --pull never and no network.
+rust_test() {
+  local state
+  # shellcheck source=tests/redis-image.sh
+  . tests/redis-image.sh
+  state=$(bash .github/ciod/prefetch.sh)
+  echo "$state" | tee "$CI_EVIDENCE/redis-image.txt"
+  docker load -i "$CI_CACHE_DIR/web-proxy/tools/$REDIS_ARCHIVE_NAME"
+  REDIS_IMAGE=$REDIS_LOCAL_IMAGE REDIS_PULL=never sh tests/allowance-redis.sh \
+    cargo nextest run --locked --profile "$NEXTEST_PROFILE" --target-dir target/test --run-ignored all
+}
+
 node_test() {
   local files=(tests/*.test.mjs)
   if [ "$MODE" = full ]; then
@@ -60,7 +75,7 @@ part actionlint "$ACTIONLINT"
 if [ "$SCOPE_RUST" = true ]; then
   part fmt cargo fmt --check
   part clippy cargo clippy --all-targets --locked --target-dir target/clippy -- -D warnings
-  part rust-test cargo nextest run --locked --profile "$NEXTEST_PROFILE" --target-dir target/test
+  part rust-test rust_test
 else
   rule=$rust_rule
   if [ "$SCOPE_NODE" != true ]; then rule=$docs_rule; fi
@@ -90,4 +105,14 @@ else
   part_skip npm-audit "$package_rule"
 fi
 
-parts_wait
+status=0
+parts_wait || status=$?
+# rust-test's Redis image line goes to the job summary; a download on this run is a warning.
+if [ -s "$CI_EVIDENCE/redis-image.txt" ]; then
+  state=$(cat "$CI_EVIDENCE/redis-image.txt")
+  printf '\n%s\n' "$state" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
+  if [[ $state == "redis image: installed now"* ]]; then
+    echo "::warning::$state: this server image was built without .github/ciod/prefetch.sh"
+  fi
+fi
+exit "$status"

.github/workflows/ci.yml

@@ -2,8 +2,13 @@
 # One job, tier1, on one fresh CI-SelfHosted server per run ([self-hosted, ci-8cpu]; a dispatch
 # may pick ci-16cpu or ci-32cpu). Its "Checks and tests in parallel" step runs every part at the
 # same time through .github/scripts/ci-run.sh (parts runner: .github/scripts/ci-parts.sh): policy,
-# docs, actionlint, fmt, clippy, rust-test (cargo nextest), npm-ci, then node-test, then vendor,
+# docs, actionlint, fmt, clippy, rust-test (cargo nextest, with the real-Redis store tests on a
+# private Redis from tests/allowance-redis.sh), npm-ci, then node-test, then vendor,
 # and npm-audit. The job summary holds the parts table; a failed part's log is printed in a group.
+# rust-test's Redis image is never pulled from a registry: the server image prefetched it into
+# CI_CACHE_DIR (.github/ciod/prefetch.sh, verified by digest), docker loads it and Redis runs with
+# --pull never. On a server image built without the hook, the hook downloads it once with a warning;
+# the summary says "redis image: prefetched" or "redis image: installed now".
 #
 # Triggers and modes:
 # - pull_request (ready, not draft): fast mode. Rust tests named slow_* and tests/slow/*.test.mjs
@@ -22,8 +27,7 @@
 # The server's job-started guard refuses forks, pull requests by anyone but the owner and every
 # event not listed above, before any step runs.
 # Not run here: the Docker stack suites, browser tests, live smoke tests, the credential scan
-# (tests/secret-scan.py) and the 11 real-Redis store tests, which need REDIS_TEST_URL. Run them
-# locally as ENGINEERING.md "Testing and validation" describes.
+# (tests/secret-scan.py). Run them locally as ENGINEERING.md "Testing and validation" describes.
 name: ci
 
 on:

ENGINEERING.md

@@ -589,7 +589,7 @@ The gateway binds sessions to their assigned region and rejects a region mismatc
 | `actionlint` | actionlint 1.7.12 (pinned, checksum-verified download), all workflows | always |
 | `fmt` | `cargo fmt --check` | Rust inputs changed |
 | `clippy` | `cargo clippy --all-targets --locked --target-dir target/clippy -- -D warnings` | Rust inputs changed |
-| `rust-test` | `cargo nextest run --locked --profile ci` (or `full`) `--target-dir target/test` | Rust inputs changed |
+| `rust-test` | the Redis image (`.github/ciod/prefetch.sh`, then `docker load`), then `sh tests/allowance-redis.sh cargo nextest run --locked --profile ci` (or `full`) `--target-dir target/test --run-ignored all` | Rust inputs changed |
 | `npm-ci` | `npm ci --ignore-scripts --no-audit --no-fund --prefer-offline` | not documentation only |
 | `node-test` | `node --test --test-timeout=90000 tests/*.test.mjs` (plus `tests/slow/*.test.mjs` in full mode), after `npm-ci` | not documentation only |
 | `vendor` | `node scripts/vendor.mjs`, after `node-test` (it rewrites `web/runtime/vendor/`, which the tests read) | not documentation only |
@@ -615,7 +615,9 @@ Merging: pull requests merge only through the merge queue, as squash commits. Th
 
 Server size and times: `ci-8cpu` is the smallest size whose measured runs all keep the parts step under 90 seconds and the job under 4 minutes. Measured with a warm cache (three manual runs per size, November 2026): on `ci-8cpu` the parts step takes 22–24 seconds and the job 33–34 seconds, with sccache at 100% and about $0.002 per run; `ci-16cpu` takes 13–15 seconds and `ci-32cpu` 9–11 seconds for the parts step, at 1.3 to 2.4 times the cost. Getting a fresh server takes about 90–115 seconds before the job starts, whatever the size; the slowest test takes under 3 seconds.
 
-CI does not run the Docker stack tests (`integration.mjs`, `limits.mjs`, `load.mjs`, `failover.mjs`, `configuration.mjs`, `allowance-redis.sh` and the other stack scripts), the browser tests, the credential scan or the live smoke tests: they need Docker, a running stack with Chrome, or private credentials. The 11 real-Redis store tests are skipped because they need an isolated `REDIS_TEST_URL`. Run them locally as described below. The CI server refuses pull requests from forks or from anyone but the repository owner, before any step runs.
+CI runs the 11 real-Redis store tests (`src/store_tests.rs`) in `rust-test`: `tests/allowance-redis.sh` starts Redis on the server's rootless Docker with `--pull never`, `--network none`, a Unix socket in a temporary folder and no published port, runs nextest with `REDIS_TEST_URL` set and removes the container afterwards, also when tests fail. The image is the `redis:7.4-alpine` index digest pinned in `tests/redis-image.sh`, and a test run never reaches a registry for it. The nightly CI server image build runs the repository's hook `.github/ciod/prefetch.sh`, which downloads the linux/amd64 image through the registry API (Docker Hub, or `public.ecr.aws` with the same digests when Docker Hub fails; one attempt each), checks every part against its sha256 digest and stores it as an OCI archive in the runner's cache. `rust-test` runs the same hook first: on a prefetched image it does nothing, and on a server image built without the hook it downloads the image once, with a warning. It then loads the archive with `docker load`. The job summary says `redis image: prefetched` or `redis image: installed now`. A failed download, a digest mismatch, a failed load or a Redis that does not start fails the part; nothing is retried. Mark a Rust test `#[ignore]` only when it needs `REDIS_TEST_URL`; CI runs every ignored test (`--run-ignored all`), so `#[ignore]` never hides a test from CI.
+
+CI does not run the Docker stack tests (`integration.mjs`, `limits.mjs`, `load.mjs`, `failover.mjs`, `configuration.mjs` and the other stack scripts), the browser tests, the credential scan or the live smoke tests: they need a running stack with Chrome, or private credentials. Run them locally as described below. The CI server refuses pull requests from forks or from anyone but the repository owner, before any step runs.
 
 Local equivalents of the CI commands:
 
@@ -624,7 +626,7 @@ bash .github/scripts/repo-policy.sh
 python3 scripts/render-readme.py && git diff --exit-code -- docs/readme.html docs/engineering.html
 cargo fmt --check
 cargo clippy --all-targets --locked -- -D warnings
-cargo nextest run --locked --profile ci   # or: cargo test --locked
+sh tests/allowance-redis.sh cargo nextest run --locked --profile ci --run-ignored all   # Linux Docker
 npm ci --ignore-scripts
 node --test tests/*.test.mjs
 node scripts/vendor.mjs