diff --git a/Cargo.lock b/Cargo.lock index 2b3d65d0..18b16fe4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,7 @@ name = "avocado-cli" version = "1.0.0-rc.1" dependencies = [ "anyhow", + "axum", "base64", "blake3", "bytes", @@ -150,9 +151,11 @@ dependencies = [ "libc", "num_cpus", "rand 0.10.1", + "rcgen", "regex", "reqwest", "rpassword", + "rustls", "semver", "serde", "serde_jcs", @@ -164,8 +167,12 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "tokio-rustls", "tokio-test", + "tokio-tungstenite", + "tokio-util", "tough", + "tower", "uuid", "walkdir", ] @@ -193,6 +200,58 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base64" version = "0.22.1" @@ -216,7 +275,16 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", ] [[package]] @@ -293,7 +361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -477,6 +545,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -522,6 +599,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.1" @@ -559,6 +646,12 @@ version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b10589d1a5e400d61f9f38f12f884cfd080ff345de8f17efda36fe0e4a02aa8" +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + [[package]] name = "deranged" version = "0.5.8" @@ -590,15 +683,25 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" dependencies = [ - "block-buffer", + "block-buffer 0.12.0", "const-oid", - "crypto-common", + "crypto-common 0.2.1", ] [[package]] @@ -840,6 +943,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -982,6 +1095,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hybrid-array" version = "0.4.11" @@ -1004,6 +1123,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1383,6 +1503,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.0" @@ -1755,6 +1881,19 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2144,6 +2283,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_plain" version = "1.0.2" @@ -2153,6 +2303,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -2192,6 +2354,17 @@ dependencies = [ "syn", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -2205,8 +2378,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] @@ -2534,6 +2707,22 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -2594,6 +2783,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -2632,6 +2822,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-core", ] @@ -2651,6 +2842,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.4", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + [[package]] name = "typed-path" version = "0.9.3" @@ -2738,6 +2948,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3373,6 +3589,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 5d9b57f4..275d775d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,10 +31,15 @@ tokio = { version = "1.0", features = [ "rt-multi-thread", "process", "io-util", + "fs", "signal", "time", "sync", ] } +# `io` only, for ReaderStream: the registry streams stored blobs off disk instead +# of sizing an allocation by the blob. Already in the tree transitively (axum, +# tokio-tungstenite), so this adds an explicit edge rather than a new dependency. +tokio-util = { version = "0.7", features = ["io"] } thiserror = "2.0" directories = "6.0" reqwest = { version = "0.13", default-features = false, features = [ @@ -70,7 +75,25 @@ tough = { version = "0.22", default-features = false } semver = "1" crossterm = "0.29" num_cpus = "1.16" +# Container Dev Mode embedded registry server surface. Pure-Rust HTTP/TLS/WS +# stack pinned to the aws-lc-rs rustls provider already linked via reqwest, so +# no new C toolchain requirement is introduced. rcgen and tokio-tungstenite +# both default to aws_lc_rs; do not enable a `ring` feature here or a second +# crypto provider would be linked. +axum = "0.8" +rustls = "0.23" +tokio-rustls = "0.26" +rcgen = { version = "0.13", default-features = false, features = [ + "crypto", + "pem", + "aws_lc_rs", +] } +tokio-tungstenite = { version = "0.26", features = ["rustls-tls-native-roots"] } [dev-dependencies] +# `oneshot` drives the write router directly, so a body stream that ERRORS can be +# injected deterministically - a truncated real HTTP body just makes the server +# wait for bytes that never arrive. +tower = { version = "0.5", features = ["util"] } tokio-test = "0.4" serial_test = "3.0" diff --git a/docs/container-dev/lab/README.md b/docs/container-dev/lab/README.md new file mode 100644 index 00000000..e6c0c7fd --- /dev/null +++ b/docs/container-dev/lab/README.md @@ -0,0 +1,67 @@ +# Container Dev Mode - local VM write-path lab + +A from-scratch harness to exercise Container Dev Mode's authenticated VM write +path (task 7.1) on Linux, without hardware. It stands up a disposable Debian 12 +"engine VM" under QEMU user-mode networking so the guest reaches the host at +`10.0.2.2` exactly like the macOS `avocado-vm`, then runs the end-to-end verify. + +This is the setup the 2026-07-23 field note ("Container Dev Mode VM push") was +written from; it validated the path 8/8 and caught two real bugs (a plain-HTTP +write listener where the guest required HTTPS, and a 2 MiB body limit that +413'd real layers). + +## What's here + +- `setup-lab.sh` - idempotent provisioner: ssh keypair, cloud-init seed + (`docker.io` + root login), a copy-on-write overlay off the Debian base, a + QEMU SLIRP boot with an ssh hostfwd, a forward of the guest dockerd to the + socket `is_vm_routing_active()` resolves, and an `env.sh` for the verify step. +- `avocado.yaml` - the minimal `container_dev` runtime config the lab uses. +- `../verify-vm-write-path.sh` - the actual end-to-end assertion (built image -> + authenticated push over `10.0.2.2` HTTPS -> single-layer sync). Sourced env + comes from the generated `env.sh`. + +Generated state (the qcow2 overlay, ssh key, cloud-init seed, `env.sh`) is +written to a work dir OUTSIDE this repo (`$AVOCADO_CDM_LAB_WORK`, default +`~/.cache/avocado-cdm-lab`) so a ~900 MB overlay never lands in git. + +## Prerequisites + +Host packages: `qemu-system-x86_64`, `qemu-img`, `cloud-image-utils` +(`cloud-localds`), `ssh`/`ssh-keygen`, and the `docker` client. + +## Run it (from scratch) + +```bash +# 1. one-time: download a Debian 12 generic-cloud base image +WORK="${AVOCADO_CDM_LAB_WORK:-$HOME/.cache/avocado-cdm-lab}" +mkdir -p "$WORK" +curl -L -o "$WORK/debian12.qcow2" \ + https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2 + +# 2. build the avocado CLI (the lab points AVOCADO_BIN at target/debug/avocado) +cargo build --bin avocado + +# 3. stand up the engine VM (interactive: it runs ssh-keygen + touches ~/.ssh) +bash docs/container-dev/lab/setup-lab.sh + +# 4. run the end-to-end verify +source "$WORK/env.sh" +docs/container-dev/verify-vm-write-path.sh +``` + +Optional: set `BBAPPEND` to the meta-avocado +`meta-avocado-qemu/recipes-core/base-files/base-files_%.bbappend` path so the +verify script also checks the guest trust-store-dir overlay; leave it empty to +skip that check. + +## Tear down + +```bash +WORK="${AVOCADO_CDM_LAB_WORK:-$HOME/.cache/avocado-cdm-lab}" +kill "$(cat "$WORK/qemu.pid")" 2>/dev/null || true +pkill -f "$HOME/.avocado/vm/docker.sock:" 2>/dev/null || true +``` + +Deleting `$WORK/engine.qcow2` gives a clean VM on the next run; the base image +and ssh key are reused. diff --git a/docs/container-dev/lab/avocado.yaml b/docs/container-dev/lab/avocado.yaml new file mode 100644 index 00000000..7fd2c1a2 --- /dev/null +++ b/docs/container-dev/lab/avocado.yaml @@ -0,0 +1,24 @@ +# HOST-side session config for the Container Dev Mode lab. +# +# There are deliberately two avocado.yaml files in this lab and they do different +# jobs. This one is read by the host CLI (`container dev up` runs from this +# directory) and only has to carry the `container_dev` block. The one setup-lab.sh +# renders into $AVOCADO_CDM_LAB_WORK/hitl is the RUNTIME BUILD config - it composes +# the Avocado OS image with avocado-ext-docker and avocado-ext-container-agent-dev. +# That one is generated rather than tracked because the agent extension is not in +# the published feed and has to be sourced by absolute path. +# +# A runtime carrying a `container_dev` block is all that enables the feature +# (see src/utils/container_dev/config.rs). The watched image ref must match the +# image the verify script builds on the VM engine (TEST_IMAGE, default +# my-app:dev). `registry.port` is the bulk *read* listener; the authenticated +# *write* listener uses AVOCADO_CONTAINER_DEV_WRITE_PORT (default 5601). +runtimes: + dev: + target: qemux86-64 + container_dev: + images: + - ref: my-app:dev + service: app + registry: + port: 5599 diff --git a/docs/container-dev/lab/demo.sh b/docs/container-dev/lab/demo.sh new file mode 100755 index 00000000..d8aee66b --- /dev/null +++ b/docs/container-dev/lab/demo.sh @@ -0,0 +1,691 @@ +#!/usr/bin/env bash +# +# Container Dev Mode demo driver - one entry point for the whole lab. +# +# demo.sh setup boot the lab VM (delegates to setup-lab.sh) +# demo.sh verify run the Part A push-path verify (8 checks) +# demo.sh app [version] build the demo app on the TARGET engine + install its unit +# demo.sh seed ship what the host holds; reads the version off the image +# demo.sh up start `container dev up`, backgrounded +# demo.sh agent (re)start the device agent on the target +# demo.sh reload [version] rebuild only, then wait for the hot reload to land +# demo.sh sync re-push + notify now, without waiting on an event +# demo.sh status where everything is right now +# demo.sh logs session | agent | app (what/where each one is) +# demo.sh down stop the session, agent and app; leave the VM warm +# demo.sh reset full wipe, back to a pre-demo state +# demo.sh all [v1] [v2] setup -> app -> up -> agent -> reload, end to end +# +# Every action prints a context header naming WHICH MACHINE it runs against and +# WHAT it touches, because this lab has two docker daemons - your workstation's and +# the target's - and picking the wrong one fails silently in both directions. +# +# MODE selects the topology, and it is the difference between a clear demo and a +# confusing one: +# +# MODE=native (default) Build on THIS workstation's engine. The target only runs +# the app and the agent, reached solely over ssh. Two +# machines, one job each, no ambiguity. This is the real +# topology for a Linux dev with a board, and it is what +# makes the pull an actual network transfer. +# +# MODE=vm Build on the target's own engine through the forwarded +# socket, emulating macOS/Windows where docker runs in a +# helper VM. The target then plays BOTH roles, which is +# what made "which docker am I talking to" ambiguous. +# `verify` needs this, because the VM write path is what +# it tests. +# +# The CLI picks the topology off DOCKER_HOST alone: is_vm_routing_active() +# (container.rs:79-89) is true iff DOCKER_HOST equals the avocado-vm socket. So +# native mode is a config choice, not a second VM. +# +# Environment: +# MODE native | vm (default: native) +# TARGET_PLATFORM e.g. linux/arm64 (default: empty = same arch as the +# build engine). Setting it switches the build to buildx, +# which emits NO tag event, so the sync is triggered +# explicitly instead of waiting on the watcher. +# LAB_VM 1 = the target is this repo's QEMU lab VM (default: 1). +# Set 0 for real hardware: `setup`/`verify` then refuse +# rather than trying to boot or test a VM that is not there. +# AVOCADO_CDM_LAB_WORK generated lab state (default: ~/repos/work/peridio-container-dev/lab) +# AVOCADO_CLI avocado-cli checkout (default: derived from this script's path) +# SSH_ALIAS ssh alias for the target (default: avocado-hitl) +# TEST_IMAGE watched image ref (default: my-app:dev) +# APP_SERVICE unit owning it (default: app.service) +# +# Pointing this at a Raspberry Pi 5 (or any real board) is env only: +# +# export LAB_VM=0 # no VM to boot or verify +# export SSH_ALIAS=pi5 # your ssh alias for the board +# export TARGET_PLATFORM=linux/arm64 # cross-build from an x86-64 host +# unset AVOCADO_CONTAINER_DEV_HOST # let the CLI detect your LAN address +# demo.sh app v1 && demo.sh up && demo.sh agent && demo.sh reload v2 +# +# Two things genuinely differ on arm64 and both are handled above rather than left +# as a surprise. The arch guard REFUSES a wrong-arch push, so an amd64 image built +# on your laptop never silently reaches an arm64 board - TARGET_PLATFORM is what +# keeps that guard satisfied. And a cross-build needs buildx, which is BuildKit and +# therefore emits no tag event, so the watcher cannot see it; the script triggers +# `container dev sync` itself in that case. +# +# Still manual for a real board: the agent binary must exist on it. Either it ships +# in the runtime as avocado-ext-container-agent-dev, or cross-compile it for +# aarch64-unknown-linux-musl (runbook B1, swapping the target triple) and copy it in. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +AVOCADO_CLI="${AVOCADO_CLI:-$(cd "$SCRIPT_DIR/../../.." && pwd)}" +# Generated lab state, matching setup-lab.sh's own default. It lives outside any +# repo checkout because the target's disk image is ~1 GB. +LAB="${AVOCADO_CDM_LAB_WORK:-$HOME/.cache/avocado-cdm-lab}" +# Where the bootable target artifacts and its qemu pidfile live. +VMDIR="${VMDIR:-$LAB/hitl-vm}" +SSH_ALIAS="${SSH_ALIAS:-avocado-hitl}" +# The agent ships in the runtime as a real unit (avocado-ext-container-agent-dev), +# so there is nothing to cross-compile and copy any more. It used to be started as +# a transient `cdm-agent` via systemd-run against a hand-placed binary. +AGENT_UNIT="${AGENT_UNIT:-container-agent-dev}" +# The target's docker daemon reports its own hostname, which is the Avocado image's +# hostname (avocado-) and NOT the ssh alias. setup-lab.sh exports the real +# value; fall back to asking the target so a bare run still works. +TARGET_HOSTNAME="${TARGET_HOSTNAME:-}" +# The lab alias uses UserKnownHostsFile=/dev/null, so ssh prints "Permanently +# added ..." on every connection. That noise ends up inside captured command output +# and reads as if it came from the app, so quiet it at the source. +SSH_Q=(ssh -o LogLevel=ERROR) +TEST_IMAGE="${TEST_IMAGE:-my-app:dev}" +APP_SERVICE="${APP_SERVICE:-app.service}" +CONTAINER="${APP_SERVICE%.service}" +BUILD_CTX="${BUILD_CTX:-/tmp/cdm-app}" +DOCK_SOCK="${DOCK_SOCK:-$HOME/.avocado/vm/docker.sock}" +UP_LOG="${UP_LOG:-/tmp/cdm-up.log}" +MODE="${MODE:-native}" +TARGET_PLATFORM="${TARGET_PLATFORM:-}" +LAB_VM="${LAB_VM:-1}" +case "$MODE" in native|vm) ;; *) echo "MODE must be native or vm, got '$MODE'" >&2; exit 1 ;; esac + +B=$'\033[1m'; R=$'\033[0m' + +# --------------------------------------------------------------------------- +# Context reporting. The whole point of this script: never run a docker command +# without first saying which daemon it lands on. +# --------------------------------------------------------------------------- + +ctx() { + printf '\n%s== %s ==%s\n' "$B" "$1" "$R" + shift + while [ $# -gt 0 ]; do printf ' %-11s %s\n' "${1%%|*}" "${1#*|}"; shift; done +} + +die() { printf '\n!! %s\n' "$*" >&2; exit 1; } + +# `grep -c` prints 0 AND exits 1 on no-match, so a naive `|| echo 0` prints twice. +count_in() { local n; n="$(grep -c "$1" "$2" 2>/dev/null)"; echo "${n:-0}"; } + +# Which daemon does a given DOCKER_HOST answer as? The name is the daemon's own +# hostname, so matching it against the target's hostname says which box answered. +daemon_name() { DOCKER_HOST="$1" docker info --format '{{.Name}}' 2>/dev/null || true; } + +# The target's own hostname, resolved once and cached. NOT the ssh alias: an +# Avocado OS image is named avocado-, so comparing a daemon's reported +# name against the alias would never match and every socket check would fail. +target_hostname() { + if [ -z "$TARGET_HOSTNAME" ]; then + TARGET_HOSTNAME="$("${SSH_Q[@]}" "$SSH_ALIAS" hostname 2>/dev/null || true)" + fi + echo "$TARGET_HOSTNAME" +} + +# The two engines, as two named functions. Every docker call in this script goes +# through one of them, so no command can quietly land on the wrong machine. + +# The engine that BUILDS. native: this workstation. vm: the target's, forwarded. +build_engine() { + if [ "$MODE" = native ]; then + env -u DOCKER_HOST docker "$@" + else + target_engine "$@" + fi +} + +# The engine that RUNS the app - always the target's, reached differently per mode. +# native has no forwarded socket by design, so it goes over ssh. +target_engine() { + if [ "$MODE" = native ]; then + "${SSH_Q[@]}" "$SSH_ALIAS" docker "$@" + else + [ -S "$DOCK_SOCK" ] || die "no forwarded target engine socket at $DOCK_SOCK - run: $0 setup" + local name want; name="$(daemon_name "unix://$DOCK_SOCK")"; want="$(target_hostname)" + [ "$name" = "$want" ] || die "socket $DOCK_SOCK answers as '$name', expected the target '$want'" + DOCKER_HOST="unix://$DOCK_SOCK" docker "$@" + fi +} + +# Human-readable description of where each engine lives, for the ctx headers. +# Which builder build_image will pick, and why - for the context header. +build_desc() { + if [ -n "$TARGET_PLATFORM" ]; then echo "buildx cross-arch -> no tag event, sync triggered explicitly"; return; fi + local srv major + srv="$(build_engine version --format '{{.Server.Version}}' 2>/dev/null || echo 0)" + major="${srv%%.*}"; case "$major" in ''|*[!0-9]*) major=0 ;; esac + if [ "$major" -ge 23 ]; then echo "BuildKit (docker $srv emits the tag event the watcher needs)" + else echo "classic, DOCKER_BUILDKIT=0 (docker $srv emits no event for BuildKit builds)"; fi +} + +build_engine_where() { + if [ "$MODE" = native ]; then echo "THIS WORKSTATION's engine ($(env -u DOCKER_HOST docker info --format '{{.Name}}' 2>/dev/null || echo unreachable))" + else echo "the HITL TARGET's engine ($SSH_ALIAS) via $DOCK_SOCK"; fi +} +target_engine_where() { + if [ "$MODE" = native ]; then echo "the HITL TARGET's engine ($SSH_ALIAS) over ssh" + else echo "the HITL TARGET's engine ($SSH_ALIAS) via $DOCK_SOCK"; fi +} + +# Build the demo image. Returns 0 and sets EMITS_TAG_EVENT to 1/0 so callers know +# whether the watcher can see the rebuild or whether it must be triggered. +EMITS_TAG_EVENT=0 +write_ctx() { + local version="$1" + mkdir -p "$BUILD_CTX" + cat >"$BUILD_CTX/Dockerfile" < /base.bin +RUN printf '$version\\n' > /version +# Same version as a label so \`seed\` can read it with \`image inspect\`, which does +# not execute the image. A cross-arch build (TARGET_PLATFORM) is not runnable on +# the build host unless binfmt is registered there, so reading /version by running +# the image would break the very flow this lab advertises. +LABEL org.avocado.demo.version="$version" +CMD ["sh","-c","while true; do echo \\"app \$(cat /version) base=\$(wc -c /dev/null || die "cross-build for $TARGET_PLATFORM failed (is buildx + binfmt set up?)" + elif [ "$major" -ge 23 ]; then + # Modern daemon: BuildKit is fine, and the watcher sees the tag event. + EMITS_TAG_EVENT=1 + build_engine build -q -t "$TEST_IMAGE" "$BUILD_CTX" >/dev/null || die "build failed" + else + # Old daemon (<23): BuildKit emits no image event at all, so fall back to the + # classic builder, which does. + EMITS_TAG_EVENT=1 + DOCKER_BUILDKIT=0 build_engine build -q -t "$TEST_IMAGE" "$BUILD_CTX" >/dev/null || die "build failed" + fi +} + +# Is the watched image present on the TARGET's engine? +# Does the TARGET hold exactly the image the HOST currently has under the watched +# tag? Presence of the tag is NOT the question, and testing it was a real bug: a +# tag left behind by an earlier run points at a different image, passes a presence +# check, and makes the demo assert a version that was never delivered. +# +# Comparing image IDs is sound here because an ID is the digest of the image +# config, which a push/pull round trip preserves - verified by finding the +# target's running image present on the host under the previous session's +# registry tags, same ID. +host_image_id() { build_engine image inspect "$TEST_IMAGE" --format '{{.Id}}' 2>/dev/null; } +target_image_id() { target_engine image inspect "$TEST_IMAGE" --format '{{.Id}}' 2>/dev/null; } +target_has_host_image() { + local h t + h="$(host_image_id)" + t="$(target_image_id)" + [ -n "$h" ] && [ "$h" = "$t" ] +} + +# Find the running `container dev up` session. +# +# NOT by `pgrep -f "container dev up"`: that matches ANY process whose argv happens +# to contain the phrase, including the very shell running this script if the phrase +# appears anywhere in its command line - which kills the caller. Match the process +# NAME instead (comm is `avocado`; a wrapper shell's is not) and confirm via +# /proc//cmdline. +session_pids() { + local pid cmd + for pid in $(pgrep -x avocado 2>/dev/null); do + cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null)" + case "$cmd" in *"container dev up"*) echo "$pid" ;; esac + done +} +session_running() { [ -n "$(session_pids)" ]; } + +# The write listener's actual host:port, read from what the session reported. +# +# It is NOT the configured port: the session binds an EPHEMERAL loopback port +# (37633 and 41753 across two observed runs). It is also not the guest-facing +# 10.0.2.2 - the push goes to 127.0.0.1, which is what the pushed tag shows +# (`The push refers to repository [127.0.0.1:41753/my-app]`). Both were previously +# hardcoded as 10.0.2.2:5601, which described a path that never existed. This +# matters beyond cosmetics: the push credential is keyed on the tagged host:port +# byte-for-byte, so a reader debugging an auth failure needs the real pair. +write_endpoint() { + local wport + wport="$(sed -n 's/.*write listener loopback-only on 127\.0\.0\.1:\([0-9]\+\).*/\1/p' \ + "$UP_LOG" 2>/dev/null | tail -1)" + if [ -n "$wport" ]; then echo "127.0.0.1:$wport" + else echo "127.0.0.1:${AVOCADO_CONTAINER_DEV_WRITE_PORT:-5601} (configured; no session has bound one yet)"; fi +} + +registry_endpoints() { + local host="${AVOCADO_CONTAINER_DEV_HOST:-10.0.2.2}" + printf 'bulk read %s:5599 (target pulls) | write %s (host pushes, loopback-bound) | control WS %s:5600' \ + "$host" "$(write_endpoint)" "$host" +} + +# --------------------------------------------------------------------------- + +cmd_setup() { + [ "${LAB_VM:-1}" = 1 ] || die "LAB_VM=0: the target is real hardware, there is no VM to boot" + ctx "SETUP the lab VM" \ + "runs on|this workstation" \ + "creates|QEMU VM '$SSH_ALIAS', ssh 127.0.0.1:2222, forwarded engine socket $DOCK_SOCK" \ + "note|first run builds and provisions a real Avocado OS runtime: minutes, needs network" + AVOCADO_CDM_LAB_WORK="$LAB" AVOCADO_CLI="$AVOCADO_CLI" bash "$SCRIPT_DIR/setup-lab.sh" || die "setup-lab.sh failed" +} + +cmd_verify() { + [ "${LAB_VM:-1}" = 1 ] || die "LAB_VM=0: verify-vm-write-path.sh tests the QEMU VM write path only" + [ "$MODE" = vm ] || die "verify tests the VM write path - re-run as: MODE=vm $0 verify" + ctx "VERIFY the authenticated push path" \ + "runs on|this workstation" \ + "reaches|$SSH_ALIAS over ssh, and its engine over $DOCK_SOCK" \ + "note|starts and tears down its OWN session, and rotates the target's bootstrap token" + # shellcheck source=/dev/null + source "$LAB/env.sh" + ( cd "$AVOCADO_CLI" && ./docs/container-dev/verify-vm-write-path.sh ) +} + +cmd_app() { + local version="${1:-v1}" + ctx "BUILD the demo app" \ + "mode|$MODE" \ + "builds on|$(build_engine_where)" \ + "image|$TEST_IMAGE version=$version${TARGET_PLATFORM:+ platform=$TARGET_PLATFORM}" \ + "builder|$(build_desc)" \ + "runs on|$(target_engine_where)" + + build_image "$version" + + ctx "INSTALL the owning service" \ + "installs on|the HITL TARGET ($SSH_ALIAS), over ssh" \ + "unit|/etc/systemd/system/$APP_SERVICE -> docker run --name $CONTAINER $TEST_IMAGE" \ + "why|the agent restarts this UNIT; an engine 'restart' would re-run the pinned image ID and silently keep the old code" + + # shellcheck disable=SC2087 # local expansion is intended: bake the refs in + ssh "$SSH_ALIAS" "cat > /etc/systemd/system/$APP_SERVICE" </dev/null 2>&1" \ + || die "could not install $APP_SERVICE on $SSH_ALIAS" + + # In native mode the image was built HERE, so nothing on the target is that image + # until `seed` ships it - delivery IS the loop. Starting the unit and asserting a + # baseline here can only work in vm mode, where the build happened on the target. + # + # There is deliberately no "unless the target already has it" escape. That escape + # existed and was wrong: it tested whether the TAG was present, which a previous + # run leaves behind pointing at a DIFFERENT image, so the demo restarted the unit + # on a stale image and then failed asserting the version it had just built but + # never delivered. + if [ "$MODE" = vm ]; then + ssh "$SSH_ALIAS" "systemctl restart $APP_SERVICE" || die "could not start $APP_SERVICE" + sleep 4 + local line; line="$(target_engine logs --tail 1 "$CONTAINER" 2>&1)" + ctx "APP is up" "reading|$(target_engine_where)" "says|$line" + case "$line" in + "app $version "*) printf ' %-11s %s\n' "result" "baseline $version confirmed on the target" ;; + *) die "app is not reporting '$version' - ssh $SSH_ALIAS 'journalctl -u $APP_SERVICE -n 20'" ;; + esac + else + printf ' %-11s %s\n' "unit" "installed and enabled, NOT started" + printf ' %-11s %s\n' "why" "the host built $TEST_IMAGE; '$0 seed' delivers it over the loop" + if [ -n "$(target_image_id)" ]; then + printf ' %-11s %s\n' "note" "the target holds an older $TEST_IMAGE from a previous run; seed replaces it" + fi + fi +} + +# Deliver the built image to the target through the real path, then start the unit. +# +# This is the step that proves delivery works at all. It needs a live session AND a +# running agent, because the push goes to the host's write listener and only the +# agent can pull it back down over the control WS. +cmd_seed() { + # No version argument. It used to take one and assert it, but `seed` does not + # build - it ships whatever the host holds under the watched tag - so the + # argument was a claim about content that this step never established. Passing + # `seed v1` after a `reload v2` failed with "app is not reporting 'v1'" while + # delivery had in fact worked perfectly. The version is a property of the + # artifact, so read it out of the artifact instead. + # Gate first, then note. Printing the note before the session check emitted a + # contextless line and then died, so `demo.sh seed v1` with no session led with + # advice about an argument instead of the actual problem. + session_running || die "no session - run '$0 up' first" + local ignored="${1:-}" + + # Read the version from the image's LABEL, not by running the image. `seed` is a + # ship-only step and must stay build-only: with TARGET_PLATFORM set the image is + # a foreign architecture, and a `docker-container` buildx driver carries its + # emulation inside the builder - so `demo.sh app v1` succeeds while `docker run` + # of that same image fails "exec format error" unless binfmt/qemu-user happens to + # be registered on the host. `image inspect` never executes anything, is cheaper, + # and works over the forwarded socket in MODE=vm. + local want + want="$(build_engine image inspect --format '{{index .Config.Labels "org.avocado.demo.version"}}' "$TEST_IMAGE" 2>/dev/null | tr -d '\r\n')" + # Both empty and the literal `` mean "no such label". Measured on docker + # 29.7.1: a missing key yields EMPTY, whether or not the image carries other + # labels, and an absent image exits non-zero with empty stdout - so the empty arm + # is the one that fires here. `` is text/template's older output for a + # missing map key; it is kept because this script deliberately supports daemons + # back to 20.10 (see the builder-version gate in build_image), and it is NOT + # verified on one. Do not drop it on the strength of a 29.x run alone. + case "$want" in + ''|'') + die "$TEST_IMAGE on the build engine carries no org.avocado.demo.version label - rebuild it with '$0 app '" ;; + esac + + ctx "SEED the target with the baseline image" \ + "runs on|this workstation, then the target pulls" \ + "shipping|$TEST_IMAGE containing version $want" \ + "path|host build -> write listener $(write_endpoint) -> control WS -> agent pulls by digest -> $APP_SERVICE" \ + "why|native mode builds HERE, so the target has no image until the loop ships one" + if [ -n "$ignored" ]; then + printf ' %-11s %s\n' "note" "ignoring '$ignored' - seed ships what the host holds and reads the version off the image" + fi + + ( cd "$SCRIPT_DIR" && "${AVOCADO_BIN:-avocado}" container dev sync >/dev/null 2>&1 ) \ + || die "container dev sync failed - is a session up? ($0 up)" + + # Wait for the target to hold the HOST's image, not merely a tag of that name. + # A stale tag from a previous run satisfies a presence check instantly, so this + # loop used to fall through on the first tick and then restart the unit on the + # old image. + printf ' %-11s ' "waiting" + for _ in $(seq 1 30); do + sleep 2; printf '.' + target_has_host_image && break + done + printf '\n' + target_has_host_image || die "the host's $TEST_IMAGE never reached the target (host $(host_image_id | cut -c8-19), target $(target_image_id | cut -c8-19 || echo none)) - check: $0 logs session ; $0 logs agent" + + ssh "$SSH_ALIAS" "systemctl restart $APP_SERVICE" || die "could not start $APP_SERVICE" + sleep 4 + local line; line="$(target_engine logs --tail 1 "$CONTAINER" 2>&1)" + ctx "APP is up" "reading|$(target_engine_where)" "says|$line" + # Still a real check even though the image IDs already match: it is the + # difference between the image having landed and the SERVICE having adopted it. + # + # Anchor on the leading `app ` field rather than substring-matching the + # whole line. write_ctx puts `image=$TEST_IMAGE` into the same line, so a bare + # `*"$want"*` passes whenever the version is a substring of the image ref - and + # TEST_IMAGE defaults to my-app:dev, so `want=dev` matched unconditionally. It + # also let a prefix satisfy its own extension (v1 accepted while v10 ran). + case "$line" in + "app $want "*) printf ' %-11s %s\n' "result" "$want delivered over the loop and running on the target" ;; + *) die "the target holds the host's image but its container still reports something else - ssh $SSH_ALIAS 'journalctl -u $APP_SERVICE -n 20'" ;; + esac +} + +cmd_up() { + session_running && die "a session is already running - $0 down first" + # Clear the log BEFORE the header reads it. write_endpoint() parses the bound + # write port out of this file, and the session below truncates it on start - so + # on a second run the header was reporting the PREVIOUS session's port as though + # it were current. Truncating here makes write_endpoint fall back to saying no + # session has bound one yet, which is the truth at this point in the run. + : >"$UP_LOG" + ctx "START the dev session" \ + "runs on|this workstation" \ + "serves|$(registry_endpoints)" \ + "store|$HOME/.avocado/container-dev//registry/" \ + "reaches|$SSH_ALIAS once over ssh to bootstrap it, then never again" \ + "log|$UP_LOG (every push shows up here)" + # The CLI reads ./avocado.yaml from the cwd, not $AVOCADO_CONFIG. + # shellcheck source=/dev/null + [ -f "$LAB/env.sh" ] && source "$LAB/env.sh" + if [ "$MODE" = native ]; then + # env.sh points DOCKER_HOST at the VM socket, and is_vm_routing_active() keys + # on exactly that. Unset it or the CLI takes the vm path and builds/pushes + # through the target's engine - the topology native mode exists to avoid. + unset DOCKER_HOST + fi + # `setsid --fork`, with no trailing `&` and no subshell job. + # + # The previous form was `( setsid nohup CMD ... & )`. setsid execs in place when + # it can, so the session stayed a child of this script and a bash job; the script + # then blocked at exit waiting on it. Every step of `up` ran and printed, but the + # script never returned - and when its output is piped (`demo.sh up | tail`) the + # reader sees NOTHING at all, because the pipe's write end is still held. That + # reads as "up hangs" when the session is in fact healthy. + # + # --fork makes setsid fork unconditionally, so the session is reparented away and + # is never a job of this shell. Redirecting all three streams is still required: + # an inherited stdout would keep the caller's pipe open on its own. + # `${AVOCADO_BIN:-avocado}`, matching every other call site. Bare "$AVOCADO_BIN" + # was an unbound variable under `set -u` whenever $LAB/env.sh was absent (its + # source above is `[ -f ]`-guarded), and bash aborts the subshell BEFORE + # performing the >"$UP_LOG" redirection - so the log kept a PREVIOUS session's + # contents, and the `bulk listener` grep below then passed for a session that + # never started. Verified: the redirect does not run, and with no `set -e` the + # failing subshell does not stop the script either. + ( cd "$SCRIPT_DIR" && setsid --fork "${AVOCADO_BIN:-avocado}" container dev up >"$UP_LOG" 2>&1 the host's bulk listener" \ + "restarts|$APP_SERVICE (AVOCADO_CONTAINER_DEV_SERVICE, from the setup-lab drop-in)" \ + "gate|ConditionPathExists=/var/lib/avocado/container-dev/bootstrap.json, so '$0 up' must run first" + + # The unit stays inert until `container dev up` has delivered the bootstrap, so a + # start before that is not an error - it is the condition doing its job. Say so + # rather than reporting a failure the operator cannot act on. + ssh "$SSH_ALIAS" "test -f /var/lib/avocado/container-dev/bootstrap.json" 2>/dev/null \ + || die "no bootstrap on the target yet - run '$0 up' first (the unit's ConditionPathExists gates on it)" + + # Restart rather than start: a re-run after a new session must not keep an agent + # holding the previous session's pinned CA. + ssh "$SSH_ALIAS" "systemctl restart $AGENT_UNIT" \ + || die "could not start $AGENT_UNIT - ssh $SSH_ALIAS 'journalctl -u $AGENT_UNIT -n 30'" + sleep 5 + local active; active="$(ssh "$SSH_ALIAS" "systemctl is-active $AGENT_UNIT" 2>/dev/null || echo unknown)" + [ "$active" = active ] || die "$AGENT_UNIT is '$active' - ssh $SSH_ALIAS 'journalctl -u $AGENT_UNIT -n 30'" + ssh "$SSH_ALIAS" "journalctl -u $AGENT_UNIT --no-pager -n 3 -o cat" 2>/dev/null | sed -e 's/^/ /' +} + +cmd_reload() { + local version="${1:-v2-RELOADED}" + local before; before="$(target_engine logs --tail 1 "$CONTAINER" 2>&1)" + ctx "RELOAD: rebuild only, let the loop do the rest" \ + "mode|$MODE" \ + "builds on|$(build_engine_where)" \ + "version|$version${TARGET_PLATFORM:+ platform=$TARGET_PLATFORM}" \ + "pushes to|the host's write listener $(write_endpoint), tagged $(write_endpoint)/${TEST_IMAGE%%:*}" \ + "then|control WS notifies the target, which pulls by digest and restarts $APP_SERVICE" \ + "note|the unit is NOT touched here, so only the watcher path can move the container" \ + "before|$before" + + build_image "$version" + if [ "$EMITS_TAG_EVENT" = 0 ]; then + printf ' %-11s %s\n' "trigger" "buildx emits no tag event, so triggering the sync explicitly" + ( cd "$SCRIPT_DIR" && "${AVOCADO_BIN:-avocado}" container dev sync >/dev/null 2>&1 ) \ + || die "container dev sync failed - is a session up? ($0 up)" + fi + + printf ' %-11s ' "waiting" + local line="" + for _ in $(seq 1 30); do + sleep 2; printf '.' + line="$(target_engine logs --tail 1 "$CONTAINER" 2>&1)" + case "$line" in "app $version "*) break ;; esac + done + printf '\n' + + ctx "RESULT" \ + "reading|$(target_engine_where)" \ + "after|$line" \ + "pushes|$(count_in 'The push refers' "$UP_LOG") in $UP_LOG, $(count_in 'no basic auth credentials' "$UP_LOG") auth failures" + case "$line" in + "app $version "*) printf ' %-11s %s\n' "result" "hot reload landed: the watcher moved the target to $version" ;; + *) die "no reload after 60s - check: $0 logs session ; $0 logs agent" ;; + esac +} + +cmd_sync() { + ctx "SYNC: re-push and notify, without waiting on an event" \ + "runs on|this workstation" \ + "pushes|whatever the BUILD engine currently holds under $TEST_IMAGE" \ + "caveat|if your image went to the other engine, this pushes the stale one and reports success" + # shellcheck source=/dev/null + [ -f "$LAB/env.sh" ] && source "$LAB/env.sh" + [ "$MODE" = native ] && unset DOCKER_HOST + ( cd "$SCRIPT_DIR" && "${AVOCADO_BIN:-avocado}" container dev sync ) || die "sync failed - is a session up?" +} + +cmd_status() { + local host_daemon target_daemon + host_daemon="$(env -u DOCKER_HOST docker info --format '{{.Name}}' 2>/dev/null || echo '(unreachable)')" + target_daemon="$(daemon_name "unix://$DOCK_SOCK")"; : "${target_daemon:=(unreachable)}" + + ctx "WHERE THINGS ARE" \ + "workstation|engine '$host_daemon' <- your builds land here if DOCKER_HOST is unset" \ + "HITL target|engine '$target_daemon' via $DOCK_SOCK, shell via 'ssh $SSH_ALIAS'" \ + "registry|$(registry_endpoints)" \ + "store|$HOME/.avocado/container-dev/" + + local up_pid; up_pid="$(session_pids | head -1)" + ctx "SESSION (workstation)" \ + "up|$([ -n "$up_pid" ] && echo "running (pid $up_pid)" || echo 'not running')" \ + "log|$UP_LOG" \ + "pushes|$(count_in 'The push refers' "$UP_LOG"), auth failures $(count_in 'no basic auth credentials' "$UP_LOG")" + + # Gate on ssh, not on the forwarded socket. Native mode has no forwarded socket by + # design, so keying this block on the socket hid the target's whole state in the + # default topology. + if "${SSH_Q[@]}" -o ConnectTimeout=5 "$SSH_ALIAS" true 2>/dev/null; then + ctx "TARGET ($SSH_ALIAS = $(target_hostname))" \ + "agent|$AGENT_UNIT $(ssh "$SSH_ALIAS" "systemctl is-active $AGENT_UNIT" 2>/dev/null || echo unknown)" \ + "service|$APP_SERVICE $(ssh "$SSH_ALIAS" "systemctl is-active $APP_SERVICE" 2>/dev/null || echo unknown)" \ + "app says|$(target_engine logs --tail 1 "$CONTAINER" 2>&1 | tail -1)" \ + "running|$(ssh "$SSH_ALIAS" 'cat /var/lib/avocado/container-dev/active-image.json 2>/dev/null | tr -d "\n " ' 2>/dev/null || echo '(no pointer yet)')" + else + ctx "TARGET ($SSH_ALIAS)" "state|unreachable over ssh - run '$0 setup'" + fi +} + +cmd_logs() { + case "${1:-}" in + session) + ctx "SESSION LOG" "from|this workstation" "file|$UP_LOG" + tail -30 "$UP_LOG" ;; + agent) + ctx "AGENT LOG" "from|the HITL TARGET ($SSH_ALIAS)" "source|journalctl -u $AGENT_UNIT" + ssh "$SSH_ALIAS" "journalctl -u $AGENT_UNIT --no-pager -n 30 -o cat" ;; + app) + ctx "APP LOG" "from|the HITL TARGET's engine ($SSH_ALIAS) via $DOCK_SOCK" "source|docker logs $CONTAINER" + target_engine logs --tail 30 "$CONTAINER" ;; + *) die "usage: $0 logs session|agent|app" ;; + esac +} + +cmd_down() { + ctx "STOP the demo" "affects|this workstation (session) and the target (agent, app)" "keeps|the VM running and warm" + # shellcheck source=/dev/null + [ -f "$LAB/env.sh" ] && source "$LAB/env.sh" + ( cd "$SCRIPT_DIR" && "${AVOCADO_BIN:-avocado}" container dev down 2>/dev/null | sed -e 's/^/ /' ) || true + session_pids | while read -r p; do kill "$p" 2>/dev/null; done + ssh "$SSH_ALIAS" "systemctl stop $AGENT_UNIT $APP_SERVICE 2>/dev/null; docker stop $CONTAINER 2>/dev/null; docker rm $CONTAINER 2>/dev/null; true" >/dev/null 2>&1 + printf ' %-11s %s\n' "done" "session, agent and app stopped" +} + +cmd_reset() { + ctx "RESET to a pre-demo state" \ + "deletes|the HITL target's disk image and u-boot.rom under $VMDIR" \ + "deletes|the host registry store and the demo build context" \ + "keeps|the built runtime in the SDK volume - 'setup' reprovisions from it without a rebuild" + session_pids | while read -r p; do kill "$p" 2>/dev/null; done + sleep 2 + pkill -f "$DOCK_SOCK:" 2>/dev/null; rm -f "$DOCK_SOCK" + if [ -f "$VMDIR/qemu.pid" ]; then + local qp; qp="$(cat "$VMDIR/qemu.pid")" + kill "$qp" 2>/dev/null + for _ in $(seq 1 10); do kill -0 "$qp" 2>/dev/null || break; sleep 1; done + kill -9 "$qp" 2>/dev/null + fi + rm -f "$VMDIR/avocado-os-"*.img "$VMDIR/u-boot.rom" "$VMDIR/console.log" "$VMDIR/qemu.pid" + rm -rf "$HOME/.avocado/container-dev" "$BUILD_CTX" + printf ' %-11s %s\n' "done" "start again with: $0 all" +} + +cmd_all() { + local v1="${1:-v1}" v2="${2:-v2-RELOADED}" + [ "${LAB_VM:-1}" = 1 ] && cmd_setup + # Order matters and is not arbitrary: the unit can only run an image the target + # actually has, and in native mode only the session+agent can put one there. So + # build and install first, bring the loop up, then seed through it, then reload. + cmd_app "$v1" + cmd_up + cmd_agent + # No version passed: seed reads it out of the image cmd_app just built. + cmd_seed + cmd_reload "$v2" + cmd_status +} + +case "${1:-}" in + setup) shift; cmd_setup "$@" ;; + verify) shift; cmd_verify "$@" ;; + app) shift; cmd_app "$@" ;; + seed) shift; cmd_seed "$@" ;; + up) shift; cmd_up "$@" ;; + agent) shift; cmd_agent "$@" ;; + reload) shift; cmd_reload "$@" ;; + sync) shift; cmd_sync "$@" ;; + status) shift; cmd_status "$@" ;; + logs) shift; cmd_logs "$@" ;; + down) shift; cmd_down "$@" ;; + reset) shift; cmd_reset "$@" ;; + all) shift; cmd_all "$@" ;; + ""|-h|--help|help) + # Print the header comment block: from line 3 until the first non-comment line. + awk 'NR>=3 && /^#/ { sub(/^# ?/, ""); print; next } NR>=3 { exit }' "${BASH_SOURCE[0]}" + ;; + *) die "unknown command '${1}' - run '$0 help'" ;; +esac diff --git a/docs/container-dev/lab/setup-lab.sh b/docs/container-dev/lab/setup-lab.sh new file mode 100644 index 00000000..5dde658f --- /dev/null +++ b/docs/container-dev/lab/setup-lab.sh @@ -0,0 +1,397 @@ +#!/usr/bin/env bash +# +# setup-lab.sh - Stand up a real Avocado OS HITL target for Container Dev Mode. +# +# The target is Avocado OS, because Avocado OS is the OS the feature ships on and +# every board is expected to run it. This script previously booted a Debian cloud +# image as a stand-in; that stand-in produced a false finding (its docker 20.10.24 +# emits no tag event for BuildKit builds, which was generalised into a property of +# BuildKit rather than of that daemon), so it is gone. Do not reintroduce it. +# +# What it does, all idempotent: +# +# 1. Renders a runtime config into $WORK/hitl and builds it from the published +# 2024/edge feed plus the SDK container. Two extensions matter: +# avocado-ext-docker - published in the feed. +# avocado-ext-container-agent-dev - NOT published; sourced from the local +# avocado-os checkout and compiled by the +# SDK for x86_64-avocado-linux-gnu. +# 2. Provisions it with the default `img` profile (fwup: gpt_write + raw_write). +# The `direct` profile is NOT the path - its own header says "No fwup archive, +# no GPT, no A/B slots, no bootloader", which is why a hand-rolled direct boot +# has no GPT partition UUID for /var to wait on and lands in emergency mode. +# 3. Copies the disk image and u-boot.rom out of the SDK docker volume onto the +# host, and boots them under host QEMU. Running on the host rather than inside +# the SDK container is deliberate: the target has to be a genuinely separate +# machine reached only over ssh, which is the entire point of the topology. +# 4. Adds an ssh alias, and installs the agent drop-in the device needs. +# 5. Writes an env file the demo driver and verify script source. +# +# Prerequisites on the host: docker (for the SDK container), qemu-system-x86_64, +# qemu-img, ssh, python3, an `avocado` on PATH carrying `container dev`, and a +# checkout of avocado-os on a branch that has extensions/container-agent-dev. +# +# Run it yourself (it touches ~/.ssh/config, so run interactively, not from an +# agent): +# bash docs/container-dev/lab/setup-lab.sh +# Then: +# source "${AVOCADO_CDM_LAB_WORK:-$HOME/.cache/avocado-cdm-lab}/env.sh" +# docs/container-dev/lab/demo.sh all +# +# Tunables (env overrides): AVOCADO_CDM_LAB_WORK (generated-state dir), +# AVOCADO_CLI (avocado-cli repo root), AVOCADO_OS (avocado-os repo root), +# TARGET (avocado target, default qemux86-64), DISK_SIZE, SSH_PORT, MEM, SMP. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Generated state (disk image, rendered config, env.sh) lives OUTSIDE the repo +# checkout so a ~1 GB image never lands in git. +WORK="${AVOCADO_CDM_LAB_WORK:-$HOME/.cache/avocado-cdm-lab}" +# This script sits at docs/container-dev/lab/, so ../../.. is the crate root. +AVOCADO_CLI="${AVOCADO_CLI:-$(cd "$SCRIPT_DIR/../../.." && pwd)}" +# avocado-os is a sibling checkout of avocado-cli in the peridio workspace. +AVOCADO_OS="${AVOCADO_OS:-$(cd "$AVOCADO_CLI/.." && pwd)/avocado-os}" + +TARGET="${TARGET:-qemux86-64}" +PROJ="$WORK/hitl" +VMDIR="$WORK/hitl-vm" +IMG="$VMDIR/avocado-os-$TARGET.img" +BIOS="$VMDIR/u-boot.rom" +DISK_SIZE="${DISK_SIZE:-8192M}" +SSH_PORT="${SSH_PORT:-2222}" +MEM="${MEM:-2048}" +SMP="${SMP:-2}" +SSH_ALIAS="${SSH_ALIAS:-avocado-hitl}" +APP_SERVICE="${APP_SERVICE:-app.service}" +CONSOLE="$VMDIR/console.log" +PIDFILE="$VMDIR/qemu.pid" +# is_vm_routing_active() keys on exactly this socket path, so MODE=vm needs it here. +VMROOT="$HOME/.avocado/vm" +DOCK_SOCK="$VMROOT/docker.sock" +WRITE_PORT=5601 + +say() { echo ">> $*"; } +die() { echo "$*" >&2; exit 1; } + +AGENT_EXT="$AVOCADO_OS/extensions/container-agent-dev" +[ -d "$AGENT_EXT" ] || die "missing $AGENT_EXT - set AVOCADO_OS to an avocado-os checkout carrying extensions/container-agent-dev" + +command -v avocado >/dev/null || die "no 'avocado' on PATH" +avocado container dev --help >/dev/null 2>&1 \ + || die "the 'avocado' on PATH has no 'container dev' subcommand - rebuild it from the working branch" + +mkdir -p "$PROJ" "$VMDIR" + +# --------------------------------------------------------------------------- +# 1. Render the runtime config. +# +# Generated rather than tracked because the container-agent-dev extension is +# sourced by ABSOLUTE path - it is not in the published feed, so it has to point +# at wherever avocado-os is checked out on this machine. +# --------------------------------------------------------------------------- +say "rendering $PROJ/avocado.yaml (target $TARGET, agent ext from $AGENT_EXT)" +# QUOTED heredoc, with @PLACEHOLDER@ substituted afterwards. +# +# It used to be unquoted so $TARGET and $AGENT_EXT would expand, which also made +# every backtick in the prose below a command substitution. That shipped: a comment +# mentioning `docker build ...` and `x509: ...` ran both as commands on every run, +# printing "requires 1 argument" and "x509:: command not found" and silently +# emptying the text from the generated file. Quoting the delimiter makes the whole +# block inert, so no future comment can execute, and the two values that genuinely +# vary are injected explicitly below where they are easy to see. +cat >"$PROJ/avocado.yaml" <<'EOF' +# GENERATED by setup-lab.sh - edit the generator, not this file. +# +# Shape follows the published qemu-quickstart reference, minus connect/tunnels +# (they need org credentials this lab has no use for), plus the two extensions +# Container Dev Mode requires. +default_target: @TARGET@ + +supported_targets: + - @TARGET@ + +distro: + release: 2024 + channel: edge + +runtimes: + dev: + extensions: + - avocado-ext-dev + - avocado-ext-sshd-dev + - avocado-bsp-{{ avocado.target.board }} + - avocado-ext-ca-certificates + - avocado-ext-docker + - avocado-ext-container-agent-dev + - config + packages: + avocado-runtime: "*" + +extensions: + avocado-ext-dev: + source: {type: package, version: "*"} + + # Public root CAs. Without these the target's engine cannot verify TLS to any + # public registry: a guest-side `docker build FROM busybox:latest` dies with + # `x509: certificate signed by unknown authority`, which is what took Part A to + # 7/8. Container Dev Mode's own traffic does not need this - it pins the + # per-project CA - but anything that pulls a public base image does. + avocado-ext-ca-certificates: + source: {type: package, version: "*"} + + avocado-ext-sshd-dev: + source: {type: package, version: "*"} + + avocado-bsp-{{ avocado.target.board }}: + source: {type: package, version: "*"} + + avocado-ext-docker: + source: {type: package, version: "*"} + + # Not published in the feed - built from source by the SDK via the extension's + # own cad-compile.sh / cad-install.sh, which target x86_64-avocado-linux-gnu. + avocado-ext-container-agent-dev: + source: + type: path + path: @AGENT_EXT@ + + # Empty root password so the lab can ssh in without provisioning a key. + # Dev target only - this is what avocado-ext-sshd-dev exists for. + config: + types: + - confext + version: "0.1.0" + users: + root: + password: "" + +sdk: + image: "docker.io/avocadolinux/sdk:{{ avocado.distro.release }}-{{ avocado.distro.channel }}" + container_args: + - --privileged + - --network=host + - -v /dev:/dev + - -v /sys:/sys + packages: + avocado-sdk-toolchain: "*" +EOF + +# Inject the two values the template leaves open. `|` as the delimiter because +# AGENT_EXT is a path and would otherwise need its slashes escaped. +sed -i \ + -e "s|@TARGET@|$TARGET|g" \ + -e "s|@AGENT_EXT@|$AGENT_EXT|g" \ + "$PROJ/avocado.yaml" + +# Fail loudly rather than handing avocado a config with an unfilled slot. +if grep -q '@[A-Z_]\+@' "$PROJ/avocado.yaml"; then + die "unsubstituted placeholder left in $PROJ/avocado.yaml: $(grep -o '@[A-Z_]\+@' "$PROJ/avocado.yaml" | sort -u | tr '\n' ' ')" +fi + +# --------------------------------------------------------------------------- +# 2. Build + provision. +# +# DOCKER_HOST must be unset for all of these: the SDK container runs on the HOST +# daemon. A DOCKER_HOST left pointing at the target's socket sends the build to +# the wrong daemon, and dockerd then auto-creates the missing bind source as an +# empty directory - which surfaces as a baffling "could not find Cargo.toml". +# --------------------------------------------------------------------------- +if [ ! -f "$IMG" ]; then + say "installing SDK + extension deps (first run pulls the SDK image: minutes)" + ( cd "$PROJ" && env -u DOCKER_HOST avocado install -f ) + + say "building the runtime (compiles the agent for the target ABI)" + ( cd "$PROJ" && env -u DOCKER_HOST avocado build ) + + say "provisioning with the default 'img' profile" + ( cd "$PROJ" && env -u DOCKER_HOST avocado provision -f dev ) + + # 3. Copy the image + BIOS out of the SDK volume onto the host. + say "copying the disk image and u-boot.rom out of the SDK volume" + vol="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["volume_name"])' "$PROJ/.avocado-state")" + stone="/opt/_avocado/$TARGET/output/runtimes/dev/stone" + env -u DOCKER_HOST docker run --rm \ + -v "$vol":/opt/_avocado -v "$VMDIR":/out alpine:3 sh -c " + set -e + cp $stone/_build/avocado-os-$TARGET.img /out/ + cp $stone/u-boot.rom /out/ + chown -R $(id -u):$(id -g) /out + " +else + say "image already present at $IMG (delete it to rebuild from scratch)" +fi + +# --------------------------------------------------------------------------- +# 4. Boot it. +# --------------------------------------------------------------------------- +if [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then + say "HITL target already running (pid $(cat "$PIDFILE"))" +else + # Grow only, never shrink. The SDK's own vm script runs an unconditional + # `qemu-img resize -f raw 1024M`, which is a SHRINK for any image over + # 1024M; qemu-img refuses it and the script dies under `set -e` before qemu + # starts. Growing also gives /var room for the container images the demo pulls + # (avocado-grow-var.service expands /var to fill the disk on boot). + cur="$(qemu-img info --output=json "$IMG" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["virtual-size"])')" + want="$(numfmt --from=iec "${DISK_SIZE%B}")" + if [ "$cur" -lt "$want" ]; then + say "growing the disk to $DISK_SIZE" + qemu-img resize -f raw "$IMG" "$DISK_SIZE" + fi + + # TCG, not KVM: `-cpu host -enable-kvm` faults the u-boot BIOS with + # "Exception 13 executing option rom". The disk attaches as an SD card + # (sdhci-pci + sd-card) because that is where this u-boot looks for its boot + # partition. The guest reaches the host at 10.0.2.2 under SLIRP, which is how + # the agent dials the host's control WS and registry - only ssh needs a + # hostfwd, every Container Dev Mode connection is guest-initiated. + say "booting the HITL target (TCG, ssh hostfwd $SSH_PORT->22, console -> $CONSOLE)" + qemu-system-x86_64 \ + -bios "$BIOS" \ + -device sdhci-pci -device sd-card,drive=mmc \ + -drive file="$IMG",if=none,format=raw,id=mmc \ + -m "$MEM" -smp "$SMP" -cpu max \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \ + -device e1000,netdev=net0 \ + -display none -serial file:"$CONSOLE" -monitor none \ + -daemonize -pidfile "$PIDFILE" +fi + +# --------------------------------------------------------------------------- +# 5. ssh alias, PREPENDED so its host-key policy wins. +# +# ssh uses the FIRST value seen for each keyword, so a global "Host *" block +# earlier in the file would otherwise force its StrictHostKeyChecking and +# UserKnownHostsFile onto this alias. Our block at the top makes accept-new + +# /dev/null win, so a throwaway target whose host key changes on every reprovision +# never triggers a refusal. +# --------------------------------------------------------------------------- +mkdir -p "$HOME/.ssh" +CFG="$HOME/.ssh/config" +touch "$CFG" +say "prepending ssh alias '$SSH_ALIAS' to ~/.ssh/config" +STRIPPED="$(awk ' + /^Host '"$SSH_ALIAS"'$/ {skip=1; next} + skip && /^[ \t]/ {next} + {skip=0; print} +' "$CFG")" +# No backticks in the heredoc below: it is unquoted so the $VARs expand, which +# means a backtick would run as command substitution on the host instead of +# landing as text. (That bug shipped once and ran `config` as a command.) +{ + cat <"$CFG" +chmod 600 "$CFG" + +say "waiting for ssh + docker on the target (first boot ~60-90s under TCG)" +ok=0 +for _ in $(seq 1 60); do + if ssh -o ConnectTimeout=3 "$SSH_ALIAS" 'docker version >/dev/null 2>&1' 2>/dev/null; then + ok=1 + break + fi + sleep 3 +done +[ "$ok" = 1 ] || { echo "target never became ready; see $CONSOLE" >&2; exit 1; } +say "target ready: ssh + docker" + +# --------------------------------------------------------------------------- +# 6. Agent drop-in. +# +# The agent learns which unit owns the container ONLY from +# $AVOCADO_CONTAINER_DEV_SERVICE (agent/src/sync.rs service_from_env). The +# `service:` field under container_dev.images is host-side config: DeviceBootstrap +# carries bulk_endpoint, read_token, ca_cert_pem and ws_endpoint - no service - so +# nothing delivers it to the device. Without this drop-in the agent falls back to +# `docker restart `, which re-executes the container's pinned image ID, +# so a freshly pulled image is ignored and every sync silently no-ops while +# reporting success. Install it here rather than leaving it to the operator. +# --------------------------------------------------------------------------- +say "installing the agent drop-in (AVOCADO_CONTAINER_DEV_SERVICE=$APP_SERVICE)" +# shellcheck disable=SC2087 # client-side expansion is intended: bake APP_SERVICE in +# mkdir -p, not install -d: the target's coreutils are BusyBox and it has no +# `install`. Anything this script runs on the target must stay inside BusyBox's +# subset (same reason `head -n N` is required over `head -N`). +ssh "$SSH_ALIAS" "mkdir -p /etc/systemd/system/container-agent-dev.service.d && \ + cat > /etc/systemd/system/container-agent-dev.service.d/10-service.conf" < $DOCK_SOCK" +mkdir -p "$VMROOT" +pkill -f "${DOCK_SOCK}:/var/run/docker.sock" 2>/dev/null || true +rm -f "$DOCK_SOCK" +ssh -f -N -L "${DOCK_SOCK}:/var/run/docker.sock" "$SSH_ALIAS" +for _ in $(seq 1 10); do + [ -S "$DOCK_SOCK" ] && break + sleep 1 +done +if DOCKER_HOST="unix://$DOCK_SOCK" docker version >/dev/null 2>&1; then + say "target engine reachable via $DOCK_SOCK" +else + die "target engine not reachable via $DOCK_SOCK" +fi + +# --------------------------------------------------------------------------- +# 8. env file for the demo driver and the verify script. +# --------------------------------------------------------------------------- +AVOCADO_BIN="${AVOCADO_BIN:-$(command -v avocado)}" +TARGET_HOSTNAME="$(ssh "$SSH_ALIAS" 'hostname' 2>/dev/null || echo "$SSH_ALIAS")" + +cat >"$WORK/env.sh" <> to tear down: $SCRIPT_DIR/demo.sh reset" diff --git a/docs/container-dev/phase0-findings.md b/docs/container-dev/phase0-findings.md new file mode 100644 index 00000000..5051ebd9 --- /dev/null +++ b/docs/container-dev/phase0-findings.md @@ -0,0 +1,171 @@ +# Container Dev Mode — Phase 0 de-risk findings + +Phase 0 is the recorded de-risk gate for the `container-dev-mode` devspec change. +Phase 1 (the embedded registry, watcher, and device agent) is blocked until a GO +is recorded here (task 1.7). Each task below is a spike whose result is recorded, +not production code. + +## Status + +| Task | What it proves | Status | +|------|----------------|--------| +| 1.1 | Free layer-delta on the device runtime (warm-cache path) | DONE — maintainer-attested in-lab (2026-07-21) | +| 1.2 | Five-host-path sync-latency matrix (macOS split) | DONE — maintainer-attested in-lab (2026-07-21) | +| 1.3 | INGEST digest preservation per image store | DONE — maintainer-attested in-lab (2026-07-21) | +| 1.4 | Native-Linux loopback push + CLI-injected Basic credential, both engines | DONE — docker arm in-session; podman arm maintainer-attested (2026-07-21) | +| 1.5 | Loopback proxy over a production-shaped (TLS+token) bulk leg | DONE — maintainer-attested in-lab (2026-07-21) | +| 1.6 | macOS firewall + non-conflicting default port | DONE — maintainer-attested in-lab (2026-07-21) | +| 1.7 | Recorded GO/NO-GO decision | **GO (2026-07-21)** | +| 1.8 | Authenticated VM push with delivered CA + IP-SAN | DONE — maintainer-attested in-lab (2026-07-21) | +| 1.9 | Agent TLS stack cross-compile across SDK targets | DONE — maintainer-attested in-lab (2026-07-21) | +| 1.10 | Rootless-no-socket podman tag-event emission | DONE — maintainer-attested in-lab (2026-07-21) | +| **1.11** | **Two-socket separation + defense-in-depth auth matrix** | **DONE — in-session (cargo test)** | + +## GO/NO-GO decision (task 1.7) — GO, 2026-07-21 + +**Decision: GO.** Phase 1 (groups 2-8) is unblocked. + +Evidence provenance (recorded honestly, per the safety-critical tier): + +- **In-session, tool-verified:** 1.11 (axum spike, `cargo test` green + negative-control + mutation) and the **docker arm of 1.4** (live `docker push` cases: A2 127/8 exemption, + A10 ephemeral-`DOCKER_CONFIG` credential, H-3 auth-key-must-match). +- **Maintainer-attested in-lab (2026-07-21):** the remaining spikes — 1.1 layer-delta, + 1.2 five-path latency matrix, 1.3 digest preservation, the 1.4 podman arm, 1.5 loopback + proxy, 1.6 macOS firewall/port, 1.8 VM push, 1.9 cross-compile, 1.10 podman-events — were + run in the maintainer's lab and confirmed passing. Per-spike measurements are not + transcribed into this file; the maintainer holds the raw results. These are attested, not + independently re-verified in-session. + +The GO rests on that attestation for 1.1-1.10; the two in-session results stand on their own +tool output above. + +## 1.6 — default registry port (recorded for task 2.2) + +Task 1.6 chose a non-conflicting default port on stock macOS: **5599**. `5000` is +avoided because the macOS AirPlay Receiver binds it. This is the literal the typed +`container_dev` config uses as `RegistryConfig::DEFAULT_REGISTRY_PORT` when +`registry.port` is omitted (task 2.2), and it matches the loopback registry port +used in the 1.4 spike (`127.0.0.1:5599`). + +## 1.4 — Native-Linux loopback push + CLI-injected credential — PARTIAL (docker arm GO) + +**Claim under test.** A2 (docker treats a `127.0.0.0/8` registry as trust-free, no cert +config) and A10's docker arm (the CLI supplies the write token via an ephemeral +`DOCKER_CONFIG` forwarded as `X-Registry-Auth`, no persisted `docker login`), plus H-3 +(the auth-entry key must be byte-identical to the tagged registry host:port, or docker +omits the credential and the push 401s). + +**Setup.** `registry:2` with htpasswd Basic auth (bcrypt, generated via `httpd:2.4-alpine`), +published on `127.0.0.1:5599` (loopback-only) over plain HTTP; `hello-world` tagged +`127.0.0.1:5599/test:dev`; three ephemeral `DOCKER_CONFIG` dirs (matching key, wrong-host +key, empty). Host: docker 29.6.2, no podman. + +**Result: docker arm GO.** + +| Case | `DOCKER_CONFIG` auth entry | Result | +|------|---------------------------|--------| +| A — anonymous | `{}` (none) | `exit 1`, "no basic auth credentials" — write refused | +| C — wrong host key (H-3) | keyed `localhost:5599`, tag `127.0.0.1:5599` | `exit 1`, "no basic auth credentials" — docker sent NO credential | +| B — matching key (A10) | keyed `127.0.0.1:5599` | `exit 0`, `digest: sha256:c766679d…` pushed | + +- **A2 (docker):** case B pushed over plain-HTTP loopback with no `insecure-registries` entry + and no certs — docker's built-in `127.0.0.0/8` exemption holds. +- **A10 (docker arm):** the Basic credential from an ephemeral `DOCKER_CONFIG` was accepted + with no `docker login`; nothing was persisted (each push used an isolated `DOCKER_CONFIG`). +- **H-3:** case C proves the auth-entry key must equal the tagged host:port exactly — a + `localhost` vs `127.0.0.1` mismatch made docker silently omit `X-Registry-Auth`, degrading + to anonymous → 401. The implementation MUST key the ephemeral auth entry on the exact + tagged host:port. + +**PENDING (podman arm, M-1).** podman is not installed on this host, so the podman side — +`podman push --creds`/`REGISTRY_AUTH_FILE` and, critically, whether podman transmits Basic +over a plaintext loopback under `--tls-verify=false` (M-1) — is unverified. Run on a host +with podman before the overall GO. + +**Conclusion.** GO on 1.4's docker arm; 1.4 is NOT complete until the podman arm runs. + +## 1.11 — Two-socket separation + defense-in-depth auth matrix — GO + +**Claim under test.** The load-bearing security invariant established across cold-review +rounds 3-4: the compromised-device write class is closed *primarily* by route-class = +listener identity (write routes on a loopback-only listener distinct from the +device-reachable bulk read listener, its address never disclosed to a device), with the +Basic/Bearer per-route-class token gate as defense-in-depth. The concern C-1/H-1 raised +was whether this is realizable without depending on registry-middleware per-method +authorization. + +**Result: realizable and enforced.** A throwaway `axum` 0.8 crate binds two independent +listeners and enforces the credential-type split. `cargo test` is green; a negative-control +mutation (making the write guard permissive) correctly fails cell 2, so the test is not +vacuous. + +Proven: + +- **Primary — socket separation.** The write router binds `127.0.0.1:0` and its resolved + address `is_loopback()`; the read router binds `0.0.0.0:0` and `is_unspecified()`; the two + addresses differ. Route-class is a listener property, trivially expressible on our own + axum server — no third-party middleware per-method capability is needed (dissolves the + round-3 C-1 NO-GO risk). +- **Defense-in-depth — the six auth cells:** + + | Credential | Write route | Read route | + |-----------|-------------|------------| + | Basic write token (correct) | 200 accept (cell 1) | 401 refuse (cell 6) | + | Bearer read/control token | 401 refuse (cell 2) | 200 accept (cell 5) | + | anonymous (no `Authorization`) | 401 refuse (cell 3) | — | + | Basic, wrong password | 401 refuse (cell 4) | — | + +- **L-1.** The read listener challenges with a bare `WWW-Authenticate: Bearer` (no + `realm`/token-endpoint redirect that would send a client to a nonexistent auth server). + +**Command + output.** + +``` +$ cargo test --manifest-path /tmp/cdm-1.11-spike/Cargo.toml +running 1 test +test tests::two_socket_separation_and_auth_matrix ... ok +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out +``` + +Negative control (mutation: write guard accepts everything): + +``` +test tests::two_socket_separation_and_auth_matrix ... FAILED +assertion `left == right` failed: cell 2: Bearer read/control on write route refused +``` + +**Reproducible source** (throwaway spike; toolchain rustc/cargo 1.97.1, axum 0.8, plain +HTTP — TLS is a separate Phase-0 concern, not this spike's subject): + +`Cargo.toml` + +``` +[package] +name = "cdm-1-11-spike" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +axum = "0.8" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "net"] } +base64 = "0.22" + +[dev-dependencies] +reqwest = { version = "0.12", default-features = false } +``` + +`src/lib.rs` — two `axum::Router`s behind `middleware::from_fn` guards: +`require_basic_write` accepts only Basic `avocado:` (else 401 `Basic`); +`require_bearer_read` accepts only `Bearer ` (else 401 bare `Bearer`). +`write_router()` serves `PUT /v2/{name}/manifests/{reference}` + `POST .../blobs/uploads/`; +`read_router()` serves `GET /v2/{name}/manifests/{reference}` + `GET .../blobs/{digest}`. +The test spawns each on an ephemeral port (write on `127.0.0.1:0`, read on `0.0.0.0:0`), +asserts the address properties above, then drives the six cells + the L-1 challenge with +a `reqwest` client. + +**Conclusion.** GO on the 1.11 invariant: the two-socket model is realizable on axum and +the per-route-class credential-type gate holds. This de-risks the security model the plan +centers on. It does NOT constitute the overall Phase-0 GO (1.7) — the hardware spikes +(1.1-1.10) remain. diff --git a/docs/container-dev/verify-vm-write-path.sh b/docs/container-dev/verify-vm-write-path.sh new file mode 100755 index 00000000..af587321 --- /dev/null +++ b/docs/container-dev/verify-vm-write-path.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# +# verify-vm-write-path.sh - Validate Container Dev Mode task 7.1: +# the authenticated VM write path + CA delivery (design D2/H4). +# +# Run this on the HOST (the machine running `avocado container dev up`), with a +# booted avocado-vm engine reachable over SSH and the host docker CLI routed at +# it (DOCKER_HOST -> avocado-vm dockerd, so is_vm_routing_active() is true). +# +# It asserts the four falsifiable properties task 7.1 requires: +# 1. The per-project CA is DELIVERED at `up` into the VM engine's per-connection +# docker trust store (/etc/docker/certs.d/10.0.2.2:/ca.crt), not +# baked. (falsifier: VM CA is a build-time static overlay file) +# 2. A guest push to 10.0.2.2: over authenticated HTTPS SUCCEEDS. +# 3. An unauthenticated write to that listener is REFUSED (401), so the write +# path is not anonymous. (falsifier: guest write path unauthenticated / A3) +# 4. The avocado-vm overlay bakes NO CA. (falsifier: a cert in the image) +# +# Every step prints PASS/FAIL; a non-zero exit means the verify failed. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Config - override via env. The :? entries are required; the rest have defaults. +# --------------------------------------------------------------------------- +AVOCADO_BIN="${AVOCADO_BIN:-avocado}" +: "${AVOCADO_CONTAINER_DEV_VM:?set to of the avocado-vm engine guest}" +: "${AVOCADO_CONTAINER_DEV_DEVICE:?set to of the QEMU device}" +: "${DOCKER_HOST:?set to the avocado-vm dockerd socket so is_vm_routing_active() is true}" +WRITE_PORT="${AVOCADO_CONTAINER_DEV_WRITE_PORT:-5601}" +CONFIG="${AVOCADO_CONFIG:-avocado.yaml}" +# A trivial watched image whose ref matches runtimes..container_dev.images[].ref +TEST_IMAGE="${TEST_IMAGE:-my-app:dev}" +# Path to the meta-avocado base-files bbappend, read only for the "no static CA +# baked" source check. Adjust to your checkout. +BBAPPEND="${BBAPPEND:-$HOME/repos/work/peridio-scarthgap-build/meta-avocado/meta-avocado-qemu/recipes-core/base-files/base-files_%.bbappend}" + +VM_REGISTRY="10.0.2.2:${WRITE_PORT}" +GUEST_CA="/etc/docker/certs.d/${VM_REGISTRY}/ca.crt" +# The host-side registry store the write listener persists blobs/manifests into. +STORE_ROOT="${AVOCADO_CONTAINER_DEV_STORE:-$HOME/.avocado/container-dev}" + +# The CLI loads its config as the relative path "avocado.yaml" from the working +# directory (it does not honor $AVOCADO_CONFIG), so run every avocado invocation +# from the directory that holds the config. +CONFIG="$(readlink -f "$CONFIG")" +cd "$(dirname "$CONFIG")" + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"; "$AVOCADO_BIN" container dev down >/dev/null 2>&1 || true' EXIT + +pass=0 +fail=0 +ok() { + echo " PASS: $*" + pass=$((pass + 1)) +} +bad() { + echo " FAIL: $*" + fail=$((fail + 1)) +} +step() { + echo + echo "== $* ==" +} + +# --------------------------------------------------------------------------- +step "0. Preflight" +# --------------------------------------------------------------------------- +command -v "$AVOCADO_BIN" >/dev/null || { + echo "avocado binary '$AVOCADO_BIN' not found" + exit 2 +} +# Hermetic run: clear any prior registry store so a manifest present after sync +# proves THIS run's guest push landed (the store persists across runs). +"$AVOCADO_BIN" container dev down >/dev/null 2>&1 || true +rm -rf "$STORE_ROOT" +if ssh -o BatchMode=yes "$AVOCADO_CONTAINER_DEV_VM" 'docker version >/dev/null 2>&1'; then + ok "avocado-vm reachable and docker responds" +else + bad "avocado-vm unreachable or docker not running on it" +fi +if grep -q 'container_dev' "$CONFIG"; then + ok "$CONFIG carries a container_dev block" +else + bad "$CONFIG has no container_dev block (feature off)" +fi + +# --------------------------------------------------------------------------- +step "4. No static CA baked into the avocado-vm overlay (design D8/H4)" +# --------------------------------------------------------------------------- +# A source check, and deliberately a negative one: the overlay must install no +# cert. Where the CA actually lands is asserted at run time by step 1/4 against +# the live guest, not by reading build metadata. +# +# There is no positive counterpart here on purpose. A grep for a path string in +# this same file only proves the file contains that string - it passes for a +# typo'd path, for the wrong directory tree, and for a comment with the install +# line deleted. The CLI creates the trust dir itself with `mkdir -p` at `up` +# time, so there is nothing for the image to provision in the first place. +if [ -f "$BBAPPEND" ]; then + if grep -Eq 'install .*(\.crt|\.pem|ca-cert|ca\.crt)' "$BBAPPEND"; then + bad "the base-files bbappend installs a certificate - a CA is baked ($BBAPPEND)" + else + ok "the base-files bbappend bakes no CA" + fi +else + echo " SKIP: bbappend not found at $BBAPPEND (set BBAPPEND to your checkout)" +fi + +# --------------------------------------------------------------------------- +step "1. Bring the session up (delivers the CA, binds the write listener)" +# --------------------------------------------------------------------------- +echo " running: $AVOCADO_BIN container dev up (background)" +"$AVOCADO_BIN" container dev up >"$TMP/up.log" 2>&1 & +UP_PID=$! +# Wait for the write listener + CA delivery to settle (bootstrap is one-shot at up). +for _ in $(seq 1 30); do + if ssh -o BatchMode=yes "$AVOCADO_CONTAINER_DEV_VM" "test -f '$GUEST_CA'" 2>/dev/null; then + break + fi + kill -0 "$UP_PID" 2>/dev/null || { + echo " up exited early; log:" + sed 's/^/ /' "$TMP/up.log" + exit 2 + } + sleep 1 +done + +# --------------------------------------------------------------------------- +step "1/4. CA delivered into the VM engine trust store at run time" +# --------------------------------------------------------------------------- +if ssh -o BatchMode=yes "$AVOCADO_CONTAINER_DEV_VM" \ + "openssl x509 -in '$GUEST_CA' -noout -subject" >"$TMP/ca.txt" 2>/dev/null; then + ok "delivered CA present + valid at $GUEST_CA on the guest ($(cat "$TMP/ca.txt"))" +else + bad "no valid CA at $GUEST_CA on the guest - deliver_vm_ca did not run" +fi + +# --------------------------------------------------------------------------- +step "3. Write path is authenticated - an unauthenticated write is refused" +# --------------------------------------------------------------------------- +# The write listener is loopback-bound on the host at 127.0.0.1: +# (the guest reaches the same socket via 10.0.2.2). An unauthenticated manifest +# PUT must be refused (Basic write token required, not anonymous / A3). +code="$(curl -sk -o /dev/null -w '%{http_code}' -X PUT \ + "https://127.0.0.1:${WRITE_PORT}/v2/verify-7-1/manifests/dev" 2>/dev/null || echo 000)" +if [ "$code" = "401" ]; then + ok "unauthenticated write refused with 401 (Basic write token required)" +else + bad "unauthenticated write returned $code, expected 401 (write path not authenticated)" +fi + +# --------------------------------------------------------------------------- +step "2. Guest push over authenticated HTTPS SUCCEEDS" +# --------------------------------------------------------------------------- +# Build the watched image on the VM engine, then let the CLI push it to the +# routable HTTPS write listener with the delivered CA + Basic write token. +printf 'FROM busybox:latest\nRUN echo verify-7.1 > /marker\n' >"$TMP/Dockerfile" +if docker build -t "$TEST_IMAGE" "$TMP" >"$TMP/build.log" 2>&1; then + ok "built watched image $TEST_IMAGE on the VM engine" +else + bad "failed to build $TEST_IMAGE (see below)" + sed 's/^/ /' "$TMP/build.log" +fi +echo " running: $AVOCADO_BIN container dev sync" +"$AVOCADO_BIN" container dev sync >"$TMP/sync.log" 2>&1 +# `sync` only SIGNALs the running `up` to re-push; the guest `docker push` over +# HTTPS then runs asynchronously in `up`. A zero exit from `sync` proves the +# signal was sent, NOT that a blob landed - so wait for the manifest tag to +# appear in the registry store (cleared at preflight), which is the real proof +# the authenticated HTTPS push to $VM_REGISTRY succeeded. +tag="${TEST_IMAGE##*:}" +landed=0 +for _ in $(seq 1 25); do + # `*/tags/*/$tag`, not `*/tags/$tag`: tags are stored under the repository + # name (`manifests/tags//`), so a pattern ending in `/tags/$tag` + # matches nothing and this loop reports failure on a run where the push + # actually succeeded - the one script that would catch a regression on this + # path failing on a healthy one. + if find "$STORE_ROOT" -path "*/registry/manifests/tags/*/$tag" 2>/dev/null | grep -q .; then + landed=1 + break + fi + sleep 1 +done +if [ "$landed" = 1 ]; then + ok "guest push landed: manifest tag '$tag' present in the registry store (authenticated HTTPS push to $VM_REGISTRY succeeded)" +else + bad "guest push did not land: no manifest tag '$tag' in $STORE_ROOT after sync" + echo " -- up.log tail --" + tail -15 "$TMP/up.log" 2>/dev/null | sed 's/^/ /' +fi + +# --------------------------------------------------------------------------- +step "Verdict" +# --------------------------------------------------------------------------- +echo " passed: $pass failed: $fail" +if [ "$fail" -eq 0 ]; then + echo " RESULT: 7.1 VM write path VERIFIED" + exit 0 +fi +echo " RESULT: 7.1 VM write path FAILED - see the FAIL lines above" +exit 1 diff --git a/src/commands/container/dev.rs b/src/commands/container/dev.rs new file mode 100644 index 00000000..79dbf8a6 --- /dev/null +++ b/src/commands/container/dev.rs @@ -0,0 +1,1700 @@ +//! `avocado container dev` orchestration: `up`/`down`/`status` + per-`up` +//! bootstrap (task 5.2). +//! +//! `up` mints BOTH session tokens (task 3.6), starts the embedded registry (the +//! dedicated bulk read listener + the distinct write listener), the engine-driver +//! watcher (task 4.x), and the control WebSocket (task 5.1); resolves the host +//! endpoint (reusing `get_local_ip_for_remote` + the `AVOCADO_CONTAINER_DEV_HOST` +//! / `AVOCADO_CONTAINER_DEV_PORT` overrides, design L2); and writes ONCE per `up` +//! to the device writable partition the BULK-LISTENER endpoint (never the write +//! listener, design G-4), the READ/CONTROL token (never the write token), and the +//! CA certificate. Steady-state sync then rides the control WS with no further +//! SSH (design D5). +//! +//! `down` stops all listeners AND tears down the write listener through a +//! guaranteed-cleanup guard +//! ([`crate::utils::container_dev::bootstrap::WriteListenerGuard`]), so an unclean +//! exit never leaves an authenticated write port bound (design L-1). The write +//! listener binds `127.0.0.1` only; the VM push path reaches it through QEMU's +//! `10.0.2.2` host alias rather than a routable bind, so there is no LAN-facing +//! write port to leak. +//! +//! `status` reports the registry/watcher/last-sync state recorded at `up` time, +//! and reports "not running" when no live `up` owns the session (proved by the +//! session lock, not by the recorded pid). +//! +//! NOT YET LIVE, despite being implemented and tested in `bootstrap.rs`: the +//! per-device `status.devices` list and the drain-based +//! [`crate::utils::container_dev::bootstrap::TokenRegistry`] rotation behind +//! `needs_rebootstrap()` (design H-2). `up` writes `session.json` once and never +//! updates it, so `devices` stays empty and `needs_rebootstrap()` is +//! structurally false; token rotation cannot cross an `up` either, because +//! `TokenRegistry::rotate` needs `&mut self` and a re-`up` is a NEW process that +//! starts from a fresh registry. Making both live needs `up` to keep publishing +//! session state while it runs, which is a change in its own right rather than a +//! missing call here. Until then `status` reports a live-or-not answer and the +//! per-device detail is absent, not stale. + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{bail, Context, Result}; +use tokio::net::TcpListener; +use tokio::task::JoinHandle; +use tokio_rustls::TlsAcceptor; + +use crate::utils::config::{Config, RuntimeConfig}; +use crate::utils::container_dev::bootstrap::{ + bootstrap_path, host_override, port_override, resolve_endpoint, write_port_override, + ws_port_override, DevStatus, DeviceBootstrap, VmWriteSetup, WriteListenerGuard, + DEFAULT_WRITE_PORT, DEFAULT_WS_PORT, WRITABLE_PARTITION, +}; +use crate::utils::container_dev::commands::{prune_store, run_one_shot_sync}; +use crate::utils::container_dev::config::ContainerDevConfig; +use crate::utils::container_dev::engine::{ + driver_for, resolve_image_id, watch_tag_events, TagEvent, +}; +use crate::utils::container_dev::image_ref; +use crate::utils::container_dev::registry::{serve_write_router_tls, write_router, BulkListener}; +use crate::utils::container_dev::store::{BlobStore, SessionActivity}; +use crate::utils::container_dev::tls::DevSession; +use crate::utils::container_dev::watcher::{ + arch_guard::{ArchGuardSyncer, EngineArchProbe, HelloArchBook, ImageArchBook}, + run_watcher, EngineSyncer, HostTopology, SyncMode, Syncer, WatchSet, DEBOUNCE, +}; +use crate::utils::container_dev::ws::{ControlServer, DesiredState}; +use crate::utils::output::{print_info, print_success, print_warning, OutputLevel}; +use crate::utils::remote::{get_local_ip_for_remote, RemoteHost, SshClient}; + +/// Default config file, matching the rest of the CLI (`-C/--config`). +const DEFAULT_CONFIG: &str = "avocado.yaml"; + +/// The device SSH target `up` bootstraps and the endpoint auto-detection resolves +/// the reachable host IP against (design A6/L2). The `up`/`down`/`status` +/// subcommands take no positional arguments (task 2.3), so the device is sourced +/// here. +const DEVICE_ENV: &str = "AVOCADO_CONTAINER_DEV_DEVICE"; + +/// The avocado-vm engine guest SSH target the per-project CA is delivered into on +/// the VM write path (design D2/H4, task 7.1). Only consulted when the host +/// topology selects the avocado-vm push path; the native-Linux loopback push +/// never uses it. +const VM_ENV: &str = "AVOCADO_CONTAINER_DEV_VM"; + +/// The default engine CLI when none is configured. +const DEFAULT_ENGINE: &str = "docker"; + +pub struct DevUpCommand; +pub struct DevSyncCommand; +pub struct DevStatusCommand; +pub struct DevDownCommand; +pub struct DevPruneCommand; + +/// The resolved dev context: the runtime that carries the `container_dev` block, +/// its config, and the per-project namespace derived from the runtime name +/// (design D8 per-project store/CA/token/port namespacing). +struct DevContext { + project: String, + dev: ContainerDevConfig, +} + +/// Load the config and select the runtime whose `container_dev` block enables the +/// feature (design D7 — presence of the block is the gate). +fn load_dev_context() -> Result { + let config = Config::load(DEFAULT_CONFIG) + .with_context(|| format!("loading Container Dev Mode config from {DEFAULT_CONFIG}"))?; + let runtimes = config.runtimes.unwrap_or_default(); + + let mut enabled: Vec<(String, RuntimeConfig)> = runtimes + .into_iter() + .filter(|(_, rt)| rt.container_dev.is_some()) + .collect(); + enabled.sort_by(|a, b| a.0.cmp(&b.0)); + + match enabled.len() { + 0 => bail!( + "no runtime has a `container_dev` block; add `runtimes..container_dev` to \ + {DEFAULT_CONFIG} to enable Container Dev Mode" + ), + 1 => { + let (project, rt) = enabled.into_iter().next().unwrap(); + let dev = rt + .container_dev + .expect("filtered runtimes carry a container_dev block"); + Ok(DevContext { project, dev }) + } + _ => { + let names: Vec = enabled.into_iter().map(|(name, _)| name).collect(); + bail!( + "multiple runtimes enable Container Dev Mode ({}); v1 supports a single dev \ + runtime per config", + names.join(", ") + ) + } + } +} + +/// The per-project dir holding the session state and lock files. +fn session_dir(store: &BlobStore) -> &Path { + store + .root() + .parent() + .expect("the registry store root sits under the per-project dir") +} + +/// The path to the per-`up` session state file, a sibling of the per-project +/// registry store (`~/.avocado/container-dev//session.json`). `down` and +/// `status` read it; `up` writes it on start and clears it on teardown. +fn session_state_path(store: &BlobStore) -> PathBuf { + session_dir(store).join("session.json") +} + +/// The path to the per-`up` lock file, a sibling of the state file. +/// +/// Deliberately NOT the state file itself. `flock` is held on an inode, and the +/// teardown paths unlink `session.json` - so locking that inode would mean the +/// next `up` locks a freshly created one and mutual exclusion would not survive +/// a single `down`. This file is created once and never removed, so the inode +/// every `up` contends on is stable for the life of the project dir. +fn session_lock_path(store: &BlobStore) -> PathBuf { + session_dir(store).join("session.lock") +} + +impl DevUpCommand { + pub async fn execute(self) -> Result<()> { + let ctx = load_dev_context()?; + let store = Arc::new( + BlobStore::for_project(&ctx.project) + .with_context(|| format!("opening the dev store for project `{}`", ctx.project))?, + ); + + // Claim the project's session BEFORE anything observable happens. Every + // step below is a side effect a second `up` must not interleave with: + // minting fresh tokens, binding three listeners, spawning the watcher, + // and SSHing a new bootstrap over the device's existing one. Taken at the + // end instead, the lock would only report a collision that had already + // occurred - the loser would have already repointed the device at itself + // and overwritten the winner's state file before finding out it lost. + let state_path = session_state_path(&store); + let lock_path = session_lock_path(&store); + let _session_lock = SessionLock::acquire(&lock_path)?; + // Register the signal handlers BEFORE the pid is published, not where + // the tasks that consume them are spawned. + // + // Publishing the pid is what makes this process a signalable target: + // liveness is proved by the flock, held since above, so from the moment + // the record lands a concurrent `sync` or `down` will signal it. The + // default disposition of both SIGUSR1 and SIGTERM is Term, and the work + // between here and the spawns below is slow and observable - a TLS mint, + // a DNS plus UDP host probe that takes seconds when the device is a + // hostname, three binds, and a `docker events` fork - so a `sync` in that + // window killed `up` mid-startup with no error and no teardown. The + // SIGTERM window ran further still, past both SSH round trips, so a + // `down` skipped WriteListenerGuard::drop, the write-guard teardown, and + // the events child kill. + // + // tokio registers the handler when the stream is created, so creating + // both here closes the window; the tasks below just consume them. + let early_signals = register_early_signals(); + + // Publish OUR pid the instant the lock is ours, before any of the slow + // work below. Moving the lock to the top of `up` decoupled it from the + // state file, and that reopened the hazard the lock exists to close: a + // predecessor SIGKILLed before its teardown leaves its pid in + // `session.json`, so between acquiring the lock here and writing the + // record after the binds and the SSH, `load_live_session` would report + // the session live while handing out a DEAD pid. `sync` would then send + // SIGUSR1 - default disposition terminate - to whatever recycled that + // number. Overwriting the record now restores the invariant that a live + // lock implies the recorded pid is the holder's; the fuller status is + // written again once the listeners are up. + write_session_state( + &state_path, + &SessionState { + pid: std::process::id(), + status: DevStatus { + registry_running: false, + watcher_running: false, + last_sync: None, + devices: Vec::new(), + }, + }, + )?; + + // Source the device SSH target: needed to deliver the bootstrap and, when + // no host override is set, to auto-detect the reachable host IP. + let device_spec = std::env::var(DEVICE_ENV) + .ok() + .filter(|s| !s.trim().is_empty()); + let Some(device_spec) = device_spec else { + bail!( + "set {DEVICE_ENV}= to the dev device so `up` can bootstrap it \ + (the subcommands take no positional arguments)" + ); + }; + let device = RemoteHost::parse(&device_spec)?; + + // Resolve the BULK-LISTENER endpoint the device pulls from (design L2): + // AVOCADO_CONTAINER_DEV_HOST overrides host auto-detection; + // AVOCADO_CONTAINER_DEV_PORT overrides the configured port. + let configured_port = ctx.dev.registry.port; + let auto_host = match host_override() { + Some(_) => String::new(), + None => get_local_ip_for_remote(&device.host) + .await + .with_context(|| { + format!( + "auto-detecting the host IP reachable from `{}`", + device.host + ) + })? + .to_string(), + }; + let bulk_endpoint = resolve_endpoint( + host_override().as_deref(), + &auto_host, + port_override(), + configured_port, + ); + + // The host the device will actually dial, and therefore the name its TLS + // stack verifies against. `bulk_host` reads it back off the resolved + // endpoint so this is the same value the bootstrap carries rather than a + // second derivation that could drift from it. + let device_facing_host = bulk_host(&bulk_endpoint, &auto_host).to_string(); + + // Mint fresh TLS material + BOTH tokens for this `up` (design D2/D8). + // + // Minted AFTER the endpoint is resolved, not before, because the leaf has + // to carry the address the bootstrap advertises. The device agent builds + // a rustls ClientConfig with the pinned CA and no custom verifier, so + // `ServerName::try_from()` demands a matching SAN; with + // only {runtime, 10.0.2.2, 127.0.0.1} in the set, a real board on a LAN + // failed hostname verification with NotValidForName on both the bulk + // listener and the control WS. The lab never caught it because + // `setup-lab.sh` pins AVOCADO_CONTAINER_DEV_HOST=10.0.2.2, the one + // address that was already in the set. + let session = DevSession::mint(&ctx.project, std::slice::from_ref(&device_facing_host)) + .with_context(|| format!("minting the dev session for `{}`", ctx.project))?; + let tls_config = session.tls.server_config(); + let read_token = session.read_token.clone(); + let write_token = session.write_token.clone(); + + // The bulk read listener binds the resolved port on all interfaces so the + // device (or its loopback proxy) can reach it over TLS. The write listener + // is bound SEPARATELY and loopback-only (design D9/G-4). + let bulk_bind: SocketAddr = format!("0.0.0.0:{}", endpoint_port(&bulk_endpoint)?) + .parse() + .expect("a host:port endpoint yields a valid bind address"); + let bulk = BulkListener::bind( + bulk_bind, + Arc::clone(&store), + read_token.clone(), + tls_config, + ) + .await + .context("binding the dedicated bulk read listener")?; + let bulk_addr = bulk.local_addr(); + + // Detect the host topology once (design D1): it selects PUSH vs INGEST AND, + // on the avocado-vm push path, drives the write listener onto a KNOWN port + // with a routable 10.0.2.2 registry + guest CA delivery (task 7.1). + let topo = HostTopology::detect(); + + // On the VM push path the per-project CA must be delivered into the + // avocado-vm engine guest's trust store; require its SSH target up front so + // `up` fails fast rather than after binding listeners (design H4). + let vm_target = if topo.vm_routing { + let spec = std::env::var(VM_ENV) + .ok() + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| { + anyhow::anyhow!( + "the host topology selected the avocado-vm push path; set \ + {VM_ENV}= to the avocado-vm engine guest so `up` can deliver \ + the per-project CA into its docker trust store" + ) + })?; + Some(RemoteHost::parse(&spec)?) + } else { + None + }; + + // The DISTINCT write listener: loopback-BOUND (design D9/H-1) so a device + // (handed only the bulk endpoint) can never reach a write route; its + // address is NEVER disclosed to a device. On the avocado-vm push path the + // port must be KNOWN (not ephemeral) so the guest's certs.d dir and the + // pushed tag are both keyed on 10.0.2.2: (H-3) — a QEMU-SLIRP guest + // reaches this loopback listener through the 10.0.2.2 host alias. Native + // Linux keeps an ephemeral loopback port. + let write_port = write_port_override().unwrap_or(DEFAULT_WRITE_PORT); + let write_bind: SocketAddr = if topo.vm_routing { + format!("127.0.0.1:{write_port}") + .parse() + .expect("a known write port yields a valid loopback bind") + } else { + "127.0.0.1:0".parse().expect("loopback write bind is valid") + }; + let write_listener = TcpListener::bind(write_bind) + .await + .context("binding the loopback write listener")?; + let write_addr = write_listener.local_addr()?; + + // On the VM push path compose the guest write-path plan (task 7.1): the + // routable 10.0.2.2: registry the guest daemon connects to, plus the + // CA to deliver into its trust store. Native Linux pushes to the loopback + // listener directly, so no guest plan is needed. + let vm_setup = topo + .vm_routing + .then(|| VmWriteSetup::docker(&session, write_addr.port())); + // The registry the syncer tags + authenticates against: the routable VM + // registry on the VM path, else the loopback write listener itself. + let syncer_registry = match &vm_setup { + Some(setup) => setup.registry.clone(), + None => write_addr.to_string(), + }; + // On the VM push path the guest reaches this listener via 10.0.2.2 — not a + // docker-trusted 127.0.0.0/8 loopback — so it must terminate the same + // per-project leaf TLS the bulk and control listeners do; the guest's + // delivered certs.d CA pins it (design A2/H4). The native loopback path + // keeps plain HTTP under docker's built-in 127.0.0.0/8 insecure exemption. + let write_task: JoinHandle<()> = if topo.vm_routing { + serve_write_router_tls( + write_listener, + session.tls.server_config(), + Arc::clone(&store), + write_token.clone(), + ) + } else { + let write_router = write_router(Arc::clone(&store), write_token.clone()); + tokio::spawn(async move { + let _ = axum::serve(write_listener, write_router).await; + }) + }; + + // Guaranteed-cleanup guard for the loopback write listener (design L-1): + // aborting the serve task tears the listener down on ANY exit path, clean + // or unclean, so no authenticated write port lingers. + let mut write_guard = WriteListenerGuard::new(move || { + write_task.abort(); + }); + + // The control WS (task 5.1) shares the read/control-token validator with + // the bulk listener (design G-5) AND terminates the SAME per-project + // pinned-CA TLS the bulk listener does (design D8/D9): the device agent + // dials `wss://` and pins the session CA, so the control channel is never + // plaintext. Its desired state is RE-DERIVED at `up` from the engine's + // current watched tags (design D5) — the watcher's first events populate + // it; we start empty and let hellos reconcile. + // The arch book is shared: the control server writes each device's + // `hello.arch` into it, and the cross-arch guard below reads the snapshot + // before every sync. + let arch_book = HelloArchBook::new(); + // The image-arch book runs the other direction: the guard writes what it + // probed, the control server reads it in `notify` so the arch is stored + // beside the digest and a later `reconcile` can refuse a wrong-arch + // delivery the guard could not, having had no connected device to + // compare against at push time. + let image_arches = ImageArchBook::new(); + // The unit that consumes each watched image, keyed the way the push path + // keys it: `image_ref::split` strips any registry prefix and defaults the + // tag, which is what `build_push_plan`'s retag and `notify`'s own key + // both land on. Keying by the raw config `ref` instead would miss for any + // entry written as `localhost/my-app:dev`. + let mut desired = DesiredState::default(); + desired.set_services(service_map(&ctx.dev.images)); + let control = ControlServer::new( + read_token.clone(), + desired, + arch_book.clone(), + image_arches.clone(), + // The notify path resolves a tag to the registry manifest digest here. + Some(store.clone()), + ); + // Bind the control WS on a RESOLVED, discoverable port (design D9), NOT an + // ephemeral `0.0.0.0:0` the device could never learn: the device agent is + // handed `ws_endpoint` at bootstrap and must be able to dial it. The port + // is the configured/overridden WS port (AVOCADO_CONTAINER_DEV_WS_PORT), + // distinct from the bulk listener's port; the host component is the same + // device-reachable host the bulk endpoint resolves to. + let ws_port = ws_port_override().unwrap_or(DEFAULT_WS_PORT); + let ws_bind: SocketAddr = format!("0.0.0.0:{ws_port}") + .parse() + .expect("a ws port yields a valid bind address"); + let ws_listener = TcpListener::bind(ws_bind) + .await + .context("binding the control WS listener")?; + let ws_addr = ws_listener.local_addr()?; + // `tls_config` was moved into `BulkListener::bind` above; `server_config()` + // returns a fresh `Arc::clone` of the same leaf-backed config for the + // control acceptor. + let control_acceptor = TlsAcceptor::from(session.tls.server_config()); + let control_serve = Arc::clone(&control); + let ws_task: JoinHandle<()> = + tokio::spawn( + async move { control_serve.serve_tls(ws_listener, control_acceptor).await }, + ); + + // The engine-driver watcher (task 4.x): tag events over the engine CLI + // subprocess (never an API socket), topology-selected PUSH/INGEST, then a + // control-WS notify — no SSH per sync (design D5). + let engine = DEFAULT_ENGINE; + let driver = + driver_for(engine).with_context(|| format!("no engine driver for `{engine}`"))?; + let mode = topo.sync_mode(); + let project_dir = store + .root() + .parent() + .expect("store root has a per-project parent") + .to_path_buf(); + let engine_syncer = Arc::new(EngineSyncer::new( + driver_for(engine).expect("engine driver resolves"), + syncer_registry, + write_token.clone(), + project_dir, + )); + // Set just before teardown kills the events child, so the EOF that kill + // produces is not reported as a watcher that died on its own. + let watcher_shutdown = Arc::new(AtomicBool::new(false)); + let (events_rx, mut events_child, watcher_ended) = + watch_tag_events(driver, Arc::clone(&watcher_shutdown)) + .await + .context("starting the engine event watcher")?; + let notifier = Arc::clone(&control); + // Wrap the real syncer in the cross-arch guard (task 4.3) BEFORE anything + // can push through it. `control` already records every device's + // `hello.arch` into `arch_book`; without this decorator nothing ever reads + // that book, so an amd64 host targeting an aarch64 device would build, + // push and notify a wrong-arch image the device cannot run — the exact + // silent delivery the guard exists to refuse. A refusal returns `Err`, so + // the notify is skipped too. + let syncer: Arc = Arc::new(ArchGuardSyncer::new( + engine_syncer, + // A fresh driver handle: `driver` itself was moved into the event + // watcher above. The probe keeps only the engine binary name. + Arc::new(EngineArchProbe::new( + driver_for(engine).expect("engine driver resolves").as_ref(), + )), + Arc::new(arch_book), + image_arches, + )); + // The watcher and the manual `sync` trigger share the SAME push+notify + // primitives (design D5): clone the syncer + control for the trigger + // before the watcher takes ownership of its copies. + let trigger_syncer = Arc::clone(&syncer); + let trigger_notifier = Arc::clone(&control); + // The declared watch list scopes the watcher too, not just the manual + // `sync` trigger below: the engine reports every tag on the daemon, + // including the registry retag each push performs, so an unscoped watcher + // syncs in response to its own side effect (see `WatchSet`). + let watched_images: Vec = + ctx.dev.images.iter().map(|i| i.image_ref.clone()).collect(); + let watch_set = WatchSet::new(watched_images.clone()); + let watcher_task: JoinHandle<()> = tokio::spawn(async move { + run_watcher(events_rx, mode, syncer, notifier, DEBOUNCE, watch_set).await; + }); + + // The `container dev sync` trigger (task 5.3): a separate `sync` + // invocation signals this process (SIGUSR1), and each signal drives ONE + // re-push + notify of every configured watched image through the SAME + // pipeline the watcher uses — exactly once per signal, never a second + // watch loop. Reusing the running session's syncer + control WS is what + // lets the notify reach a connected device with no extra SSH. + let sync_signal = early_signals.sync; + let sync_trigger_task: JoinHandle<()> = tokio::spawn(async move { + run_sync_trigger( + mode, + trigger_syncer, + trigger_notifier, + watched_images, + engine, + sync_signal, + ) + .await; + }); + + // Deliver the bootstrap ONCE per `up` (design D5): the bulk endpoint (the + // device-reachable address of the bulk listener), the read/control token, + // and the CA cert — never the write token, never the write-listener + // address (design G-4). Steady-state sync never re-opens SSH. + let device_host = bulk_host(&bulk_endpoint, &auto_host); + let device_bulk_endpoint = format!("{}:{}", device_host, bulk_addr.port()); + // The control-WS endpoint the device dials: the same device-reachable host + // as the bulk endpoint, on the resolved WS port (design D9/G-4). NEVER the + // write-listener address, which is never disclosed to a device. + let device_ws_endpoint = format!("{}:{}", device_host, ws_addr.port()); + let payload = + DeviceBootstrap::from_session(&session, device_bulk_endpoint, device_ws_endpoint); + deliver_bootstrap(&device, &payload).await?; + + // On the VM push path, deliver the per-project CA into the avocado-vm + // engine guest's docker trust store so its daemon trusts the host write + // listener's leaf per connection (design H4). Delivered at `up` over SSH, + // NEVER baked into the VM overlay (design D8). `vm_setup` and `vm_target` + // are both `Some` iff the topology selected the VM push path. + if let (Some(setup), Some(vm)) = (&vm_setup, &vm_target) { + deliver_vm_ca(vm, setup).await?; + } + + // Record the running session (with this process's pid) so `status`/`down` + // in a separate invocation can find and signal it. + let state = SessionState { + pid: std::process::id(), + status: DevStatus { + registry_running: true, + watcher_running: true, + last_sync: None, + // Empty for as long as `up` writes this record once and never + // revisits it; see the per-device caveat in the module docs. + devices: Vec::new(), + }, + }; + // The lock claimed at the top of `up` is still held; this only publishes + // the pid and status for a separate `status`/`down` to read. + write_session_state(&state_path, &state)?; + + print_success( + &format!( + "container dev up: bulk listener on {bulk_addr}, write listener loopback-only on \ + {write_addr}, control WS on {ws_addr}; device `{}` bootstrapped", + device.host + ), + OutputLevel::Normal, + ); + print_info( + "Watching for image rebuilds; press Ctrl-C or run `container dev down` to tear down.", + OutputLevel::Normal, + ); + + // Run foreground until interrupted by Ctrl-C (SIGINT) or by a separate + // `down` (SIGTERM). On ANY exit — including a panic or early return — the + // write guard tears down the write listener via Drop (design L-1); the + // other listeners' tasks are aborted and the state file is cleared. + // A watcher that dies mid-session has to correct the record it published, + // or `status` from a second terminal - which is how anyone actually + // checks - keeps reporting `watcher_running=true`. The warning from the + // forwarder reaches only the terminal holding `up`, which is the one + // place the operator is not looking. + let shutdown_fut = wait_for_shutdown(early_signals.shutdown); + tokio::pin!(shutdown_fut); + tokio::select! { + () = &mut shutdown_fut => {} + _ = watcher_ended => { + if let Err(e) = mark_watcher_stopped(&state_path) { + print_warning( + &format!( + "container dev: the watcher stopped, but recording that in the \ + session file failed ({e}); `avocado container dev status` may \ + still report it running" + ), + OutputLevel::Normal, + ); + } + // Keep serving: the registry, the control WS and manual `sync` + // all still work without the event stream, so this is a degraded + // session rather than a finished one. + shutdown_fut.await; + } + } + + // Before the kill below, not after: the kill is what closes the child's + // stdout, so the forwarder can observe this flag only if it is already set. + watcher_shutdown.store(true, Ordering::SeqCst); + write_guard.teardown(); + ws_task.abort(); + watcher_task.abort(); + sync_trigger_task.abort(); + let _ = events_child.kill().await; + drop(bulk); + let _ = std::fs::remove_file(&state_path); + + print_info( + "container dev down: listeners torn down.", + OutputLevel::Normal, + ); + Ok(()) + } +} + +impl DevSyncCommand { + /// One-shot re-push + notify of the current watched tag (task 5.3, design + /// M4): NO long-running watcher. `sync` finds the running `up` session and + /// signals it (SIGUSR1) to drive ONE pass of the same push+notify pipeline + /// the watcher uses — reusing the session's registry write listener, engine + /// syncer, and control WS so the notify reaches a connected device with no + /// extra SSH. With no active session there is nothing holding those + /// listeners, so `sync` reports that `up` must run first rather than silently + /// doing nothing. + pub async fn execute(self) -> Result<()> { + let ctx = load_dev_context()?; + let store = BlobStore::for_project(&ctx.project) + .with_context(|| format!("opening the dev store for project `{}`", ctx.project))?; + let state_path = session_state_path(&store); + + let Some(state) = load_live_session(&state_path, &session_lock_path(&store))? else { + bail!( + "container dev: no active `up` session to sync; run `avocado container dev up` \ + first, then `sync` re-pushes the current watched image" + ); + }; + + // Trigger exactly one re-push + notify in the running `up` process. The + // liveness check above is what makes this safe: signalling a stale pid + // would deliver SIGUSR1 to whatever process recycled that number, and + // SIGUSR1 terminates by default. + signal_sync(state.pid); + print_info( + "container dev sync: triggered a one-shot re-push + notify of the watched image(s).", + OutputLevel::Normal, + ); + Ok(()) + } +} + +impl DevStatusCommand { + pub async fn execute(self) -> Result<()> { + let ctx = load_dev_context()?; + let store = BlobStore::for_project(&ctx.project) + .with_context(|| format!("opening the dev store for project `{}`", ctx.project))?; + let state_path = session_state_path(&store); + + // A session file whose owner is gone would otherwise be reported verbatim, + // i.e. registry_running=true for listeners that died with the process. + let Some(state) = load_live_session(&state_path, &session_lock_path(&store))? else { + print_info( + "container dev: not running (no active `up` session).", + OutputLevel::Normal, + ); + return Ok(()); + }; + + let status = &state.status; + print_info( + &format!( + "container dev status: registry_running={}, watcher_running={}, last_sync={}", + status.registry_running, + status.watcher_running, + status.last_sync.as_deref().unwrap_or(""), + ), + OutputLevel::Normal, + ); + // Surface the re-bootstrap state when any device presented a stale token + // (design H-2) — a stale token yields a status, never a silent loop. + if status.needs_rebootstrap() { + print_warning( + "a device presented a stale token; re-run `avocado container dev up` to \ + re-bootstrap it", + OutputLevel::Normal, + ); + } + Ok(()) + } +} + +impl DevDownCommand { + pub async fn execute(self) -> Result<()> { + let ctx = load_dev_context()?; + let store = BlobStore::for_project(&ctx.project) + .with_context(|| format!("opening the dev store for project `{}`", ctx.project))?; + let state_path = session_state_path(&store); + + let Some(state) = load_live_session(&state_path, &session_lock_path(&store))? else { + print_info( + "container dev: nothing to tear down (no active `up` session).", + OutputLevel::Normal, + ); + return Ok(()); + }; + + // Signal the foreground `up` process to shut down. It handles SIGTERM the + // same as Ctrl-C, tearing down ALL listeners — including the write + // listener via the guaranteed-cleanup guard (design L-1) — so no + // authenticated write port survives `down`. + signal_shutdown(state.pid); + // The `up` process removes its own state file on graceful exit; remove it + // here too so a `down` against an already-dead process still clears stale + // state. + let _ = std::fs::remove_file(&state_path); + print_info( + "container dev down: signaled the dev session to stop; listeners torn down.", + OutputLevel::Normal, + ); + Ok(()) + } +} + +impl DevPruneCommand { + /// Garbage-collect THIS project's Container Dev Mode store only (task 5.3, + /// design M4): sweep blobs no currently-tagged manifest references, via the + /// group-3.5 GC ([`prune_store`]). It touches only store blobs — never the + /// per-session token or the per-project CA material — and refuses while an + /// `up` session is live rather than sweeping a blob a transfer still needs. + pub async fn execute(self) -> Result<()> { + let ctx = load_dev_context()?; + let store = BlobStore::for_project(&ctx.project) + .with_context(|| format!("opening the dev store for project `{}`", ctx.project))?; + + // The flock is the only cross-process proof there is. `prune` runs in a + // different process from `up`, so anything the store counted for itself + // was always zero here - the store now takes this as an argument for + // exactly that reason. A live `up` may be streaming a blob to a device + // or holding an upload's staging file open, and `sweep_uploads` unlinks + // that file by path. + let session = if session_is_live(&session_lock_path(&store))? { + SessionActivity::Live + } else { + SessionActivity::Idle + }; + + let swept = prune_store(&store, session).with_context(|| { + format!( + "pruning the Container Dev Mode store for project `{}`", + ctx.project + ) + })?; + + print_success( + &format!( + "container dev prune: swept {} unreferenced blob(s) from the `{}` store; the \ + session token and CA material are left intact.", + swept.len(), + ctx.project + ), + OutputLevel::Normal, + ); + Ok(()) + } +} + +/// The remote shell command that writes the bootstrap file at mode 0600. +/// +/// Split out of [`deliver_bootstrap`] so the one property that matters here is +/// assertable rather than reviewed by eye: this file carries the Bearer +/// read/control token, and the token must never exist world-readable. +/// +/// The umask is in force when the redirect creates the file, so the mode is +/// right from the first byte. Writing the file and then correcting it with +/// `chmod 0600` - which is what this did before - leaves the token on disk at +/// the remote shell's umask (0644 under a default 0022) for the width of two +/// commands, readable by any local user on the device. A subshell keeps the +/// umask change from leaking into anything else `run_command` might later chain. +/// +/// The payload arrives on stdin rather than embedded in the command. It used to +/// be base64 in argv, decoded with `base64 -d` on the device - which assumes +/// coreutils. Avocado OS has no `base64`, so this failed on a real device with +/// `sh: base64: not found`. Reading stdin needs nothing but the shell, and it +/// also keeps the token out of the device's process list. +fn bootstrap_delivery_command(remote_dir: &str, remote_path: &str) -> String { + format!("mkdir -p {remote_dir} && (umask 077 && cat > {remote_path})") +} + +/// Deliver the bootstrap payload to the device writable partition ONCE (design +/// D5). Renders the JSON and streams it over SSH stdin into +/// `WRITABLE_PARTITION/container-dev/bootstrap.json`, at mode 0600 from creation. +/// +/// Streaming rather than encoding into the command keeps the device free of any +/// decoder dependency and keeps the Bearer token out of its argv. +async fn deliver_bootstrap(device: &RemoteHost, payload: &DeviceBootstrap) -> Result<()> { + let json = payload + .to_json() + .context("rendering the bootstrap payload")?; + let remote_path = bootstrap_path(std::path::Path::new(WRITABLE_PARTITION)); + let remote_path = remote_path.to_string_lossy(); + let remote_dir = std::path::Path::new(WRITABLE_PARTITION).join("container-dev"); + let remote_dir = remote_dir.to_string_lossy(); + + let ssh = SshClient::new(device.clone()); + let command = bootstrap_delivery_command(&remote_dir, &remote_path); + ssh.run_command_with_stdin(&command, json.as_bytes()) + .await + .context("writing the bootstrap file to the device writable partition")?; + Ok(()) +} + +/// Deliver the per-project CA into the avocado-vm engine guest's docker trust +/// store over SSH (task 7.1, design H4). +/// +/// Base64-decodes the CA PEM into the guest's `certs.d//ca.crt` so its +/// docker daemon trusts the host write listener's leaf per connection (no daemon +/// reload — phase-0 task 1.8). The CA cert is public material (mode 0644); the CA +/// private key is never delivered (design D8), and only the CA *cert* travels in +/// [`VmWriteSetup`]. Delivered at `up`, NEVER baked into the VM overlay. +async fn deliver_vm_ca(vm: &RemoteHost, setup: &VmWriteSetup) -> Result<()> { + let ca_path = &setup.ca_trust_path; + let ca_dir = std::path::Path::new(ca_path) + .parent() + .expect("the CA trust path has a parent directory") + .to_string_lossy(); + + let ssh = SshClient::new(vm.clone()); + // Streamed over stdin for the same reason as the bootstrap: `base64 -d` is not + // present on an Avocado OS engine guest, and the PEM never needs to survive + // shell quoting if it never enters the command. + let command = format!("mkdir -p {ca_dir} && cat > {ca_path} && chmod 0644 {ca_path}"); + ssh.run_command_with_stdin(&command, setup.ca_cert_pem.as_bytes()) + .await + .context("delivering the per-project CA into the avocado-vm engine trust store")?; + Ok(()) +} + +/// The persisted per-`up` session record: the foreground `up` process id (so a +/// separate `down` can signal it to stop its listeners) plus the reported +/// [`DevStatus`]. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +struct SessionState { + /// PID of the foreground `up` process. + pid: u32, + /// The status `status` reports. + status: DevStatus, +} + +/// Try to take `flag` on `file`, returning `true` when the lock was acquired and +/// `false` when a conflicting lock is already held. +#[cfg(unix)] +fn try_flock(file: &std::fs::File, flag: libc::c_int) -> Result { + use std::os::unix::io::AsRawFd; + // SAFETY: `flock` takes a raw fd plus a flag word and has no memory-safety + // hazard; `file` owns a valid open fd for the duration of the call. + if unsafe { libc::flock(file.as_raw_fd(), flag | libc::LOCK_NB) } == 0 { + return Ok(true); + } + let err = std::io::Error::last_os_error(); + // EWOULDBLOCK (== EAGAIN on Linux and macOS) is the "someone else holds it" + // answer, which is a result here rather than a failure. + match err.raw_os_error() { + Some(libc::EWOULDBLOCK) => Ok(false), + _ => Err(err).context("locking the session lock file"), + } +} + +/// How long `up` waits for the session lock before declaring a competing `up`. +/// +/// Long enough to outlast the liveness probe's shared hold (microseconds), short +/// enough that a genuine collision is reported promptly rather than hanging. +const LOCK_ACQUIRE_WAIT: std::time::Duration = std::time::Duration::from_millis(250); +/// Gap between acquire attempts within [`LOCK_ACQUIRE_WAIT`]. +const LOCK_ACQUIRE_POLL: std::time::Duration = std::time::Duration::from_millis(10); + +/// Take the exclusive lock (`up`'s ownership claim). +#[cfg(unix)] +fn try_lock_exclusive(file: &std::fs::File) -> Result { + try_flock(file, libc::LOCK_EX) +} + +/// Take a shared lock (the read-only liveness probe). +#[cfg(unix)] +fn try_lock_shared(file: &std::fs::File) -> Result { + try_flock(file, libc::LOCK_SH) +} + +/// An advisory exclusive lock on the session file, held for the whole life of +/// the foreground `up` process. +/// +/// `up` removes `session.json` only on the graceful teardown path, so a panic or +/// a SIGKILL leaves the file behind carrying a pid that is no longer `up`. +/// Signalling that pid is not harmless: pids get recycled, so `sync` would +/// deliver SIGUSR1 — whose default disposition is *terminate* — to whatever +/// unrelated process inherited the number, and `down` would SIGTERM it. A +/// liveness check on the pid alone cannot tell a recycled pid from the original. +/// +/// The kernel releases this lock when the holder dies by ANY route, including +/// SIGKILL, so "can I take the lock?" answers the question the pid cannot: no +/// live `up` owns this file. It doubles as the guard against two concurrent +/// `up`s on one project. +#[cfg(unix)] +struct SessionLock { + _file: std::fs::File, +} + +#[cfg(not(unix))] +struct SessionLock; + +#[cfg(unix)] +impl SessionLock { + /// Lock the project's session for this `up`. Fails when another `up` holds it. + /// + /// Creates the lock file when absent: `up` takes this before it has written + /// any state, so requiring the file to pre-exist would make the very first + /// `up` in a project fail. + fn acquire(path: &std::path::Path) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating the session dir {parent:?}"))?; + } + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path) + .with_context(|| format!("opening the session lock at {path:?}"))?; + // Retry briefly instead of failing on the first EWOULDBLOCK. `LOCK_EX` + // conflicts with a held `LOCK_SH` just as it does with another `LOCK_EX`, + // so the read-only liveness probe - which holds a shared lock for + // microseconds - could make a legitimate `up` abort with "another `up` is + // already running" when none was. Switching the probe to shared fixed + // probe-vs-probe only; this is what fixes probe-vs-up. + // + // The wait separates the two cases on duration rather than guessing: a + // probe's hold is over almost immediately, while a real competing `up` + // holds the lock for its entire lifetime and will still be holding it + // when the window expires. + let deadline = Instant::now() + LOCK_ACQUIRE_WAIT; + loop { + if try_lock_exclusive(&file)? { + return Ok(Self { _file: file }); + } + if Instant::now() >= deadline { + bail!( + "another `avocado container dev up` is already running for this project; \ + run `avocado container dev down` first" + ); + } + std::thread::sleep(LOCK_ACQUIRE_POLL); + } + } +} + +#[cfg(not(unix))] +impl SessionLock { + fn acquire(_path: &std::path::Path) -> Result { + Ok(Self) + } +} + +/// Whether a live `up` process still owns `path`'s session lock. +/// +/// Taking a SHARED lock proves no `up` holds the exclusive one, because the two +/// are mutually exclusive; the lock is dropped immediately since only the answer +/// was wanted. Shared rather than exclusive on purpose: this is a read-only +/// probe, so it must not block a concurrent probe, and it must not make `up`'s +/// own `acquire` fail with "another `up` is already running" merely because a +/// `status` held an exclusive lock for that instant. Opened read-only for the +/// same reason - a read-only mount, or a file this user cannot write, is not a +/// reason for `status` to fail. A missing lock file means no `up` ever ran here. +#[cfg(unix)] +fn session_is_live(path: &std::path::Path) -> Result { + let file = match std::fs::OpenOptions::new().read(true).open(path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(e).with_context(|| format!("opening the session lock at {path:?}")), + }; + Ok(!try_lock_shared(&file)?) +} + +/// Without `flock` there is no ownership proof — but `signal_shutdown` and +/// `signal_sync` are no-ops off unix, so nothing can be mis-signalled either. +#[cfg(not(unix))] +fn session_is_live(path: &std::path::Path) -> Result { + Ok(path.exists()) +} + +/// The recorded session, but only when a live `up` still owns it. +/// +/// `sync`, `status` and `down` all need the same three-step policy - probe the +/// lock, read the state, discard and clear a record whose owner is gone - and +/// each then signals or reports on the result. Keeping it in one place means the +/// "is this pid safe to signal?" rule has a single home rather than three copies +/// to keep in agreement. +/// +/// Clears only the state file; the lock file is never removed (see +/// [`session_lock_path`]). +fn load_live_session( + state_path: &std::path::Path, + lock_path: &std::path::Path, +) -> Result> { + if !session_is_live(lock_path)? { + // No owner: a leftover record describes a process that is gone, and its + // pid may since have been recycled onto something unrelated. + let _ = std::fs::remove_file(state_path); + return Ok(None); + } + read_session_state(state_path) +} + +/// Persist the session state so `status`/`down` in a separate invocation can find +/// the running `up`. +fn write_session_state(path: &std::path::Path, state: &SessionState) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating the session state dir {parent:?}"))?; + } + let json = serde_json::to_string_pretty(state).context("serializing the session state")?; + std::fs::write(path, json).with_context(|| format!("writing session state to {path:?}"))?; + Ok(()) +} + +/// The `(image, tag) -> owning unit` map for a project's watched images. +/// +/// Keyed through `image_ref::split`, which strips any registry prefix and +/// defaults the tag - the same derivation `build_push_plan`'s retag and +/// `notify`'s own key both land on. Keying by the raw config `ref` instead looks +/// correct and silently misses for an entry written `localhost/my-app:dev`, and a +/// missed lookup is indistinguishable from "no service declared": the device +/// falls back to restarting the container, which is the no-op sending the unit +/// exists to end. +/// +/// A function rather than an inline closure at the call site so that agreement +/// has a test. Inline, the only test possible was one that re-derived the key +/// itself and so passed with the production keying broken. +fn service_map( + images: &[crate::utils::container_dev::config::ContainerDevImage], +) -> Vec<((String, String), String)> { + images + .iter() + .map(|image| { + let (repo, tag) = image_ref::split(&image.image_ref); + ((repo, tag), image.service.clone()) + }) + .collect() +} + +/// Rewrite the published record with `watcher_running: false`. +/// +/// Read-modify-write rather than reconstructing the record, so the pid and +/// everything else `status` reports survive - rebuilding it here would silently +/// reset `last_sync` and the per-device token state. +/// +/// A missing file is not an error: `down` removes it, so losing the race with a +/// concurrent teardown just means there is nothing left to correct. +fn mark_watcher_stopped(path: &std::path::Path) -> Result<()> { + let Some(mut state) = read_session_state(path)? else { + return Ok(()); + }; + state.status.watcher_running = false; + write_session_state(path, &state) +} + +/// Read the session state, or `None` when no `up` session is recorded. +fn read_session_state(path: &std::path::Path) -> Result> { + match std::fs::read_to_string(path) { + Ok(content) => { + let state: SessionState = serde_json::from_str(&content) + .with_context(|| format!("parsing the session state at {path:?}"))?; + Ok(Some(state)) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e).with_context(|| format!("reading the session state at {path:?}")), + } +} + +/// The SIGUSR1 and SIGTERM streams, registered before the pid is published. +/// +/// Both are created up front and carried to the tasks that consume them rather +/// than created where those tasks are spawned. Registration is what changes the +/// signal's disposition away from Term, and the pid becomes signalable the +/// moment `up` writes its session record - so creating them at the point of use +/// left a window in which a concurrent `sync` or `down` killed `up` outright. +/// +/// Either may be `None`: a platform that refuses the handler leaves the +/// corresponding path inert rather than failing `up`, which is what the previous +/// per-task `Err(_) => return` did. +#[cfg(unix)] +struct EarlySignals { + sync: Option, + shutdown: Option, +} + +#[cfg(unix)] +fn register_early_signals() -> EarlySignals { + use tokio::signal::unix::{signal, SignalKind}; + EarlySignals { + sync: signal(SignalKind::user_defined1()).ok(), + shutdown: signal(SignalKind::terminate()).ok(), + } +} + +/// Off unix there is nothing to register: `signal_shutdown` and `signal_sync` +/// are both no-ops, so no process can be signalled into the window either. +#[cfg(not(unix))] +struct EarlySignals { + sync: (), + shutdown: (), +} + +#[cfg(not(unix))] +fn register_early_signals() -> EarlySignals { + EarlySignals { + sync: (), + shutdown: (), + } +} + +/// Block until the process receives SIGINT (Ctrl-C) or SIGTERM (a separate +/// `down`), so both a foreground Ctrl-C and `down` reach the same graceful +/// teardown path. +#[cfg(unix)] +async fn wait_for_shutdown(term: Option) { + let Some(mut term) = term else { + // No SIGTERM handler was available at registration: fall back to Ctrl-C. + let _ = tokio::signal::ctrl_c().await; + return; + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = term.recv() => {} + } +} + +#[cfg(not(unix))] +async fn wait_for_shutdown(_shutdown: ()) { + let _ = tokio::signal::ctrl_c().await; +} + +/// Signal the recorded `up` process to shut down (SIGTERM), driving its graceful +/// teardown (and, on any unclean exit, its [`WriteListenerGuard`]). +#[cfg(unix)] +fn signal_shutdown(pid: u32) { + // SAFETY: `kill` with a plain signal number has no memory-safety hazard; a + // stale pid simply yields ESRCH, which is ignored (the process already exited). + unsafe { + libc::kill(pid as libc::pid_t, libc::SIGTERM); + } +} + +#[cfg(not(unix))] +fn signal_shutdown(_pid: u32) {} + +/// Serve the `container dev sync` trigger: each SIGUSR1 (sent by a separate +/// `sync` invocation, [`signal_sync`]) drives ONE re-push + notify of every +/// configured watched image through the shared push+notify pipeline +/// ([`run_one_shot_sync`]) — exactly one pass per signal, never a second watch +/// loop. Runs until the task is aborted on teardown. A per-image failure is +/// surfaced as a warning and does not stop the trigger (a later `sync` retries). +#[cfg(unix)] +async fn run_sync_trigger( + mode: SyncMode, + syncer: Arc, + notifier: Arc, + images: Vec, + engine: &'static str, + usr1: Option, +) { + // Registered in `register_early_signals` before the pid was published; a + // platform that refused the handler leaves the trigger inert. + let Some(mut usr1) = usr1 else { + return; + }; + while usr1.recv().await.is_some() { + for image in &images { + // Ask the engine for the image id. A signal carries no event, so + // unlike the watcher this path has nothing to read it from - and + // passing `None` here is not harmless: the notifier turns it into an + // empty desired digest, which then compares equal to the empty + // `running_digest` a device reports before its first pull, so the + // device is silently never told to pull. + let image_id = match resolve_image_id(engine, image).await { + Ok(Some(id)) => id, + Ok(None) => { + print_warning( + &format!( + "container dev sync: `{engine}` does not know image `{image}`; \ + build it first" + ), + OutputLevel::Normal, + ); + continue; + } + Err(e) => { + print_warning( + &format!("container dev sync: resolving `{image}` failed: {e:#}"), + OutputLevel::Normal, + ); + continue; + } + }; + let event = TagEvent { + image: image.clone(), + image_id: Some(image_id), + }; + if let Err(e) = + run_one_shot_sync(mode, syncer.as_ref(), notifier.as_ref(), &event).await + { + print_warning( + &format!("container dev sync of `{image}` failed: {e:#}"), + OutputLevel::Normal, + ); + } + } + } +} + +#[cfg(not(unix))] +async fn run_sync_trigger( + _mode: SyncMode, + _syncer: Arc, + _notifier: Arc, + _images: Vec, + _engine: &'static str, + _usr1: (), +) { +} + +/// Signal the recorded `up` process to perform one manual sync (SIGUSR1), +/// driving its [`run_sync_trigger`] through a single re-push + notify pass. +#[cfg(unix)] +fn signal_sync(pid: u32) { + // SAFETY: `kill` with a plain signal number has no memory-safety hazard; a + // stale pid simply yields ESRCH, which is ignored (the process already exited). + unsafe { + libc::kill(pid as libc::pid_t, libc::SIGUSR1); + } +} + +#[cfg(not(unix))] +fn signal_sync(_pid: u32) {} + +/// The port component of a `host:port` endpoint. +fn endpoint_port(endpoint: &str) -> Result { + endpoint + .rsplit_once(':') + .and_then(|(_, port)| port.parse().ok()) + .with_context(|| format!("`{endpoint}` is not a valid host:port endpoint")) +} + +/// The host component the device uses to reach the bulk listener: the endpoint's +/// host (an override or the auto-detected reachable IP). +fn bulk_host<'a>(endpoint: &'a str, auto_host: &'a str) -> &'a str { + match endpoint.rsplit_once(':') { + Some((host, _)) if !host.is_empty() => host, + _ => auto_host, + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + fn session_with_watcher(running: bool) -> SessionState { + SessionState { + pid: 4242, + status: DevStatus { + registry_running: true, + watcher_running: running, + last_sync: Some("sha256:abc".to_string()), + devices: Vec::new(), + }, + } + } + + #[test] + fn the_service_map_is_keyed_the_way_the_push_path_keys_it() { + // The mutation this exists for: keying by the raw config `ref` passes + // any test that derives the key itself, and silently misses in production + // for a registry-prefixed entry - which reads as "no service declared" + // and falls back to the container restart that never adopts the image. + use crate::utils::container_dev::config::ContainerDevImage; + + let images = vec![ + ContainerDevImage { + image_ref: "localhost/my-app:dev".to_string(), + service: "app.service".to_string(), + }, + ContainerDevImage { + image_ref: "sidecar".to_string(), + service: "sidecar.service".to_string(), + }, + ]; + + let map = service_map(&images); + + assert_eq!( + map, + vec![ + ( + ("my-app".to_string(), "dev".to_string()), + "app.service".to_string() + ), + ( + ("sidecar".to_string(), "latest".to_string()), + "sidecar.service".to_string() + ), + ], + "the registry prefix must be stripped and a missing tag defaulted, \ + matching what the frame carries" + ); + } + + #[test] + fn a_stopped_watcher_is_recorded_where_status_reads_it() { + // `up` wrote `watcher_running: true` once and never revisited it, so + // after the watcher died a `status` from a second terminal - which is how + // anyone actually checks - still reported a live watcher. The forwarder's + // warning only reaches the terminal holding `up`, which is the one place + // the operator is not looking. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("session.json"); + write_session_state(&path, &session_with_watcher(true)).unwrap(); + + mark_watcher_stopped(&path).unwrap(); + + let after = read_session_state(&path) + .unwrap() + .expect("the record must still exist"); + assert!( + !after.status.watcher_running, + "status must report the watcher as stopped" + ); + // Read-modify-write, not a rebuild: reconstructing the record here would + // silently reset everything else `status` reports. + assert_eq!(after.pid, 4242, "the pid must survive the correction"); + assert_eq!( + after.status.last_sync.as_deref(), + Some("sha256:abc"), + "last_sync must survive the correction" + ); + assert!(after.status.registry_running, "the registry is still up"); + } + + #[test] + fn recording_a_stopped_watcher_tolerates_a_removed_record() { + // `down` removes the file, so a watcher dying concurrently with teardown + // finds nothing to correct. That is a race with no consequence, not an + // error worth surfacing. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("gone.json"); + + mark_watcher_stopped(&path).expect("a missing record must not be an error"); + assert!(read_session_state(&path).unwrap().is_none()); + } + + /// The bootstrap file carries the Bearer read/control token, so it must never + /// exist world-readable - not even briefly. + /// + /// The delivery used to write the file and then `chmod 0600` it, which leaves + /// the token on disk at the remote shell's umask (0644 on a default 0022) for + /// the width of two commands. The window cannot be observed from a test + /// without racing the shell, so this asserts the shape that makes it + /// impossible instead: the mode is established by a umask in force when the + /// file is created, and there is no separate correcting step afterwards. + /// The delivery must not require ANY decoder on the device. + /// + /// It used to `printf %s '' | base64 -d`, which assumes coreutils on the + /// target. Avocado OS - the OS this feature ships on - has no `base64`, so + /// `container dev up` failed at bootstrap on a real device with + /// `sh: base64: not found`. A Debian stand-in hid it because Debian has + /// coreutils. The payload now travels over ssh stdin, so the device needs no + /// decoder and the JSON never passes through argv or shell quoting. + #[test] + fn bootstrap_delivery_needs_no_decoder_on_the_device() { + let command = bootstrap_delivery_command("/tmp/d", "/tmp/d/bootstrap.json"); + + assert!( + !command.contains("base64"), + "the device may not need a base64 decoder: {command}" + ); + // Nor any other decoder that is absent from a minimal target. + for tool in ["openssl", "xxd", "python3", "perl", "uudecode", "od"] { + assert!( + !command.contains(tool), + "the device may not need `{tool}`: {command}" + ); + } + // The payload arrives on stdin, so the command only redirects it into place. + assert!( + command.contains("cat >"), + "the payload must be piped from stdin: {command}" + ); + } + + #[test] + fn bootstrap_delivery_never_creates_a_world_readable_token() { + let command = bootstrap_delivery_command("/tmp/d", "/tmp/d/bootstrap.json"); + + assert!( + command.contains("umask 077"), + "the mode must be set by a umask in force at creation: {command}" + ); + // A chmod means the file existed at some other mode first, which is the + // whole defect - so its absence is the assertion, not a style preference. + assert!( + !command.contains("chmod"), + "a correcting chmod means the file was created at the wrong mode: {command}" + ); + // The umask has to precede the redirect to govern it at all. + let umask_at = command.find("umask 077").expect("umask present"); + let redirect_at = command.find('>').expect("redirect present"); + assert!( + umask_at < redirect_at, + "the umask must be in force before the write: {command}" + ); + } + + /// The generated command is plain POSIX shell, so running it locally proves + /// the mode it actually produces rather than only its shape. + #[test] + fn bootstrap_delivery_command_produces_a_0600_file() { + use std::os::unix::fs::PermissionsExt; + + use std::io::Write as _; + + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("container-dev").join("bootstrap.json"); + let command = bootstrap_delivery_command( + &dir.path().join("container-dev").to_string_lossy(), + &target.to_string_lossy(), + ); + + // A permissive umask in the parent: if the command relied on inheriting a + // strict one, this would catch it. The payload goes in on stdin, exactly as + // `deliver_bootstrap` feeds it to ssh. + let mut child = std::process::Command::new("sh") + .arg("-c") + .arg(format!("umask 0022 && {command}")) + .stdin(std::process::Stdio::piped()) + .spawn() + .expect("spawning the delivery command"); + child + .stdin + .as_mut() + .expect("stdin is piped") + .write_all(b"{\"t\":1}") + .expect("writing the payload"); + let status = child.wait().expect("running the delivery command"); + assert!(status.success(), "delivery command failed: {command}"); + + let mode = std::fs::metadata(&target).unwrap().permissions().mode() & 0o777; + assert_eq!( + mode, 0o600, + "the token file must be created 0600, got {mode:o}" + ); + assert_eq!(std::fs::read_to_string(&target).unwrap(), "{\"t\":1}"); + } + + /// A lock nobody holds must read as dead, so `down`/`sync` never signal the + /// recorded pid. Without this, a `session.json` surviving an unclean exit is + /// indistinguishable from a running `up` - and the pid it carries may since + /// have been recycled onto an unrelated process, which SIGUSR1 would + /// terminate. + #[test] + fn an_unheld_lock_is_not_live() { + let dir = tempfile::tempdir().unwrap(); + let lock = dir.path().join("session.lock"); + std::fs::write(&lock, "").unwrap(); + + assert!( + !session_is_live(&lock).unwrap(), + "a lock file nobody holds must read as dead" + ); + } + + /// The lock is what proves liveness, and it is held for as long as the + /// process that took it lives. + #[test] + fn a_held_lock_is_live_and_excludes_a_second_up() { + let dir = tempfile::tempdir().unwrap(); + let lock = dir.path().join("session.lock"); + + // No pre-created file: `up` takes the lock before writing any state, so + // acquire has to create it. + let held = SessionLock::acquire(&lock).expect("the first acquire succeeds"); + assert!(lock.exists(), "acquire must create the lock file"); + assert!( + session_is_live(&lock).unwrap(), + "a held session lock must read as live" + ); + + // A second `up` on the same project must be refused rather than racing + // the first one's listeners. + assert!( + SessionLock::acquire(&lock).is_err(), + "a second acquire must be refused while the first is held" + ); + + drop(held); + assert!( + !session_is_live(&lock).unwrap(), + "releasing the lock must make the session read as dead again" + ); + } + + /// A missing lock file is simply "no session", not an error. + #[test] + fn a_missing_lock_is_not_live() { + let dir = tempfile::tempdir().unwrap(); + assert!(!session_is_live(&dir.path().join("absent.lock")).unwrap()); + } + + /// The probe must not disturb the thing it observes. + /// + /// This needs TWO CONCURRENT holders to mean anything, which is what the + /// earlier version of this test lacked: it ran two sequential probes plus an + /// acquire on one thread, and `session_is_live` drops its `File` (releasing + /// the flock) on every return - so every assertion passed with `LOCK_EX` + /// restored, leaving the whole shared-lock mechanism unverified. + /// + /// Holds a real shared lock open across the acquire instead. `LOCK_EX` + /// conflicts with a held `LOCK_SH`, so without the bounded retry in + /// `acquire` this is exactly the case that made a legitimate `up` abort while + /// an IDE task polled `status`. + #[test] + fn a_concurrent_probe_does_not_make_up_abort() { + let dir = tempfile::tempdir().unwrap(); + let lock = dir.path().join("session.lock"); + std::fs::write(&lock, "").unwrap(); + + // Two concurrent shared holders coexist - the half that switching the + // probe to LOCK_SH did fix. + let probe = std::fs::OpenOptions::new().read(true).open(&lock).unwrap(); + assert!(try_lock_shared(&probe).unwrap()); + let probe2 = std::fs::OpenOptions::new().read(true).open(&lock).unwrap(); + assert!( + try_lock_shared(&probe2).unwrap(), + "two concurrent probes must not block each other" + ); + drop(probe); + drop(probe2); + + // Now the half it did NOT fix. A probe holds its shared lock briefly, as + // `session_is_live` does - open, flock, drop - and `up` starts while it is + // held. `LOCK_EX` conflicts with a held `LOCK_SH`, so without the bounded + // retry `acquire` fails on the first EWOULDBLOCK and reports a competing + // `up` that does not exist. + let holding = std::fs::OpenOptions::new().read(true).open(&lock).unwrap(); + assert!(try_lock_shared(&holding).unwrap()); + let releaser = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(40)); + drop(holding); + }); + + let held = SessionLock::acquire(&lock); + releaser.join().unwrap(); + assert!( + held.is_ok(), + "a transient probe must not make `up` report a competing `up`: {:?}", + held.err() + ); + } + + /// `session_is_live` must not report a live session merely because ANOTHER + /// probe is reading at the same instant. + /// + /// This is the assertion that actually distinguishes `LOCK_SH` from + /// `LOCK_EX`, and its absence is why the mechanism went unverified: with an + /// exclusive probe, a concurrent shared holder makes the flock fail, and + /// `session_is_live` maps that failure to "someone holds it" - a FALSE + /// POSITIVE. `status` would report a session running with no `up` alive, and + /// `down`/`sync` would then signal whatever pid the stale record carried. + #[test] + fn a_concurrent_probe_does_not_make_the_session_look_live() { + let dir = tempfile::tempdir().unwrap(); + let lock = dir.path().join("session.lock"); + std::fs::write(&lock, "").unwrap(); + + // Another probe reading concurrently - nobody owns the session. + let other = std::fs::OpenOptions::new().read(true).open(&lock).unwrap(); + assert!(try_lock_shared(&other).unwrap()); + + assert!( + !session_is_live(&lock).unwrap(), + "a concurrent reader must not be mistaken for a live `up`" + ); + + drop(other); + // And the true-positive direction still holds. + let _held = SessionLock::acquire(&lock).expect("acquire succeeds"); + assert!( + session_is_live(&lock).unwrap(), + "a genuinely held lock must still read as live" + ); + } + + /// The retry must not paper over a REAL collision: a live `up` holds the lock + /// for its whole lifetime, so a second `up` must still be refused - promptly, + /// not after a hang. + #[test] + fn a_live_up_still_excludes_a_second_up_promptly() { + let dir = tempfile::tempdir().unwrap(); + let lock = dir.path().join("session.lock"); + + let _first = SessionLock::acquire(&lock).expect("the first acquire succeeds"); + + let started = Instant::now(); + let second = SessionLock::acquire(&lock); + let waited = started.elapsed(); + + assert!( + second.is_err(), + "a second `up` must be refused while the first holds the lock" + ); + assert!( + waited >= LOCK_ACQUIRE_WAIT, + "it must actually wait out the window before giving up, waited {waited:?}" + ); + assert!( + waited < LOCK_ACQUIRE_WAIT * 4, + "it must give up promptly rather than hang, waited {waited:?}" + ); + } + + /// Mutual exclusion has to survive a `down`. The teardown paths unlink + /// `session.json`, so locking that inode would hand the next `up` a brand + /// new one and silently drop the guarantee. + #[test] + fn clearing_the_state_file_does_not_release_the_lock() { + let dir = tempfile::tempdir().unwrap(); + let state = dir.path().join("session.json"); + let lock = dir.path().join("session.lock"); + std::fs::write(&state, "{}").unwrap(); + + let held = SessionLock::acquire(&lock).expect("acquire succeeds"); + // What `down` does to a session it is tearing down. + std::fs::remove_file(&state).unwrap(); + + assert!( + session_is_live(&lock).unwrap(), + "unlinking the state file must not release the owner's lock" + ); + assert!( + SessionLock::acquire(&lock).is_err(), + "a second `up` must still be excluded after the state file is cleared" + ); + drop(held); + } + + /// `load_live_session` is the single place the stale-record policy lives: + /// an owner-less record is discarded AND cleared, so no caller signals its + /// pid. + #[test] + fn load_live_session_discards_and_clears_an_ownerless_record() { + let dir = tempfile::tempdir().unwrap(); + let state = dir.path().join("session.json"); + let lock = dir.path().join("session.lock"); + std::fs::write(&state, r#"{"pid":999999,"status":{"registry_running":true,"watcher_running":true,"last_sync":null,"devices":[]}}"#).unwrap(); + std::fs::write(&lock, "").unwrap(); + + let loaded = load_live_session(&state, &lock).expect("load succeeds"); + assert!( + loaded.is_none(), + "a record whose owner is gone must not be returned" + ); + assert!( + !state.exists(), + "the stale record must be cleared, not left for the next caller" + ); + assert!( + lock.exists(), + "the lock inode must survive so exclusion holds for the next `up`" + ); + } + + /// The live case: an owned record is returned intact. + #[test] + fn load_live_session_returns_an_owned_record() { + let dir = tempfile::tempdir().unwrap(); + let state = dir.path().join("session.json"); + let lock = dir.path().join("session.lock"); + std::fs::write(&state, r#"{"pid":4242,"status":{"registry_running":true,"watcher_running":true,"last_sync":null,"devices":[]}}"#).unwrap(); + + let _held = SessionLock::acquire(&lock).expect("acquire succeeds"); + let loaded = load_live_session(&state, &lock).expect("load succeeds"); + assert_eq!( + loaded.map(|s| s.pid), + Some(4242), + "a record with a live owner must be returned as-is" + ); + } +} diff --git a/src/commands/container/mod.rs b/src/commands/container/mod.rs new file mode 100644 index 00000000..b8e24f14 --- /dev/null +++ b/src/commands/container/mod.rs @@ -0,0 +1,7 @@ +//! `avocado container` subcommands. +//! +//! Top-level noun for Container Dev Mode. v1 exposes only the `dev` command +//! family (`up`/`sync`/`status`/`down`/`prune`); dev-to-prod graduation is +//! deliberately out of scope for v1 (see the design doc). + +pub mod dev; diff --git a/src/commands/mod.rs b/src/commands/mod.rs index b9f4d3d2..821f1c9f 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod build; pub mod clean; pub mod config_show; pub mod connect; +pub mod container; pub mod ext; pub mod fetch; pub mod hitl; diff --git a/src/main.rs b/src/main.rs index 183c72af..1638e141 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,9 @@ use commands::connect::trust::{ ConnectTrustPromoteRootCommand, ConnectTrustRotateServerKeyCommand, ConnectTrustStatusCommand, }; use commands::connect::upload::ConnectUploadCommand; +use commands::container::dev::{ + DevDownCommand, DevPruneCommand, DevStatusCommand, DevSyncCommand, DevUpCommand, +}; use commands::ext::{ ExtBuildCommand, ExtCheckoutCommand, ExtCleanCommand, ExtDepsCommand, ExtDnfCommand, ExtFetchCommand, ExtImageCommand, ExtInstallCommand, ExtListCommand, ExtPackageCommand, @@ -190,6 +193,11 @@ enum Commands { #[command(subcommand)] command: VmCommands, }, + /// Container Dev Mode: iterate on containers running on a device. + Container { + #[command(subcommand)] + command: ContainerCommands, + }, /// Project configuration introspection (read-only). Config { #[command(subcommand)] @@ -2018,6 +2026,25 @@ fn needs_vm_routing(cmd: &Commands) -> bool { | Commands::Connect { command: ConnectCommands::Upload { .. } } + // `container dev` drives the engine directly: `up` watches + // `docker events` and pushes to the embedded registry, `sync` + // re-pushes, `down` and `prune` inspect and remove. Without an arm + // here `ensure_routed_for_process` never ran for them, so DOCKER_HOST + // was never set and `HostTopology::sync_mode` could not select the VM + // push path no matter how many times the user started the VM — the + // ingest error told them to start it and routing never followed. + // + // `status` is deliberately excluded: it only reads the session record + // written by `up`, and routing may AUTO-START the VM, so gating it + // here would make a read-only status query boot a virtual machine. + | Commands::Container { + command: ContainerCommands::Dev { + command: ContainerDevCommands::Up + | ContainerDevCommands::Sync + | ContainerDevCommands::Down + | ContainerDevCommands::Prune + } + } ) } @@ -3173,6 +3200,15 @@ async fn main() -> Result<()> { Ok(()) } }, + Commands::Container { command } => match command { + ContainerCommands::Dev { command } => match command { + ContainerDevCommands::Up => DevUpCommand.execute().await, + ContainerDevCommands::Sync => DevSyncCommand.execute().await, + ContainerDevCommands::Status => DevStatusCommand.execute().await, + ContainerDevCommands::Down => DevDownCommand.execute().await, + ContainerDevCommands::Prune => DevPruneCommand.execute().await, + }, + }, Commands::Vm { command } => match command { VmCommands::Start { vm_source, @@ -4793,6 +4829,30 @@ enum VmCommands { }, } +#[derive(Subcommand)] +enum ContainerCommands { + /// Layer-aware hot-reload loop for a container running on a device. + Dev { + #[command(subcommand)] + command: ContainerDevCommands, + }, +} + +#[derive(Subcommand)] +enum ContainerDevCommands { + /// Start the dev registry + watcher and bootstrap the device. + Up, + /// One-shot re-push of the current watched image + notify the device. + Sync, + /// Report registry/watcher/last-sync state for the dev loop. + Status, + /// Stop the dev registry + watcher and tear down listeners. + Down, + /// Garbage-collect this project's Container Dev Mode registry store + /// (distinct from the top-level `prune`, which removes Docker volumes). + Prune, +} + #[derive(Subcommand)] enum VmConfigCommands { /// Print the value of a dotted key (e.g. `network.dns`). Silent on @@ -5008,4 +5068,37 @@ mod tests { "qemux86-64", ]))); } + + /// The engine-driving `container dev` subcommands must route, or + /// `ensure_routed_for_process` never runs for them, `DOCKER_HOST` is never + /// set, and `HostTopology::sync_mode` cannot select the VM push path however + /// many times the operator starts the VM. That made the ingest error's own + /// remedy ("start the avocado-vm and route it") unreachable: following it + /// produced the identical error, and the only escape was a manual + /// `DOCKER_HOST` export. + /// + /// `status` must NOT route: it only reads the record `up` published, and + /// routing may auto-start the VM, so gating it would make a read-only query + /// boot a virtual machine. + #[test] + fn needs_vm_routing_gates_engine_driving_container_dev_subcommands() { + let cmd = |args: &[&str]| { + Cli::try_parse_from(args) + .expect("args should parse") + .command + }; + + for sub in ["up", "sync", "down", "prune"] { + assert!( + needs_vm_routing(&cmd(&["avocado", "container", "dev", sub])), + "`container dev {sub}` drives the engine and must route" + ); + } + + assert!( + !needs_vm_routing(&cmd(&["avocado", "container", "dev", "status"])), + "`container dev status` only reads the session record; routing it \ + would let a status query auto-start the VM" + ); + } } diff --git a/src/utils/config.rs b/src/utils/config.rs index 14d736d6..c085543d 100644 --- a/src/utils/config.rs +++ b/src/utils/config.rs @@ -7,6 +7,7 @@ use std::env; use std::fs; use std::path::{Path, PathBuf}; +use crate::utils::container_dev::config::ContainerDevConfig; use crate::utils::kernel_version::KernelVersionSpec; use crate::utils::output::{print_warning, OutputLevel}; @@ -679,6 +680,9 @@ pub struct RuntimeConfig { pub initramfs: Option, /// Var partition configuration: default compression, subvolume definitions. pub var: Option, + /// Container Dev Mode configuration. Presence of this block enables the + /// feature for this runtime; an absent block means the feature is off. + pub container_dev: Option, } /// SDK configuration section @@ -2368,7 +2372,9 @@ impl Config { /// - `ext..dependencies..config` /// /// Returns a list of (extension_name, config_path) tuples. - fn discover_external_config_refs(config: &serde_yaml::Value) -> Vec<(String, String)> { + pub(crate) fn discover_external_config_refs( + config: &serde_yaml::Value, + ) -> Vec<(String, String)> { let mut refs = Vec::new(); let mut visited = std::collections::HashSet::new(); @@ -2394,6 +2400,7 @@ impl Config { "signing", "var_files", "var", + "container_dev", ] .contains(&key_str) { @@ -3510,6 +3517,7 @@ impl Config { rootfs: rootfs_ref, initramfs: initramfs_ref, var: None, + container_dev: None, }; let mut map = self.runtimes.take().unwrap_or_default(); map.insert("default".to_string(), synth); diff --git a/src/utils/container_dev/auth.rs b/src/utils/container_dev/auth.rs new file mode 100644 index 00000000..277c7fee --- /dev/null +++ b/src/utils/container_dev/auth.rs @@ -0,0 +1,493 @@ +//! Authentication for the Container Dev Mode registry. +//! +//! Two credential *types* exist, structurally distinct (design D2): +//! +//! - the host-only WRITE token, presented as an HTTP **Basic** credential +//! (fixed username, password = the write token) on the write listener; +//! - the READ/CONTROL token, a **Bearer** value delivered to devices (task 3.4). +//! +//! This module owns both the write-side Basic validator (task 3.3) and the +//! read/control Bearer validator (task 3.4). The write validator also REJECTS a +//! Bearer credential presented on a write route (a Bearer scheme is not Basic, +//! so it never satisfies [`basic_write_is_valid`]); the read validator likewise +//! rejects the Basic write token on a read route (M-2). The read/control token +//! is authorized through ONE seam ([`read_request_authorized`]) that both the +//! bulk read listener ([`require_bearer_read`]) and the control-WS upgrade (task +//! 5.1) call, so the WS is not a second, separately-implemented auth surface +//! (G-5). + +use std::sync::Arc; + +use axum::{ + extract::{Request, State}, + http::{header, HeaderMap, StatusCode}, + middleware::Next, + response::{IntoResponse, Response}, +}; +use base64::Engine as _; + +/// Fixed Basic-auth username paired with the write token as the password. +/// +/// docker/podman inject a `user:password` pair on push; the username carries no +/// authority (only the password — the write token — is checked) but it must be +/// a fixed, known value so the injected credential form is deterministic. +pub const WRITE_USERNAME: &str = "avocado"; + +/// Realm advertised in the write listener's `WWW-Authenticate: Basic` challenge. +const WRITE_REALM: &str = "avocado-container-dev"; + +/// The host-only write token gating every write route (design D2). +/// +/// Presented by the engine push as an HTTP Basic password. It never leaves the +/// host and is never delivered to a device. +#[derive(Clone)] +pub struct WriteToken(Arc); + +impl WriteToken { + /// Wrap a freshly minted write token. + pub fn new(token: impl Into) -> Self { + Self(Arc::new(token.into())) + } + + /// The raw token value, for host-side comparison only. Never logged. + pub fn secret(&self) -> &str { + &self.0 + } +} + +/// Whether `header_value` is a Basic credential whose username is +/// [`WRITE_USERNAME`] and whose password equals `expected_token`. +/// +/// Returns `false` for an absent header, a non-Basic scheme (e.g. the Bearer +/// read/control token), undecodable base64, a missing `:` separator, a wrong +/// username, or a wrong password. This is the entire accept predicate for a +/// write route. +/// +/// The password comparison is a plain byte equality, not constant-time: the +/// threat model scopes the write listener to loopback on native Linux (or a +/// routable HTTPS listener never disclosed to a device) on a single-developer +/// host, so a timing side channel is not in scope (design D2, threat-model +/// residual assumption). +pub fn basic_write_is_valid(header_value: Option<&str>, expected_token: &str) -> bool { + let Some(raw) = header_value else { + return false; + }; + // The scheme must be Basic (case-insensitive per RFC 7617); a Bearer + // read/control token is rejected right here. + let Some(encoded) = scheme_payload(raw, "basic") else { + return false; + }; + let Ok(decoded) = base64::engine::general_purpose::STANDARD.decode(encoded) else { + return false; + }; + let Ok(pair) = std::str::from_utf8(&decoded) else { + return false; + }; + let Some((user, pass)) = pair.split_once(':') else { + return false; + }; + user == WRITE_USERNAME && pass == expected_token +} + +/// Split an `Authorization` header into its scheme and payload, returning the +/// trimmed payload only when the scheme matches `scheme` case-insensitively. +fn scheme_payload<'a>(header: &'a str, scheme: &str) -> Option<&'a str> { + let (got, rest) = header.split_once(' ')?; + got.eq_ignore_ascii_case(scheme).then(|| rest.trim()) +} + +/// axum middleware gating every write route on a valid Basic write credential. +/// +/// On failure it returns `401 Unauthorized` with a `WWW-Authenticate: Basic` +/// challenge — never a Bearer challenge (design D2/L-2): issuing a Basic +/// challenge is what makes docker/podman send a Basic credential on push. +pub async fn require_basic_write( + State(token): State, + request: Request, + next: Next, +) -> Response { + let header = request + .headers() + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()); + if basic_write_is_valid(header, token.secret()) { + next.run(request).await + } else { + write_unauthorized() + } +} + +/// A `401` carrying the Basic challenge for the write path. +fn write_unauthorized() -> Response { + let body = serde_json::json!({ + "errors": [{ "code": "UNAUTHORIZED", "message": "write token required" }] + }) + .to_string(); + ( + StatusCode::UNAUTHORIZED, + [ + ( + header::WWW_AUTHENTICATE, + format!("Basic realm=\"{WRITE_REALM}\""), + ), + (header::CONTENT_TYPE, "application/json".to_string()), + ], + body, + ) + .into_response() +} + +/// The per-session read/control token (design D2 split). +/// +/// A **Bearer** value delivered to a device at bootstrap; it is the ONLY token +/// a device holds and authorizes both bulk pulls (the read listener) and the +/// control-WS upgrade (task 5.1). Structurally distinct from the Basic +/// [`WriteToken`]: a Bearer credential can never satisfy a write route, and the +/// Basic write token can never satisfy a read route (M-2). +#[derive(Clone)] +pub struct ReadToken(Arc); + +impl ReadToken { + /// Wrap a freshly minted read/control token. + pub fn new(token: impl Into) -> Self { + Self(Arc::new(token.into())) + } + + /// The raw token value, for host-side comparison only. Never logged. + pub fn secret(&self) -> &str { + &self.0 + } +} + +/// Whether `header_value` is a Bearer credential whose token equals +/// `expected_token`. +/// +/// Returns `false` for an absent header, a non-Bearer scheme (crucially the +/// Basic write token, which is rejected on a read route per M-2), or a wrong +/// token. This is the entire accept predicate for a read/control route. +/// +/// The comparison is a plain byte equality, not constant-time: the read +/// listener is served over TLS to a device on a single-developer host, so a +/// timing side channel is out of scope (design D2, threat-model residual +/// assumption), matching [`basic_write_is_valid`]. +pub fn bearer_read_is_valid(header_value: Option<&str>, expected_token: &str) -> bool { + let Some(raw) = header_value else { + return false; + }; + // The scheme must be Bearer (case-insensitive per RFC 6750); a Basic write + // credential is rejected right here (M-2). + let Some(token) = scheme_payload(raw, "bearer") else { + return false; + }; + token == expected_token +} + +/// Authorize a request against the read/control `token` by reading its +/// `Authorization` header. +/// +/// This is the ONE seam both the bulk read listener ([`require_bearer_read`]) +/// and the control-WS upgrade (task 5.1) call, so the two auth surfaces cannot +/// diverge (G-5). A WebSocket upgrade is an HTTP `GET` carrying the same +/// `Authorization` header, so the upgrade handler authorizes through this exact +/// function rather than re-implementing the check. +pub fn read_request_authorized(headers: &HeaderMap, token: &ReadToken) -> bool { + let header = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()); + bearer_read_is_valid(header, token.secret()) +} + +/// axum middleware gating every bulk read route on a valid Bearer read/control +/// credential, delegating to the shared [`read_request_authorized`] seam. +/// +/// On failure it returns `401 Unauthorized` with a BARE `Bearer` challenge — +/// no `realm` or token-endpoint parameters (design L-1): a token-endpoint +/// redirect would send a stray client off to a phantom auth server that does +/// not exist. +pub async fn require_bearer_read( + State(token): State, + request: Request, + next: Next, +) -> Response { + if read_request_authorized(request.headers(), &token) { + next.run(request).await + } else { + read_unauthorized() + } +} + +/// A `401` carrying a BARE `Bearer` challenge for the read path (design L-1). +/// +/// The challenge is the single word `Bearer` with no `realm`/token-endpoint +/// parameters, so a client that stumbles onto the read listener is told the +/// scheme without being redirected to an auth server that does not exist. +fn read_unauthorized() -> Response { + let body = serde_json::json!({ + "errors": [{ "code": "UNAUTHORIZED", "message": "read/control token required" }] + }) + .to_string(); + ( + StatusCode::UNAUTHORIZED, + [ + (header::WWW_AUTHENTICATE, "Bearer".to_string()), + (header::CONTENT_TYPE, "application/json".to_string()), + ], + body, + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Encode a `user:pass` pair as a Basic `Authorization` header value. + fn basic(user: &str, pass: &str) -> String { + let encoded = base64::engine::general_purpose::STANDARD.encode(format!("{user}:{pass}")); + format!("Basic {encoded}") + } + + #[test] + fn correct_username_and_password_are_accepted() { + let header = basic(WRITE_USERNAME, "s3cret"); + assert!(basic_write_is_valid(Some(&header), "s3cret")); + } + + #[test] + fn wrong_password_is_rejected() { + let header = basic(WRITE_USERNAME, "wrong"); + assert!(!basic_write_is_valid(Some(&header), "s3cret")); + } + + #[test] + fn wrong_username_is_rejected() { + let header = basic("intruder", "s3cret"); + assert!(!basic_write_is_valid(Some(&header), "s3cret")); + } + + #[test] + fn a_bearer_token_is_not_a_basic_credential() { + // Even if the Bearer value equals the write token, the scheme is wrong. + let header = "Bearer s3cret"; + assert!(!basic_write_is_valid(Some(header), "s3cret")); + } + + #[test] + fn absent_header_is_rejected() { + assert!(!basic_write_is_valid(None, "s3cret")); + } + + #[test] + fn undecodable_base64_is_rejected() { + assert!(!basic_write_is_valid( + Some("Basic !!!not-base64!!!"), + "s3cret" + )); + } + + #[test] + fn a_credential_without_a_colon_is_rejected() { + let encoded = base64::engine::general_purpose::STANDARD.encode("no-colon-here"); + let header = format!("Basic {encoded}"); + assert!(!basic_write_is_valid(Some(&header), "s3cret")); + } + + // ---- read/control Bearer validator (task 3.4) ---- + + const READ_TOKEN: &str = "read-control-token"; + const A_WRITE_TOKEN: &str = "write-token-secret"; + + use axum::{middleware, routing::get, routing::put, Router}; + + /// A trivial handler standing in for a real read route or write route; the + /// auth middleware runs before it, so reaching it means the request passed. + async fn ok() -> &'static str { + "ok" + } + + /// Serve a Bearer-gated read router (the bulk-listener shape) over + /// [`require_bearer_read`]; return its base URL. + async fn spawn_read(token: &str) -> String { + let app = Router::new() + .route("/v2/", get(ok)) + .route("/v2/{*rest}", get(ok)) + .layer(middleware::from_fn_with_state( + ReadToken::new(token), + require_bearer_read, + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{addr}") + } + + /// Serve a Basic-gated write route over [`require_basic_write`]; return its + /// base URL. Used to assert the Bearer read/control token is refused here. + async fn spawn_write(token: &str) -> String { + let app = + Router::new() + .route("/v2/{*rest}", put(ok)) + .layer(middleware::from_fn_with_state( + WriteToken::new(token), + require_basic_write, + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{addr}") + } + + /// Build a `HeaderMap` carrying a single `Authorization` header. + fn headers_with(auth: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, auth.parse().unwrap()); + headers + } + + /// The same `Authorization` header a WebSocket upgrade carries: a `GET` with + /// the `Connection: Upgrade` / `Upgrade: websocket` handshake headers added. + fn ws_upgrade_headers(auth: &str) -> HeaderMap { + let mut headers = headers_with(auth); + headers.insert(header::CONNECTION, "Upgrade".parse().unwrap()); + headers.insert(header::UPGRADE, "websocket".parse().unwrap()); + headers + } + + #[test] + fn correct_bearer_read_token_is_accepted() { + let header = format!("Bearer {READ_TOKEN}"); + assert!(bearer_read_is_valid(Some(&header), READ_TOKEN)); + } + + #[test] + fn wrong_bearer_read_token_is_rejected() { + assert!(!bearer_read_is_valid( + Some("Bearer not-the-token"), + READ_TOKEN + )); + } + + #[test] + fn absent_header_is_rejected_on_a_read_route() { + assert!(!bearer_read_is_valid(None, READ_TOKEN)); + } + + #[test] + fn a_basic_credential_is_not_a_bearer_read_token() { + // M-2: even if the Basic password equals the read token, the scheme is + // Basic, so it cannot satisfy a read route. + let header = basic(WRITE_USERNAME, READ_TOKEN); + assert!(!bearer_read_is_valid(Some(&header), READ_TOKEN)); + } + + #[test] + fn bearer_scheme_matching_is_case_insensitive() { + let header = format!("bearer {READ_TOKEN}"); + assert!(bearer_read_is_valid(Some(&header), READ_TOKEN)); + } + + #[test] + fn bulk_and_ws_upgrade_authorize_through_the_same_seam() { + let token = ReadToken::new(READ_TOKEN); + let good = format!("Bearer {READ_TOKEN}"); + // A Basic credential is the write token's transport form. + let write_basic = basic(WRITE_USERNAME, A_WRITE_TOKEN); + + // A bulk GET and a WS upgrade carrying the SAME credential get the SAME + // decision because both authorize through read_request_authorized (G-5); + // the WS upgrade (task 5.1) is not a divergent auth surface. + assert!(read_request_authorized(&headers_with(&good), &token)); + assert!(read_request_authorized(&ws_upgrade_headers(&good), &token)); + assert!(!read_request_authorized( + &headers_with(&write_basic), + &token + )); + assert!(!read_request_authorized( + &ws_upgrade_headers(&write_basic), + &token + )); + } + + #[tokio::test] + async fn a_read_request_without_the_token_is_rejected_with_a_bare_bearer_challenge() { + let base = spawn_read(READ_TOKEN).await; + let resp = reqwest::get(format!("{base}/v2/")).await.unwrap(); + assert_eq!( + resp.status().as_u16(), + 401, + "an unauthenticated read must be refused" + ); + let challenge = resp + .headers() + .get("www-authenticate") + .and_then(|h| h.to_str().ok()) + .unwrap_or(""); + // A bare `Bearer` challenge: exactly the scheme, no realm/token-endpoint + // redirect that would send a stray client to a phantom auth server (L-1). + assert_eq!( + challenge.trim(), + "Bearer", + "the read challenge must be a bare Bearer, got {challenge:?}" + ); + assert!( + !challenge.to_ascii_lowercase().contains("realm"), + "the read challenge must not carry a realm/token-endpoint redirect" + ); + } + + #[tokio::test] + async fn the_basic_write_token_is_rejected_on_a_read_route() { + let base = spawn_read(READ_TOKEN).await; + // The write token presented in its Basic transport form on the read + // listener must be refused (M-2 — read routes accept only Bearer). + let resp = reqwest::Client::new() + .get(format!("{base}/v2/my-app/blobs/sha256:aa")) + .basic_auth(WRITE_USERNAME, Some(A_WRITE_TOKEN)) + .send() + .await + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 401, + "a Basic write credential must not authorize a read route" + ); + } + + #[tokio::test] + async fn a_valid_bearer_read_token_is_accepted_on_a_read_route() { + let base = spawn_read(READ_TOKEN).await; + let resp = reqwest::Client::new() + .get(format!("{base}/v2/")) + .bearer_auth(READ_TOKEN) + .send() + .await + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 200, + "a valid Bearer read/control token must be accepted" + ); + } + + #[tokio::test] + async fn the_bearer_read_token_is_rejected_on_a_write_route() { + let base = spawn_write(A_WRITE_TOKEN).await; + // The device-held Bearer read/control token must never authorize a write + // — even when its value equals the write token's secret. + let resp = reqwest::Client::new() + .put(format!("{base}/v2/my-app/manifests/dev")) + .bearer_auth(A_WRITE_TOKEN) + .send() + .await + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 401, + "the Bearer read/control token must not authorize a write" + ); + } +} diff --git a/src/utils/container_dev/bootstrap.rs b/src/utils/container_dev/bootstrap.rs new file mode 100644 index 00000000..5db24df4 --- /dev/null +++ b/src/utils/container_dev/bootstrap.rs @@ -0,0 +1,917 @@ +//! Per-`up` device bootstrap, teardown guard, drain-based token rotation, and +//! `status` reporting for Container Dev Mode (task 5.2). +//! +//! This module carries the load-bearing, testable core of the `up`/`down`/ +//! `status` lifecycle; the imperative glue that binds listeners and drives a +//! device over SSH lives in [`crate::commands::container::dev`]. Four guarantees +//! from the design + threat model are realized here as unit-testable primitives: +//! +//! - **Bootstrap non-disclosure (design G-4 / D2 / D8).** [`DeviceBootstrap`] +//! carries EXACTLY the four things a device needs — the BULK-LISTENER endpoint, +//! the control-WS endpoint, the Bearer read/control token, and the per-project +//! CA certificate. It has no field for the host-only Basic write token or the +//! write-listener address, so a serialization can never leak either — the +//! control-WS endpoint is a device-reachable control channel, NOT the write +//! listener whose address is never disclosed. [`write_bootstrap`] always lands +//! the file INSIDE the device writable partition (A7). +//! - **Guaranteed write-listener teardown (design L-1).** [`WriteListenerGuard`] +//! runs its teardown from `Drop`, so an unclean exit (panic, early `?` return, +//! dropped `up` future) still tears down the write listener — no authenticated +//! write port survives the process. The listener is loopback-bound, so this is +//! about not stranding a port for the next `up`, not about a LAN exposure. +//! - **Drain-based read/control rotation (design D5 / G-2 / H-2).** +//! [`TokenRegistry`] keeps a rotated-out token valid until its in-flight bulk +//! pulls drain to zero OR a hard ceiling elapses — NOT a fixed timer, which +//! would 401 an in-flight pull of the largest supported image on a throttled +//! link (there is no OCI/HTTP "terminal, do not retry" wire signal, so a +//! mid-stream 401 is re-pulled forever). +//! - **Stale-token surfacing (design H-2).** A device presenting a token that is +//! neither current nor a still-draining prior token is classified +//! [`TokenStatus::NeedsReBootstrap`] and surfaced by [`DevStatus`], never looped +//! on silently. + +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +use super::auth::{ReadToken, WRITE_USERNAME}; +use super::engine::WriteCredential; +use super::tls::{DevSession, VM_HOST_IP}; + +/// The device writable-partition root the bootstrap file lands under (design D5, +/// assumption A7: the dev runtime mounts this rw before bootstrap runs). +pub const WRITABLE_PARTITION: &str = "/var/lib/avocado"; + +/// The bootstrap file path RELATIVE to the writable-partition root. +pub const BOOTSTRAP_RELATIVE_PATH: &str = "container-dev/bootstrap.json"; + +/// Environment override for the host endpoint the device reaches the host on +/// (mirrors `avocado deploy`'s `AVOCADO_DEPLOY_REPO_HOST`; design A6/L2). When +/// set it overrides host auto-detection. +pub const HOST_ENV: &str = "AVOCADO_CONTAINER_DEV_HOST"; + +/// Environment override for the bulk-listener port (design L2). When set it +/// overrides the configured `registry.port`. +pub const PORT_ENV: &str = "AVOCADO_CONTAINER_DEV_PORT"; + +/// Environment override for the control WS port (design D9/L2), consistent with +/// [`PORT_ENV`]. When set it overrides [`DEFAULT_WS_PORT`]. +pub const WS_PORT_ENV: &str = "AVOCADO_CONTAINER_DEV_WS_PORT"; + +/// Default port the control WS binds when [`WS_PORT_ENV`] is unset. The control +/// WS is a listener DISTINCT from the bulk read listener (design D9), so it +/// takes its own port; the device dials it at the `ws_endpoint` from bootstrap. +/// Kept off 5000 (macOS AirPlay, design 1.6). +pub const DEFAULT_WS_PORT: u16 = 5600; + +/// The device-delivery bootstrap payload written once per `up` (design D5). +/// +/// It carries EXACTLY four fields — and deliberately no field for the host-only +/// write token or the write-listener endpoint (design G-4/D2). A device is +/// handed only the bulk read listener and control-WS endpoints, so it cannot +/// reach the write listener on any topology; and it never receives the Basic +/// write secret, so a compromised device cannot forge a push. The absence is +/// structural: there is no field to populate, so a serialization can never leak +/// either value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeviceBootstrap { + /// The BULK read listener endpoint (`host:port`) the device pulls from — the + /// only PULL endpoint a device is handed (design G-4). NEVER the + /// write-listener address. + pub bulk_endpoint: String, + /// The Bearer read/control token the device authenticates pulls and the + /// control WS with. NEVER the Basic host-only write token (design D2). + pub read_token: String, + /// The per-project CA certificate (PEM) the device pins the host TLS leaf + /// against. NEVER the CA private key (design D8). + pub ca_cert_pem: String, + /// The control-WS endpoint (`host:port`) the device agent dials for `sync` + /// notifications (design D9). A DISTINCT listener from both the bulk read + /// listener and the write listener; it carries only control frames, never + /// blob bytes and never write authority. NEVER the write-listener address + /// (design G-4). + pub ws_endpoint: String, +} + +impl DeviceBootstrap { + /// Assemble the payload from a minted session plus the resolved bulk + /// endpoint. + /// + /// The read token and CA cert come from the session's device-delivery subset + /// ([`DevSession::bootstrap_payload`]), which by construction excludes the + /// write token and the CA private key. The bulk and control-WS endpoints are + /// supplied by the caller (task 5.2 resolves them); each must be its own + /// listener's address, never the write listener's (design G-4). + pub fn from_session( + session: &DevSession, + bulk_endpoint: impl Into, + ws_endpoint: impl Into, + ) -> Self { + let payload = session.bootstrap_payload(); + Self { + bulk_endpoint: bulk_endpoint.into(), + read_token: payload.read_token, + ca_cert_pem: payload.ca_cert_pem, + ws_endpoint: ws_endpoint.into(), + } + } + + /// Render the on-device JSON form. + pub fn to_json(&self) -> Result { + serde_json::to_string_pretty(self) + } +} + +/// The absolute on-device path the bootstrap file lands at, always under +/// `writable_root` (design D5 / A7). +pub fn bootstrap_path(writable_root: &Path) -> PathBuf { + writable_root.join(BOOTSTRAP_RELATIVE_PATH) +} + +/// Write the bootstrap file under the device writable-partition root, creating +/// the parent directory, and return the path written. +/// +/// One-shot per `up`: task 5.2 calls this exactly once per `up`, never per sync +/// (steady-state sync rides the control WS with no SSH, design D5). The file +/// always lands inside `writable_root`. +pub fn write_bootstrap(writable_root: &Path, bootstrap: &DeviceBootstrap) -> io::Result { + let path = bootstrap_path(writable_root); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let json = bootstrap.to_json().map_err(io::Error::other)?; + std::fs::write(&path, json)?; + Ok(path) +} + +/// Pure endpoint resolution (design L2): apply the host + port overrides over the +/// auto-detected host and configured port. +/// +/// Kept free of env reads and networking so the precedence is unit-testable; the +/// caller supplies the override values (from [`host_override`] / [`port_override`]) +/// and the auto-detected host (from `get_local_ip_for_remote`). +pub fn resolve_endpoint( + host_override: Option<&str>, + auto_host: &str, + port_override: Option, + configured_port: u16, +) -> String { + let host = host_override.unwrap_or(auto_host); + let port = port_override.unwrap_or(configured_port); + format!("{host}:{port}") +} + +/// The `AVOCADO_CONTAINER_DEV_HOST` override, if set and non-empty. +pub fn host_override() -> Option { + std::env::var(HOST_ENV) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +/// The `AVOCADO_CONTAINER_DEV_PORT` override, if set and a valid port. +pub fn port_override() -> Option { + std::env::var(PORT_ENV) + .ok() + .and_then(|s| s.trim().parse().ok()) +} + +/// The `AVOCADO_CONTAINER_DEV_WS_PORT` override, if set and a valid port. +pub fn ws_port_override() -> Option { + std::env::var(WS_PORT_ENV) + .ok() + .and_then(|s| s.trim().parse().ok()) +} + +// --------------------------------------------------------------------------- +// Authenticated VM write path + CA delivery (task 7.1, design D2/H4). +// +// On the avocado-vm fast path the container engine runs INSIDE the VM and pushes +// to the host's write listener over HTTPS. Two host-authorable pieces make that +// work: the guest engine must trust the per-project CA (delivered per-connection +// into its `certs.d`), and the push must target the routable write registry with +// the Basic WRITE token. This section is the pure, testable core; the thin SSH +// glue that drops the CA into the guest lives in +// [`crate::commands::container::dev`]. +// +// Per design D1 the VM PUSH path is docker-only: a podman-machine takes INGEST +// (which never reaches the write listener), and the avocado-vm runs dockerd — so +// there is no podman variant here. +// --------------------------------------------------------------------------- + +/// Environment override for the write-listener port on the VM path. +/// +/// On the VM path the port must be KNOWN (not ephemeral) so the guest's +/// `certs.d` trust dir and the pushed image tag can BOTH be keyed byte-identically +/// on `10.0.2.2:` (design H-3). Native-Linux loopback push keeps an +/// ephemeral port. +pub const WRITE_PORT_ENV: &str = "AVOCADO_CONTAINER_DEV_WRITE_PORT"; + +/// Default write-listener port on the VM path when [`WRITE_PORT_ENV`] is unset. +/// Distinct from the bulk-listener default (`config::DEFAULT_REGISTRY_PORT` = +/// 5599) and the control-WS default ([`DEFAULT_WS_PORT`] = 5600); kept off 5000 +/// (macOS AirPlay, design 1.6). +pub const DEFAULT_WRITE_PORT: u16 = 5601; + +/// The `AVOCADO_CONTAINER_DEV_WRITE_PORT` override, if set and a valid port. +pub fn write_port_override() -> Option { + std::env::var(WRITE_PORT_ENV) + .ok() + .and_then(|s| s.trim().parse().ok()) +} + +/// The routable write registry `host:port` the VM guest engine pushes to: the +/// QEMU user-networking host alias `10.0.2.2` (a leaf IP SAN, [`VM_HOST_IP`]) on +/// the known write port (design D2/H4, phase-0 task 1.8). +/// +/// NEVER `127.0.0.1`: the guest is a separate network namespace and reaches the +/// host's loopback-bound write listener through the `10.0.2.2` alias (QEMU SLIRP +/// maps it to the host loopback), so the tag host, the delivered CA's SAN, and +/// the injected `DOCKER_CONFIG` auth key all agree on the one IP (design H-3). +pub fn vm_write_registry(write_port: u16) -> String { + format!("{VM_HOST_IP}:{write_port}") +} + +/// The in-guest docker per-connection CA trust path for `registry`: +/// `/etc/docker/certs.d//ca.crt`. +/// +/// docker reads this fresh per connection, so dropping the CA here needs NO +/// daemon reload (phase-0 task 1.8) — the reload IS specified: none. +pub fn docker_ca_trust_path(registry: &str) -> String { + format!("/etc/docker/certs.d/{registry}/ca.crt") +} + +/// The pure, testable plan for the docker avocado-vm write path (task 7.1). +/// +/// It composes the routable write `registry`, the guest CA trust path the +/// per-project CA is delivered to, the CA PEM itself, and the Basic write +/// credential the push authenticates with — the host-only WRITE token, NEVER the +/// device-delivered read/control token (design D2). The CA travels here to be +/// delivered at `up`; it is NEVER baked into the VM overlay (design D8/H4). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VmWriteSetup { + /// The routable write registry `10.0.2.2:` the guest pushes to. + pub registry: String, + /// The in-guest path the per-project CA is delivered to (docker `certs.d`). + pub ca_trust_path: String, + /// The per-project CA certificate (PEM) delivered into the guest trust store. + pub ca_cert_pem: String, + /// The Basic write credential (fixed username + host-only write token) the + /// guest push authenticates with — never the read/control token (design D2). + pub credential: WriteCredential, +} + +impl VmWriteSetup { + /// Compose the docker VM write-path plan from a minted session and the known + /// write port. + pub fn docker(session: &DevSession, write_port: u16) -> Self { + let registry = vm_write_registry(write_port); + let ca_trust_path = docker_ca_trust_path(®istry); + let credential = WriteCredential::DockerConfigEnv { + registry: registry.clone(), + username: WRITE_USERNAME.to_string(), + token: session.write_token.secret().to_string(), + }; + Self { + registry, + ca_trust_path, + ca_cert_pem: session.tls.ca_cert_pem().to_string(), + credential, + } + } +} + +/// A guaranteed-cleanup guard for the write listener (design L-1). +/// +/// `down` calls [`teardown`](Self::teardown) to stop the write listener on the +/// clean path. But an UNCLEAN exit — a panic, an early `?` return, or a dropped +/// `up` future — would skip that call, leaving an authenticated write port bound +/// after the process is gone. The listener binds `127.0.0.1` only, so the +/// exposure is device-local rather than LAN-wide, but a stale bound port still +/// collides with the next `up`. Running the teardown from `Drop` closes that +/// hole: whether `up` returns normally or unwinds, the closure runs exactly +/// once, so no authenticated write port survives the process. +pub struct WriteListenerGuard { + on_teardown: Option>, +} + +impl WriteListenerGuard { + /// Wrap a teardown closure that stops the write listener. + pub fn new(teardown: F) -> Self { + Self { + on_teardown: Some(Box::new(teardown)), + } + } + + /// Run the teardown now (idempotent). Safe to call on the clean `down` path; + /// the `Drop` impl then does nothing because the closure was already taken. + pub fn teardown(&mut self) { + if let Some(f) = self.on_teardown.take() { + f(); + } + } + + /// Whether the teardown has already run. + pub fn is_torn_down(&self) -> bool { + self.on_teardown.is_none() + } +} + +impl Drop for WriteListenerGuard { + fn drop(&mut self) { + self.teardown(); + } +} + +/// The hard ceiling above the worst-case single-blob pull on a throttled link. +/// +/// The drain-based grace window (design D5/G-2) never keeps a rotated-out token +/// valid past this, even if its connection count never reaches zero. Sized well +/// above a large-image pull on a slow link so a legitimate in-flight pull is +/// never cut, but bounded so a wedged connection cannot pin the old token open. +pub const DEFAULT_DRAIN_CEILING: Duration = Duration::from_secs(15 * 60); + +/// A prior read/control token kept valid while its in-flight bulk pulls drain +/// (design G-2 / H-2). +struct DrainingToken { + token: ReadToken, + /// Per-token count of open bulk connections authenticated with this token on + /// the read listener. The registry keeps the token valid while this is > 0. + open_connections: Arc, + /// When the rotation happened, for the hard-ceiling arm. + since: Instant, +} + +/// The device-presented token classification produced by [`TokenRegistry`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TokenStatus { + /// The presented token is the current one, or a still-draining prior token. + Accepted, + /// The device presented a STALE token; the operator must re-run `up` to + /// re-bootstrap the device (design H-2). Surfaced by `status`, never looped + /// on silently. + NeedsReBootstrap, +} + +/// Tracks the current read/control token plus one prior token still draining +/// in-flight pulls, and classifies a device-presented token (design D5). +/// +/// Rotation at re-`up` is DRAIN-BASED, not a fixed timer: the prior token stays +/// valid until its open bulk connections reach zero OR the hard ceiling elapses. +/// A fixed timer would 401 an in-flight pull of the largest supported image on a +/// throttled link — and because there is no OCI/HTTP "terminal, do not retry" +/// wire signal, that mid-stream 401 is re-pulled forever (design H-2). The drain +/// overlap makes the mid-pull 401 not occur. +pub struct TokenRegistry { + current: ReadToken, + draining: Option, + ceiling: Duration, +} + +impl TokenRegistry { + /// A registry seeded with the initial `up` read/control token and the + /// default drain ceiling. + pub fn new(current: ReadToken) -> Self { + Self::with_ceiling(current, DEFAULT_DRAIN_CEILING) + } + + /// A registry with an explicit drain ceiling (used by tests to exercise the + /// hard-ceiling arm deterministically). + pub fn with_ceiling(current: ReadToken, ceiling: Duration) -> Self { + Self { + current, + draining: None, + ceiling, + } + } + + /// The current read/control token. + pub fn current(&self) -> &ReadToken { + &self.current + } + + /// Rotate to `next` on re-`up`, moving the prior token into the draining slot + /// with its live open-connection counter (`prior_open`). + /// + /// The prior token stays valid until `prior_open` reaches zero (all in-flight + /// pulls drained) OR the ceiling elapses — never a fixed timer. + pub fn rotate(&mut self, next: ReadToken, prior_open: Arc) { + let prev = std::mem::replace(&mut self.current, next); + self.draining = Some(DrainingToken { + token: prev, + open_connections: prior_open, + since: Instant::now(), + }); + } + + /// Classify a presented token secret at instant `now`. + /// + /// A secret matching the current token is always accepted. A secret matching + /// the draining prior token is accepted only while it has NOT yet drained + /// (open connections > 0) AND is within the ceiling; once drained OR past the + /// ceiling it is stale. Anything else is stale. + pub fn classify_at(&self, secret: &str, now: Instant) -> TokenStatus { + if self.current.secret() == secret { + return TokenStatus::Accepted; + } + if let Some(d) = &self.draining { + if d.token.secret() == secret { + let drained = d.open_connections.load(Ordering::SeqCst) == 0; + let expired = now.duration_since(d.since) >= self.ceiling; + return if drained || expired { + TokenStatus::NeedsReBootstrap + } else { + TokenStatus::Accepted + }; + } + } + TokenStatus::NeedsReBootstrap + } + + /// Classify a presented token secret at the current instant. + pub fn classify(&self, secret: &str) -> TokenStatus { + self.classify_at(secret, Instant::now()) + } +} + +/// A single device's state in a [`DevStatus`] report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeviceStatus { + /// The reporting device's stable id. + pub device_id: String, + /// Whether the token the device presented is accepted or stale. + pub token: TokenStatus, +} + +/// The `container dev status` report (design D5): registry/watcher/last-sync +/// state plus per-device token classification. +/// +/// [`needs_rebootstrap`](Self::needs_rebootstrap) is the surfaced "re-run +/// `up`/bootstrap" signal: it is true when any connected device presented a +/// stale token, so the operator sees a status rather than a silent retry loop +/// (design H-2). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DevStatus { + /// Whether the embedded registry (bulk + write listeners) is running. + pub registry_running: bool, + /// Whether the engine-driver watcher is running. + pub watcher_running: bool, + /// The digest last synced to the device, or `None` if nothing synced yet. + pub last_sync: Option, + /// Per-device token state. + pub devices: Vec, +} + +impl DevStatus { + /// Whether any device presented a stale token, so the operator should re-run + /// `up` to re-bootstrap it (design H-2). + pub fn needs_rebootstrap(&self) -> bool { + self.devices + .iter() + .any(|d| d.token == TokenStatus::NeedsReBootstrap) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RUNTIME: &str = "dev-runtime"; + const BULK_ENDPOINT: &str = "192.168.1.10:5599"; + const WS_ENDPOINT: &str = "192.168.1.10:5600"; + /// A representative write-listener address: loopback-only, its own ephemeral + /// port (design D9/G-4). The bootstrap must never carry it, and the disclosed + /// `ws_endpoint` must be distinct from it. + const WRITE_LISTENER_ADDR: &str = "127.0.0.1:34567"; + + // ---- bootstrap payload: bulk endpoint + read token + CA, never the write + // token and never the write-listener address (design G-4/D2/D8) ---- + + #[test] + fn bootstrap_payload_carries_bulk_endpoint_read_token_and_ca_cert() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let bootstrap = DeviceBootstrap::from_session(&session, BULK_ENDPOINT, WS_ENDPOINT); + + assert_eq!(bootstrap.bulk_endpoint, BULK_ENDPOINT); + assert_eq!(bootstrap.ws_endpoint, WS_ENDPOINT); + assert_eq!(bootstrap.read_token, session.read_token.secret()); + assert_eq!(bootstrap.ca_cert_pem, session.tls.ca_cert_pem()); + + let json = bootstrap.to_json().expect("payload serializes"); + assert!( + json.contains(BULK_ENDPOINT), + "the payload must deliver the bulk-listener endpoint" + ); + assert!( + json.contains(WS_ENDPOINT), + "the payload must deliver the control-WS endpoint" + ); + assert!( + json.contains(session.read_token.secret()), + "the payload must deliver the read/control token" + ); + assert!( + json.contains("BEGIN CERTIFICATE"), + "the payload must deliver the CA certificate" + ); + } + + #[test] + fn bootstrap_payload_never_carries_the_write_token() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let bootstrap = DeviceBootstrap::from_session(&session, BULK_ENDPOINT, WS_ENDPOINT); + let json = bootstrap.to_json().expect("payload serializes"); + assert!( + !json.contains(session.write_token.secret()), + "the bootstrap payload must NEVER contain the host-only write token (design D2/G-4)" + ); + } + + #[test] + fn bootstrap_payload_never_carries_the_ca_private_key() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let json = DeviceBootstrap::from_session(&session, BULK_ENDPOINT, WS_ENDPOINT) + .to_json() + .expect("payload serializes"); + assert!( + !json.contains("PRIVATE KEY"), + "the bootstrap payload must NEVER contain CA private key material (design D8)" + ); + } + + #[test] + fn bootstrap_payload_has_no_field_for_a_write_endpoint() { + // Structural guarantee: the only endpoint keys are `bulk_endpoint` (pull) + // and `ws_endpoint` (control). A write-listener address has no field to + // land in, so it cannot leak (design G-4). Pin the exact key set. + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let bootstrap = DeviceBootstrap::from_session(&session, BULK_ENDPOINT, WS_ENDPOINT); + let value: serde_json::Value = + serde_json::to_value(&bootstrap).expect("payload serializes to a value"); + let keys: std::collections::BTreeSet<&str> = value + .as_object() + .expect("payload is a JSON object") + .keys() + .map(String::as_str) + .collect(); + assert_eq!( + keys, + ["bulk_endpoint", "ca_cert_pem", "read_token", "ws_endpoint"] + .into_iter() + .collect::>(), + "the payload must expose exactly the bulk endpoint, control-WS endpoint, read token, \ + and CA cert - no write-listener endpoint field" + ); + // The disclosed control-WS endpoint must never be the write-listener + // address: it is a control channel, not a write route (design G-4/D9). + assert_ne!( + bootstrap.ws_endpoint, WRITE_LISTENER_ADDR, + "the control-WS endpoint must be distinct from the write-listener address" + ); + assert_ne!( + bootstrap.bulk_endpoint, bootstrap.ws_endpoint, + "the bulk (pull) and control-WS endpoints are distinct listeners" + ); + } + + // ---- write_bootstrap always lands inside the writable partition (A7) ---- + + #[test] + fn write_bootstrap_lands_under_the_writable_partition_root() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let bootstrap = DeviceBootstrap::from_session(&session, BULK_ENDPOINT, WS_ENDPOINT); + let root = tempfile::tempdir().expect("tempdir"); + + let path = write_bootstrap(root.path(), &bootstrap).expect("bootstrap writes"); + + assert!( + path.starts_with(root.path()), + "the bootstrap file must land INSIDE the writable-partition root: {path:?}" + ); + assert_eq!(path, bootstrap_path(root.path())); + assert!(path.exists(), "the bootstrap file must exist after writing"); + + let written = std::fs::read_to_string(&path).expect("read back"); + let round: DeviceBootstrap = + serde_json::from_str(&written).expect("written payload round-trips"); + assert_eq!(round, bootstrap); + } + + #[test] + fn bootstrap_path_is_relative_to_the_writable_partition() { + let path = bootstrap_path(Path::new(WRITABLE_PARTITION)); + assert_eq!( + path, + Path::new(WRITABLE_PARTITION).join(BOOTSTRAP_RELATIVE_PATH), + "the on-device path must sit under the writable partition" + ); + assert!(path.starts_with(WRITABLE_PARTITION)); + } + + // ---- endpoint resolution precedence (design L2) ---- + + #[test] + fn resolve_endpoint_uses_auto_host_and_configured_port_by_default() { + assert_eq!( + resolve_endpoint(None, "10.0.0.5", None, 5599), + "10.0.0.5:5599" + ); + } + + #[test] + fn resolve_endpoint_applies_host_and_port_overrides() { + assert_eq!( + resolve_endpoint(Some("host.override"), "10.0.0.5", Some(6001), 5599), + "host.override:6001", + "the host and port overrides must take precedence over auto-detection" + ); + } + + // ---- guaranteed write-listener teardown (design L-1) ---- + + #[test] + fn write_listener_guard_tears_down_on_explicit_teardown() { + let torn = Arc::new(AtomicUsize::new(0)); + let flag = Arc::clone(&torn); + let mut guard = WriteListenerGuard::new(move || { + flag.fetch_add(1, Ordering::SeqCst); + }); + assert!(!guard.is_torn_down()); + guard.teardown(); + assert!(guard.is_torn_down()); + assert_eq!(torn.load(Ordering::SeqCst), 1); + } + + #[test] + fn write_listener_guard_tears_down_even_on_an_error_path() { + // Simulate `up` failing partway through after the routable write listener + // was bound. The guard is dropped on the early `?` return, and its + // teardown MUST still run so no authenticated LAN write port survives. + let torn = Arc::new(AtomicUsize::new(0)); + + fn faulty_up(torn: Arc) -> Result<(), &'static str> { + let flag = Arc::clone(&torn); + let _guard = WriteListenerGuard::new(move || { + flag.fetch_add(1, Ordering::SeqCst); + }); + // Fail after the write listener is up: the `?`-style early return + // drops the guard without an explicit teardown call. + Err("bootstrap delivery failed")?; + Ok(()) + } + + let result = faulty_up(Arc::clone(&torn)); + assert!(result.is_err(), "the simulated up must fail"); + assert_eq!( + torn.load(Ordering::SeqCst), + 1, + "the write listener must be torn down on the error path via Drop (design L-1)" + ); + } + + #[test] + fn write_listener_guard_runs_teardown_exactly_once() { + let torn = Arc::new(AtomicUsize::new(0)); + let flag = Arc::clone(&torn); + { + let mut guard = WriteListenerGuard::new(move || { + flag.fetch_add(1, Ordering::SeqCst); + }); + guard.teardown(); + // Dropping after an explicit teardown must not run it a second time. + } + assert_eq!( + torn.load(Ordering::SeqCst), + 1, + "teardown must run exactly once across an explicit call plus Drop" + ); + } + + // ---- stale-token surfacing (design H-2) ---- + + #[test] + fn an_unknown_token_is_classified_needs_rebootstrap() { + let registry = TokenRegistry::new(ReadToken::new("current-token")); + assert_eq!( + registry.classify("current-token"), + TokenStatus::Accepted, + "the current token must be accepted" + ); + assert_eq!( + registry.classify("some-old-token"), + TokenStatus::NeedsReBootstrap, + "a device presenting a stale token must surface a re-bootstrap status, not loop" + ); + } + + // ---- drain-based read/control rotation (design D5/G-2/H-2) ---- + + #[test] + fn rotation_holds_the_old_token_until_in_flight_pulls_drain() { + let mut registry = TokenRegistry::new(ReadToken::new("token-a")); + // One in-flight bulk pull is authenticated with token-a on the read + // listener. + let open = Arc::new(AtomicUsize::new(1)); + + registry.rotate(ReadToken::new("token-b"), Arc::clone(&open)); + + // The new token is current; the old token is STILL valid because a pull + // is in flight (draining, not yet zero). + assert_eq!(registry.classify("token-b"), TokenStatus::Accepted); + assert_eq!( + registry.classify("token-a"), + TokenStatus::Accepted, + "the prior token must stay valid while an in-flight pull has not drained" + ); + + // The in-flight pull completes: the connection count drains to zero. + open.store(0, Ordering::SeqCst); + assert_eq!( + registry.classify("token-a"), + TokenStatus::NeedsReBootstrap, + "the prior token must retire once its in-flight pulls have drained to zero" + ); + } + + #[test] + fn rotation_is_drain_based_not_a_fixed_timer() { + // A large ceiling stands in for "well past any fixed timer would fire". + // With a pull still in flight, the old token must remain valid regardless + // of elapsed time - proving the overlap is keyed on drain, not a timer + // that would 401 the largest in-flight image on a slow link. + let mut registry = TokenRegistry::new(ReadToken::new("token-a")); + let open = Arc::new(AtomicUsize::new(1)); + registry.rotate(ReadToken::new("token-b"), Arc::clone(&open)); + + let long_after = Instant::now() + Duration::from_secs(10 * 60); + assert_eq!( + registry.classify_at("token-a", long_after), + TokenStatus::Accepted, + "with a pull still in flight the old token must remain valid regardless of elapsed \ + time - a fixed timer would have 401'd the in-flight pull" + ); + } + + #[test] + fn a_hard_ceiling_retires_a_wedged_prior_token_even_if_connections_remain() { + // A short ceiling: even though a connection never drains (count stays 1), + // the ceiling forces the prior token to retire so a wedged connection + // cannot pin the old credential open forever (design D5, the OR arm). + let ceiling = Duration::from_secs(60); + let mut registry = TokenRegistry::with_ceiling(ReadToken::new("token-a"), ceiling); + let open = Arc::new(AtomicUsize::new(1)); + registry.rotate(ReadToken::new("token-b"), Arc::clone(&open)); + + // Within the ceiling: still valid (drain overlap active). + assert_eq!(registry.classify("token-a"), TokenStatus::Accepted); + + // Past the ceiling with the connection still open: forced retirement. + let past_ceiling = Instant::now() + ceiling + Duration::from_secs(1); + assert_eq!( + registry.classify_at("token-a", past_ceiling), + TokenStatus::NeedsReBootstrap, + "the hard ceiling must retire a prior token even if its connections never drain" + ); + } + + // ---- status surfacing (design D5/H-2) ---- + + #[test] + fn dev_status_surfaces_rebootstrap_when_any_device_is_stale() { + let stale = DevStatus { + registry_running: true, + watcher_running: true, + last_sync: Some("sha256:abc".to_string()), + devices: vec![ + DeviceStatus { + device_id: "dev-1".to_string(), + token: TokenStatus::Accepted, + }, + DeviceStatus { + device_id: "dev-2".to_string(), + token: TokenStatus::NeedsReBootstrap, + }, + ], + }; + assert!( + stale.needs_rebootstrap(), + "a status with any stale-token device must surface the re-bootstrap state" + ); + + let json = serde_json::to_string(&stale).expect("status serializes"); + assert!(json.contains("registry_running"), "status reports registry"); + assert!(json.contains("watcher_running"), "status reports watcher"); + assert!(json.contains("last_sync"), "status reports last-sync"); + assert!( + json.contains("needs_re_bootstrap"), + "the stale device's token state must serialize the re-bootstrap variant: {json}" + ); + } + + #[test] + fn dev_status_is_clean_when_all_devices_are_accepted() { + let clean = DevStatus { + registry_running: true, + watcher_running: true, + last_sync: None, + devices: vec![DeviceStatus { + device_id: "dev-1".to_string(), + token: TokenStatus::Accepted, + }], + }; + assert!( + !clean.needs_rebootstrap(), + "a status with only accepted-token devices must not signal a re-bootstrap" + ); + } + + // ---- authenticated VM write path + CA delivery (task 7.1, design D2/H4) ---- + + #[test] + fn vm_write_registry_targets_10_0_2_2_on_the_known_write_port() { + assert_eq!(vm_write_registry(5601), "10.0.2.2:5601"); + assert_eq!(vm_write_registry(6001), "10.0.2.2:6001"); + // NEVER a loopback target: the guest reaches the host via the 10.0.2.2 + // alias, not 127.0.0.1 (a distinct network namespace). + assert!(!vm_write_registry(5601).starts_with("127.0.0.1")); + } + + #[test] + fn docker_ca_trust_path_is_the_per_connection_certs_d_ca() { + // docker reads this per connection — no reload needed (phase-0 1.8). + assert_eq!( + docker_ca_trust_path("10.0.2.2:5601"), + "/etc/docker/certs.d/10.0.2.2:5601/ca.crt" + ); + } + + #[test] + fn vm_write_setup_uses_the_write_token_not_the_read_token() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let setup = VmWriteSetup::docker(&session, 5601); + + assert_eq!( + setup.registry, "10.0.2.2:5601", + "the target is the routable registry" + ); + assert_eq!( + setup.ca_trust_path, "/etc/docker/certs.d/10.0.2.2:5601/ca.crt", + "the CA is delivered to the docker per-connection trust path" + ); + match &setup.credential { + WriteCredential::DockerConfigEnv { + registry, + username, + token, + } => { + // H-3: the auth-entry key is byte-identical to the routable registry. + assert_eq!(registry, "10.0.2.2:5601"); + assert_eq!(username, WRITE_USERNAME); + // The Basic WRITE token gates the guest push... + assert_eq!(token, session.write_token.secret()); + // ...NEVER the device-delivered read/control token (design D2). + assert_ne!( + token.as_str(), + session.read_token.secret(), + "the VM guest push must authenticate with the host-only write token" + ); + } + other => panic!("the VM write path must use a Basic write credential, got {other:?}"), + } + } + + #[test] + fn vm_write_ca_is_delivered_material_never_the_private_key() { + // The CA PEM is carried in the plan to be delivered at `up` (design H4), + // NOT a baked overlay file. It is real cert material, and never the CA + // private key (design D8). + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let setup = VmWriteSetup::docker(&session, 5601); + assert!( + setup.ca_cert_pem.contains("BEGIN CERTIFICATE"), + "the delivered CA must be real certificate material" + ); + assert!( + !setup.ca_cert_pem.contains("PRIVATE KEY"), + "the VM CA delivery must NEVER carry CA private key material (design D8)" + ); + } + + #[test] + fn default_write_port_is_distinct_from_the_ws_and_bulk_defaults() { + assert_ne!(DEFAULT_WRITE_PORT, DEFAULT_WS_PORT); + assert_ne!( + DEFAULT_WRITE_PORT, 5599, + "the write port must not collide with the bulk-listener default" + ); + assert_ne!( + DEFAULT_WRITE_PORT, 5000, + "the write port must not be 5000 (AirPlay)" + ); + } +} diff --git a/src/utils/container_dev/commands.rs b/src/utils/container_dev/commands.rs new file mode 100644 index 00000000..522fb1d5 --- /dev/null +++ b/src/utils/container_dev/commands.rs @@ -0,0 +1,341 @@ +//! Testable core of `container dev sync` and `container dev prune` (task 5.3, +//! design M4). +//! +//! Both subcommands are thin passes over primitives that already exist: +//! +//! - **`sync`** is a ONE-SHOT re-push + notify of the current watched tag, NOT a +//! long-running watcher. [`run_one_shot_sync`] drives the exact same +//! [`Syncer`]/[`Notifier`] seams the watcher (task 4.2) uses per rebuild — the +//! topology-selected PUSH/INGEST transfer followed by a control-WS notify — +//! but exactly once, then returns. It never enters the `run_watcher` receive +//! loop, so a `sync` invocation performs one transfer and one notification and +//! is done. +//! - **`prune`** garbage-collects the per-project store ONLY, via the group-3.5 +//! GC ([`BlobStore::prune`]). [`prune_store`] reuses that policy verbatim: it +//! sweeps blobs no currently-tagged manifest references and refuses while a +//! device is mid-pull. It touches nothing but blobs under the store's +//! `registry/` tree — never the per-session token or the CA material (which +//! live in memory for the session and, where persisted, sit OUTSIDE the +//! `registry/` tree the GC walks). + +use anyhow::{Context, Result}; + +use super::engine::TagEvent; +use super::store::{BlobStore, SessionActivity, StoreError}; +use super::watcher::{Notifier, SyncMode, Syncer}; + +/// Perform ONE re-push + notify of a watched tag and return — the `container dev +/// sync` core (design M4). +/// +/// This reuses the group-4 sync pipeline (`Syncer` then `Notifier`), the same +/// two seams the watcher drives on every rebuild, run exactly once: transfer the +/// image's changed layers (PUSH into the embedded registry, or the INGEST +/// fallback, per `mode`), then notify the device over the control WS. Unlike +/// [`super::watcher::run_watcher`] there is no receive loop — a single pass, then +/// this returns, so a manual `sync` is one transfer + one notification, never a +/// persistent watch. +/// +/// A failed re-push short-circuits before the notify (propagated as `Err`), so a +/// device is never told an image is ready when the push did not land — mirroring +/// the watcher's push-then-notify ordering, but surfacing the failure to the CLI +/// caller rather than swallowing it as a warning. +pub async fn run_one_shot_sync( + mode: SyncMode, + syncer: &dyn Syncer, + notifier: &dyn Notifier, + event: &TagEvent, +) -> Result<()> { + syncer + .sync(mode, event) + .await + .with_context(|| format!("re-pushing `{}`", event.image))?; + notifier + .notify(event) + .await + .with_context(|| format!("notifying the device that `{}` is ready", event.image))?; + Ok(()) +} + +/// Garbage-collect the per-project store — the `container dev prune` core (design +/// M4, task 3.5). +/// +/// This delegates to [`BlobStore::prune`], reusing the single GC policy verbatim: +/// it retains every blob a currently-tagged manifest references, sweeps the rest, +/// and refuses (rather than sweeping a blob a transfer still needs) while an `up` +/// session is live. It operates ONLY on blobs under the store's `registry/` tree, +/// so it never removes the per-session read/control or write token, nor the +/// per-project CA material — those are session state, not store blobs, and prune +/// has no path to them. +/// +/// `session` comes from the caller because only the command layer can probe the +/// session flock, and that flock is the sole cross-process proof of liveness. +pub fn prune_store(store: &BlobStore, session: SessionActivity) -> Result, StoreError> { + store.prune(session) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::future::Future; + use std::pin::Pin; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Mutex; + use std::time::Duration; + + use serde_json::json; + use tempfile::TempDir; + + // ---- sync: a recording double for the Syncer + Notifier seams ---- + + /// Records every `sync`/`notify` call so a test can assert the one-shot + /// pipeline runs each exactly once, in order, and stops. + #[derive(Default)] + struct Recorder { + /// Ordered log of `sync::` / `notify:`. + log: Mutex>, + sync_calls: AtomicUsize, + notify_calls: AtomicUsize, + /// When true, the push fails so the notify must be skipped. + fail_sync: bool, + } + + impl Syncer for Recorder { + fn sync<'a>( + &'a self, + mode: SyncMode, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.sync_calls.fetch_add(1, Ordering::SeqCst); + self.log + .lock() + .unwrap() + .push(format!("sync:{}:{mode:?}", event.image)); + if self.fail_sync { + anyhow::bail!("push to the embedded registry failed"); + } + Ok(()) + }) + } + } + + impl Notifier for Recorder { + fn notify<'a>( + &'a self, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.notify_calls.fetch_add(1, Ordering::SeqCst); + self.log + .lock() + .unwrap() + .push(format!("notify:{}", event.image)); + Ok(()) + }) + } + } + + fn ev(image: &str) -> TagEvent { + TagEvent { + image: image.to_string(), + image_id: None, + } + } + + #[tokio::test] + async fn sync_re_pushes_then_notifies_exactly_once() { + let rec = Recorder::default(); + run_one_shot_sync(SyncMode::Push, &rec, &rec, &ev("my-app:dev")) + .await + .expect("a one-shot sync succeeds"); + + assert_eq!( + rec.sync_calls.load(Ordering::SeqCst), + 1, + "sync must re-push exactly once" + ); + assert_eq!( + rec.notify_calls.load(Ordering::SeqCst), + 1, + "sync must notify exactly once" + ); + assert_eq!( + *rec.log.lock().unwrap(), + vec![ + "sync:my-app:dev:Push".to_string(), + "notify:my-app:dev".to_string(), + ], + "sync must re-push (delta) THEN notify, in that order" + ); + } + + #[tokio::test] + async fn sync_is_one_shot_not_a_persistent_watch_loop() { + // A watcher loop would block awaiting further tag events; a one-shot sync + // returns after a single pass. A generous timeout that still resolves + // proves it is not a persistent watch, and the counts prove it did not + // repeat. + let rec = Recorder::default(); + tokio::time::timeout( + Duration::from_secs(2), + run_one_shot_sync(SyncMode::Push, &rec, &rec, &ev("my-app:dev")), + ) + .await + .expect("a one-shot sync returns promptly; it is not a persistent watch loop") + .expect("the sync succeeds"); + + assert_eq!( + rec.sync_calls.load(Ordering::SeqCst), + 1, + "a one-shot sync re-pushes once, not repeatedly like a watcher" + ); + assert_eq!(rec.notify_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn a_failed_re_push_propagates_and_skips_the_notify() { + let rec = Recorder { + fail_sync: true, + ..Default::default() + }; + let err = run_one_shot_sync(SyncMode::Push, &rec, &rec, &ev("my-app:dev")) + .await + .expect_err("a failed re-push must surface as an error"); + + assert!( + err.to_string().contains("re-pushing"), + "the error must name the failed re-push: {err:#}" + ); + assert_eq!( + rec.sync_calls.load(Ordering::SeqCst), + 1, + "the push was attempted once" + ); + assert_eq!( + rec.notify_calls.load(Ordering::SeqCst), + 0, + "a failed re-push must NOT notify the device that an image is ready" + ); + } + + // ---- prune: GC the per-project store ONLY, never the token/CA ---- + + const MANIFEST: &str = + "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + const CONFIG: &str = "sha256:2222222222222222222222222222222222222222222222222222222222222222"; + const LAYER: &str = "sha256:3333333333333333333333333333333333333333333333333333333333333333"; + const ORPHAN: &str = "sha256:5555555555555555555555555555555555555555555555555555555555555555"; + + /// Bytes of a single-platform image manifest referencing `config` + `layer`. + fn image_manifest(config: &str, layer: &str) -> Vec { + json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": config}, + "layers": [ + {"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": layer} + ], + }) + .to_string() + .into_bytes() + } + + /// A store with a tagged image (manifest + config + layer) plus one orphan. + fn store_with_tagged_image_and_orphan(dir: &TempDir) -> BlobStore { + let store = BlobStore::at(dir.path(), "alpha").expect("store opens"); + store.write_blob(CONFIG, b"config-bytes").unwrap(); + store.write_blob(LAYER, b"layer-bytes").unwrap(); + store + .write_blob(MANIFEST, &image_manifest(CONFIG, LAYER)) + .unwrap(); + store.set_tag("my-app", "dev", MANIFEST).unwrap(); + store.write_blob(ORPHAN, b"unreferenced").unwrap(); + store + } + + #[test] + fn prune_sweeps_orphan_store_blobs_but_retains_tagged_ones() { + let dir = TempDir::new().unwrap(); + let store = store_with_tagged_image_and_orphan(&dir); + + let swept = prune_store(&store, SessionActivity::Idle) + .expect("prune succeeds with no live session"); + + assert_eq!( + swept, + vec![ORPHAN.to_string()], + "prune must sweep exactly the unreferenced orphan blob" + ); + assert!(!store.has_blob(ORPHAN).unwrap(), "the orphan is gone"); + for kept in [MANIFEST, CONFIG, LAYER] { + assert!( + store.has_blob(kept).unwrap(), + "a blob referenced by the tagged manifest must survive prune: {kept}" + ); + } + } + + #[test] + fn prune_never_touches_the_token_or_ca_material() { + let dir = TempDir::new().unwrap(); + let store = store_with_tagged_image_and_orphan(&dir); + + // The per-project dir is the store root's parent + // (`/container-dev//`); the session's token and CA + // material are siblings of the `registry/` tree prune walks. Stand in + // for them with files prune must leave untouched. + let project_dir = store + .root() + .parent() + .expect("the store root sits under the per-project dir") + .to_path_buf(); + let ca = project_dir.join("ca.pem"); + let read_token = project_dir.join("read-token"); + let write_token = project_dir.join("write-token"); + let ca_pem = "-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----"; + std::fs::write(&ca, ca_pem).unwrap(); + std::fs::write(&read_token, "read-secret").unwrap(); + std::fs::write(&write_token, "write-secret").unwrap(); + + let swept = prune_store(&store, SessionActivity::Idle).expect("prune succeeds"); + assert_eq!(swept, vec![ORPHAN.to_string()], "prune only sweeps blobs"); + + // The token and CA material must be byte-for-byte intact after prune. + assert!(ca.exists(), "prune must NOT delete the CA material"); + assert!(read_token.exists(), "prune must NOT delete the read token"); + assert!( + write_token.exists(), + "prune must NOT delete the write token" + ); + assert_eq!( + std::fs::read_to_string(&ca).unwrap(), + ca_pem, + "the CA material must be unchanged" + ); + assert_eq!(std::fs::read_to_string(&read_token).unwrap(), "read-secret"); + assert_eq!( + std::fs::read_to_string(&write_token).unwrap(), + "write-secret" + ); + } + + #[test] + fn prune_refuses_while_an_up_session_is_live() { + let dir = TempDir::new().unwrap(); + let store = store_with_tagged_image_and_orphan(&dir); + + let result = prune_store(&store, SessionActivity::Live); + assert!( + matches!(result, Err(StoreError::PruneWhileSessionLive)), + "prune must refuse while an `up` session is live, got {result:?}" + ); + assert!( + store.has_blob(ORPHAN).unwrap(), + "a refused prune must not sweep anything" + ); + + let swept = + prune_store(&store, SessionActivity::Idle).expect("prune proceeds once `up` is down"); + assert_eq!(swept, vec![ORPHAN.to_string()]); + } +} diff --git a/src/utils/container_dev/config.rs b/src/utils/container_dev/config.rs new file mode 100644 index 00000000..55a3ea39 --- /dev/null +++ b/src/utils/container_dev/config.rs @@ -0,0 +1,188 @@ +//! Typed configuration for a runtime's `container_dev` block. +//! +//! The feature is gated structurally under the runtime: presence of a +//! `runtimes..container_dev` block enables Container Dev Mode for that +//! runtime; an absent block means the feature is off. A `container_dev` block +//! placed anywhere other than under a runtime is not honored — only the typed +//! [`RuntimeConfig::container_dev`] field enables the feature. + +use serde::{Deserialize, Serialize}; + +/// Default registry port for Container Dev Mode. +/// +/// Phase 0 task 1.6 chose a non-conflicting default: `5000` collides with the +/// macOS AirPlay Receiver, so it is explicitly avoided. Recorded in +/// `docs/container-dev/phase0-findings.md`. +pub const DEFAULT_REGISTRY_PORT: u16 = 5599; + +fn default_registry_port() -> u16 { + DEFAULT_REGISTRY_PORT +} + +/// Container Dev Mode configuration for a runtime. +/// +/// Parsed from `runtimes..container_dev`. Its mere presence enables the +/// feature for the owning runtime. +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct ContainerDevConfig { + /// Images to watch on the host engine and hot-reload on the device. + #[serde(default)] + pub images: Vec, + /// Embedded registry settings. + #[serde(default)] + pub registry: RegistryConfig, +} + +/// A single watched image and the device service that consumes it. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ContainerDevImage { + /// Image reference (`repository[:tag]`) watched on the host engine. + #[serde(rename = "ref")] + pub image_ref: String, + /// Device service consuming the image. + pub service: String, +} + +/// Embedded registry settings for Container Dev Mode. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct RegistryConfig { + /// Port the bulk read listener binds. Defaults to + /// [`DEFAULT_REGISTRY_PORT`] when omitted. + #[serde(default = "default_registry_port")] + pub port: u16, +} + +impl Default for RegistryConfig { + fn default() -> Self { + Self { + port: DEFAULT_REGISTRY_PORT, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::config::{Config, RuntimeConfig}; + + fn runtime_from(yaml: &str) -> RuntimeConfig { + serde_yaml::from_str(yaml).expect("runtime config parses") + } + + #[test] + fn absent_block_leaves_feature_off() { + let runtime = runtime_from("target: qemux86-64\n"); + assert!( + runtime.container_dev.is_none(), + "a runtime with no container_dev block must leave the feature off" + ); + } + + #[test] + fn present_block_enables_feature_and_parses_images() { + let runtime = runtime_from( + r#" +target: qemux86-64 +container_dev: + images: + - ref: my-app:dev + service: app + - ref: sidecar:latest + service: sidecar + registry: + port: 6001 +"#, + ); + + let cd = runtime + .container_dev + .expect("a present container_dev block enables the feature"); + assert_eq!(cd.images.len(), 2); + assert_eq!(cd.images[0].image_ref, "my-app:dev"); + assert_eq!(cd.images[0].service, "app"); + assert_eq!(cd.images[1].image_ref, "sidecar:latest"); + assert_eq!(cd.images[1].service, "sidecar"); + assert_eq!(cd.registry.port, 6001); + } + + #[test] + fn registry_port_defaults_to_phase0_literal_not_5000() { + // registry block present but port omitted + let runtime = runtime_from( + r#" +container_dev: + images: [] + registry: {} +"#, + ); + let cd = runtime.container_dev.unwrap(); + assert_eq!(cd.registry.port, DEFAULT_REGISTRY_PORT); + assert_ne!(cd.registry.port, 5000, "default port must not be 5000"); + + // registry block entirely absent + let runtime = runtime_from("container_dev:\n images: []\n"); + let cd = runtime.container_dev.unwrap(); + assert_eq!(cd.registry.port, DEFAULT_REGISTRY_PORT); + assert_ne!(cd.registry.port, 5000, "default port must not be 5000"); + } + + #[test] + fn default_registry_port_is_not_5000() { + assert_ne!(DEFAULT_REGISTRY_PORT, 5000); + } + + #[test] + fn top_level_block_does_not_enable_the_feature() { + // A container_dev block placed at the top level (not under a runtime) + // must NOT enable the feature for any runtime. + let config_content = r#" +container_dev: + images: + - ref: my-app:dev + service: app +runtimes: + dev: + target: qemux86-64 +"#; + let parsed: serde_yaml::Value = serde_yaml::from_str(config_content).unwrap(); + let runtimes = parsed + .get("runtimes") + .and_then(|r| r.as_mapping()) + .expect("runtimes present"); + for (_name, runtime_value) in runtimes { + let runtime: RuntimeConfig = + serde_yaml::from_value(runtime_value.clone()).expect("runtime parses"); + assert!( + runtime.container_dev.is_none(), + "a top-level container_dev block must not enable the feature for a runtime" + ); + } + } + + #[test] + fn container_dev_is_registered_as_a_known_runtime_key() { + // The ref-scanner must NOT recurse into container_dev.images looking + // for dependency refs. We embed a spec_map shaped like an external + // extension reference inside container_dev; if the scanner recursed + // into it, that ref would be discovered. + let config_content = r#" +runtimes: + dev: + target: qemux86-64 + container_dev: + images: + - ref: my-app:dev + service: app + packages: + poison: + extensions: leaked-ext + config: leaked/path +"#; + let parsed: serde_yaml::Value = serde_yaml::from_str(config_content).unwrap(); + let refs = Config::discover_external_config_refs(&parsed); + assert!( + !refs.iter().any(|(ext, _)| ext == "leaked-ext"), + "container_dev must be a known-runtime-key so the ref scanner does not recurse into it" + ); + } +} diff --git a/src/utils/container_dev/engine.rs b/src/utils/container_dev/engine.rs new file mode 100644 index 00000000..945afdb3 --- /dev/null +++ b/src/utils/container_dev/engine.rs @@ -0,0 +1,642 @@ +//! Engine-driver trait for the Container Dev Mode watcher (design D4). +//! +//! The host watches its container engine for image *tag* events and, on a +//! rebuild, re-tags and syncs the changed layers to the device. This module +//! defines the engine abstraction those tasks build on: a driver per engine +//! (docker + podman) that +//! +//! 1. streams tag events over the engine **CLI subprocess** (`docker events` / +//! `podman events --format json`), NEVER the API socket — so a rootless +//! podman with no `podman.socket` still works (design D4, assumption A4); +//! 2. parses one engine-specific JSON event line into a structured +//! [`TagEvent`]; and +//! 3. describes the per-engine write-credential injection used on push (docker: +//! an ephemeral `DOCKER_CONFIG`; podman: `--creds`), because A10 couples +//! credential injection to the engine (design M-3). +//! +//! Podman *conformance* is a droppable Phase 0 gate outcome (design D4): the +//! trait ships with both drivers regardless; the podman driver is a real +//! CLI-event path, not a stub. The subprocess plumbing ([`watch_tag_events`]) +//! is engine-agnostic — it drives whichever driver it is handed through +//! [`EngineDriver::events_argv`] and [`EngineDriver::parse_tag_event`], so the +//! push wiring and watcher orchestration (tasks 4.2/4.3) reuse it unchanged. + +use std::process::Stdio; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use serde::Deserialize; +use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader}; +use tokio::process::Command; +use tokio::sync::mpsc; + +use super::auth::{WriteToken, WRITE_USERNAME}; +use crate::utils::output::{print_warning, OutputLevel}; + +/// A parsed image *tag* event from the engine's CLI event stream. +/// +/// This is the engine-agnostic shape both drivers normalize their +/// (structurally different) JSON events into: docker carries the name under +/// `Actor.Attributes.name`, podman under `Name`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TagEvent { + /// The image reference that was (re)tagged, e.g. `my-app:dev` (docker) or + /// `localhost/my-app:dev` (podman qualifies the registry). Reported + /// verbatim as the engine emitted it; ref normalization/matching against a + /// configured `ref` is the watcher's concern (task 4.2), not the parser's. + pub image: String, + /// The image content id (digest) the event carried, when present. + pub image_id: Option, +} + +/// How an engine receives a per-invocation, non-persisted write credential on +/// push (design D2/A10, M-3). +/// +/// This is the per-engine credential-injection *shape*. The actual mechanics — +/// writing the ephemeral `DOCKER_CONFIG` dir 0600 under the per-project +/// directory and deleting it after the push (docker), or threading `--creds` +/// into the push argv (podman) — land with the push wiring in task 4.2. Neither +/// path ever runs `docker login` against the user's real `~/.docker/config.json` +/// (design M-E). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteCredential { + /// docker: point `DOCKER_CONFIG` at an ephemeral dir whose `auths` entry is + /// keyed byte-identically to `registry` (the tagged `host:port`, H-3), so + /// the docker CLI resolves it locally and forwards it as `X-Registry-Auth` + /// — surviving `DOCKER_HOST`/VM routing. A key that does not byte-match the + /// tag host makes docker attach no auth and the push 401s with no prompt. + DockerConfigEnv { + /// The registry `host:port`, byte-identical to the image tag host (H-3). + registry: String, + /// Fixed Basic username paired with the write token. + username: String, + /// The host-only write token (Basic password). + token: String, + }, + /// podman: pass `--creds :` per push invocation. + PodmanCreds { + /// Fixed Basic username paired with the write token. + username: String, + /// The host-only write token. + token: String, + }, +} + +/// An engine driver: everything engine-specific about watching for tag events +/// and injecting a push credential. +/// +/// Implementors MUST drive events through the engine CLI subprocess only; a +/// driver that reaches for the API socket violates the design (D4) and the +/// falsifier for task 4.1. +pub trait EngineDriver: Send + Sync { + /// The engine CLI binary name (`docker` / `podman`). + fn binary(&self) -> &'static str; + + /// The argv (after the binary) that streams image tag events as + /// newline-delimited JSON over the engine CLI subprocess. + /// + /// This is ` events …` in every case — never a socket dial — which + /// is precisely what lets a rootless podman with no `podman.socket` work + /// (A4). The stream is filtered to image tag events so the watcher does not + /// have to discard unrelated container/network/volume traffic. + fn events_argv(&self) -> Vec; + + /// Parse a single JSON event line emitted by [`Self::events_argv`] into a + /// [`TagEvent`], or `None` when the line is not an image tag event + /// (a different event type/action, or an unparseable line). + fn parse_tag_event(&self, line: &str) -> Option; + + /// The per-engine write-credential injection shape for a push to + /// `registry` (design D2/A10/M-3). The value describes HOW the credential + /// is delivered; task 4.2 realizes it on the push subprocess. + fn write_credential(&self, registry: &str, token: &WriteToken) -> WriteCredential; +} + +/// The docker engine driver. +/// +/// Events: `docker events --filter type=image --filter event=tag --format +/// {{json .}}`. docker's event JSON capitalizes `Type`/`Action` and nests the +/// image name under `Actor.Attributes.name`. +#[derive(Debug, Clone, Copy, Default)] +pub struct DockerDriver; + +/// docker's event JSON shape (the fields we read from `{{json .}}`). +#[derive(Debug, Deserialize)] +struct DockerEvent { + #[serde(rename = "Type")] + typ: Option, + #[serde(rename = "Action")] + action: Option, + #[serde(rename = "Actor")] + actor: Option, + /// Deprecated top-level id, retained by docker for compatibility; used as a + /// fallback for the image digest when `Actor.ID` is absent. + id: Option, +} + +#[derive(Debug, Deserialize)] +struct DockerActor { + #[serde(rename = "ID")] + id: Option, + #[serde(rename = "Attributes")] + attributes: Option>, +} + +impl EngineDriver for DockerDriver { + fn binary(&self) -> &'static str { + "docker" + } + + fn events_argv(&self) -> Vec { + [ + "events", + "--filter", + "type=image", + "--filter", + "event=tag", + "--format", + "{{json .}}", + ] + .iter() + .map(|s| s.to_string()) + .collect() + } + + fn parse_tag_event(&self, line: &str) -> Option { + let event: DockerEvent = serde_json::from_str(line.trim()).ok()?; + // Only an image `tag` action is a tag event. + if event.typ.as_deref() != Some("image") || event.action.as_deref() != Some("tag") { + return None; + } + let actor = event.actor.as_ref(); + let image = actor + .and_then(|a| a.attributes.as_ref()) + .and_then(|attrs| attrs.get("name")) + .cloned()?; + let image_id = actor + .and_then(|a| a.id.clone()) + .or(event.id) + .filter(|s| !s.is_empty()); + Some(TagEvent { image, image_id }) + } + + fn write_credential(&self, registry: &str, token: &WriteToken) -> WriteCredential { + WriteCredential::DockerConfigEnv { + registry: registry.to_string(), + username: WRITE_USERNAME.to_string(), + token: token.secret().to_string(), + } + } +} + +/// The podman engine driver. +/// +/// Events: `podman events --filter type=image --filter event=tag --format +/// json`. podman's event JSON uses `Status` for the action and carries the +/// image name in `Name`. Rootless podman emits these over its `events_backend` +/// (journald or file) with NO API socket (A4). +#[derive(Debug, Clone, Copy, Default)] +pub struct PodmanDriver; + +/// podman's event JSON shape (the fields we read from `--format json`). +#[derive(Debug, Deserialize)] +struct PodmanEvent { + #[serde(rename = "Type")] + typ: Option, + #[serde(rename = "Status")] + status: Option, + #[serde(rename = "Name")] + name: Option, + #[serde(rename = "Image")] + image: Option, + #[serde(rename = "ID")] + id: Option, +} + +impl EngineDriver for PodmanDriver { + fn binary(&self) -> &'static str { + "podman" + } + + fn events_argv(&self) -> Vec { + [ + "events", + "--filter", + "type=image", + "--filter", + "event=tag", + "--format", + "json", + ] + .iter() + .map(|s| s.to_string()) + .collect() + } + + fn parse_tag_event(&self, line: &str) -> Option { + let event: PodmanEvent = serde_json::from_str(line.trim()).ok()?; + if event.typ.as_deref() != Some("image") || event.status.as_deref() != Some("tag") { + return None; + } + // podman reports the tagged reference under `Name`; fall back to `Image`. + let image = event + .name + .filter(|s| !s.is_empty()) + .or(event.image) + .filter(|s| !s.is_empty())?; + let image_id = event.id.filter(|s| !s.is_empty()); + Some(TagEvent { image, image_id }) + } + + fn write_credential(&self, _registry: &str, token: &WriteToken) -> WriteCredential { + WriteCredential::PodmanCreds { + username: WRITE_USERNAME.to_string(), + token: token.secret().to_string(), + } + } +} + +/// Resolve an engine driver by CLI tool name (`docker` / `podman`). +/// +/// Returns `None` for an unknown tool. Both drivers are real CLI-event paths; +/// podman is not a stub (design D4). +pub fn driver_for(tool: &str) -> Option> { + match tool { + "docker" => Some(Box::new(DockerDriver)), + "podman" => Some(Box::new(PodmanDriver)), + _ => None, + } +} + +/// Read newline-delimited JSON events from `reader`, parse each through +/// `driver.parse_tag_event`, and hand every recognized [`TagEvent`] to `sink`. +/// +/// Non-tag and unparseable lines are skipped, so a driver that emits unfiltered +/// events (or a stray log line) never breaks the stream. This is the +/// engine-agnostic core of the CLI-subprocess event loop: [`watch_tag_events`] +/// pipes a live subprocess stdout in here, and tests drive it with captured +/// fixtures — the event source is the CLI byte stream either way, never an API +/// socket. +pub async fn forward_tag_events( + driver: &dyn EngineDriver, + reader: R, + mut sink: F, +) -> std::io::Result<()> +where + R: AsyncBufRead + Unpin, + F: FnMut(TagEvent), +{ + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + if line.trim().is_empty() { + continue; + } + if let Some(event) = driver.parse_tag_event(&line) { + sink(event); + } + } + Ok(()) +} + +/// Spawn ` events …` as a subprocess and stream parsed [`TagEvent`]s +/// over the returned channel. +/// +/// The events come ONLY from the engine CLI subprocess (design D4) — no API +/// socket is opened — so a rootless podman without `podman.socket` works. The +/// caller owns the returned [`tokio::process::Child`] and kills it to stop +/// watching (e.g. on `down`); dropping the receiver ends the forwarding task. +/// `shutting_down` must be set by the caller BEFORE it kills the returned child, +/// so an expected EOF during teardown is not reported as a dead watcher. The +/// third return value resolves when the forwarder stops, for whatever reason. +pub async fn watch_tag_events( + driver: Box, + shutting_down: Arc, +) -> Result<( + mpsc::Receiver, + tokio::process::Child, + tokio::sync::oneshot::Receiver<()>, +)> { + let argv = driver.events_argv(); + let mut child = Command::new(driver.binary()) + .args(&argv) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .with_context(|| format!("failed to spawn `{} events`", driver.binary()))?; + + let stdout = child + .stdout + .take() + .context("engine events subprocess produced no stdout handle")?; + + let (tx, rx) = mpsc::channel(64); + let (ended_tx, ended_rx) = tokio::sync::oneshot::channel(); + let engine_binary = driver.binary(); + tokio::spawn(async move { + let reader = BufReader::new(stdout); + let outcome = forward_tag_events(driver.as_ref(), reader, |event| { + // A closed receiver means the watcher stopped; blocking_send is not + // available in async, so use try_send and drop on a full/closed + // channel — the watcher (task 4.2) debounces, so a dropped burst + // event is coalesced by the next one. + let _ = tx.try_send(event); + }) + .await; + + if let Some(message) = stream_end_report( + &outcome, + shutting_down.load(Ordering::SeqCst), + engine_binary, + ) { + print_warning(&message, OutputLevel::Normal); + } + // Signal regardless of whether anything was printed. The warning reaches + // only the terminal holding `up`; the caller uses this to correct the + // published session record, which is what a `status` from a second + // terminal actually reads. + let _ = ended_tx.send(()); + }); + + Ok((rx, child, ended_rx)) +} + +/// What to tell the operator when the event stream ends, or `None` when the end +/// was expected. +/// +/// Say so when the stream ends unexpectedly. Swallowing it made a dead watcher +/// indistinguishable from an idle one: restarting the engine daemon +/// (`systemctl restart docker`, or Docker Desktop) kills the `events` child, the +/// forwarder ends, `run_watcher` returns - and `up` stays in the foreground still +/// printing "Watching for image rebuilds..." while every later rebuild goes +/// undetected. Manual `sync` keeps working, so it reads as "auto-reload broke" +/// rather than as a stopped watcher. +/// +/// `shutting_down` is the caller's own teardown flag, and without it this warned +/// on every clean `down`. Teardown kills the events child, and that kill is +/// precisely what closes the child's stdout and makes the forwarder runnable - so +/// the EOF arrives mid-teardown and is byte-for-byte identical to a daemon +/// restart. Nothing in the stream can tell them apart; only the caller knows +/// which one it did. +/// +/// An `Err` is always reported. A read failure is not what a kill produces, so it +/// is news even during teardown. +pub(crate) fn stream_end_report( + outcome: &std::io::Result<()>, + shutting_down: bool, + engine_binary: &str, +) -> Option { + match outcome { + Ok(()) if shutting_down => None, + Ok(()) => Some(format!( + "container dev: the `{engine_binary} events` stream ended, so image rebuilds \ + are no longer detected automatically. This usually means the engine daemon \ + restarted. Run `avocado container dev down` and `up` again to resume \ + watching; `avocado container dev sync` still works in the meantime." + )), + Err(e) => Some(format!( + "container dev: reading the `{engine_binary} events` stream failed ({e}), so \ + image rebuilds are no longer detected automatically. Run `avocado container \ + dev down` and `up` again to resume watching." + )), + } +} + +/// Resolve an image reference to the engine's content ID for it. +/// +/// The watcher reads `image_id` off the event stream, but a manual `sync` has +/// no event to read - it is driven by a signal, not by the engine. Without this +/// it would build a `TagEvent` with `image_id: None`, which the notifier turns +/// into an empty digest and records as the desired state, and an empty desired +/// digest matches the empty `running_digest` a fresh device reports - so the +/// device is never told to pull anything. +/// +/// Returns `None` when the engine does not know the image, which the caller +/// treats as "nothing to sync" rather than as a digest. +pub async fn resolve_image_id(binary: &str, image: &str) -> Result> { + let output = Command::new(binary) + .args(["image", "inspect", "--format", "{{.Id}}", image]) + .output() + .await + .with_context(|| format!("running `{binary} image inspect {image}`"))?; + if !output.status.success() { + return Ok(None); + } + let id = String::from_utf8_lossy(&output.stdout).trim().to_string(); + Ok(if id.is_empty() { None } else { Some(id) }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn a_stream_end_during_teardown_is_not_reported() { + // `down` kills the events child, and that kill is exactly what closes + // the child's stdout and makes the forwarder runnable - so a clean + // teardown produced an EOF indistinguishable from a daemon restart, and + // the warning told the operator to run the `down` they were already + // running. + assert_eq!(stream_end_report(&Ok(()), true, "docker"), None); + } + + #[test] + fn a_stream_end_outside_teardown_is_reported() { + // The mirror: the whole point is still to surface a watcher that died on + // its own. Without this, silencing the teardown case could silence + // everything and both tests would pass. + let message = stream_end_report(&Ok(()), false, "docker") + .expect("an unexpected stream end must be reported"); + assert!(message.contains("docker events"), "{message}"); + assert!( + message.contains("no longer detected automatically"), + "{message}" + ); + } + + #[test] + fn a_read_failure_is_reported_even_during_teardown() { + // A kill produces EOF, not an error - so an Err arriving during teardown + // is news either way, and suppressing it would hide a real fault behind + // an unrelated flag. + let err = std::io::Error::other("boom"); + let message = stream_end_report(&Err(err), true, "podman") + .expect("a read failure must be reported regardless of teardown"); + assert!(message.contains("podman events"), "{message}"); + assert!(message.contains("boom"), "{message}"); + } + + // ---- docker fixtures (captured `docker events --format '{{json .}}'`) ---- + + const DOCKER_TAG_EVENT: &str = r#"{"status":"tag","id":"sha256:1111aaaa","Type":"image","Action":"tag","Actor":{"ID":"sha256:1111aaaa","Attributes":{"name":"my-app:dev"}},"scope":"local","time":1718030000,"timeNano":1718030000000000000}"#; + + const DOCKER_CONTAINER_START: &str = r#"{"status":"start","id":"c0ffee","Type":"container","Action":"start","Actor":{"ID":"c0ffee","Attributes":{"image":"my-app:dev","name":"web"}},"scope":"local","time":1718030001}"#; + + // ---- podman fixtures (captured `podman events --format json`) ---- + + const PODMAN_TAG_EVENT: &str = r#"{"ID":"2222bbbbcccc","Image":"localhost/my-app:dev","Name":"localhost/my-app:dev","Status":"tag","Time":"2024-06-10T12:00:00.000000000-06:00","Type":"image","Attributes":null}"#; + + const PODMAN_CONTAINER_START: &str = r#"{"ID":"deadbeef","Image":"localhost/my-app:dev","Name":"web","Status":"start","Time":"2024-06-10T12:00:01.000000000-06:00","Type":"container","Attributes":null}"#; + + #[test] + fn docker_driver_parses_a_tag_event_from_the_cli_json_line() { + let event = DockerDriver + .parse_tag_event(DOCKER_TAG_EVENT) + .expect("a docker image tag event parses"); + assert_eq!(event.image, "my-app:dev"); + assert_eq!(event.image_id.as_deref(), Some("sha256:1111aaaa")); + } + + #[test] + fn podman_driver_parses_a_tag_event_from_the_cli_json_line() { + let event = PodmanDriver + .parse_tag_event(PODMAN_TAG_EVENT) + .expect("a podman image tag event parses"); + // podman qualifies the ref with the registry; the parser reports it + // verbatim (matching/normalization is the watcher's job). + assert_eq!(event.image, "localhost/my-app:dev"); + assert_eq!(event.image_id.as_deref(), Some("2222bbbbcccc")); + } + + #[test] + fn docker_driver_ignores_a_non_tag_event() { + assert!( + DockerDriver + .parse_tag_event(DOCKER_CONTAINER_START) + .is_none(), + "a container start is not an image tag event" + ); + } + + #[test] + fn podman_driver_ignores_a_non_tag_event() { + assert!( + PodmanDriver + .parse_tag_event(PODMAN_CONTAINER_START) + .is_none(), + "a container start is not an image tag event" + ); + } + + #[test] + fn a_driver_returns_none_on_an_unparseable_line() { + assert!(DockerDriver.parse_tag_event("not json").is_none()); + assert!(PodmanDriver.parse_tag_event("").is_none()); + } + + #[test] + fn docker_drives_events_over_the_cli_not_the_api_socket() { + let driver = DockerDriver; + assert_eq!(driver.binary(), "docker"); + let argv = driver.events_argv(); + // The event source is the `docker events` CLI subcommand — not a socket. + assert_eq!(argv.first().map(String::as_str), Some("events")); + assert!( + !argv.iter().any(|a| a.contains("--host") + || a.contains("-H") + || a.contains(".sock") + || a.contains("unix://")), + "the driver must not dial the API socket: {argv:?}" + ); + } + + #[test] + fn podman_drives_events_over_the_cli_with_json_and_no_socket() { + let driver = PodmanDriver; + assert_eq!(driver.binary(), "podman"); + let argv = driver.events_argv(); + assert_eq!(argv.first().map(String::as_str), Some("events")); + // The task pins `podman events --format json`. + let format_idx = argv + .iter() + .position(|a| a == "--format") + .expect("podman events must request an explicit format"); + assert_eq!(argv.get(format_idx + 1).map(String::as_str), Some("json")); + assert!( + !argv.iter().any(|a| a.contains("--url") + || a.contains(".sock") + || a.contains("unix://") + || a.contains("--remote")), + "rootless podman must be driven with no API socket: {argv:?}" + ); + } + + #[test] + fn both_docker_and_podman_drivers_resolve_and_podman_is_not_a_stub() { + let docker = driver_for("docker").expect("docker driver exists"); + assert_eq!(docker.binary(), "docker"); + + let podman = driver_for("podman").expect("podman driver exists"); + assert_eq!(podman.binary(), "podman"); + // podman is a real CLI-event path, not a bare stub: it both drives + // `events` and parses a real tag event. + assert_eq!( + podman.events_argv().first().map(String::as_str), + Some("events") + ); + assert!( + podman.parse_tag_event(PODMAN_TAG_EVENT).is_some(), + "the podman driver must parse a real CLI tag event, not stub out" + ); + + assert!(driver_for("nerdctl").is_none()); + } + + #[tokio::test] + async fn forward_tag_events_streams_only_tag_events_from_the_cli_byte_stream() { + // A captured multi-line event stream, as it would arrive on the engine + // subprocess stdout: two tag events interleaved with noise the driver + // must skip. + let stream = format!( + "{DOCKER_TAG_EVENT}\n\ + {DOCKER_CONTAINER_START}\n\ + garbage-not-json\n\ + {}\n", + DOCKER_TAG_EVENT.replace("my-app:dev", "sidecar:latest") + ); + let reader = BufReader::new(Cursor::new(stream.into_bytes())); + + let mut collected: Vec = Vec::new(); + forward_tag_events(&DockerDriver, reader, |event| collected.push(event)) + .await + .expect("forwarding over a byte-stream reader succeeds"); + + // Only the two image tag lines surface, in order; the container start + // and the garbage line are dropped. + assert_eq!(collected.len(), 2, "only tag events are forwarded"); + assert_eq!(collected[0].image, "my-app:dev"); + assert_eq!(collected[1].image, "sidecar:latest"); + } + + #[test] + fn docker_write_credential_is_an_ephemeral_docker_config_keyed_to_the_registry() { + let cred = DockerDriver.write_credential("127.0.0.1:5599", &WriteToken::new("wtok")); + match cred { + WriteCredential::DockerConfigEnv { + registry, + username, + token, + } => { + // The auth-entry key must be byte-identical to the tagged + // registry host:port (H-3). + assert_eq!(registry, "127.0.0.1:5599"); + assert_eq!(username, WRITE_USERNAME); + assert_eq!(token, "wtok"); + } + other => panic!("docker must inject via an ephemeral DOCKER_CONFIG, got {other:?}"), + } + } + + #[test] + fn podman_write_credential_is_per_invocation_creds() { + let cred = PodmanDriver.write_credential("127.0.0.1:5599", &WriteToken::new("wtok")); + match cred { + WriteCredential::PodmanCreds { username, token } => { + assert_eq!(username, WRITE_USERNAME); + assert_eq!(token, "wtok"); + } + other => panic!("podman must inject via --creds, got {other:?}"), + } + } +} diff --git a/src/utils/container_dev/image_ref.rs b/src/utils/container_dev/image_ref.rs new file mode 100644 index 00000000..48d7d975 --- /dev/null +++ b/src/utils/container_dev/image_ref.rs @@ -0,0 +1,137 @@ +//! One place that decides what an image reference means. +//! +//! Three copies of this logic used to live in `watcher.rs` (`repo_and_tag`, +//! `with_default_tag`) and `ws.rs` (`split_image_tag`), and they drifted: the +//! cross-arch guard recorded an image's architecture under the RAW event ref and +//! looked it up under the registry-stripped one, so `arch_for` returned `None` +//! for every registry-qualified ref and the broadcast filter fell through to its +//! permissive arm. podman qualifies local refs as `localhost/my-app:dev`, so on +//! podman that was every ref - an amd64 image reached an aarch64 device, which is +//! the case the guard exists to prevent. +//! +//! Two normal forms, deliberately distinct, because they answer different +//! questions: +//! +//! - [`canonical`] strips the registry and applies the default tag. It is the +//! identity of an image as a THING, and the right key for anything that has to +//! agree across the watcher and the control WS. +//! - [`with_default_tag`] alone leaves the registry in place. It is the identity +//! of a ref as CONFIGURED, and `WatchSet` must keep using it: the push retags +//! to `/:` on the way to every push, so a watch set keyed +//! on the canonical form would match the watcher's own side effect and drive a +//! retag -> event -> sync -> retag loop. + +/// Strip a leading registry component (`localhost/…`, `host.tld/…`, +/// `host:port/…`), leaving `repo[:tag]`. +/// +/// A first path segment is a registry only if it looks like a host: podman's +/// `localhost`, or something carrying a dot or a port colon. `library/alpine` has +/// neither, so it stays whole. +pub fn strip_registry(image: &str) -> &str { + match image.split_once('/') { + Some((first, rest)) + if first == "localhost" || first.contains('.') || first.contains(':') => + { + rest + } + _ => image, + } +} + +/// `repo` -> `repo:latest`, leaving an already-tagged ref alone. +/// +/// Only a colon AFTER the last `/` is a tag separator; a colon before it belongs +/// to a registry `host:port` (`host:5601/repo`). +pub fn with_default_tag(image: &str) -> String { + let name_start = image.rfind('/').map_or(0, |i| i + 1); + if image[name_start..].contains(':') { + image.to_string() + } else { + format!("{image}:latest") + } +} + +/// The normal form used as a cross-module key: registry stripped, tag defaulted. +/// +/// `localhost/my-app:dev`, `my-app:dev` and `10.0.2.2:5000/my-app:dev` all +/// canonicalize to `my-app:dev`, so a value recorded on one path is found on +/// another regardless of which engine produced the ref. +pub fn canonical(image: &str) -> String { + with_default_tag(strip_registry(image)) +} + +/// Split an image reference into `(repo, tag)` in [`canonical`] form. +pub fn split(image: &str) -> (String, String) { + let canonical = canonical(image); + match canonical.rsplit_once(':') { + Some((repo, tag)) => (repo.to_string(), tag.to_string()), + // `canonical` always appends a tag, so this is unreachable in practice. + None => (canonical, "latest".to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_agrees_across_engine_ref_shapes() { + // The bug this module exists for: the arch book recorded under the raw + // ref and read back under the stripped one, so the two never matched for + // a podman user. Every shape an engine can report has to land on one key. + for raw in [ + "my-app:dev", + "localhost/my-app:dev", + "10.0.2.2:5000/my-app:dev", + "registry.example.com/my-app:dev", + ] { + assert_eq!(canonical(raw), "my-app:dev", "canonical({raw:?})"); + } + } + + #[test] + fn canonical_applies_the_default_tag() { + assert_eq!(canonical("my-app"), "my-app:latest"); + assert_eq!(canonical("localhost/my-app"), "my-app:latest"); + } + + #[test] + fn a_registry_port_colon_is_not_a_tag_separator() { + assert_eq!(with_default_tag("host:5601/repo"), "host:5601/repo:latest"); + } + + #[test] + fn a_bare_namespace_is_not_a_registry() { + // `library` has no dot, no colon, and is not `localhost`, so stripping it + // would silently rewrite the image the user asked for. + assert_eq!(strip_registry("library/alpine"), "library/alpine"); + assert_eq!(canonical("library/alpine"), "library/alpine:latest"); + } + + #[test] + fn with_default_tag_keeps_the_registry_that_watchset_needs() { + // WatchSet keys on this form, NOT on `canonical`. The push retags to + // `/:`, so canonicalizing here would make the + // watcher match its own retag and re-enter its sync path forever. + assert_eq!( + with_default_tag("10.0.2.2:5000/my-app:dev"), + "10.0.2.2:5000/my-app:dev" + ); + assert_ne!( + with_default_tag("10.0.2.2:5000/my-app:dev"), + canonical("10.0.2.2:5000/my-app:dev") + ); + } + + #[test] + fn split_returns_canonical_components() { + assert_eq!( + split("localhost/my-app:dev"), + ("my-app".to_string(), "dev".to_string()) + ); + assert_eq!( + split("my-app"), + ("my-app".to_string(), "latest".to_string()) + ); + } +} diff --git a/src/utils/container_dev/mod.rs b/src/utils/container_dev/mod.rs new file mode 100644 index 00000000..4b64a7af --- /dev/null +++ b/src/utils/container_dev/mod.rs @@ -0,0 +1,55 @@ +//! Container Dev Mode: embedded OCI Distribution registry and engine-driver +//! dev loop for iterating on containers running on Avocado devices. +//! +//! Scaffolding at this stage. TLS material, the remaining registry listeners, +//! the engine-driver watcher, and sync orchestration are added by later tasks +//! in the `container-dev-mode` change. + +// The write-side Basic validator (3.3) and the read/control Bearer validator +// (3.4). +#[allow(dead_code)] +pub mod auth; +// Per-`up` device bootstrap, guaranteed write-listener teardown guard, +// drain-based read/control token rotation, and `status` reporting (task 5.2). +// The `up`/`down`/`status` glue in `commands::container::dev` binds these to the +// live listeners; some helpers are exercised only from that glue, hence +// dead_code here. +#[allow(dead_code)] +pub mod bootstrap; +// One-shot `sync` (re-push + notify, no watcher loop) and `prune` (per-project +// store GC only) command cores (task 5.3); wired to the live listeners/syncer/WS +// by the `commands::container::dev` glue. +pub mod commands; +pub mod config; +// The engine-driver trait + docker/podman drivers (4.1): tag events via the +// engine CLI subprocess (never the API socket). The watcher (4.2/4.3) that +// consumes the event stream and the push wiring that uses the credential hook +// are added later, hence dead_code here. +#[allow(dead_code)] +pub mod engine; +// The single normal form for an image reference, shared by the watcher and the +// control WS so a key recorded on one path is found on the other. +pub mod image_ref; +// The store (3.1), OCI read handlers (3.2), and write handlers + auth layer +// (3.3) land before the listeners that bind them: the read router is bound onto +// the dedicated bulk listener by 3.7, the write router onto the distinct write +// listener by 3.6/3.7. +#[allow(dead_code)] +pub mod registry; +#[allow(dead_code)] +pub mod store; +// Per-project CA + leaf, the rustls server config, and the per-session token +// mint (3.6). Bound onto the bulk/WS listeners by 3.7/5.2, hence dead_code here. +#[allow(dead_code)] +pub mod tls; +// Engine-driver watcher + sync orchestration (4.2): topology-selected PUSH/INGEST +// on a debounced tag event, then notify over the control-WS seam. Wired into the +// `up` orchestration (5.2) and the control WS (5.1) later, hence dead_code here. +#[allow(dead_code)] +pub mod watcher; +// Control-only WebSocket channel (5.1): host->device `sync`, device->host +// `hello`/`progress`/`status`; the WS upgrade authenticates through the shared +// read/control-token validator (3.4). Wired into the `up` orchestration (5.2) +// later, hence dead_code here. +#[allow(dead_code)] +pub mod ws; diff --git a/src/utils/container_dev/registry.rs b/src/utils/container_dev/registry.rs new file mode 100644 index 00000000..b8dea12f --- /dev/null +++ b/src/utils/container_dev/registry.rs @@ -0,0 +1,2318 @@ +//! OCI Distribution read handlers for the Container Dev Mode registry. +//! +//! These handlers implement the read half of the OCI Distribution spec that a +//! device engine exercises on a pull: +//! +//! - `GET /v2/` — the API version check. +//! - `GET|HEAD /v2//manifests/` — a manifest by tag or by +//! digest, including a multi-arch image index. +//! - `GET|HEAD /v2//blobs/` — a blob, honoring a `Range:` +//! request with a `206 Partial Content` response. +//! +//! Content is read from the per-project [`BlobStore`] built in task 3.1; this +//! module never re-implements storage. The read routes are gated by the +//! per-session Bearer read/control token (task 3.4) via [`read_router`] and +//! bound onto the dedicated bulk read listener by task 3.7. +//! +//! Task 3.3 adds the write half — blob upload (`POST`/`PATCH`/`PUT +//! .../blobs/uploads/...`), manifest `PUT`, and blob `HEAD` dedup — assembled +//! into a SEPARATE [`write_router`] gated by the host-only Basic write token +//! ([`super::auth`]). Those write routes live on a DISTINCT write listener +//! (design D9/H-1), bound by tasks 3.6/3.7; a device is only ever handed the +//! bulk-listener endpoint, so it cannot reach a write route on any topology. +//! The TLS/listener sockets (tasks 3.6/3.7) remain out of scope here. +//! +//! HEAD requests are served by the same handler as GET: axum routes HEAD to the +//! GET handler and strips the response body while preserving the headers, so a +//! HEAD carries the resource's `Content-Length` and `Docker-Content-Digest` +//! with an empty body. + +use std::collections::HashMap; +use std::io; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use axum::{ + body::Body, + extract::{Path, Query, State}, + http::{header, HeaderMap, HeaderValue, StatusCode}, + middleware, + response::{IntoResponse, Response}, + routing::{get, post}, + Router, +}; +use futures_util::StreamExt as _; +use rustls::ServerConfig; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinHandle; +use tokio_rustls::server::TlsStream; +use tokio_rustls::TlsAcceptor; +use tokio_util::io::ReaderStream; +use uuid::Uuid; + +use super::auth::{require_basic_write, require_bearer_read, ReadToken, WriteToken}; +use super::store::{BlobStore, BlobUpload, StoreError}; + +/// Non-standard OCI response header carrying the content digest of the served +/// manifest or blob. +const DOCKER_CONTENT_DIGEST: &str = "docker-content-digest"; + +/// How long an upload session may sit untouched before it is evicted. +/// +/// Bounds how long an abandoned push can hold a staging file open. An upload that +/// is never finalized - an interrupted push, a killed `docker` - leaves a session +/// in the map with a `NamedTempFile` behind it; eviction drops the session, which +/// unlinks the file. +/// +/// What "untouched" means, precisely: `touched` is refreshed on every `PATCH` and +/// on the finalizing `PUT`. Both handlers take the session OUT of the map for the +/// duration of the transfer, so `evict_expired` cannot see a session that is +/// actively being streamed into at all - a slow transfer is not evictable, however +/// long it runs, and the only sessions the sweep can reach are ones no request is +/// touching. +/// +/// This replaces an earlier comment describing the buffered implementation, whose +/// premises this path no longer has: there is no `Bytes` extractor buffering a +/// chunk before the handler runs, no in-flight session visible to the sweep, and +/// no request-size limit to derive a throughput bound from. The resource at risk +/// is disk, not memory - see [`crate::utils::container_dev::store::MAX_BLOB_BYTES`] +/// for the ceiling that bounds it. +const UPLOAD_SESSION_TTL: Duration = Duration::from_secs(600); + +/// Upper bound on a manifest body. +/// +/// Blobs are streamed to disk and need no limit, but a manifest is parsed as a +/// whole document, so this one path still reads into memory - under a cap +/// chosen to be far above any real manifest and far below anything that +/// threatens the host. +const MAX_MANIFEST_BYTES: usize = 32 * 1024 * 1024; + +/// Default media type used when a stored manifest omits its `mediaType` field. +const DEFAULT_MANIFEST_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; + +/// Shared state for the registry handlers: the backing content-addressed store. +#[derive(Clone)] +pub struct RegistryState { + store: Arc, +} + +impl RegistryState { + /// Build registry state over an existing store. + pub fn new(store: Arc) -> Self { + Self { store } + } +} + +/// Build the ungated OCI read route assembly over `store`. +/// +/// These are the read handlers only — `GET /v2/`, manifest reads, and blob +/// reads (GET + HEAD). It is a composition primitive: [`read_router`] wraps it +/// with the Bearer read/control gate. The read-semantics tests exercise this +/// assembly directly so they test handler behavior without auth noise. +fn read_routes(store: Arc) -> Router { + Router::new() + .route("/v2/", get(base)) + // A single wildcard route captures `/manifests/` and + // `/blobs/`; `` may itself contain `/`, so it + // cannot be a fixed path segment. The suffix is dispatched by hand. + .route("/v2/{*rest}", get(read)) + .with_state(RegistryState::new(store)) +} + +/// Build the device-facing OCI read router over `store`, gated by the +/// per-session Bearer `read_token` (task 3.4). +/// +/// Every read route sits behind [`require_bearer_read`] — the SAME validator +/// the control-WS upgrade (task 5.1) authorizes through (G-5) — so an +/// unauthenticated pull, or one presenting the Basic write token, is refused +/// with a bare `Bearer` challenge before any handler runs (M-2). This is the +/// only read entry point a device is handed; it is bound onto the dedicated +/// bulk read listener in task 3.7. +pub fn read_router(store: Arc, read_token: ReadToken) -> Router { + read_routes(store).layer(middleware::from_fn_with_state( + read_token, + require_bearer_read, + )) +} + +/// A TLS-terminating [`axum::serve::Listener`] over a bound [`TcpListener`]. +/// +/// Every accepted TCP connection is handshaked with the per-project leaf +/// (task 3.6) before the OCI read router sees a byte, so the dedicated bulk +/// listener speaks only TLS. The axum `Listener` contract forbids surfacing an +/// accept error, so a failed TCP accept or TLS handshake is dropped and the +/// loop continues; a persistent TCP accept error backs off briefly to avoid a +/// busy-spin. +struct TlsListener { + tcp: TcpListener, + acceptor: TlsAcceptor, +} + +impl axum::serve::Listener for TlsListener { + type Io = TlsStream; + type Addr = SocketAddr; + + async fn accept(&mut self) -> (Self::Io, Self::Addr) { + loop { + let (stream, addr) = match self.tcp.accept().await { + Ok(pair) => pair, + Err(_) => { + // Transient accept errors (e.g. fd exhaustion) must not be + // surfaced; back off so we do not busy-spin on a persistent + // one, then retry. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + continue; + } + }; + // A handshake failure is a per-connection concern (a client that + // does not trust the CA, or a probe); drop it and keep serving. + if let Ok(tls) = self.acceptor.accept(stream).await { + return (tls, addr); + } + } + } + + fn local_addr(&self) -> io::Result { + self.tcp.local_addr() + } +} + +/// A bound, running bulk read listener: the dedicated TLS socket that serves the +/// OCI read router (task 3.2) gated by the Bearer read/control token (task 3.4). +/// +/// This is the bulk-read leg of the three-listener model (design D9/H-1). A +/// listener's identity IS a socket, so bulk pulls live on their OWN socket, +/// separate from the write listener (task 3.3, [`write_router`]) and the control +/// WebSocket (task 5.1). Bulk transfers therefore never share the control WS +/// byte stream: a blob GET is an ordinary HTTP request on this dedicated TLS +/// socket, so a large pull can never head-of-line-block a control frame. A +/// device is only ever handed this listener's endpoint (task 5.2), so it cannot +/// reach the write listener on any topology. +pub struct BulkListener { + local_addr: SocketAddr, + task: JoinHandle<()>, +} + +impl BulkListener { + /// Bind the dedicated bulk read listener at `addr` and start serving the + /// token-gated OCI read router over TLS with the session leaf. + /// + /// Pass a `0` port to let the OS choose one; [`local_addr`](Self::local_addr) + /// then reports the concrete socket. The server runs on a spawned task that + /// is aborted when the returned handle is dropped. + pub async fn bind( + addr: SocketAddr, + store: Arc, + read_token: ReadToken, + tls_config: Arc, + ) -> io::Result { + let tcp = TcpListener::bind(addr).await?; + let local_addr = tcp.local_addr()?; + let listener = TlsListener { + tcp, + acceptor: TlsAcceptor::from(tls_config), + }; + let router = read_router(store, read_token); + let task = tokio::spawn(async move { + // `axum::serve` only returns on shutdown; the dev session drops the + // handle (aborting this task) when the registry is torn down. + let _ = axum::serve(listener, router).await; + }); + Ok(Self { local_addr, task }) + } + + /// The socket this bulk listener is bound to — its listener identity + /// (design H-1), distinct from the write listener's and the control WS's. + pub fn local_addr(&self) -> SocketAddr { + self.local_addr + } +} + +impl Drop for BulkListener { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// One in-flight chunked upload: the on-disk staging handle plus when it last +/// grew. +struct UploadSession { + upload: BlobUpload, + touched: Instant, +} + +impl UploadSession { + fn new(upload: BlobUpload) -> Self { + Self { + upload, + touched: Instant::now(), + } + } +} + +/// In-flight chunked-upload sessions, keyed by upload UUID. +/// +/// The OCI blob-upload protocol is stateful: `POST` opens a session, `PATCH` +/// appends chunks, and `PUT` finalizes with the expected digest. A session holds a +/// [`BlobUpload`] staging the bytes on disk and hashing them incrementally, so no +/// layer is ever held whole in memory. +/// +/// Nothing in the protocol obliges a client to finish what it starts: a `POST` +/// followed by `PATCH`es and no `PUT` - an interrupted push, a killed `docker` - +/// abandons its staging file here. [`evict_expired`] reclaims those, and +/// `BlobStore::sweep_uploads` catches the ones whose process died before any +/// eviction could run. +#[derive(Default)] +struct UploadSessions { + inner: Mutex>, +} + +/// Drop sessions untouched for longer than [`UPLOAD_SESSION_TTL`] as of `now`. +/// +/// Called when a new session opens, which is both the moment a fresh buffer is +/// about to be allocated and the only point an abandoned one can be noticed - no +/// client ever tells us it gave up. +/// +/// `now` is a parameter rather than a call to `Instant::now()` inside so a test +/// can place a session at a chosen distance from the TTL boundary. Reading the +/// clock internally left the only available test a sweep milliseconds after the +/// POST, which passes whether or not the mechanism works at all. +fn evict_expired(sessions: &mut HashMap, now: Instant) { + sessions.retain(|_uuid, session| { + now.saturating_duration_since(session.touched) < UPLOAD_SESSION_TTL + }); +} + +/// Shared state for the write handlers: the backing store plus upload sessions. +#[derive(Clone)] +struct WriteState { + store: Arc, + uploads: Arc, +} + +/// Build the OCI WRITE router over `store`, gated by the host-only Basic +/// `write_token`. +/// +/// The router serves blob upload (`POST`/`PATCH`/`PUT .../blobs/uploads/...`), +/// manifest `PUT`, blob `HEAD` dedup, and the `GET /v2/` ping — every route +/// behind [`require_basic_write`], so an anonymous request (including the ping) +/// receives a `401` with a Basic challenge. This router is bound onto the +/// DISTINCT write listener (design D9); it is never merged onto the bulk read +/// listener. +pub fn write_router(store: Arc, write_token: WriteToken) -> Router { + write_router_with_uploads(store, write_token, Arc::new(UploadSessions::default())) +} + +/// [`write_router`], but over a caller-supplied session map. +/// +/// Exists so a test can hold the same `Arc` the handlers mutate and place a +/// session at a chosen age. Without it the TTL is only reachable through a real +/// 10-minute wait, which is why the first attempt at a TTL test asserted nothing. +fn write_router_with_uploads( + store: Arc, + write_token: WriteToken, + uploads: Arc, +) -> Router { + let state = WriteState { store, uploads }; + Router::new() + .route("/v2/", get(base)) + .route( + "/v2/{*rest}", + post(post_route) + .patch(patch_route) + .put(put_route) + .head(head_route), + ) + // The auth layer wraps the whole router, so it runs before routing: an + // unauthenticated request to any path (or an unrouted method) is + // rejected with the Basic challenge before a handler is reached. + .layer(middleware::from_fn_with_state( + write_token, + require_basic_write, + )) + // No DefaultBodyLimit layer at all. It would be inert: that limit is + // consumed by the `Bytes`/`String` extractors, and every write handler + // now takes `Body` and streams it, so the layer would gate nothing while + // reading as though it did. Blob bodies never exist whole in memory, and + // the one path that does buffer - manifest PUT - applies + // MAX_MANIFEST_BYTES explicitly where the read happens. + .with_state(state) +} + +/// Serve the write router over TLS with the per-project session leaf, spawned on +/// its own task (aborted when the returned handle is dropped). +/// +/// Used for the VM push path only: a QEMU-SLIRP guest reaches the loopback write +/// listener through the `10.0.2.2` host alias, which is NOT inside docker's +/// built-in `127.0.0.0/8` insecure exemption (design A2). The guest daemon is +/// therefore configured for HTTPS via a delivered `certs.d//ca.crt` +/// (design H4), so the listener must terminate the same leaf TLS the bulk and +/// control listeners do. The native loopback path keeps plain HTTP under docker's +/// exemption and does not call this. +pub fn serve_write_router_tls( + tcp: TcpListener, + tls_config: Arc, + store: Arc, + write_token: WriteToken, +) -> JoinHandle<()> { + let listener = TlsListener { + tcp, + acceptor: TlsAcceptor::from(tls_config), + }; + let router = write_router(store, write_token); + tokio::spawn(async move { + // `axum::serve` only returns on shutdown; the session aborts this task + // via the returned handle when the write listener is torn down. + let _ = axum::serve(listener, router).await; + }) +} + +/// `POST /v2//blobs/uploads/[?digest=]` — open a chunked upload, +/// or complete a monolithic upload when a `digest` query is present. +async fn post_route( + State(state): State, + Path(rest): Path, + Query(q): Query>, + body: Body, +) -> Response { + let Some(name) = rest + .strip_suffix("/blobs/uploads/") + .or_else(|| rest.strip_suffix("/blobs/uploads")) + else { + return oci_error( + StatusCode::NOT_FOUND, + "UNSUPPORTED", + "unsupported write path", + ); + }; + let name = name.to_string(); + + if let Some(digest) = q.get("digest") { + // Monolithic upload: the whole blob arrives with the POST. Still + // streamed - "monolithic" describes the protocol, not how much of it we + // are willing to hold at once. + let mut upload = match state.store.begin_blob_upload() { + Ok(upload) => upload, + Err(e) => return store_error(&e), + }; + if let Err(resp) = stream_into(body, &mut upload).await { + return resp; + } + return finish_upload(&name, digest, upload); + } + + let uuid = Uuid::new_v4().to_string(); + let upload = match state.store.begin_blob_upload() { + Ok(upload) => upload, + Err(e) => return store_error(&e), + }; + let mut sessions = state + .uploads + .inner + .lock() + .expect("upload sessions mutex is not poisoned"); + // Reclaim sessions from pushes that opened one and never finalized it, + // before starting another alongside them. Their temp files go with them. + evict_expired(&mut sessions, Instant::now()); + sessions.insert(uuid.clone(), UploadSession::new(upload)); + drop(sessions); + upload_accepted(&name, &uuid, 0) +} + +/// Drain `body` into `upload`, mapping a transport error to an OCI response. +async fn stream_into(body: Body, upload: &mut BlobUpload) -> Result<(), Response> { + let mut stream = body.into_data_stream(); + while let Some(chunk) = stream.next().await { + let chunk = match chunk { + Ok(chunk) => chunk, + Err(_) => { + return Err(oci_error( + StatusCode::BAD_REQUEST, + "BLOB_UPLOAD_INVALID", + "upload stream ended early", + )) + } + }; + if let Err(e) = upload.append(&chunk) { + return Err(store_error(&e)); + } + } + Ok(()) +} + +/// Verify and store a completed upload, returning the OCI response. +fn finish_upload(name: &str, digest: &str, upload: BlobUpload) -> Response { + match upload.finish(digest) { + Ok(true) => blob_created(name, digest), + Ok(false) => oci_error( + StatusCode::BAD_REQUEST, + "DIGEST_INVALID", + "uploaded content does not match the supplied digest", + ), + Err(e) => store_error(&e), + } +} + +/// `PATCH /v2//blobs/uploads/` — append a chunk to a session. +async fn patch_route( + State(state): State, + Path(rest): Path, + body: Body, +) -> Response { + let Some((name, uuid)) = split_upload(&rest) else { + return oci_error( + StatusCode::NOT_FOUND, + "UNSUPPORTED", + "unsupported write path", + ); + }; + let (name, uuid) = (name.to_string(), uuid.to_string()); + + // Take the session OUT of the map for the duration of the transfer. The + // mutex cannot be held across the await, and a session being streamed into + // is not a session a concurrent sweep should be able to reclaim - removing + // it makes both true at once. + let Some(mut session) = state + .uploads + .inner + .lock() + .expect("upload sessions mutex is not poisoned") + .remove(&uuid) + else { + return oci_error( + StatusCode::NOT_FOUND, + "BLOB_UPLOAD_UNKNOWN", + "upload session unknown", + ); + }; + + let start = session.upload.written(); + let streamed = stream_into(body, &mut session.upload).await; + // Re-insert on BOTH paths. Taking the session out of the map is what keeps a + // concurrent sweep from reclaiming it mid-transfer, but returning early on an + // error would drop the `BlobUpload` here - unlinking the staging file and + // every chunk already accepted. The buffered implementation got resumability + // for free: the `Bytes` extractor rejected a truncated body before the + // handler ran, so the session and its bytes survived and `Range` told the + // client where to continue. Streaming has to restore that explicitly, or a + // dropped connection on chunk 6 of 8 restarts the layer from byte 0. + let end = session.upload.written(); + session.touched = Instant::now(); + state + .uploads + .inner + .lock() + .expect("upload sessions mutex is not poisoned") + .insert(uuid.clone(), session); + if let Err(resp) = streamed { + return resp; + } + upload_range_accepted(&name, &uuid, start, end) +} + +/// `PUT` on the write listener: finalize a blob upload +/// (`.../blobs/uploads/?digest=`) or store a manifest +/// (`.../manifests/`). +async fn put_route( + State(state): State, + Path(rest): Path, + Query(q): Query>, + body: Body, +) -> Response { + if let Some((name, reference)) = rest.split_once("/manifests/") { + // Manifests are small JSON documents and are parsed as a whole, so this + // one path still buffers - under an explicit cap, not an unbounded read. + let bytes = match axum::body::to_bytes(body, MAX_MANIFEST_BYTES).await { + Ok(bytes) => bytes, + Err(_) => { + return oci_error( + StatusCode::BAD_REQUEST, + "MANIFEST_INVALID", + "manifest exceeds the maximum accepted size", + ) + } + }; + return put_manifest(&state, name, reference, &bytes); + } + if let Some((name, uuid)) = split_upload(&rest) { + let (name, uuid) = (name.to_string(), uuid.to_string()); + let Some(digest) = q.get("digest").map(String::as_str) else { + return oci_error( + StatusCode::BAD_REQUEST, + "DIGEST_INVALID", + "digest query parameter required to finalize an upload", + ); + }; + let Some(mut session) = state + .uploads + .inner + .lock() + .expect("upload sessions mutex is not poisoned") + .remove(&uuid) + else { + return oci_error( + StatusCode::NOT_FOUND, + "BLOB_UPLOAD_UNKNOWN", + "upload session unknown", + ); + }; + // The PUT may carry a final chunk of its own. On a truncated one, put the + // session back rather than discarding every previously accepted chunk - + // the client can retry the finalize against the same Location. + if let Err(resp) = stream_into(body, &mut session.upload).await { + session.touched = Instant::now(); + state + .uploads + .inner + .lock() + .expect("upload sessions mutex is not poisoned") + .insert(uuid.clone(), session); + return resp; + } + return finish_upload(&name, digest, session.upload); + } + oci_error( + StatusCode::NOT_FOUND, + "UNSUPPORTED", + "unsupported write path", + ) +} + +/// `HEAD /v2//blobs/` — the push-side dedup probe: `200` when the +/// blob already exists so the engine skips re-uploading it, else `404`. +async fn head_route(State(state): State, Path(rest): Path) -> Response { + let Some((_name, digest)) = rest.split_once("/blobs/") else { + return oci_error( + StatusCode::NOT_FOUND, + "UNSUPPORTED", + "unsupported write path", + ); + }; + // `blob_size` stats the entry instead of reading it: the probe reports only + // a length, and an engine HEADs every layer before pushing, so reading each + // existing layer into memory to discard it would put the whole image on the + // heap just to answer "do you already have this?". + match state.store.blob_size(digest) { + Ok(Some(len)) => Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_LENGTH, len.to_string()) + .header(DOCKER_CONTENT_DIGEST, digest) + .body(Body::empty()) + .expect("blob-head response is always valid"), + Ok(None) => blob_unknown(), + Err(e) => store_error(&e), + } +} + +/// `PUT /v2//manifests/` — store a manifest and, when +/// `reference` is a tag (not a digest), point that tag at it. +fn put_manifest(state: &WriteState, name: &str, reference: &str, body: &[u8]) -> Response { + let digest = compute_digest(body); + if let Err(e) = state.store.write_blob(&digest, body) { + return store_error(&e); + } + if !looks_like_digest(reference) { + if let Err(e) = state.store.set_tag(name, reference, &digest) { + return store_error(&e); + } + } + manifest_created(name, reference, &digest) +} + +/// Split `/blobs/uploads/` into `(name, uuid)`. +fn split_upload(rest: &str) -> Option<(&str, &str)> { + let (name, uuid) = rest.split_once("/blobs/uploads/")?; + if name.is_empty() || uuid.is_empty() || uuid.contains('/') { + return None; + } + Some((name, uuid)) +} + +/// Compute the OCI digest (`sha256:`) of `bytes`. +fn compute_digest(bytes: &[u8]) -> String { + use sha2::{Digest as _, Sha256}; + let hex: String = Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("sha256:{hex}") +} + +/// `202 Accepted` opening a chunked upload session. +fn upload_accepted(name: &str, uuid: &str, offset: u64) -> Response { + Response::builder() + .status(StatusCode::ACCEPTED) + .header(header::LOCATION, format!("/v2/{name}/blobs/uploads/{uuid}")) + .header("docker-upload-uuid", uuid) + .header(header::RANGE, format!("0-{offset}")) + .body(Body::empty()) + .expect("upload-accepted response is always valid") +} + +/// `202 Accepted` acknowledging an appended chunk, reporting the new byte range. +fn upload_range_accepted(name: &str, uuid: &str, start: u64, end: u64) -> Response { + // An empty session reports `0-0`; otherwise the last written byte index. + let last = end.saturating_sub(1).max(start); + Response::builder() + .status(StatusCode::ACCEPTED) + .header(header::LOCATION, format!("/v2/{name}/blobs/uploads/{uuid}")) + .header("docker-upload-uuid", uuid) + .header(header::RANGE, format!("0-{last}")) + .body(Body::empty()) + .expect("upload-range response is always valid") +} + +/// `201 Created` for a completed blob upload. +fn blob_created(name: &str, digest: &str) -> Response { + Response::builder() + .status(StatusCode::CREATED) + .header(header::LOCATION, format!("/v2/{name}/blobs/{digest}")) + .header(DOCKER_CONTENT_DIGEST, digest) + .body(Body::empty()) + .expect("blob-created response is always valid") +} + +/// `201 Created` for a stored manifest. +fn manifest_created(name: &str, reference: &str, digest: &str) -> Response { + Response::builder() + .status(StatusCode::CREATED) + .header( + header::LOCATION, + format!("/v2/{name}/manifests/{reference}"), + ) + .header(DOCKER_CONTENT_DIGEST, digest) + .body(Body::empty()) + .expect("manifest-created response is always valid") +} + +/// Map a [`StoreError`] to an OCI error response. +fn store_error(err: &StoreError) -> Response { + match err { + StoreError::InvalidDigest(_) => { + oci_error(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "invalid digest") + } + StoreError::InvalidTag(_) => { + oci_error(StatusCode::BAD_REQUEST, "TAG_INVALID", "invalid tag") + } + // A real OCI error the engine can act on, not a bare axum rejection: + // 413 with BLOB_UPLOAD_INVALID tells the client the layer is too big + // rather than leaving it to guess from a closed connection. + StoreError::BlobTooLarge { .. } => oci_error( + StatusCode::PAYLOAD_TOO_LARGE, + "BLOB_UPLOAD_INVALID", + "blob exceeds the registry's size ceiling", + ), + StoreError::NoHome + | StoreError::Io(_) + | StoreError::InvalidName(_) + | StoreError::PruneWhileSessionLive => oci_error( + StatusCode::INTERNAL_SERVER_ERROR, + "UNKNOWN", + "registry storage error", + ), + } +} + +/// `GET /v2/` — advertise OCI Distribution v2 support. +async fn base() -> impl IntoResponse { + ( + StatusCode::OK, + [ + ("docker-distribution-api-version", "registry/2.0"), + (header::CONTENT_TYPE.as_str(), "application/json"), + ], + "{}", + ) +} + +/// Dispatch a `/v2/` read to the manifest or blob handler. +async fn read( + State(state): State, + headers: HeaderMap, + Path(rest): Path, +) -> Response { + if let Some((name, reference)) = rest.split_once("/manifests/") { + // `name` is part of the tag key, not just the Location header: two + // watched images sharing a tag resolve to each other's manifest in a + // flat namespace. + serve_manifest(&state, name, reference) + } else if let Some((_name, digest)) = rest.split_once("/blobs/") { + serve_blob(&state, &headers, digest).await + } else { + oci_error( + StatusCode::NOT_FOUND, + "NAME_UNKNOWN", + "unsupported registry path", + ) + } +} + +/// Serve a manifest identified by `reference`, which is either a digest +/// (`:`) or a tag that resolves to a manifest digest. +fn serve_manifest(state: &RegistryState, name: &str, reference: &str) -> Response { + let digest = if looks_like_digest(reference) { + reference.to_string() + } else { + match state.store.resolve_tag(name, reference) { + Ok(Some(d)) => d, + _ => return manifest_unknown(), + } + }; + + let bytes = match state.store.read_blob(&digest) { + Ok(Some(b)) => b, + _ => return manifest_unknown(), + }; + + let media_type = manifest_media_type(&bytes); + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, media_type) + .header(DOCKER_CONTENT_DIGEST, digest) + .body(Body::from(bytes)) + .expect("static manifest response is always valid") +} + +/// Serve a blob by `digest`, honoring a single `Range:` request. +/// +/// Streamed off disk rather than read whole. Uploads land on disk without ever +/// existing complete in memory, so the store can hold a layer larger than host +/// RAM - and a read that sized one allocation by the blob turned a single +/// oversized push into an OOM on every later pull, taking every listener and the +/// session's TLS material with it. Reading incrementally makes the served size +/// independent of available memory, and a ranged read serves its window from the +/// same handle instead of copying the slice back out of a full-blob buffer. +async fn serve_blob(state: &RegistryState, headers: &HeaderMap, digest: &str) -> Response { + let (file, total) = match state.store.open_blob(digest) { + Ok(Some(open)) => open, + _ => return blob_unknown(), + }; + let mut file = tokio::fs::File::from_std(file); + + if let Some(range) = headers.get(header::RANGE) { + return match parse_range(range, total) { + Some((start, end)) => { + if tokio::io::AsyncSeekExt::seek(&mut file, io::SeekFrom::Start(start)) + .await + .is_err() + { + return blob_unknown(); + } + // `end` is inclusive, matching Content-Range. + let window = tokio::io::AsyncReadExt::take(file, end - start + 1); + Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header(header::CONTENT_TYPE, "application/octet-stream") + .header(header::ACCEPT_RANGES, "bytes") + .header( + header::CONTENT_RANGE, + format!("bytes {start}-{end}/{total}"), + ) + .header(DOCKER_CONTENT_DIGEST, digest) + .body(Body::from_stream(ReaderStream::new(window))) + .expect("range response is always valid") + } + None => Response::builder() + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(header::CONTENT_RANGE, format!("bytes */{total}")) + .body(Body::empty()) + .expect("unsatisfiable-range response is always valid"), + }; + } + + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/octet-stream") + .header(header::ACCEPT_RANGES, "bytes") + .header(header::CONTENT_LENGTH, total) + .header(DOCKER_CONTENT_DIGEST, digest) + .body(Body::from_stream(ReaderStream::new(file))) + .expect("full-blob response is always valid") +} + +/// Read the `mediaType` field from a stored manifest, falling back to the +/// default OCI image-manifest type when it is absent or the body is not JSON. +/// +/// A multi-arch image index carries its own index `mediaType` +/// (`application/vnd.oci.image.index.v1+json` or the Docker manifest-list type), +/// so echoing it back is what lets the engine recognize an index versus a +/// single-platform manifest. +fn manifest_media_type(bytes: &[u8]) -> String { + serde_json::from_slice::(bytes) + .ok() + .and_then(|v| { + v.get("mediaType") + .and_then(|m| m.as_str()) + .map(str::to_string) + }) + .unwrap_or_else(|| DEFAULT_MANIFEST_MEDIA_TYPE.to_string()) +} + +/// Whether `reference` is shaped like an OCI digest (`:`), +/// distinguishing a by-digest reference from a tag. +fn looks_like_digest(reference: &str) -> bool { + match reference.split_once(':') { + Some((algorithm, hex)) => { + !algorithm.is_empty() + && !hex.is_empty() + && algorithm.chars().all(|c| c.is_ascii_alphanumeric()) + && hex.chars().all(|c| c.is_ascii_hexdigit()) + } + None => false, + } +} + +/// Parse a single-range `Range: bytes=...` header against a resource of +/// `total` bytes, returning an inclusive `(start, end)` clamped to bounds, or +/// `None` when the range is malformed, multi-range, or unsatisfiable. +fn parse_range(value: &HeaderValue, total: u64) -> Option<(u64, u64)> { + let spec = value.to_str().ok()?.strip_prefix("bytes=")?; + // Multi-range is not supported; treat it as unsatisfiable. + if spec.contains(',') { + return None; + } + let (start_s, end_s) = spec.split_once('-')?; + + if start_s.is_empty() { + // Suffix range: the last `n` bytes. + let suffix: u64 = end_s.parse().ok()?; + if suffix == 0 || total == 0 { + return None; + } + let len = suffix.min(total); + return Some((total - len, total - 1)); + } + + let start: u64 = start_s.parse().ok()?; + if start >= total { + return None; + } + let end = if end_s.is_empty() { + total - 1 + } else { + end_s.parse::().ok()?.min(total - 1) + }; + if end < start { + return None; + } + Some((start, end)) +} + +fn manifest_unknown() -> Response { + oci_error( + StatusCode::NOT_FOUND, + "MANIFEST_UNKNOWN", + "manifest unknown", + ) +} + +fn blob_unknown() -> Response { + oci_error(StatusCode::NOT_FOUND, "BLOB_UNKNOWN", "blob unknown") +} + +/// Build an OCI error response (`{"errors":[{"code","message"}]}`). +fn oci_error(status: StatusCode, code: &str, message: &str) -> Response { + let body = serde_json::json!({ "errors": [{ "code": code, "message": message }] }).to_string(); + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body)) + .expect("oci error response is always valid") +} + +#[cfg(test)] +mod read { + use super::*; + use crate::utils::container_dev::store::BlobStore; + use sha2::{Digest as _, Sha256}; + use tempfile::TempDir; + + /// Compute the OCI digest (`sha256:`) of `bytes`. + fn digest_of(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let hex: String = hasher + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("sha256:{hex}") + } + + /// Start the ungated read route assembly over a fresh per-project store and + /// return the base URL plus a handle keeping the store's temp dir alive. + /// + /// These tests exercise read semantics (ranges, media types, dedup); the + /// Bearer read/control gate on the public [`read_router`] is covered by the + /// `container_dev::auth` tests, so the assembly is served ungated here. + async fn spawn() -> (String, Arc, TempDir) { + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let app = read_routes(store.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{addr}"), store, dir) + } + + /// A minimal single-platform image manifest. + fn image_manifest() -> Vec { + serde_json::to_vec(&serde_json::json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "size": 7, + }, + "layers": [], + })) + .unwrap() + } + + /// A multi-arch image index referencing per-platform manifests. + fn image_index() -> Vec { + serde_json::to_vec(&serde_json::json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "size": 100, + "platform": { "architecture": "amd64", "os": "linux" }, + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "size": 100, + "platform": { "architecture": "arm64", "os": "linux" }, + }, + ], + })) + .unwrap() + } + + #[tokio::test] + async fn v2_base_returns_200_with_api_version() { + let (base, _store, _dir) = spawn().await; + let resp = reqwest::get(format!("{base}/v2/")).await.unwrap(); + assert_eq!(resp.status().as_u16(), 200); + assert_eq!( + resp.headers() + .get("docker-distribution-api-version") + .and_then(|h| h.to_str().ok()), + Some("registry/2.0"), + ); + } + + #[tokio::test] + async fn manifest_by_tag_returns_stored_manifest() { + let (base, store, _dir) = spawn().await; + let manifest = image_manifest(); + let digest = digest_of(&manifest); + store.write_blob(&digest, &manifest).unwrap(); + store.set_tag("my-app", "dev", &digest).unwrap(); + + let resp = reqwest::get(format!("{base}/v2/my-app/manifests/dev")) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + assert_eq!( + resp.headers() + .get("content-type") + .and_then(|h| h.to_str().ok()), + Some("application/vnd.oci.image.manifest.v1+json"), + ); + assert_eq!( + resp.headers() + .get("docker-content-digest") + .and_then(|h| h.to_str().ok()), + Some(digest.as_str()), + ); + assert_eq!(resp.bytes().await.unwrap().as_ref(), manifest.as_slice()); + } + + #[tokio::test] + async fn manifest_by_digest_returns_stored_manifest() { + let (base, store, _dir) = spawn().await; + let manifest = image_manifest(); + let digest = digest_of(&manifest); + store.write_blob(&digest, &manifest).unwrap(); + + // No tag set: fetching by digest must still resolve. + let resp = reqwest::get(format!("{base}/v2/my-app/manifests/{digest}")) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + assert_eq!( + resp.headers() + .get("docker-content-digest") + .and_then(|h| h.to_str().ok()), + Some(digest.as_str()), + ); + assert_eq!(resp.bytes().await.unwrap().as_ref(), manifest.as_slice()); + } + + #[tokio::test] + async fn multi_arch_index_is_served_with_index_media_type() { + let (base, store, _dir) = spawn().await; + let index = image_index(); + let digest = digest_of(&index); + store.write_blob(&digest, &index).unwrap(); + store.set_tag("my-app", "multi", &digest).unwrap(); + + let resp = reqwest::get(format!("{base}/v2/my-app/manifests/multi")) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + // The index media type — not a single-platform manifest type — is what + // lets the engine recognize a multi-arch index and pick a platform. + assert_eq!( + resp.headers() + .get("content-type") + .and_then(|h| h.to_str().ok()), + Some("application/vnd.oci.image.index.v1+json"), + ); + assert_eq!(resp.bytes().await.unwrap().as_ref(), index.as_slice()); + } + + #[tokio::test] + async fn unknown_manifest_returns_404() { + let (base, _store, _dir) = spawn().await; + let resp = reqwest::get(format!("{base}/v2/my-app/manifests/nope")) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 404); + } + + #[tokio::test] + async fn full_blob_get_returns_whole_body() { + let (base, store, _dir) = spawn().await; + let blob: Vec = (0u8..=255).collect(); + let digest = digest_of(&blob); + store.write_blob(&digest, &blob).unwrap(); + + let resp = reqwest::get(format!("{base}/v2/my-app/blobs/{digest}")) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + assert_eq!( + resp.headers() + .get("docker-content-digest") + .and_then(|h| h.to_str().ok()), + Some(digest.as_str()), + ); + assert_eq!(resp.bytes().await.unwrap().as_ref(), blob.as_slice()); + } + + #[tokio::test] + async fn ranged_blob_get_returns_206_with_only_the_requested_bytes() { + let (base, store, _dir) = spawn().await; + let blob: Vec = (0u8..=255).collect(); + let digest = digest_of(&blob); + store.write_blob(&digest, &blob).unwrap(); + + let resp = reqwest::Client::new() + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .header(reqwest::header::RANGE, "bytes=10-19") + .send() + .await + .unwrap(); + + assert_eq!( + resp.status().as_u16(), + 206, + "a Range request must return 206" + ); + assert_eq!( + resp.headers() + .get("content-range") + .and_then(|h| h.to_str().ok()), + Some("bytes 10-19/256"), + ); + let body = resp.bytes().await.unwrap(); + // Exactly the requested slice, not the whole blob. + assert_eq!(body.len(), 10); + assert_eq!(body.as_ref(), &blob[10..=19]); + } + + #[tokio::test] + async fn suffix_range_returns_last_bytes() { + let (base, store, _dir) = spawn().await; + let blob: Vec = (0u8..=99).collect(); + let digest = digest_of(&blob); + store.write_blob(&digest, &blob).unwrap(); + + let resp = reqwest::Client::new() + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .header(reqwest::header::RANGE, "bytes=-5") + .send() + .await + .unwrap(); + + assert_eq!(resp.status().as_u16(), 206); + assert_eq!( + resp.headers() + .get("content-range") + .and_then(|h| h.to_str().ok()), + Some("bytes 95-99/100"), + ); + assert_eq!(resp.bytes().await.unwrap().as_ref(), &blob[95..=99]); + } + + #[tokio::test] + async fn unsatisfiable_range_returns_416() { + let (base, store, _dir) = spawn().await; + let blob: Vec = vec![1, 2, 3, 4]; + let digest = digest_of(&blob); + store.write_blob(&digest, &blob).unwrap(); + + let resp = reqwest::Client::new() + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .header(reqwest::header::RANGE, "bytes=100-200") + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 416); + } + + /// Collect a response body, returning how many data frames it arrived in. + /// + /// Frame count is the discriminator these two tests need: a `Body` built from + /// one `Vec` carries exactly one data frame however large it is, while a + /// body streamed off disk carries one per read. Driven through `oneshot` + /// rather than a real request on purpose - over TCP the chunk boundaries a + /// client observes come from coalescing, not from how the handler built the + /// body, so counting socket reads would prove nothing about buffering. + async fn collect_frames(body: Body) -> (usize, Vec) { + use futures_util::StreamExt as _; + + let mut frames = 0usize; + let mut bytes: Vec = Vec::new(); + let mut stream = body.into_data_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.expect("body stream must not error"); + frames += 1; + bytes.extend_from_slice(&chunk); + } + (frames, bytes) + } + + /// A blob is read off disk incrementally, never sized into one allocation. + /// + /// The store accepts a layer larger than host RAM (the upload streams + /// straight to disk), so a read path that buffers the whole object turns one + /// oversized push into an OOM on every subsequent pull. Fails with `1 frame` + /// if `serve_blob` returns to reading the blob whole. + #[tokio::test] + async fn a_large_blob_is_streamed_in_many_frames_not_one_allocation() { + use tower::ServiceExt as _; + + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + // Comfortably more than one read, small enough to keep the test fast. + let blob: Vec = (0..512 * 1024).map(|i| (i % 251) as u8).collect(); + let digest = digest_of(&blob); + store.write_blob(&digest, &blob).unwrap(); + + let resp = read_routes(store) + .oneshot( + axum::http::Request::get(format!("/v2/my-app/blobs/{digest}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + + let (frames, collected) = collect_frames(resp.into_body()).await; + assert!( + frames > 1, + "a {}-byte blob must arrive in more than one frame; got {frames}, \ + so the whole blob was buffered into a single allocation", + blob.len() + ); + assert_eq!(collected, blob, "streaming must deliver the blob unchanged"); + } + + /// A ranged read streams the requested window instead of copying it. + /// + /// Slicing a buffered blob allocated the window a second time on top of the + /// whole object, so this covers the doubling specifically rather than only + /// the full-body path. + #[tokio::test] + async fn a_large_range_is_streamed_rather_than_copied() { + use tower::ServiceExt as _; + + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let blob: Vec = (0..512 * 1024).map(|i| (i % 251) as u8).collect(); + let digest = digest_of(&blob); + store.write_blob(&digest, &blob).unwrap(); + + let resp = read_routes(store) + .oneshot( + axum::http::Request::get(format!("/v2/my-app/blobs/{digest}")) + .header(header::RANGE, "bytes=1024-401023") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 206); + assert_eq!( + resp.headers() + .get(header::CONTENT_RANGE) + .and_then(|h| h.to_str().ok()), + Some("bytes 1024-401023/524288"), + ); + + let (frames, collected) = collect_frames(resp.into_body()).await; + assert!( + frames > 1, + "a 400000-byte range must arrive in more than one frame; got {frames}" + ); + assert_eq!( + collected, + &blob[1024..=401023], + "the range must be byte-exact" + ); + } + + #[tokio::test] + async fn head_manifest_returns_headers_without_body() { + let (base, store, _dir) = spawn().await; + let manifest = image_manifest(); + let digest = digest_of(&manifest); + store.write_blob(&digest, &manifest).unwrap(); + store.set_tag("my-app", "dev", &digest).unwrap(); + + let resp = reqwest::Client::new() + .head(format!("{base}/v2/my-app/manifests/dev")) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + assert_eq!( + resp.headers() + .get("docker-content-digest") + .and_then(|h| h.to_str().ok()), + Some(digest.as_str()), + ); + assert!( + resp.bytes().await.unwrap().is_empty(), + "HEAD carries no body" + ); + } + + #[tokio::test] + async fn unknown_blob_returns_404() { + let (base, _store, _dir) = spawn().await; + let missing = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + let resp = reqwest::get(format!("{base}/v2/my-app/blobs/{missing}")) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 404); + } +} + +#[cfg(test)] +mod write_auth { + use super::*; + use crate::utils::container_dev::auth::{WriteToken, WRITE_USERNAME}; + use crate::utils::container_dev::store::BlobStore; + use tempfile::TempDir; + + const WRITE_TOKEN: &str = "write-token-secret"; + + /// Start the WRITE router (gated by [`WRITE_TOKEN`]) over a fresh + /// per-project store; return the base URL plus a handle keeping the store's + /// temp dir alive. + async fn spawn() -> (String, Arc, TempDir) { + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let app = write_router(store.clone(), WriteToken::new(WRITE_TOKEN)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{addr}"), store, dir) + } + + /// A minimal single-platform image manifest. + fn manifest() -> Vec { + serde_json::to_vec(&serde_json::json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "size": 7, + }, + "layers": [], + })) + .unwrap() + } + + #[tokio::test] + async fn valid_basic_write_token_stores_a_manifest() { + let (base, store, _dir) = spawn().await; + let body = manifest(); + let digest = compute_digest(&body); + + let resp = reqwest::Client::new() + .put(format!("{base}/v2/my-app/manifests/dev")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .body(body.clone()) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status().as_u16(), + 201, + "a valid Basic write credential must be accepted on a write route" + ); + // Observable side effect: the manifest is stored and the tag points at it. + assert!(store.has_blob(&digest).unwrap()); + assert_eq!( + store.resolve_tag("my-app", "dev").unwrap().as_deref(), + Some(digest.as_str()) + ); + } + + #[tokio::test] + async fn a_blob_larger_than_the_default_body_limit_is_accepted() { + // A real image layer exceeds axum's 2 MiB DefaultBodyLimit. The write + // listener buffers the body as `Bytes`, so without lifting the cap every + // `docker push` of a non-trivial image 413s ("Failed to buffer the request + // body: length limit exceeded") and the push fails mid-stream. The store + // persists blobs to disk, so a large upload must be accepted. + let (base, store, _dir) = spawn().await; + let blob = vec![0x5au8; 3 * 1024 * 1024]; // 3 MiB > the 2 MiB default + let digest = compute_digest(&blob); + + let resp = reqwest::Client::new() + .post(format!("{base}/v2/my-app/blobs/uploads/?digest={digest}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .body(blob.clone()) + .send() + .await + .unwrap(); + + assert_ne!( + resp.status().as_u16(), + 413, + "a >2 MiB blob must not be rejected with 413 by the default body limit" + ); + assert_eq!( + resp.status().as_u16(), + 201, + "a monolithic blob upload with a valid write credential must be created" + ); + assert!( + store.has_blob(&digest).unwrap(), + "the oversized blob must be persisted to the on-disk store" + ); + } + + #[tokio::test] + async fn bearer_read_control_token_is_rejected_on_a_write_route() { + let (base, store, _dir) = spawn().await; + let body = manifest(); + let digest = compute_digest(&body); + + // The device-delivered read/control token is a Bearer value. Presenting + // it (even with the same secret string) on a write route must be + // refused — this closes the H-A compromised-device write class. + let resp = reqwest::Client::new() + .put(format!("{base}/v2/my-app/manifests/dev")) + .bearer_auth(WRITE_TOKEN) + .body(body) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status().as_u16(), + 401, + "the Bearer read/control token must not authorize a write" + ); + assert!( + !store.has_blob(&digest).unwrap(), + "a rejected write must not persist any content" + ); + assert_eq!(store.resolve_tag("my-app", "dev").unwrap(), None); + } + + #[tokio::test] + async fn anonymous_write_is_rejected() { + let (base, store, _dir) = spawn().await; + let body = manifest(); + let digest = compute_digest(&body); + + let resp = reqwest::Client::new() + .put(format!("{base}/v2/my-app/manifests/dev")) + .body(body) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status().as_u16(), + 401, + "an anonymous write must be refused" + ); + assert!(!store.has_blob(&digest).unwrap()); + assert_eq!(store.resolve_tag("my-app", "dev").unwrap(), None); + } + + #[tokio::test] + async fn wrong_password_basic_credential_is_rejected() { + let (base, store, _dir) = spawn().await; + let body = manifest(); + let digest = compute_digest(&body); + + let resp = reqwest::Client::new() + .put(format!("{base}/v2/my-app/manifests/dev")) + .basic_auth(WRITE_USERNAME, Some("not-the-write-token")) + .body(body) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status().as_u16(), + 401, + "a Basic credential with the wrong password must be refused" + ); + assert!(!store.has_blob(&digest).unwrap()); + } + + #[tokio::test] + async fn write_path_issues_a_basic_challenge_not_bearer() { + let (base, _store, _dir) = spawn().await; + + // An anonymous request to the write listener must challenge with Basic; + // a Bearer/token-endpoint challenge on the write path is a falsifier. + let resp = reqwest::get(format!("{base}/v2/")).await.unwrap(); + assert_eq!(resp.status().as_u16(), 401); + let challenge = resp + .headers() + .get("www-authenticate") + .and_then(|h| h.to_str().ok()) + .unwrap_or("") + .to_ascii_lowercase(); + assert!( + challenge.starts_with("basic"), + "the write path must issue a Basic challenge, got {challenge:?}" + ); + assert!( + !challenge.contains("bearer"), + "the write path must NOT issue a Bearer challenge" + ); + } + + #[tokio::test] + async fn valid_token_completes_a_monolithic_blob_upload() { + let (base, store, _dir) = spawn().await; + let blob = b"a-container-layer".to_vec(); + let digest = compute_digest(&blob); + + let resp = reqwest::Client::new() + .post(format!("{base}/v2/my-app/blobs/uploads/?digest={digest}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .body(blob.clone()) + .send() + .await + .unwrap(); + + assert_eq!( + resp.status().as_u16(), + 201, + "a monolithic blob upload must complete" + ); + assert_eq!( + store.read_blob(&digest).unwrap().as_deref(), + Some(blob.as_slice()), + "the uploaded blob bytes must be stored verbatim" + ); + } + + #[tokio::test] + async fn head_dedup_probe_is_gated_and_reports_presence() { + let (base, store, _dir) = spawn().await; + let blob = b"already-present".to_vec(); + let digest = compute_digest(&blob); + store.write_blob(&digest, &blob).unwrap(); + + // The dedup HEAD is a write-listener route, so it is auth-gated too. + let anon = reqwest::Client::new() + .head(format!("{base}/v2/my-app/blobs/{digest}")) + .send() + .await + .unwrap(); + assert_eq!( + anon.status().as_u16(), + 401, + "an anonymous dedup probe must be refused" + ); + + let authed = reqwest::Client::new() + .head(format!("{base}/v2/my-app/blobs/{digest}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .send() + .await + .unwrap(); + assert_eq!( + authed.status().as_u16(), + 200, + "an authenticated dedup probe must report an existing blob present" + ); + assert_eq!( + authed + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()), + Some(blob.len().to_string().as_str()), + "the probe must report the blob's real size, statted rather than read" + ); + } + + // A multi-chunk layer pushes end to end and lands intact. + // + // What this does NOT prove, stated plainly so nobody reads it as more than + // it is: the >2 GiB single-request case that motivated the streaming change + // is not reachable in a test - allocating one is impractical, and toggling + // `DefaultBodyLimit` cannot simulate it either, because that limit is + // consumed by the `Bytes` extractor and these handlers now take `Body`. + // The absence of a per-request ceiling is a property of the handler + // signatures, not something an assertion here can demonstrate. + // + // What it does prove is the chunked path over the streaming handlers: six + // PATCHes, a finalizing PUT, and the exact bytes in the store afterwards. + // `an_upload_stages_to_disk_not_memory` covers the peak-memory half. + #[tokio::test] + async fn a_multi_chunk_layer_pushes_end_to_end() { + let (base, store, _dir) = spawn().await; + let client = reqwest::Client::new(); + + let chunk = vec![0x5au8; 512 * 1024]; + let chunks = 6; // 3 MiB total + let mut whole = Vec::new(); + for _ in 0..chunks { + whole.extend_from_slice(&chunk); + } + let digest = compute_digest(&whole); + + let opened = client + .post(format!("{base}/v2/my-app/blobs/uploads/")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .send() + .await + .unwrap(); + assert_eq!(opened.status().as_u16(), 202); + let location = opened + .headers() + .get(header::LOCATION) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + for i in 0..chunks { + let patched = client + .patch(format!("{base}{location}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .body(chunk.clone()) + .send() + .await + .unwrap(); + assert_eq!( + patched.status().as_u16(), + 202, + "chunk {i} must be accepted, not 413'd" + ); + } + + let done = client + .put(format!("{base}{location}?digest={digest}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .send() + .await + .unwrap(); + assert_eq!(done.status().as_u16(), 201, "the layer must finalize"); + assert_eq!( + store.blob_size(&digest).unwrap(), + Some(whole.len() as u64), + "the whole layer must have landed in the store" + ); + } + + // The bytes must reach DISK as chunks arrive, not accumulate in memory. + // + // The previous version of this asserted `upload.written()` - a plain u64 + // counter incremented in `append` - and claimed that proved write-through. It + // did not: rewriting `BlobUpload` to accumulate into a `Vec` and only + // `write_all` inside `finish()` reintroduces exactly the whole-layer-in-memory + // behaviour this round removed, and `written`/`hasher` update identically, so + // the assertion passed unchanged. + // + // Stat the staging file mid-upload instead. That is the property, and it is + // the one a Vec-accumulating implementation cannot fake. + #[test] + fn an_upload_writes_each_chunk_through_to_disk() { + let dir = TempDir::new().unwrap(); + let store = BlobStore::at(dir.path(), "proj").expect("store opens"); + let mut upload = store.begin_blob_upload().expect("upload opens"); + + let uploads = store.root().join("uploads"); + let staged = || -> u64 { + std::fs::read_dir(&uploads) + .map(|entries| { + entries + .filter_map(Result::ok) + .filter_map(|e| e.metadata().ok()) + .filter(|m| m.is_file()) + .map(|m| m.len()) + .sum() + }) + .unwrap_or(0) + }; + + upload.append(&[1u8; 4096]).unwrap(); + let after_first = staged(); + assert_eq!( + after_first, 4096, + "the first chunk must be on disk before finish(), found {after_first} bytes" + ); + + upload.append(&[2u8; 4096]).unwrap(); + let after_second = staged(); + assert_eq!( + after_second, 8192, + "the staging file must GROW as chunks arrive, found {after_second} bytes" + ); + } + + // A mismatched digest must be refused AND leave no blob behind. Split from the + // test above, which previously asserted both in one body. + #[test] + fn a_mismatched_digest_is_refused_and_stores_nothing() { + let dir = TempDir::new().unwrap(); + let store = BlobStore::at(dir.path(), "proj").expect("store opens"); + let mut upload = store.begin_blob_upload().expect("upload opens"); + upload.append(&[7u8; 128]).unwrap(); + + let wrong = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + assert!( + !upload.finish(wrong).unwrap(), + "a mismatched digest must be rejected" + ); + assert_eq!( + store.blob_size(wrong).unwrap(), + None, + "the rejected upload must leave no blob behind" + ); + } + + // A mid-transfer failure must leave the session resumable. + // + // The buffered implementation got this for free: the `Bytes` extractor + // rejected a truncated body before the handler ran, so the session and its + // accepted chunks survived and `Range` told the client where to resume. + // Streaming takes the session OUT of the map to protect it from a concurrent + // sweep, which means an early return drops it - unlinking the staging file and + // every chunk already accepted, so the layer restarts from byte 0. Fails if + // the error-path re-insert is removed. + // + // Driven through `oneshot` with a body stream that errors, rather than a real + // truncated request: a short HTTP body just makes the server wait for bytes + // that never arrive, which hangs instead of failing. + #[tokio::test] + async fn a_failed_chunk_stream_leaves_the_session_resumable() { + use axum::body::Bytes; + use base64::Engine as _; + use tower::ServiceExt as _; + + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let uploads = Arc::new(UploadSessions::default()); + let router = + write_router_with_uploads(store, WriteToken::new(WRITE_TOKEN), uploads.clone()); + + let creds = base64::engine::general_purpose::STANDARD + .encode(format!("{WRITE_USERNAME}:{WRITE_TOKEN}")); + let auth = format!("Basic {creds}"); + + // Open a session and land one good chunk. + let opened = router + .clone() + .oneshot( + axum::http::Request::post("/v2/my-app/blobs/uploads/") + .header(header::AUTHORIZATION, &auth) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(opened.status().as_u16(), 202); + let location = opened + .headers() + .get(header::LOCATION) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + let first = router + .clone() + .oneshot( + axum::http::Request::patch(&location) + .header(header::AUTHORIZATION, &auth) + .body(Body::from(vec![0x11u8; 4096])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(first.status().as_u16(), 202); + assert_eq!( + uploads.inner.lock().unwrap().len(), + 1, + "the session should be live after a good chunk" + ); + + // Now a chunk whose stream fails partway - the transport failure this + // guards against. + let failing = Body::from_stream(futures_util::stream::iter(vec![ + Ok::<_, std::io::Error>(Bytes::from_static(&[0x22u8; 2048])), + Err(std::io::Error::other("connection reset mid-chunk")), + ])); + let broken = router + .clone() + .oneshot( + axum::http::Request::patch(&location) + .header(header::AUTHORIZATION, &auth) + .body(failing) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + broken.status().as_u16(), + 400, + "a failed stream should be reported to the client" + ); + + // The decisive assertion: the session survived, so the client can resume + // instead of restarting the layer. + assert_eq!( + uploads.inner.lock().unwrap().len(), + 1, + "the session must survive a failed chunk stream, not be discarded" + ); + let resumed = router + .oneshot( + axum::http::Request::patch(&location) + .header(header::AUTHORIZATION, &auth) + .body(Body::from(vec![0x33u8; 4096])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + resumed.status().as_u16(), + 202, + "a resumed chunk must be accepted, not 404 BLOB_UPLOAD_UNKNOWN" + ); + } + + // An upload that opens a session and never finalizes it must not pin its + // buffer forever: opening a later session reclaims it. Asserted on the map + // directly because the leak is invisible from the wire - the abandoned + // session returns nothing, it just occupies memory. + /// A staging upload backed by a throwaway store, for tests that build + /// `UploadSession`s by hand. + fn staging_upload(dir: &TempDir) -> BlobUpload { + BlobStore::at(dir.path(), "proj") + .expect("store opens") + .begin_blob_upload() + .expect("staging upload opens") + } + + #[test] + fn abandoned_upload_sessions_are_evicted_when_a_new_one_opens() { + let dir = TempDir::new().unwrap(); + let mut sessions = HashMap::new(); + sessions.insert( + "abandoned".to_string(), + UploadSession { + upload: staging_upload(&dir), + touched: Instant::now() - UPLOAD_SESSION_TTL - Duration::from_secs(1), + }, + ); + sessions.insert( + "in-progress".to_string(), + UploadSession { + // Older than the TTL as a whole, but still receiving chunks - a + // slow link must not be mistaken for an abandoned push. + upload: staging_upload(&dir), + touched: Instant::now(), + }, + ); + + evict_expired(&mut sessions, Instant::now()); + + assert!( + !sessions.contains_key("abandoned"), + "a session untouched past the TTL must be reclaimed" + ); + assert!( + sessions.contains_key("in-progress"), + "a session still receiving chunks must survive eviction" + ); + } + + // The TTL boundary itself: one tick either side must decide differently. + // Without an injected clock this is unreachable, which is what let the + // previous version of the PATCH test below assert nothing. + #[test] + fn eviction_turns_on_the_ttl_boundary() { + let dir = TempDir::new().unwrap(); + let base = Instant::now(); + let mut sessions = HashMap::new(); + sessions.insert( + "just-inside".to_string(), + UploadSession { + upload: staging_upload(&dir), + touched: base, + }, + ); + evict_expired( + &mut sessions, + base + UPLOAD_SESSION_TTL - Duration::from_millis(1), + ); + assert!( + sessions.contains_key("just-inside"), + "a session one tick inside the TTL must survive" + ); + + evict_expired(&mut sessions, base + UPLOAD_SESSION_TTL); + assert!( + sessions.is_empty(), + "a session at the TTL must be reclaimed" + ); + } + + /// Serve the write router over a session map the test also holds, so it can + /// age a live session to the TTL boundary instead of waiting ten minutes. + async fn spawn_with_uploads() -> (String, Arc, TempDir) { + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let uploads = Arc::new(UploadSessions::default()); + let app = write_router_with_uploads(store, WriteToken::new(WRITE_TOKEN), uploads.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{addr}"), uploads, dir) + } + + /// Backdate every live session by `age`, simulating time passing without + /// spending it. + fn age_sessions(uploads: &UploadSessions, age: Duration) { + let mut sessions = uploads.inner.lock().unwrap(); + for session in sessions.values_mut() { + session.touched -= age; + } + } + + // A PATCH must refresh the session clock, so a multi-chunk transfer spanning + // more than the TTL is not evicted out from under an active client. + // + // The falsifier is the backdating: the session is pushed past the TTL, then + // PATCHed, then swept. It survives ONLY if patch_route actually rewrote + // `touched`. Deleting that one line fails this test - which the previous + // version of it did not, because its sweep ran milliseconds after the POST + // and would have passed with the mechanism removed entirely. + #[tokio::test] + async fn patching_a_session_refreshes_its_ttl() { + let (base, uploads, _dir) = spawn_with_uploads().await; + let client = reqwest::Client::new(); + + let opened = client + .post(format!("{base}/v2/my-app/blobs/uploads/")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .send() + .await + .unwrap(); + assert_eq!(opened.status().as_u16(), 202); + let location = opened + .headers() + .get(header::LOCATION) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + // Push the session past the eviction boundary, then send a chunk. + age_sessions(&uploads, UPLOAD_SESSION_TTL + Duration::from_secs(60)); + let patched = client + .patch(format!("{base}{location}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .body(b"chunk".to_vec()) + .send() + .await + .unwrap(); + assert_eq!( + patched.status().as_u16(), + 202, + "an in-flight chunk must be accepted" + ); + + // Opening a second session runs the sweep. The first is only safe if the + // PATCH above reset its clock. + client + .post(format!("{base}/v2/other-app/blobs/uploads/")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .send() + .await + .unwrap(); + + let still_there = client + .patch(format!("{base}{location}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .body(b"more".to_vec()) + .send() + .await + .unwrap(); + assert_eq!( + still_there.status().as_u16(), + 202, + "the PATCH must have refreshed the TTL, so the sweep must not evict it" + ); + } + + // The other half: a session that is NOT patched past the boundary really is + // swept, and the client learns via 404 rather than silently succeeding. + // Together with the test above this pins both directions of the mechanism. + #[tokio::test] + async fn an_aged_session_is_swept_and_its_next_chunk_404s() { + let (base, uploads, _dir) = spawn_with_uploads().await; + let client = reqwest::Client::new(); + + let opened = client + .post(format!("{base}/v2/my-app/blobs/uploads/")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .send() + .await + .unwrap(); + let location = opened + .headers() + .get(header::LOCATION) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + age_sessions(&uploads, UPLOAD_SESSION_TTL + Duration::from_secs(60)); + + // No PATCH this time - the sweep on the next POST should reclaim it. + client + .post(format!("{base}/v2/other-app/blobs/uploads/")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .send() + .await + .unwrap(); + + let gone = client + .patch(format!("{base}{location}")) + .basic_auth(WRITE_USERNAME, Some(WRITE_TOKEN)) + .body(b"chunk".to_vec()) + .send() + .await + .unwrap(); + assert_eq!( + gone.status().as_u16(), + 404, + "an abandoned session must be reclaimed, and its next chunk rejected" + ); + } +} + +#[cfg(test)] +mod bulk_listener { + use super::*; + use crate::utils::container_dev::store::BlobStore; + use crate::utils::container_dev::tls::DevSession; + use sha2::{Digest as _, Sha256}; + use std::net::SocketAddr; + use tempfile::TempDir; + + const RUNTIME: &str = "dev-runtime"; + + /// Compute the OCI digest (`sha256:`) of `bytes`. + fn digest_of(bytes: &[u8]) -> String { + let hex: String = Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("sha256:{hex}") + } + + /// Bind the dedicated bulk read listener over a fresh session's TLS material + /// and a per-project store seeded with `blob`. + /// + /// Returns the loopback `https://` base URL, the minted session (whose CA + /// cert the client pins and whose read/control token it presents), the live + /// listener handle (kept alive by the caller), the seeded blob digest, and + /// the temp-dir guard. + async fn spawn_bulk(blob: &[u8]) -> (String, DevSession, BulkListener, String, TempDir) { + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let digest = digest_of(blob); + store.write_blob(&digest, blob).unwrap(); + + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let listener = BulkListener::bind( + SocketAddr::from(([127, 0, 0, 1], 0)), + store, + session.read_token.clone(), + session.tls.server_config(), + ) + .await + .expect("bulk listener binds"); + let base = format!("https://127.0.0.1:{}", listener.local_addr().port()); + (base, session, listener, digest, dir) + } + + /// A reqwest client that trusts ONLY the session CA, so it validates the + /// leaf's `127.0.0.1` IP SAN and rejects any other chain. + fn tls_client(session: &DevSession) -> reqwest::Client { + let ca = reqwest::Certificate::from_pem(session.tls.ca_cert_pem().as_bytes()) + .expect("session CA cert parses"); + reqwest::Client::builder() + .add_root_certificate(ca) + .build() + .expect("TLS client builds") + } + + #[tokio::test] + async fn token_gated_pull_succeeds_over_the_dedicated_bulk_tls_listener() { + let blob: Vec = (0u8..=255).collect(); + let (base, session, listener, digest, _dir) = spawn_bulk(&blob).await; + + // The listener owns a real bound loopback socket (its listener identity, + // design H-1): port 0 was resolved to a concrete port. + assert_ne!( + listener.local_addr().port(), + 0, + "the bulk listener must bind a concrete socket" + ); + + let resp = tls_client(&session) + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .bearer_auth(session.read_token.secret()) + .send() + .await + .expect("bulk pull request completes"); + + assert_eq!( + resp.status().as_u16(), + 200, + "a Bearer-token-gated blob pull must succeed over the dedicated bulk TLS listener" + ); + assert_eq!( + resp.headers() + .get("docker-content-digest") + .and_then(|h| h.to_str().ok()), + Some(digest.as_str()), + ); + // The exact blob bytes come back over the dedicated socket. + assert_eq!(resp.bytes().await.unwrap().as_ref(), blob.as_slice()); + } + + #[tokio::test] + async fn bulk_pull_without_the_read_token_is_refused_before_any_bytes() { + let blob = b"a-container-layer".to_vec(); + let (base, session, _listener, digest, _dir) = spawn_bulk(&blob).await; + + let resp = tls_client(&session) + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .send() + .await + .expect("anonymous bulk pull request completes"); + + assert_eq!( + resp.status().as_u16(), + 401, + "an anonymous pull on the bulk listener must be refused (fail-closed pre-stream)" + ); + // Fail-closed: the challenge is a bare Bearer, and no blob body leaks. + let challenge = resp + .headers() + .get("www-authenticate") + .and_then(|h| h.to_str().ok()) + .unwrap_or("") + .to_ascii_lowercase(); + assert!( + challenge.starts_with("bearer"), + "the bulk listener must challenge with Bearer, got {challenge:?}" + ); + assert_ne!( + resp.bytes().await.unwrap().as_ref(), + blob.as_slice(), + "a refused pull must not stream the blob body" + ); + } + + #[tokio::test] + async fn bulk_bytes_travel_as_http_not_a_control_websocket_frame() { + let blob: Vec = (0u8..200).collect(); + let (base, session, _listener, digest, _dir) = spawn_bulk(&blob).await; + + // Attempt a WebSocket upgrade on the bulk socket while pulling a blob. + // The dedicated bulk listener carries ONLY the OCI read router (no WS / + // control route), so the engine gets the blob as a plain HTTP body and + // NEVER a `101 Switching Protocols` control stream. This is the D9/H-1 + // guarantee: bulk transfers never share the control WS byte stream, so a + // large pull cannot head-of-line-block a control frame. + let resp = tls_client(&session) + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .bearer_auth(session.read_token.secret()) + .header(reqwest::header::CONNECTION, "Upgrade") + .header(reqwest::header::UPGRADE, "websocket") + .header("sec-websocket-version", "13") + .header("sec-websocket-key", "dGhlIHNhbXBsZSBub25jZQ==") + .send() + .await + .expect("bulk pull with upgrade headers completes"); + + assert_ne!( + resp.status().as_u16(), + 101, + "the bulk listener must NEVER switch to a WebSocket/control stream" + ); + assert_eq!( + resp.status().as_u16(), + 200, + "the blob must be served as an ordinary HTTP body on the bulk socket" + ); + assert_eq!( + resp.bytes().await.unwrap().as_ref(), + blob.as_slice(), + "the full blob must arrive over HTTP, not a WS frame" + ); + } + + #[tokio::test] + async fn bulk_listener_binds_a_socket_distinct_from_the_write_listener() { + // The bulk read listener seeded with a blob. + let blob = b"layer-bytes".to_vec(); + let (bulk_base, session, bulk, digest, _dir) = spawn_bulk(&blob).await; + + // A separate WRITE listener (task 3.3) on its own socket. The three- + // listener model (design D9/H-1) gives each route class its OWN socket: + // this is the write leg, distinct from the bulk read leg. + let write_dir = TempDir::new().unwrap(); + let write_store = Arc::new(BlobStore::at(write_dir.path(), "wproj").expect("store opens")); + let write_app = write_router(write_store, session.write_token.clone()); + let write_tcp = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let write_addr = write_tcp.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(write_tcp, write_app).await.unwrap(); + }); + + // Distinct sockets: the bulk read listener and the write listener never + // share a socket, so a device handed only the bulk endpoint cannot reach + // the write listener. + assert_ne!( + bulk.local_addr(), + write_addr, + "the bulk read listener and the write listener must be distinct sockets" + ); + + // The bulk socket serves the token-gated read. + let bulk_ok = tls_client(&session) + .get(format!("{bulk_base}/v2/my-app/blobs/{digest}")) + .bearer_auth(session.read_token.secret()) + .send() + .await + .expect("bulk read completes"); + assert_eq!(bulk_ok.status().as_u16(), 200); + + // The write socket is a different route class: it refuses an anonymous + // request with a Basic challenge, never a Bearer read/control token. + let write_anon = reqwest::get(format!("http://{write_addr}/v2/")) + .await + .expect("write listener responds"); + assert_eq!( + write_anon.status().as_u16(), + 401, + "the write listener gates on the Basic write token, not the read token" + ); + let write_challenge = write_anon + .headers() + .get("www-authenticate") + .and_then(|h| h.to_str().ok()) + .unwrap_or("") + .to_ascii_lowercase(); + assert!( + write_challenge.starts_with("basic"), + "the write listener must issue a Basic challenge, got {write_challenge:?}" + ); + } + + #[tokio::test] + async fn vm_write_listener_terminates_tls_not_plaintext() { + // On the VM push path the guest reaches the write listener via the QEMU + // host alias 10.0.2.2 (NOT a docker-trusted 127.0.0.0/8 loopback), so the + // listener MUST terminate the per-project leaf TLS its delivered certs.d + // CA pins (design A2/H4). A plaintext HTTP write listener there is a bug: + // the guest daemon, configured for HTTPS via certs.d, could not push. + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "wproj").expect("store opens")); + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let tcp = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = tcp.local_addr().unwrap().port(); + let _task = serve_write_router_tls( + tcp, + session.tls.server_config(), + store, + session.write_token.clone(), + ); + + // HTTPS with the session CA: the TLS handshake succeeds and an + // unauthenticated write is refused with a Basic challenge (not a + // transport error). + let resp = tls_client(&session) + .get(format!("https://127.0.0.1:{port}/v2/")) + .send() + .await + .expect("an HTTPS request over the TLS write listener completes"); + assert_eq!( + resp.status().as_u16(), + 401, + "the TLS write listener must gate an unauthenticated write with 401" + ); + + // A plaintext HTTP request to the same port must fail at the transport + // layer, proving the listener speaks TLS. Before the fix the write + // listener served plaintext and this would return an HTTP status. + let plain = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/v2/")) + .send() + .await; + assert!( + plain.is_err(), + "a plaintext HTTP request to the TLS write listener must fail at the transport, \ + got {plain:?}" + ); + } +} diff --git a/src/utils/container_dev/store.rs b/src/utils/container_dev/store.rs new file mode 100644 index 00000000..b0b05540 --- /dev/null +++ b/src/utils/container_dev/store.rs @@ -0,0 +1,1277 @@ +//! Per-project content-addressed blob store for Container Dev Mode. +//! +//! Blobs are keyed by their OCI digest (`:`) and deduplicated +//! on write: a digest that is already present is never stored a second time. +//! Tags map to the digest of the manifest they point at. +//! +//! The store is namespaced per project at +//! `~/.avocado/container-dev//registry/`, so `prune` in one project +//! can never sweep another project's blobs (design D8, M5). Garbage collection +//! runs only on `prune`/`down` (never mid-push, never on a timer), retains any +//! blob referenced by a currently-tagged manifest, and `prune` refuses while an +//! `up` session is live (design D8, threat-model M2). + +use sha2::{Digest as _, Sha256}; +use std::collections::HashSet; +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use directories::BaseDirs; +use tempfile::NamedTempFile; +use thiserror::Error; + +/// Errors returned by the blob store. +#[derive(Debug, Error)] +pub enum StoreError { + /// The user's home directory could not be resolved. + #[error("could not resolve the home directory for the container-dev store")] + NoHome, + /// A digest was not of the form `:` with a safe, + /// non-traversing algorithm and hex component. + #[error("invalid digest {0:?}: expected `:`")] + InvalidDigest(String), + /// A tag name contained a path separator or traversal component. + #[error("invalid tag {0:?}: must not contain a path separator or `..`")] + InvalidTag(String), + /// A repository name was empty or contained a traversal component. + #[error("invalid repository name {0:?}: must be non-empty and must not contain `..`")] + InvalidName(String), + /// `prune` was invoked while an `up` session was still live. + #[error( + "prune refused: an `avocado container dev up` session is running for this project \ + (it may be serving a pull or staging an upload); run `avocado container dev down` first" + )] + PruneWhileSessionLive, + /// A streamed blob grew past [`MAX_BLOB_BYTES`]. + #[error("blob exceeds the {limit}-byte ceiling (reached {attempted} bytes)")] + BlobTooLarge { limit: u64, attempted: u64 }, + /// An underlying filesystem operation failed. + #[error(transparent)] + Io(#[from] io::Error), +} + +/// Whether an `avocado container dev up` session is live for this project. +/// +/// Passed into [`BlobStore::prune`] by the caller, which is the only layer that +/// can answer it: liveness is proved by the session flock, and `prune` runs in a +/// different process from `up`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SessionActivity { + /// An `up` is running: it may be serving a pull or staging an upload. + Live, + /// No `up` holds this project's session. + Idle, +} + +/// Ceiling on a manifest read during garbage collection. +/// +/// GC walks tags to manifests to their children, and has to read a blob to find +/// out whether it IS a manifest - `manifest_child_digests` has no media-type +/// filter, so every reachable layer digest lands on the same worklist. Reading +/// those whole sized one allocation by the largest reachable layer: a 6 GB layer, +/// well under [`MAX_BLOB_BYTES`], allocated 6 GB for a `from_slice` that was +/// always going to fail, and OOM-killed `prune` on an 8 GB host with the store +/// left un-GC'd. A manifest or index is kilobytes; anything past this ceiling is +/// not one, so it can be skipped without reading it. +const MAX_MANIFEST_BYTES: u64 = 4 * 1024 * 1024; + +/// Ceiling on a single streamed blob. +/// +/// Not a memory bound - blobs stream to disk and are never held whole. This +/// bounds DISK, which streaming otherwise left completely unbounded: without it +/// a write-token holder can PATCH forever, or an accidental oversized layer can +/// fill the filesystem and take down every process on the host. 32 GiB is far +/// above any layer a dev loop produces and far below a disk-filling one. +/// +/// It also bounds the read side as a side effect: `serve_blob` still loads a blob +/// whole to serve it, so a blob that cannot be stored cannot later OOM the pull. +pub const MAX_BLOB_BYTES: u64 = 32 * 1024 * 1024 * 1024; + +/// A per-project content-addressed blob store. +/// +/// Rooted at `/container-dev//registry/` with a +/// `blobs//` layout for content and +/// `manifests/tags//` pointers holding the digest of the tagged +/// manifest. See [`BlobStore::tag_path`] for why the name is escaped into a +/// single segment. +/// +/// # Upgrading over an existing store +/// +/// Tags used to live flat at `manifests/tags/`, with no repository name. +/// There is no migration and none is possible: the name is exactly the +/// information the old layout did not record, so a flat `dev` cannot be placed +/// under the repository it belonged to. [`Self::list_tags`] skips non-directory +/// entries, so pre-existing flat tags are invisible to it - which means the +/// first `prune`/`down` after upgrading sweeps their manifests and layers as +/// unreferenced. +/// +/// That is a deliberate wipe rather than an oversight. It costs one re-push, +/// which `up` and `sync` both perform anyway, and the alternative - guessing a +/// name for an orphaned tag - would resurrect it under the wrong repository. +pub struct BlobStore { + root: PathBuf, + /// Count of [`Self::read_blob`] calls. + /// + /// Exists for one test: the GC must decide a layer-sized blob has no + /// children WITHOUT reading it, and the outcome is identical either way - + /// the layer stays reachable because the manifest's `layers` array already + /// put it on the worklist. Asserting on the outcome therefore passes with + /// the size guard deleted, so the mechanism needs its own witness. + blob_reads: AtomicUsize, +} + +impl BlobStore { + /// Open the store for `project` under the user's home directory + /// (`~/.avocado/container-dev//registry/`). + pub fn for_project(project: &str) -> Result { + let base = BaseDirs::new().ok_or(StoreError::NoHome)?; + let avocado_dir = base.home_dir().join(".avocado"); + Self::at(&avocado_dir, project) + } + + /// Open the store for `project` rooted under an explicit `avocado_dir` + /// (the `~/.avocado` equivalent). + /// + /// The per-project namespacing is derived here from `project`, which is + /// what keeps one project's store isolated from another's. + pub fn at(avocado_dir: &Path, project: &str) -> Result { + let root = avocado_dir + .join("container-dev") + .join(project) + .join("registry"); + fs::create_dir_all(root.join("blobs"))?; + fs::create_dir_all(root.join("manifests").join("tags"))?; + Ok(Self { + root, + blob_reads: AtomicUsize::new(0), + }) + } + + /// The registry root directory backing this store. + pub fn root(&self) -> &Path { + &self.root + } + + /// Write `bytes` under `digest`. + /// + /// If a blob with this digest is already present the write is skipped and + /// `Ok(false)` is returned (dedup); otherwise the blob is written + /// atomically and `Ok(true)` is returned. Because the on-disk path is + /// derived solely from the digest, a repeated digest can never produce a + /// second copy. + pub fn write_blob(&self, digest: &str, bytes: &[u8]) -> Result { + let path = self.blob_path(digest)?; + if path.exists() { + return Ok(false); + } + let dir = path + .parent() + .expect("blob path always has a parent under the store root"); + fs::create_dir_all(dir)?; + let mut tmp = NamedTempFile::new_in(dir)?; + tmp.write_all(bytes)?; + tmp.flush()?; + tmp.persist(&path).map_err(|e| e.error)?; + Ok(true) + } + + /// Begin a streaming blob upload. + /// + /// The returned [`BlobUpload`] writes straight to a temp file under the + /// store and hashes as it goes, so a layer never has to exist in memory. It + /// is what lets the write path accept a multi-gigabyte layer without a body + /// limit standing in for a memory bound - the OCI upload protocol is + /// chunked, so the same handle spans the `POST`/`PATCH`/`PUT` sequence. + pub fn begin_blob_upload(&self) -> Result { + let dir = self.root.join("uploads"); + fs::create_dir_all(&dir)?; + Ok(BlobUpload { + file: NamedTempFile::new_in(&dir)?, + hasher: Sha256::new(), + written: 0, + blobs_root: self.root.clone(), + }) + } + + /// Report whether a blob with `digest` is present (the registry HEAD path). + pub fn has_blob(&self, digest: &str) -> Result { + Ok(self.blob_path(digest)?.exists()) + } + + /// Report the size in bytes of the blob under `digest`, or `None` when + /// absent. + /// + /// The registry's HEAD dedup probe needs only the length, and `docker push` + /// issues one HEAD per layer before uploading anything. Answering that from + /// the directory entry keeps a multi-hundred-MB layer off the heap on the + /// hot push path, which `read_blob` could not. + pub fn blob_size(&self, digest: &str) -> Result, StoreError> { + let path = self.blob_path(digest)?; + match fs::metadata(&path) { + Ok(meta) => Ok(Some(meta.len())), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// Read the bytes stored under `digest`, or `None` when absent. + pub fn read_blob(&self, digest: &str) -> Result>, StoreError> { + let path = self.blob_path(digest)?; + self.blob_reads.fetch_add(1, Ordering::Relaxed); + match fs::read(&path) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// How many times [`Self::read_blob`] has been called on this store. + /// + /// Lets a test assert that the GC never pulled a layer-sized blob into + /// memory, which no assertion on the swept set can distinguish. + #[cfg(test)] + pub fn blob_read_count(&self) -> usize { + self.blob_reads.load(Ordering::Relaxed) + } + + /// Open a stored blob for incremental reading, with its size. + /// + /// The counterpart to [`Self::read_blob`] for objects whose size is not + /// bounded by anything the host chose. An upload streams to disk without + /// buffering, so the store can hold a layer larger than host RAM; reading one + /// back with [`Self::read_blob`] would then size a single allocation by the + /// blob and take the process down on every pull. Manifests keep using + /// `read_blob` - they are capped, and the media-type sniff needs the bytes. + /// + /// Returns a plain [`std::fs::File`] rather than an async handle so the store + /// stays synchronous; the caller wraps it for whichever runtime it serves on. + pub fn open_blob(&self, digest: &str) -> Result, StoreError> { + let path = self.blob_path(digest)?; + match fs::File::open(&path) { + Ok(file) => { + let len = file.metadata()?.len(); + Ok(Some((file, len))) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// Point `name`'s `tag` at the manifest identified by `manifest_digest`. + /// + /// The pointer is written atomically and overwrites any previous target for + /// that repository's tag. `name` is part of the key, not decoration: two + /// watched images sharing a tag (`api:dev` and `web:dev`, or two untagged + /// refs both defaulting to `latest`) used to overwrite one another's pointer + /// in a flat namespace, so a rebuild of one broadcast the other's digest and + /// the device ran the wrong image under the right service name. + pub fn set_tag(&self, name: &str, tag: &str, manifest_digest: &str) -> Result<(), StoreError> { + // Validate the digest so a tag never points at a malformed target. + parse_digest(manifest_digest)?; + let path = self.tag_path(name, tag)?; + let dir = path + .parent() + .expect("tag path always has a parent under the store root"); + fs::create_dir_all(dir)?; + let mut tmp = NamedTempFile::new_in(dir)?; + tmp.write_all(manifest_digest.as_bytes())?; + tmp.flush()?; + tmp.persist(&path).map_err(|e| e.error)?; + Ok(()) + } + + /// Resolve `name`'s `tag` to the digest of the manifest it points at, or + /// `None` when that repository has no such tag. + pub fn resolve_tag(&self, name: &str, tag: &str) -> Result, StoreError> { + let path = self.tag_path(name, tag)?; + match fs::read_to_string(&path) { + Ok(s) => Ok(Some(s.trim().to_string())), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// Garbage-collect blobs unreferenced by any currently-tagged manifest. + /// + /// This is the ONLY sweep path in the store; it is invoked from `down` + /// (and, via [`prune`](Self::prune), from `prune`) — never from a + /// push/sync and never on a timer. Every blob reachable from a + /// currently-set tag (the manifest, its config, its layers, and, for a + /// multi-arch index, each sub-manifest transitively) is retained; all + /// other blobs are removed. Returns the digests that were swept. + pub fn collect_garbage(&self) -> Result, StoreError> { + let reachable = self.reachable_digests()?; + let mut swept = Vec::new(); + for digest in self.present_blob_digests()? { + if reachable.contains(&digest) { + continue; + } + // Skip anything that is not a well-formed digest rather than + // propagating. `write_blob` stages its NamedTempFile inside + // blobs//, so an `up` SIGKILLed between `new_in` and `persist` + // leaves a `.tmpXXXXXX` there; `present_blob_digests` reconstructs + // it as "sha256:.tmpXXXXXX" and `?` here would make every later + // prune return InvalidDigest and sweep nothing, recoverable only by + // finding the dotfile by hand. `reachable_digests` already skips the + // same error thirty lines down. + let path = match self.blob_path(&digest) { + Ok(path) => path, + Err(StoreError::InvalidDigest(_)) => continue, + Err(e) => return Err(e), + }; + match fs::remove_file(&path) { + Ok(()) => swept.push(digest), + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + } + swept.sort(); + Ok(swept) + } + + /// `prune`: garbage-collect the per-project store, refusing while a device + /// is mid-pull. + /// + /// Single policy (design D8, threat-model M2): GC runs only on + /// `prune`/`down`, retains any blob referenced by a currently-tagged + /// manifest, and `prune` refuses (rather than sweeping a blob a transfer + /// still needs) while an `up` session is live. + /// + /// `session` is supplied by the caller rather than sampled here because the + /// only proof of liveness that holds is the session flock, and `prune` + /// always runs in a DIFFERENT PROCESS from `up`. An earlier in-process + /// counter could not work for that reason - `prune` built its own store, so + /// the counter it read was always zero and the refusal was unreachable no + /// matter what incremented it. + pub fn prune(&self, session: SessionActivity) -> Result, StoreError> { + if session == SessionActivity::Live { + return Err(StoreError::PruneWhileSessionLive); + } + // Sweep abandoned staging files too. `collect_garbage` walks `blobs/` + // only, so nothing in the tree ever looked at `uploads/`. A `NamedTempFile` + // unlinks itself on drop, which covers a clean exit and nothing else: an + // `up` SIGKILLed mid-push (OOM reaper, power loss) leaves its partial + // layer there permanently, and `prune` used to report "swept 0" while + // gigabytes sat in a directory the user had to find by hand. + // Callers that want to report the reclaimed count call `sweep_uploads` + // directly; `prune`'s return stays a digest list, since a staging file was + // never content-addressed and has no digest to name. + self.sweep_uploads()?; + self.collect_garbage() + } + + /// Remove every staged upload file, returning how many were reclaimed. + /// + /// This unlinks the very files `begin_blob_upload` streams into, so it is + /// safe ONLY because [`prune`](Self::prune) refuses while an `up` session is + /// live, and `up` is the only process that stages an upload. The previous + /// justification - that a live upload's file is held by the write router's + /// map in the same process - does not survive `prune` running in a separate + /// process: unlinking mid-`PATCH` leaves `up` writing to an fd with no name, + /// so every remaining chunk answers 202 with a growing Range and the client + /// sees a healthy upload all the way to 100%, only to have the final `PUT` + /// fail in `persist()` with ENOENT after the whole layer moved. + pub fn sweep_uploads(&self) -> Result { + let dir = self.root.join("uploads"); + let entries = match fs::read_dir(&dir) { + Ok(entries) => entries, + // Never opened an upload in this project: nothing to sweep. + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(0), + Err(e) => return Err(e.into()), + }; + let mut removed = 0; + for entry in entries { + let entry = entry?; + if entry.file_type()?.is_file() { + fs::remove_file(entry.path())?; + removed += 1; + } + } + Ok(removed) + } + + /// The set of blob digests reachable from any currently-set tag. + fn reachable_digests(&self) -> Result, StoreError> { + let mut reachable: HashSet = HashSet::new(); + let mut stack: Vec = Vec::new(); + for (name, tag) in self.list_tags()? { + if let Some(manifest_digest) = self.resolve_tag(&name, &tag)? { + stack.push(manifest_digest); + } + } + while let Some(digest) = stack.pop() { + if !reachable.insert(digest.clone()) { + continue; + } + // A manifest is itself stored as a blob; read it and, when it + // parses as a manifest or index, follow its references. An ordinary + // layer blob is not JSON and yields no children. + // + // Size first, bytes second. The worklist carries layer digests as + // well as manifest ones, so reading unconditionally sized a single + // allocation by the largest reachable layer - see MAX_MANIFEST_BYTES. + // `blob_size` answers from the directory entry. + match self.blob_size(&digest) { + Ok(Some(len)) if len > MAX_MANIFEST_BYTES => continue, + Ok(Some(_)) => {} + Ok(None) => continue, + Err(StoreError::InvalidDigest(_)) => continue, + Err(e) => return Err(e), + } + let bytes = match self.read_blob(&digest) { + Ok(Some(bytes)) => bytes, + Ok(None) => continue, + Err(StoreError::InvalidDigest(_)) => continue, + Err(e) => return Err(e), + }; + for child in manifest_child_digests(&bytes) { + if !reachable.contains(&child) { + stack.push(child); + } + } + } + Ok(reachable) + } + + /// All blob digests (`:`) currently present on disk. + fn present_blob_digests(&self) -> Result, StoreError> { + let blobs_root = self.root.join("blobs"); + let mut digests = Vec::new(); + for entry in walkdir::WalkDir::new(&blobs_root) + .into_iter() + .filter_map(Result::ok) + { + if !entry.file_type().is_file() { + continue; + } + // Layout is blobs//; reconstruct `:`. + let hex = entry.file_name().to_string_lossy().into_owned(); + let algorithm = entry + .path() + .parent() + .and_then(Path::file_name) + .map(|s| s.to_string_lossy().into_owned()); + if let Some(algorithm) = algorithm { + digests.push(format!("{algorithm}:{hex}")); + } + } + Ok(digests) + } + + /// Every `(repository, tag)` pair currently present in the store. + fn list_tags(&self) -> Result, StoreError> { + let tags_dir = self.root.join("manifests").join("tags"); + let mut tags = Vec::new(); + let names = match fs::read_dir(&tags_dir) { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(tags), + Err(e) => return Err(e.into()), + }; + for name_entry in names { + let name_entry = name_entry?; + if !name_entry.file_type()?.is_dir() { + continue; + } + let name = unescape_name(&name_entry.file_name().to_string_lossy()); + for tag_entry in fs::read_dir(name_entry.path())? { + let tag_entry = tag_entry?; + if tag_entry.file_type()?.is_file() { + tags.push(( + name.clone(), + tag_entry.file_name().to_string_lossy().into_owned(), + )); + } + } + } + Ok(tags) + } + + fn blob_path(&self, digest: &str) -> Result { + let (algorithm, hex) = parse_digest(digest)?; + Ok(self.root.join("blobs").join(algorithm).join(hex)) + } + + /// `manifests/tags//`. + /// + /// A repository name legitimately contains `/` (`library/alpine`), which the + /// tag component must never contain, so the name is escaped into a single + /// path segment rather than nested - keeping `list_tags`'s walk one level + /// deep and leaving no way for a name to collide with the tag beneath it. + fn tag_path(&self, name: &str, tag: &str) -> Result { + if tag.is_empty() || tag.contains('/') || tag.contains('\\') || tag.contains("..") { + return Err(StoreError::InvalidTag(tag.to_string())); + } + if name.is_empty() || name.contains('\\') || name.contains("..") { + return Err(StoreError::InvalidName(name.to_string())); + } + Ok(self + .root + .join("manifests") + .join("tags") + .join(escape_name(name)) + .join(tag)) + } +} + +/// Fold a repository name into one filesystem path segment. +/// +/// `/` is the only character an OCI name may carry that a path segment may not, +/// and `%` is escaped first so the mapping stays injective - without that, +/// `a%2Fb` and `a/b` would collide on disk. +fn escape_name(name: &str) -> String { + name.replace('%', "%25").replace('/', "%2F") +} + +/// Inverse of [`escape_name`]. +fn unescape_name(segment: &str) -> String { + segment.replace("%2F", "/").replace("%25", "%") +} + +/// Split an OCI digest into its `(algorithm, hex)` components, rejecting +/// anything that could traverse the filesystem. +/// An in-progress blob upload, streamed to disk and hashed as it arrives. +/// +/// Spans one OCI upload session: `POST` opens it, each `PATCH` appends, and +/// `PUT` finishes it. Nothing is buffered - the bytes go to a temp file under +/// the store and the digest is computed incrementally, so the peak memory of a +/// push is a chunk rather than a layer. +/// +/// Dropping without [`BlobUpload::finish`] discards the temp file, so an +/// abandoned upload leaves nothing behind. +pub struct BlobUpload { + file: NamedTempFile, + hasher: Sha256, + written: u64, + blobs_root: PathBuf, +} + +impl BlobUpload { + /// Append a chunk, refusing to grow the staged blob past + /// [`MAX_BLOB_BYTES`]. + /// + /// Streaming removed the accidental ceiling the old buffered path had - a + /// request over the body limit was rejected with nothing written - and + /// replaced it with none at all. Without a cap, a holder of the write token + /// (on the VM push path, the QEMU guest) can PATCH indefinitely across as + /// many sessions as it likes and fill the filesystem, taking every process + /// on the host down with it. An honest oversized layer from a bad `COPY` + /// reaches the same place by accident. + /// + /// Enforced here rather than in the handler because this is the one funnel + /// every write goes through: monolithic POST, chunked PATCH, and the final + /// PUT chunk all land on `append`. + pub fn append(&mut self, bytes: &[u8]) -> Result<(), StoreError> { + let would_be = self.written.saturating_add(bytes.len() as u64); + if would_be > MAX_BLOB_BYTES { + return Err(StoreError::BlobTooLarge { + limit: MAX_BLOB_BYTES, + attempted: would_be, + }); + } + self.file.write_all(bytes)?; + self.hasher.update(bytes); + self.written += bytes.len() as u64; + Ok(()) + } + + /// Bytes accepted so far (the OCI `Range` header the client expects). + pub fn written(&self) -> u64 { + self.written + } + + /// Verify the streamed content hashes to `expected` and move it into place. + /// + /// The digest is checked against what was actually written rather than + /// trusted from the client, exactly as the buffered path did - the + /// difference is only where the bytes lived while it was computed. A + /// mismatch discards the temp file and reports `false`. + pub fn finish(mut self, expected: &str) -> Result { + self.file.flush()?; + let hex: String = self + .hasher + .clone() + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + if format!("sha256:{hex}") != expected { + return Ok(false); + } + let (algorithm, hex) = parse_digest(expected)?; + let path = self.blobs_root.join("blobs").join(algorithm).join(hex); + if path.exists() { + // Already stored: dedup, and let the temp file drop. + return Ok(true); + } + let dir = path + .parent() + .expect("blob path always has a parent under the store root"); + fs::create_dir_all(dir)?; + self.file.persist(&path).map_err(|e| e.error)?; + Ok(true) + } +} + +fn parse_digest(digest: &str) -> Result<(&str, &str), StoreError> { + let invalid = || StoreError::InvalidDigest(digest.to_string()); + let (algorithm, hex) = digest.split_once(':').ok_or_else(invalid)?; + if algorithm.is_empty() || hex.is_empty() { + return Err(invalid()); + } + if !algorithm.chars().all(|c| c.is_ascii_alphanumeric()) { + return Err(invalid()); + } + if !hex.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(invalid()); + } + Ok((algorithm, hex)) +} + +/// Extract the child blob digests a manifest or image index references: for a +/// multi-arch index, each sub-manifest; for a single-platform image manifest, +/// its config and layers. A body that is not a recognizable manifest (an +/// ordinary layer blob) yields no children. +fn manifest_child_digests(bytes: &[u8]) -> Vec { + let Ok(value) = serde_json::from_slice::(bytes) else { + return Vec::new(); + }; + let mut children = Vec::new(); + // Multi-arch index / Docker manifest list. + if let Some(manifests) = value.get("manifests").and_then(|m| m.as_array()) { + for m in manifests { + if let Some(digest) = m.get("digest").and_then(|v| v.as_str()) { + children.push(digest.to_string()); + } + } + } + // Single-platform image manifest: config + layers. + if let Some(digest) = value + .get("config") + .and_then(|c| c.get("digest")) + .and_then(|v| v.as_str()) + { + children.push(digest.to_string()); + } + if let Some(layers) = value.get("layers").and_then(|l| l.as_array()) { + for layer in layers { + if let Some(digest) = layer.get("digest").and_then(|v| v.as_str()) { + children.push(digest.to_string()); + } + } + } + children +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + const DIGEST_A: &str = + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const DIGEST_B: &str = + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + fn store_in(dir: &TempDir, project: &str) -> BlobStore { + BlobStore::at(dir.path(), project).expect("store opens") + } + + // Streaming removed the accidental size ceiling the buffered path had and + // replaced it with none at all, so a write-token holder - or an accidental + // oversized layer - could fill the filesystem. Fails if the cap in `append` + // is removed. + #[test] + fn append_refuses_to_grow_a_blob_past_the_ceiling() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "proj"); + let mut upload = store.begin_blob_upload().expect("upload opens"); + + // Pretend most of the ceiling is already staged so one ordinary chunk + // crosses it, asserting the boundary without writing 32 GiB. + upload.written = MAX_BLOB_BYTES - 8; + upload + .append(&[0u8; 8]) + .expect("landing exactly on the ceiling is allowed"); + assert_eq!(upload.written(), MAX_BLOB_BYTES); + + let err = upload + .append(&[0u8; 1]) + .expect_err("one byte past the ceiling must be refused"); + assert!( + matches!(err, StoreError::BlobTooLarge { .. }), + "expected BlobTooLarge, got {err:?}" + ); + } + + // Nothing in the tree ever looked at `uploads/`, so an `up` killed mid-push + // left its partial layer on disk permanently while `prune` reported sweeping + // nothing. Fails if the sweep is removed from `prune`. + #[test] + fn prune_reclaims_a_staging_file_left_by_a_killed_upload() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "proj"); + + // An upload that never finished and never dropped cleanly - leaking the + // handle stops `NamedTempFile`'s unlink-on-drop, which is what SIGKILL + // does. + let mut upload = store.begin_blob_upload().expect("upload opens"); + upload.append(&[0xabu8; 4096]).unwrap(); + std::mem::forget(upload); + + let uploads = store.root().join("uploads"); + let staged = std::fs::read_dir(&uploads).unwrap().count(); + assert_eq!(staged, 1, "the staging file should be on disk"); + + store.prune(SessionActivity::Idle).expect("prune succeeds"); + + let left = std::fs::read_dir(&uploads).unwrap().count(); + assert_eq!( + left, 0, + "prune must reclaim abandoned staging files, {left} left" + ); + } + + /// Count regular files under the store's `blobs/` tree. + fn blob_file_count(store: &BlobStore) -> usize { + walkdir::WalkDir::new(store.root().join("blobs")) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .count() + } + + #[test] + fn store_path_is_per_project_not_global() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + let expected = dir + .path() + .join("container-dev") + .join("alpha") + .join("registry"); + assert_eq!(store.root(), expected.as_path()); + // The project name must appear in the path so two projects cannot + // collide on one directory. + assert!(store.root().components().any(|c| c.as_os_str() == "alpha")); + } + + #[test] + fn writing_the_same_digest_twice_stores_one_copy() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + let first = store.write_blob(DIGEST_A, b"layer-bytes").unwrap(); + assert!(first, "first write of a new digest stores the blob"); + + let second = store.write_blob(DIGEST_A, b"layer-bytes").unwrap(); + assert!(!second, "a repeated digest write must be deduplicated"); + + assert_eq!( + blob_file_count(&store), + 1, + "an existing-digest write must not store a second copy" + ); + assert_eq!( + store.read_blob(DIGEST_A).unwrap().as_deref(), + Some(&b"layer-bytes"[..]) + ); + } + + #[test] + fn dedup_does_not_clobber_existing_bytes_on_a_racing_rewrite() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + assert!(store.write_blob(DIGEST_A, b"original").unwrap()); + // A second write for the same digest is a no-op even if the caller + // passes different bytes; the stored content is unchanged. + assert!(!store.write_blob(DIGEST_A, b"different").unwrap()); + assert_eq!( + store.read_blob(DIGEST_A).unwrap().as_deref(), + Some(&b"original"[..]) + ); + } + + #[test] + fn head_reports_present_only_for_written_digests() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + assert!( + !store.has_blob(DIGEST_A).unwrap(), + "an unwritten digest must report absent" + ); + store.write_blob(DIGEST_A, b"data").unwrap(); + assert!( + store.has_blob(DIGEST_A).unwrap(), + "HEAD for an existing digest must report present" + ); + assert!( + !store.has_blob(DIGEST_B).unwrap(), + "a different, unwritten digest must still report absent" + ); + } + + #[test] + fn one_projects_blobs_are_invisible_to_another_project() { + let dir = TempDir::new().unwrap(); + let alpha = store_in(&dir, "alpha"); + let beta = store_in(&dir, "beta"); + + alpha.write_blob(DIGEST_A, b"alpha-only").unwrap(); + + assert!( + alpha.has_blob(DIGEST_A).unwrap(), + "alpha stored its own blob" + ); + assert!( + !beta.has_blob(DIGEST_A).unwrap(), + "beta must not see alpha's blob (per-project namespacing)" + ); + assert_eq!(blob_file_count(&beta), 0); + assert_ne!(alpha.root(), beta.root()); + } + + #[test] + fn tag_points_at_a_manifest_digest() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + assert_eq!(store.resolve_tag("my-app", "dev").unwrap(), None); + store.set_tag("my-app", "dev", DIGEST_A).unwrap(); + assert_eq!( + store.resolve_tag("my-app", "dev").unwrap().as_deref(), + Some(DIGEST_A) + ); + + // Retagging overwrites the pointer, it does not append. + store.set_tag("my-app", "dev", DIGEST_B).unwrap(); + assert_eq!( + store.resolve_tag("my-app", "dev").unwrap().as_deref(), + Some(DIGEST_B) + ); + } + + #[test] + fn tags_are_isolated_per_project() { + let dir = TempDir::new().unwrap(); + let alpha = store_in(&dir, "alpha"); + let beta = store_in(&dir, "beta"); + + alpha.set_tag("my-app", "dev", DIGEST_A).unwrap(); + assert_eq!(beta.resolve_tag("my-app", "dev").unwrap(), None); + } + + #[test] + fn malformed_digests_are_rejected() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + for bad in [ + "noscheme", + "sha256:", + ":abcd", + "sha256:zzzz", + "sha256:aa/bb", + ] { + assert!( + matches!( + store.write_blob(bad, b"x"), + Err(StoreError::InvalidDigest(_)) + ), + "digest {bad:?} must be rejected" + ); + assert!(matches!( + store.has_blob(bad), + Err(StoreError::InvalidDigest(_)) + )); + } + } + + #[test] + fn digest_with_path_traversal_cannot_escape_the_store() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + // A traversal attempt in the hex component is rejected outright. + assert!(matches!( + store.write_blob("sha256:../../etc/passwd", b"x"), + Err(StoreError::InvalidDigest(_)) + )); + } + + #[test] + fn tag_names_with_separators_are_rejected() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + for bad in ["../escape", "a/b", "..", ""] { + assert!( + matches!( + store.set_tag("my-app", bad, DIGEST_A), + Err(StoreError::InvalidTag(_)) + ), + "tag {bad:?} must be rejected" + ); + } + } +} + +#[cfg(test)] +mod gc { + use super::*; + use serde_json::json; + use tempfile::TempDir; + + const MANIFEST: &str = + "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + const CONFIG: &str = "sha256:2222222222222222222222222222222222222222222222222222222222222222"; + const LAYER1: &str = "sha256:3333333333333333333333333333333333333333333333333333333333333333"; + const LAYER2: &str = "sha256:4444444444444444444444444444444444444444444444444444444444444444"; + const ORPHAN: &str = "sha256:5555555555555555555555555555555555555555555555555555555555555555"; + const INDEX: &str = "sha256:6666666666666666666666666666666666666666666666666666666666666666"; + const SUBMANIFEST: &str = + "sha256:7777777777777777777777777777777777777777777777777777777777777777"; + + fn store_in(dir: &TempDir, project: &str) -> BlobStore { + BlobStore::at(dir.path(), project).expect("store opens") + } + + /// Bytes of a single-platform image manifest referencing `config` + `layers`. + fn image_manifest(config: &str, layers: &[&str]) -> Vec { + let layers: Vec<_> = layers + .iter() + .map(|l| json!({"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": l})) + .collect(); + json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": config}, + "layers": layers, + }) + .to_string() + .into_bytes() + } + + /// Bytes of a multi-arch image index referencing sub-manifest digests. + fn image_index(submanifests: &[&str]) -> Vec { + let manifests: Vec<_> = submanifests + .iter() + .map( + |m| json!({"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": m}), + ) + .collect(); + json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": manifests, + }) + .to_string() + .into_bytes() + } + + /// Populate a tagged single-platform image (manifest + config + one layer) + /// plus one unreferenced orphan layer. + fn tagged_image_with_orphan(store: &BlobStore) { + store.write_blob(CONFIG, b"config-bytes").unwrap(); + store.write_blob(LAYER1, b"layer-1-bytes").unwrap(); + store + .write_blob(MANIFEST, &image_manifest(CONFIG, &[LAYER1])) + .unwrap(); + store.set_tag("my-app", "dev", MANIFEST).unwrap(); + store.write_blob(ORPHAN, b"unreferenced").unwrap(); + } + + #[test] + fn gc_retains_blobs_referenced_by_a_currently_tagged_manifest() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + tagged_image_with_orphan(&store); + + let swept = store.collect_garbage().unwrap(); + + assert_eq!( + swept, + vec![ORPHAN.to_string()], + "only the unreferenced orphan is swept" + ); + assert!( + store.has_blob(MANIFEST).unwrap(), + "the tagged manifest survives GC" + ); + assert!( + store.has_blob(CONFIG).unwrap(), + "the manifest's config blob survives GC" + ); + assert!( + store.has_blob(LAYER1).unwrap(), + "a layer referenced by the tagged manifest survives GC" + ); + assert!( + !store.has_blob(ORPHAN).unwrap(), + "a blob no tagged manifest references is swept" + ); + } + + #[test] + fn gc_follows_a_multi_arch_index_to_its_sub_manifests() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + // dev -> index -> sub-manifest -> {config, layer1}. layer2 is an orphan. + store.write_blob(CONFIG, b"config").unwrap(); + store.write_blob(LAYER1, b"layer-1").unwrap(); + store + .write_blob(SUBMANIFEST, &image_manifest(CONFIG, &[LAYER1])) + .unwrap(); + store + .write_blob(INDEX, &image_index(&[SUBMANIFEST])) + .unwrap(); + store.set_tag("my-app", "dev", INDEX).unwrap(); + store.write_blob(LAYER2, b"orphan-layer").unwrap(); + + let swept = store.collect_garbage().unwrap(); + + assert_eq!(swept, vec![LAYER2.to_string()]); + for kept in [INDEX, SUBMANIFEST, CONFIG, LAYER1] { + assert!( + store.has_blob(kept).unwrap(), + "{kept} is reachable through the index and must survive" + ); + } + assert!(!store.has_blob(LAYER2).unwrap()); + } + + #[test] + fn a_writing_push_never_sweeps_an_unreferenced_blob() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + // An orphan left from an earlier push. + store.write_blob(ORPHAN, b"unreferenced").unwrap(); + + // A fresh push: new blobs + a retag. GC must NOT run implicitly here. + store.write_blob(CONFIG, b"config").unwrap(); + store.write_blob(LAYER1, b"layer-1").unwrap(); + store + .write_blob(MANIFEST, &image_manifest(CONFIG, &[LAYER1])) + .unwrap(); + store.set_tag("my-app", "dev", MANIFEST).unwrap(); + + assert!( + store.has_blob(ORPHAN).unwrap(), + "a push/sync must never sweep blobs; only prune/down GC does" + ); + + // The explicit GC path is what removes it. + let swept = store.collect_garbage().unwrap(); + assert_eq!(swept, vec![ORPHAN.to_string()]); + assert!(!store.has_blob(ORPHAN).unwrap()); + } + + #[test] + fn prune_refuses_while_an_up_session_is_live() { + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + tagged_image_with_orphan(&store); + + let result = store.prune(SessionActivity::Live); + assert!( + matches!(result, Err(StoreError::PruneWhileSessionLive)), + "prune must refuse while an `up` session is live, got {result:?}" + ); + assert!( + store.has_blob(ORPHAN).unwrap(), + "a refused prune must not sweep anything" + ); + + let swept = store.prune(SessionActivity::Idle).unwrap(); + assert_eq!(swept, vec![ORPHAN.to_string()]); + assert!(!store.has_blob(ORPHAN).unwrap()); + } + + #[test] + fn a_refused_prune_leaves_staged_uploads_alone() { + // The refusal has to cover `uploads/` as well as `blobs/`: sweep_uploads + // unlinks by path, and a live `up` is streaming a PATCH into exactly + // those files. Unlinking one there does not fail the push - `up` keeps + // writing to the open fd and the client sees 100% - it fails the final + // PUT's rename with ENOENT, after the whole layer has moved. + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + tagged_image_with_orphan(&store); + let upload = store.begin_blob_upload().expect("open an upload"); + + let uploads = store.root().join("uploads"); + assert_eq!(std::fs::read_dir(&uploads).unwrap().count(), 1); + + assert!(store.prune(SessionActivity::Live).is_err()); + assert_eq!( + std::fs::read_dir(&uploads).unwrap().count(), + 1, + "a refused prune must leave a live upload's staging file on disk" + ); + drop(upload); + } + + #[test] + fn down_path_gc_takes_no_session_argument() { + // `down` tears the listeners down before sweeping, so its GC is + // unconditional - `collect_garbage` has no session parameter at all. The + // live-session refusal is a `prune`-only guarantee. + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + tagged_image_with_orphan(&store); + + let swept = store.collect_garbage().unwrap(); + assert_eq!(swept, vec![ORPHAN.to_string()]); + } + + #[test] + fn two_repositories_sharing_a_tag_do_not_overwrite_each_other() { + // The flat namespace made `api:dev` and `web:dev` one pointer. A rebuild + // of `api` then broadcast whichever manifest landed last, and the device + // ran web's image as the api service - every frame correct, nothing + // logged. Two untagged refs both defaulting to `latest` collided the + // same way, and GC then swept the loser's layers as unreferenced. + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + store.set_tag("api", "dev", MANIFEST).unwrap(); + store.set_tag("web", "dev", INDEX).unwrap(); + + assert_eq!( + store.resolve_tag("api", "dev").unwrap().as_deref(), + Some(MANIFEST), + "web:dev must not have clobbered api:dev" + ); + assert_eq!( + store.resolve_tag("web", "dev").unwrap().as_deref(), + Some(INDEX) + ); + } + + #[test] + fn a_repository_name_with_a_slash_stays_one_key() { + // `library/alpine` is a legal name and `/` is the one character a tag may + // not carry, so the name is escaped into a single segment. The escape has + // to be injective, or `a%2Fb` and `a/b` would share a pointer. + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + store.set_tag("library/alpine", "dev", MANIFEST).unwrap(); + store.set_tag("library%2Falpine", "dev", INDEX).unwrap(); + + assert_eq!( + store + .resolve_tag("library/alpine", "dev") + .unwrap() + .as_deref(), + Some(MANIFEST) + ); + assert_eq!( + store + .resolve_tag("library%2Falpine", "dev") + .unwrap() + .as_deref(), + Some(INDEX) + ); + } + + #[test] + fn gc_reaches_every_repositorys_tags() { + // list_tags walks a directory per repository now; a walk that only + // looked one level deep would find no tags at all and GC would sweep + // every blob in the store. + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + store.write_blob(MANIFEST, b"api-manifest").unwrap(); + store.write_blob(INDEX, b"web-manifest").unwrap(); + store.write_blob(ORPHAN, b"unreferenced").unwrap(); + store.set_tag("api", "dev", MANIFEST).unwrap(); + store.set_tag("web", "dev", INDEX).unwrap(); + + let swept = store.collect_garbage().unwrap(); + + assert_eq!(swept, vec![ORPHAN.to_string()]); + assert!(store.has_blob(MANIFEST).unwrap(), "api's manifest retained"); + assert!(store.has_blob(INDEX).unwrap(), "web's manifest retained"); + } + + #[test] + fn gc_skips_a_stray_temp_file_instead_of_failing_forever() { + // write_blob stages its NamedTempFile inside blobs//, so an `up` + // SIGKILLed between `new_in` and `persist` leaves a dotfile there. + // present_blob_digests reads it back as "sha256:.tmpAb3xQz"; propagating + // the resulting InvalidDigest made every later prune sweep nothing, with + // no way out but finding the dotfile by hand. + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + tagged_image_with_orphan(&store); + let stray = store.root().join("blobs").join("sha256").join(".tmpAb3xQz"); + std::fs::write(&stray, b"partial").unwrap(); + + let swept = store + .collect_garbage() + .expect("a stray staging file must not fail the sweep"); + + assert_eq!(swept, vec![ORPHAN.to_string()], "the orphan is still swept"); + assert!( + stray.exists(), + "the unparseable entry is skipped, not removed" + ); + } + + #[test] + fn gc_does_not_read_a_layer_sized_blob_to_look_for_children() { + // reachable_digests walks manifest children, and manifest_child_digests + // has no media-type filter - so layer digests land on the same worklist. + // Reading those whole sized one allocation by the largest reachable + // layer, which OOM-killed prune on a host smaller than the image. + let dir = TempDir::new().unwrap(); + let store = store_in(&dir, "alpha"); + + let digest_of = |bytes: &[u8]| { + let hex: String = Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("sha256:{hex}") + }; + + let oversized = vec![0u8; (MAX_MANIFEST_BYTES + 1) as usize]; + let layer_digest = digest_of(&oversized); + store.write_blob(&layer_digest, &oversized).unwrap(); + let manifest = format!(r#"{{"schemaVersion":2,"layers":[{{"digest":"{layer_digest}"}}]}}"#); + let manifest_digest = digest_of(manifest.as_bytes()); + store + .write_blob(&manifest_digest, manifest.as_bytes()) + .unwrap(); + store.set_tag("my-app", "dev", &manifest_digest).unwrap(); + + let reads_before = store.blob_read_count(); + let swept = store.collect_garbage().unwrap(); + let reads = store.blob_read_count() - reads_before; + + // THE assertion. Both blobs stay reachable either way - the layer is on + // the worklist from the manifest's `layers` array, and with the guard + // deleted `serde_json` merely fails on its NUL bytes and yields no + // children - so `swept.is_empty()` plus both-present holds with the guard + // gone. Only the read count separates "decided without reading" from + // "read 4 MiB to decide the same thing". + assert_eq!( + reads, 1, + "the GC must read the manifest and NOT the layer-sized blob; {reads} reads" + ); + + // Still assert the outcome, so a guard that skipped the manifest too - + // losing the edge and sweeping a reachable layer - cannot pass. + assert!( + swept.is_empty(), + "nothing tagged should be swept, got {swept:?}" + ); + assert!(store.has_blob(&layer_digest).unwrap()); + assert!(store.has_blob(&manifest_digest).unwrap()); + } +} diff --git a/src/utils/container_dev/tls.rs b/src/utils/container_dev/tls.rs new file mode 100644 index 00000000..f2845675 --- /dev/null +++ b/src/utils/container_dev/tls.rs @@ -0,0 +1,446 @@ +//! Per-project TLS material and per-session token mint for Container Dev Mode +//! (task 3.6, design D2/D8). +//! +//! At `up` a session mints, in one shot: +//! +//! - a **per-project CA** and a **server leaf** signed by it. The leaf carries +//! SANs `{runtime-name, 10.0.2.2, 127.0.0.1}` so the same certificate serves +//! the native-Linux loopback path, the device loopback proxy, and the +//! `10.0.2.2` avocado-vm guest-push path. `notBefore` is BACKDATED (not the +//! generation instant): an RTC-less device that cold-boots believing it is the +//! Unix epoch (or the firmware build date) must still fall inside the validity +//! window (design D8, cert-lifecycle risk row). +//! - the two structurally distinct session tokens (design D2 split): the +//! host-only Basic [`WriteToken`] and the device-delivered Bearer +//! [`ReadToken`]. +//! +//! The [`ServerConfig`] is built from the leaf and serves the bulk-read and +//! control-WS listeners over TLS (bound by tasks 3.7 / 5.2). +//! +//! CA custody (design D8, threat model): the CA **private key** never leaves the +//! host. It is used only to sign the leaf and is then dropped — this session +//! never retains it — so it cannot be serialized into the bootstrap payload. The +//! device is delivered ONLY the CA certificate (via [`DevSession::bootstrap_payload`]) +//! plus the read/control token; never the write token and never the CA key. + +use std::net::{IpAddr, Ipv4Addr}; +use std::sync::Arc; + +use base64::Engine as _; +use rcgen::{ + BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, Ia5String, IsCa, KeyPair, + KeyUsagePurpose, SanType, +}; +use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; +use rustls::ServerConfig; +use serde::Serialize; +use thiserror::Error; + +use super::auth::{ReadToken, WriteToken}; + +/// The QEMU user-networking host alias a VM guest reaches the host by; the leaf +/// MUST carry this as an IP SAN or the `10.0.2.2` guest-push path fails cert +/// validation (design D2, macOS fast-path risk row). +pub const VM_HOST_IP: Ipv4Addr = Ipv4Addr::new(10, 0, 2, 2); + +/// Loopback address the native-Linux push path and the device-side loopback +/// proxy reach the registry by; carried as an IP SAN on the leaf. +pub const LOOPBACK_IP: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 1); + +/// Backdated `notBefore` (year, month, day). Far enough in the past that a +/// cold-booted RTC-less device's clock lands inside the validity window (D8). +const NOT_BEFORE_YMD: (i32, u8, u8) = (2000, 1, 1); + +/// `notAfter` for the long-lived per-project CA and leaf (D8). +const NOT_AFTER_YMD: (i32, u8, u8) = (2100, 1, 1); + +/// Entropy for each minted token, in bytes (256 bits). +const TOKEN_BYTES: usize = 32; + +/// Errors returned while minting TLS material or tokens. +#[derive(Debug, Error)] +pub enum TlsError { + /// Key/certificate generation via rcgen failed. + #[error("failed to generate container-dev TLS material: {0}")] + Rcgen(#[from] rcgen::Error), + /// Building the rustls server config from the leaf failed (e.g. the private + /// key did not match the certificate). + #[error("failed to build the container-dev rustls server config: {0}")] + Rustls(#[from] rustls::Error), +} + +/// Host-side TLS material for a dev session. +/// +/// Holds the CA **certificate** (PEM, for device/VM delivery) and the rustls +/// [`ServerConfig`] backed by the CA-signed leaf. The CA **private key** is +/// deliberately absent: it is dropped after the leaf is signed, so it cannot be +/// serialized anywhere (design D8). +pub struct TlsMaterial { + ca_cert_pem: String, + server_config: Arc, +} + +impl TlsMaterial { + /// Generate a per-project CA, a CA-signed server leaf carrying the + /// `{runtime-name, 10.0.2.2, 127.0.0.1}` SANs plus every entry in + /// `extra_hosts`, a backdated `notBefore`, and the rustls server config that + /// serves TLS with the leaf. + pub fn generate(runtime_name: &str, extra_hosts: &[String]) -> Result { + let chain = CertChain::build(runtime_name, extra_hosts)?; + + let cert_der = chain.leaf_cert.der().clone(); + let key_der = + PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(chain.leaf_key.serialize_der())); + // `with_single_cert` fails unless the key matches the leaf's public key, + // so a successful build is evidence the leaf and its key are consistent. + let server_config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![cert_der], key_der)?; + + // The CA key (chain.ca_key) is dropped here with `chain`: nothing retains + // it past leaf signing, so it can never reach a payload (D8). + Ok(Self { + ca_cert_pem: chain.ca_cert_pem, + server_config: Arc::new(server_config), + }) + } + + /// The CA certificate in PEM form — the ONLY CA material delivered to a + /// device or VM (design D8). + pub fn ca_cert_pem(&self) -> &str { + &self.ca_cert_pem + } + + /// The rustls server config serving the read/bulk/WS listeners with the leaf. + pub fn server_config(&self) -> Arc { + Arc::clone(&self.server_config) + } +} + +/// A minted dev session: TLS material plus the two D2 tokens. +pub struct DevSession { + /// Per-project CA cert + leaf-backed server config. + pub tls: TlsMaterial, + /// Host-only Basic write token (never delivered to a device). + pub write_token: WriteToken, + /// Device-delivered Bearer read/control token. + pub read_token: ReadToken, +} + +impl DevSession { + /// Mint fresh TLS material and both tokens for a runtime named `runtime_name`. + /// + /// Called once per `up`; the write token rotates hard and the read/control + /// token is what the bootstrap payload delivers to the device (design D5; + /// rotation orchestration lives in task 5.2). + /// `extra_hosts` are the addresses the bootstrap will advertise to the + /// device. They MUST be in the leaf's SAN set: the agent pins the CA and uses + /// rustls' stock verifier, so an advertised address absent from the set fails + /// hostname verification outright. + pub fn mint(runtime_name: &str, extra_hosts: &[String]) -> Result { + Ok(Self { + tls: TlsMaterial::generate(runtime_name, extra_hosts)?, + write_token: WriteToken::new(mint_token()), + read_token: ReadToken::new(mint_token()), + }) + } + + /// The device-delivery payload: the CA certificate and the read/control + /// token, and nothing else. + /// + /// By construction it carries neither the CA private key (which this session + /// never retains) nor the host-only write token — the two things design D8 / + /// D2 forbid ever reaching a device. Task 5.2 writes this to the device + /// writable partition (adding the resolved host endpoint); it owns the file + /// path and endpoint resolution, this owns the field set. + pub fn bootstrap_payload(&self) -> BootstrapPayload { + BootstrapPayload { + ca_cert_pem: self.tls.ca_cert_pem().to_string(), + read_token: self.read_token.secret().to_string(), + } + } +} + +/// The device-delivery subset of a session, serialized into the bootstrap +/// payload written to the device writable partition (task 5.2). +/// +/// Deliberately holds no field for the CA private key or the write token, so a +/// serialization can never leak either (design D8 / D2). +#[derive(Debug, Serialize)] +pub struct BootstrapPayload { + /// The per-project CA certificate the device pins the host TLS leaf against. + pub ca_cert_pem: String, + /// The Bearer read/control token the device authenticates pulls and the + /// control WS with. + pub read_token: String, +} + +/// A freshly generated CA + CA-signed leaf and their keys, held only long enough +/// to build the server config; the CA key is dropped with this value. +struct CertChain { + leaf_cert: rcgen::Certificate, + leaf_key: KeyPair, + ca_cert_pem: String, + // The CA key is intentionally NOT a field: it is consumed by `signed_by` + // inside `build` and never escapes, so it cannot be retained or serialized. +} + +impl CertChain { + fn build(runtime_name: &str, extra_hosts: &[String]) -> Result { + let not_before = rcgen::date_time_ymd(NOT_BEFORE_YMD.0, NOT_BEFORE_YMD.1, NOT_BEFORE_YMD.2); + let not_after = rcgen::date_time_ymd(NOT_AFTER_YMD.0, NOT_AFTER_YMD.1, NOT_AFTER_YMD.2); + + let mut ca_params = CertificateParams::new(Vec::::new())?; + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + ca_params.not_before = not_before; + ca_params.not_after = not_after; + ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + ca_params.distinguished_name.push( + DnType::CommonName, + format!("avocado container-dev CA ({runtime_name})"), + ); + let ca_key = KeyPair::generate()?; + let ca_cert = ca_params.self_signed(&ca_key)?; + + let mut leaf_params = CertificateParams::new(Vec::::new())?; + leaf_params.not_before = not_before; + leaf_params.not_after = not_after; + // The fixed three cover the runtime name, the QEMU user-net host alias + // and loopback. `extra_hosts` adds whatever address THIS `up` is about to + // advertise - typically the auto-detected LAN address of a real board's + // host, which none of the three ever matched. + let mut sans = vec![ + SanType::DnsName(Ia5String::try_from(runtime_name)?), + SanType::IpAddress(IpAddr::V4(VM_HOST_IP)), + SanType::IpAddress(IpAddr::V4(LOOPBACK_IP)), + ]; + for host in extra_hosts { + let host = host.trim(); + if host.is_empty() { + continue; + } + // An IP literal needs an iPAddress SAN; rustls will not match one + // against a dNSName, so classifying by parse rather than by shape is + // what makes both an override hostname and a probed address work. + let san = match host.parse::() { + Ok(ip) => SanType::IpAddress(ip), + Err(_) => SanType::DnsName(Ia5String::try_from(host)?), + }; + if !sans.contains(&san) { + sans.push(san); + } + } + leaf_params.subject_alt_names = sans; + leaf_params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + leaf_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + leaf_params + .distinguished_name + .push(DnType::CommonName, runtime_name.to_string()); + let leaf_key = KeyPair::generate()?; + let leaf_cert = leaf_params.signed_by(&leaf_key, &ca_cert, &ca_key)?; + + Ok(Self { + leaf_cert, + leaf_key, + ca_cert_pem: ca_cert.pem(), + }) + } +} + +/// Mint one URL-safe base64 token from [`TOKEN_BYTES`] of randomness. +fn mint_token() -> String { + use rand::RngExt; + let bytes: [u8; TOKEN_BYTES] = rand::rng().random(); + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + const RUNTIME: &str = "dev-runtime"; + + fn now_unix() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is after the Unix epoch") + .as_secs() as i64 + } + + #[test] + fn leaf_carries_the_10_0_2_2_ip_san_and_loopback_and_runtime_name() { + let chain = CertChain::build(RUNTIME, &[]).expect("cert chain builds"); + let sans = &chain.leaf_cert.params().subject_alt_names; + + assert!( + sans.contains(&SanType::IpAddress(IpAddr::V4(Ipv4Addr::new(10, 0, 2, 2)))), + "the leaf MUST carry the 10.0.2.2 IP SAN (VM guest-push path), got {sans:?}" + ); + assert!( + sans.contains(&SanType::IpAddress(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)))), + "the leaf MUST carry the 127.0.0.1 IP SAN (loopback path), got {sans:?}" + ); + assert!( + sans.contains(&SanType::DnsName( + Ia5String::try_from(RUNTIME).expect("runtime name is a valid DNS SAN") + )), + "the leaf MUST carry the runtime-name DNS SAN, got {sans:?}" + ); + } + + #[test] + fn the_advertised_lan_address_is_in_the_leafs_san_set() { + // The device pins the CA and uses rustls' stock verifier, so the address + // the bootstrap advertises must be a SAN or the handshake fails + // NotValidForName on both listeners. The fixed three never covered a real + // board's host address; the lab only ever exercised 10.0.2.2, which was + // already in the set, so nothing here failed. + let chain = + CertChain::build(RUNTIME, &["192.168.1.50".to_string()]).expect("cert chain builds"); + let sans = &chain.leaf_cert.params().subject_alt_names; + + assert!( + sans.contains(&SanType::IpAddress(IpAddr::V4(Ipv4Addr::new( + 192, 168, 1, 50 + )))), + "the advertised LAN address MUST be an iPAddress SAN, got {sans:?}" + ); + } + + #[test] + fn an_advertised_hostname_becomes_a_dns_san_not_an_ip_one() { + // AVOCADO_CONTAINER_DEV_HOST may be a name rather than a literal, and + // rustls will not match a hostname against an iPAddress SAN - so the + // classification has to be by parse, not by shape. + let chain = + CertChain::build(RUNTIME, &["dev-host.lan".to_string()]).expect("cert chain builds"); + let sans = &chain.leaf_cert.params().subject_alt_names; + + assert!( + sans.contains(&SanType::DnsName( + Ia5String::try_from("dev-host.lan").expect("valid DNS SAN") + )), + "an advertised hostname MUST be a dNSName SAN, got {sans:?}" + ); + } + + #[test] + fn extra_hosts_do_not_displace_the_fixed_sans_or_duplicate_them() { + // 10.0.2.2 is what the QEMU lab advertises, so it arrives as an extra + // host on that path too; adding it twice would be harmless but sloppy, + // and dropping the fixed set would break the VM path outright. + let chain = CertChain::build(RUNTIME, &["10.0.2.2".to_string(), String::new()]) + .expect("cert chain builds"); + let sans = &chain.leaf_cert.params().subject_alt_names; + + let vm_host = SanType::IpAddress(IpAddr::V4(Ipv4Addr::new(10, 0, 2, 2))); + assert_eq!( + sans.iter().filter(|s| **s == vm_host).count(), + 1, + "10.0.2.2 must appear exactly once, got {sans:?}" + ); + assert!( + sans.contains(&SanType::IpAddress(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)))), + "loopback must survive, got {sans:?}" + ); + assert_eq!(sans.len(), 3, "an empty extra host must add nothing"); + } + + #[test] + fn not_before_is_backdated_strictly_before_now() { + let chain = CertChain::build(RUNTIME, &[]).expect("cert chain builds"); + let now = now_unix(); + + let leaf_not_before = chain.leaf_cert.params().not_before.unix_timestamp(); + assert!( + leaf_not_before < now, + "leaf notBefore ({leaf_not_before}) must be backdated strictly before now ({now}), \ + not set to generation time" + ); + assert!( + chain.leaf_cert.params().not_before < chain.leaf_cert.params().not_after, + "leaf notBefore must precede notAfter" + ); + } + + #[test] + fn both_tokens_are_non_empty_and_distinct() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + assert!( + !session.write_token.secret().is_empty(), + "the write token must be non-empty" + ); + assert!( + !session.read_token.secret().is_empty(), + "the read/control token must be non-empty" + ); + assert_ne!( + session.write_token.secret(), + session.read_token.secret(), + "the write and read/control tokens must be distinct secrets" + ); + } + + #[test] + fn each_mint_produces_fresh_tokens() { + let a = DevSession::mint(RUNTIME, &[]).expect("first session mints"); + let b = DevSession::mint(RUNTIME, &[]).expect("second session mints"); + assert_ne!( + a.read_token.secret(), + b.read_token.secret(), + "the read/control token must rotate across mints" + ); + assert_ne!( + a.write_token.secret(), + b.write_token.secret(), + "the write token must rotate across mints" + ); + } + + #[test] + fn bootstrap_payload_carries_the_ca_cert_but_not_the_ca_private_key() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let payload = session.bootstrap_payload(); + let json = serde_json::to_string(&payload).expect("payload serializes"); + + assert!( + json.contains("BEGIN CERTIFICATE"), + "the bootstrap payload must deliver the CA certificate" + ); + assert!( + !json.contains("PRIVATE KEY"), + "the bootstrap payload must NOT contain any private key material (D8)" + ); + assert!( + json.contains(session.read_token.secret()), + "the bootstrap payload must deliver the read/control token" + ); + assert!( + !json.contains(session.write_token.secret()), + "the bootstrap payload must NEVER contain the host-only write token (D2)" + ); + } + + #[test] + fn payload_ca_cert_matches_the_session_ca_cert() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + assert_eq!( + session.bootstrap_payload().ca_cert_pem, + session.tls.ca_cert_pem(), + "the delivered CA cert must be the session's CA cert" + ); + } + + #[test] + fn mint_builds_a_server_config_from_the_leaf() { + // A successful mint means `with_single_cert` accepted the leaf and its + // key, i.e. the rustls server config is backed by the CA-signed leaf. + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let _config = session.tls.server_config(); + assert!( + session.tls.ca_cert_pem().contains("BEGIN CERTIFICATE"), + "the CA cert must be retained in PEM form for delivery" + ); + } +} diff --git a/src/utils/container_dev/watcher.rs b/src/utils/container_dev/watcher.rs new file mode 100644 index 00000000..a9f3a981 --- /dev/null +++ b/src/utils/container_dev/watcher.rs @@ -0,0 +1,1564 @@ +//! Engine-driver watcher + sync orchestration (design D1, D9; task 4.2). +//! +//! On a watched image *tag* event (streamed by [`super::engine`] over the engine +//! CLI subprocess), the watcher syncs the changed layers to the device then +//! notifies it over the control WS. Three behaviors are load-bearing: +//! +//! 1. **PUSH vs INGEST is chosen by EXPLICIT host-topology detection, never +//! emergent** (design D1). PUSH is O(delta) — re-tag + `push` into the +//! embedded registry, so the engine's pull protocol transfers only the +//! changed layers. INGEST is O(full image) — a `docker-daemon:` style export +//! — and is the fallback ONLY where PUSH is unreachable. The selector reads +//! [`is_docker_desktop`]/[`is_vm_routing_active`] (the `avocado deploy` +//! precedent): the avocado-vm and native Linux take PUSH; Docker-Desktop / +//! podman-machine WITHOUT the VM take INGEST. Per D1's note (L-A), a +//! podman-machine is invisible to both selectors, so it lands in the INGEST +//! bucket by virtue of `is_docker_desktop()` being true on macOS — the +//! correct outcome, stated explicitly rather than left implicit. +//! +//! 2. **Rapid rebuilds are debounced (300 ms).** A burst of tag events collapses +//! to a single sync of the latest tag. +//! +//! 3. **A supersede cancels an in-flight push.** A new tag event arriving while a +//! push is still running drops (cancels) that push and starts fresh. Because +//! control rides its own WS (design D9), the cancel is not blocked behind a +//! bulk transfer — it is a plain future-drop on the orchestration task. +//! +//! Notifying the device is a seam ([`Notifier`]): the control WS itself is task +//! 5.1, so this module depends only on the notify contract, never on the socket. +//! Likewise the transfer is a seam ([`Syncer`]) with a concrete engine-backed +//! implementation ([`EngineSyncer`]) that reuses the per-engine write-credential +//! injection from [`super::engine`]. + +use std::collections::HashSet; +use std::future::Future; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use base64::Engine as _; +use tokio::process::Command; +use tokio::sync::mpsc; +use tokio::time::sleep; + +use super::auth::WriteToken; +use super::engine::{EngineDriver, TagEvent, WriteCredential}; +use super::image_ref::{strip_registry, with_default_tag}; +use crate::utils::container::{is_docker_desktop, is_vm_routing_active}; +use crate::utils::output::{print_warning, OutputLevel}; + +/// Debounce window for coalescing rapid rebuilds (design task 4.2). +pub const DEBOUNCE: Duration = Duration::from_millis(300); + +/// How the host transfers a rebuilt image's layers to the device. +/// +/// The choice is made by EXPLICIT topology detection ([`HostTopology::sync_mode`]), +/// never emergent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyncMode { + /// O(delta): re-tag + `push` into the embedded registry so the device's pull + /// transfers only the changed layers. The native-Linux and avocado-vm path. + Push, + /// O(full image): a `docker-daemon:` style export. The Docker-Desktop / + /// podman-machine-without-VM fallback ONLY — never chosen on a PUSH-capable + /// endpoint. + Ingest, +} + +/// The host topology inputs that select PUSH vs INGEST (design D1). +/// +/// The two fields mirror the `avocado deploy` detectors so the selection is an +/// explicit function of DETECTED topology, not emergent behavior. Tests drive +/// the selector by constructing this directly; [`HostTopology::detect`] wires +/// the real host detectors. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HostTopology { + /// True on macOS/Windows — a Docker-Desktop or podman-machine style host + /// whose engine runs in a Linux VM ([`is_docker_desktop`]). + pub docker_desktop: bool, + /// True iff `DOCKER_HOST` points at the avocado-vm's forwarded socket, i.e. + /// the push will execute inside the avocado-vm ([`is_vm_routing_active`]). + pub vm_routing: bool, +} + +impl HostTopology { + /// Detect the host topology from the real `avocado deploy` selectors. + pub fn detect() -> Self { + Self { + docker_desktop: is_docker_desktop(), + vm_routing: is_vm_routing_active(), + } + } + + /// Select the sync mode from the detected topology (design D1). + /// + /// - avocado-vm active (`vm_routing`) -> PUSH (authenticated HTTPS push into + /// the routable write listener; the macOS fast path). + /// - Docker-Desktop / podman-machine WITHOUT the VM -> INGEST (PUSH is + /// unreachable: the engine lives in a VM whose loopback is not the host's). + /// - native Linux -> PUSH (loopback push, the common case). + /// + /// `vm_routing` is checked first so a macOS host WITH the avocado-vm routed + /// takes the PUSH fast path even though `docker_desktop` is also true. + pub fn sync_mode(&self) -> SyncMode { + if self.vm_routing { + SyncMode::Push + } else if self.docker_desktop { + SyncMode::Ingest + } else { + SyncMode::Push + } + } +} + +/// The device-notify seam (design D9): the control WS is task 5.1, so the +/// watcher depends only on this contract, never on the socket. +/// +/// The returned future is boxed and `Send` so the watcher can be spawned on the +/// multi-threaded runtime without an unstable return-type-notation Send bound. +pub trait Notifier: Send + Sync { + /// Notify the device that `event`'s image/tag/digest is now available to + /// pull. Realized over the control WS by task 5.1. + fn notify<'a>( + &'a self, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>>; +} + +/// The layer-transfer seam: PUSH (O(delta)) or INGEST (O(full image)). +/// +/// The concrete host implementation is [`EngineSyncer`]; tests substitute a +/// recording double so the watcher's debounce/supersede orchestration is +/// asserted without a real engine or registry. +pub trait Syncer: Send + Sync { + /// Transfer `event`'s image to the embedded registry using `mode`. + fn sync<'a>( + &'a self, + mode: SyncMode, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>>; +} + +/// Drive the watcher: consume tag events from `rx`, debounce, sync with `mode`, +/// then notify — superseding an in-flight sync when a newer event arrives. +/// +/// The loop runs until the event channel closes (all senders dropped, e.g. on +/// `down`): a pending debounce or an in-flight sync completes first, then the +/// loop exits. Sync/notify errors are surfaced as warnings and do not abort the +/// watcher — a later rebuild retries. +/// +/// Only events for an image in `watch` are acted on. The engine reports every tag +/// applied on the daemon, including this watcher's own registry retag, so without +/// that filter the sync path feeds itself (see [`WatchSet`]). +pub async fn run_watcher( + mut rx: mpsc::Receiver, + mode: SyncMode, + syncer: Arc, + notifier: Arc, + debounce: Duration, + watch: WatchSet, +) { + // An event carried over from a supersede that cancelled the previous sync. + let mut pending: Option = None; + // Set once the channel closes; we then stop listening for supersedes and let + // the current work finish rather than treating close as a cancel. + let mut closed = false; + + loop { + // Acquire the event to work on: a carried-over supersede, else the next + // from the channel. + let first = match pending.take() { + Some(e) => e, + None => { + if closed { + return; + } + // Drain unwatched tags (the watcher's own registry retag among + // them) without waking the sync path. + loop { + match rx.recv().await { + Some(e) if watch.is_watched(&e.image) => break e, + Some(_) => continue, + None => return, + } + } + } + }; + + // Debounce: keep only the latest event arriving within `debounce`. + let mut latest = first; + if !closed { + loop { + tokio::select! { + _ = sleep(debounce) => break, + got = rx.recv() => match got { + // supersede within the window + Some(e) if watch.is_watched(&e.image) => latest = e, + Some(_) => {} + None => { closed = true; break; } + } + } + } + } + + // Sync + notify. A superseding event (Some) cancels the in-flight work by + // dropping its future; a channel close (None) stops supersede-listening + // so the current work runs to completion. + if closed { + do_sync_and_notify(mode, syncer.as_ref(), notifier.as_ref(), &latest).await; + } else { + let work = do_sync_and_notify(mode, syncer.as_ref(), notifier.as_ref(), &latest); + tokio::pin!(work); + loop { + tokio::select! { + () = &mut work => break, + got = rx.recv(), if !closed => match got { + // Supersede: dropping `work` cancels the in-flight push. + Some(e) if watch.is_watched(&e.image) => { pending = Some(e); break; } + // An unwatched tag is not a rebuild — never cancel a push + // for one. The retag `work` itself is performing lands + // here, and cancelling on it is what orphaned the push + // against a deleted DOCKER_CONFIG. + Some(_) => {} + // Channel closed mid-work: stop listening, finish `work`. + None => { closed = true; } + } + } + } + } + } +} + +/// Run one sync + notify, surfacing (but not propagating) failures. +async fn do_sync_and_notify( + mode: SyncMode, + syncer: &dyn Syncer, + notifier: &dyn Notifier, + event: &TagEvent, +) { + if let Err(e) = syncer.sync(mode, event).await { + print_warning( + &format!("container dev: sync of `{}` failed: {e:#}", event.image), + OutputLevel::Normal, + ); + return; + } + if let Err(e) = notifier.notify(event).await { + print_warning( + &format!("container dev: notify for `{}` failed: {e:#}", event.image), + OutputLevel::Normal, + ); + } +} + +/// The PUSH command plan (O(delta)): re-tag the local image onto the embedded +/// registry and push it, injecting the host-only write credential. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PushPlan { + /// The registry-qualified target ref the image is re-tagged to and pushed. + pub target_ref: String, + /// ` tag `. + pub tag_argv: Vec, + /// ` push ` (credential injected at execution). + pub push_argv: Vec, + /// How the write credential is injected on the push (design D2/A10). + pub credential: WriteCredential, +} + +/// The INGEST command plan (O(full image)): a full-image `save` export, the +/// fallback used only where PUSH is unreachable. It never targets the embedded +/// registry — that is the whole point of the O(full-image) cost. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IngestPlan { + /// The local image exported wholesale. + pub source_ref: String, + /// ` save ` — exports every layer, not just the delta. + pub export_argv: Vec, +} + +/// The image refs `container_dev.images` declares as watched. +/// +/// The engine's tag-event stream carries EVERY tag applied on the host daemon, +/// including the `/:` retag [`EngineSyncer::push`] performs +/// itself on the way to every push. A watcher that acts on all of them re-enters +/// its own sync path: retag emits an event, that event drives a sync, that sync +/// retags. So the declared list is a filter the watcher must apply, not merely +/// documentation of intent. +#[derive(Clone, Debug, Default)] +pub struct WatchSet(HashSet); + +impl WatchSet { + /// Build the set from the configured refs. + /// + /// Applies docker's own default-tag rule, because the engine always reports a + /// fully tagged ref in a tag event: without it a legal tagless `ref: my-app` + /// would match no event and silently stop syncing. + pub fn new(refs: impl IntoIterator) -> Self { + Self(refs.into_iter().map(|r| with_default_tag(&r)).collect()) + } + + /// Whether `image`, as reported by an engine tag event, is watched. + /// + /// The event ref is normalized the same way the configured refs were, so the + /// two sides cannot disagree about an implicit `:latest`. + pub fn is_watched(&self, image: &str) -> bool { + self.0.contains(&with_default_tag(image)) + } +} + +/// Build the PUSH plan for `event` targeting `registry` (`host:port`). +pub fn build_push_plan( + driver: &dyn EngineDriver, + registry: &str, + event: &TagEvent, + token: &WriteToken, +) -> PushPlan { + let target_ref = format!("{registry}/{}", strip_registry(&event.image)); + let tag_argv = vec!["tag".to_string(), event.image.clone(), target_ref.clone()]; + let push_argv = vec!["push".to_string(), target_ref.clone()]; + let credential = driver.write_credential(registry, token); + PushPlan { + target_ref, + tag_argv, + push_argv, + credential, + } +} + +/// Build the INGEST plan for `event`: a full-image export. +/// NOTE: nothing calls this today. [`EngineSyncer::ingest`] used to run the plan +/// and drop the resulting tar on the floor; it now fails with the remedy +/// instead, so the plan is kept as the shape a real INGEST implementation needs +/// (export, transfer, load) rather than deleted and re-derived later. +pub fn build_ingest_plan(event: &TagEvent) -> IngestPlan { + IngestPlan { + source_ref: event.image.clone(), + export_argv: vec!["save".to_string(), event.image.clone()], + } +} + +/// The concrete host [`Syncer`]: drives the engine CLI to PUSH (delta) or INGEST +/// (full export), reusing the per-engine write-credential injection from +/// [`super::engine`]. +/// +/// PUSH re-tags the image onto the embedded registry and pushes it with the +/// host-only write token — injected via an ephemeral `DOCKER_CONFIG` (docker) or +/// `--creds` (podman), NEVER a persisted `docker login` against the user's real +/// config (design M-E). INGEST is the O(full-image) fallback export. +pub struct EngineSyncer { + driver: Box, + /// The write listener `host:port` — byte-identical to the tag host so docker + /// attaches the injected credential (H-3). + registry: String, + write_token: WriteToken, + /// Per-project dir the ephemeral `DOCKER_CONFIG` and export tar live under. + project_dir: PathBuf, +} + +impl EngineSyncer { + /// Construct a syncer for `driver` pushing to `registry` under `project_dir`. + pub fn new( + driver: Box, + registry: impl Into, + write_token: WriteToken, + project_dir: impl Into, + ) -> Self { + Self { + driver, + registry: registry.into(), + write_token, + project_dir: project_dir.into(), + } + } + + async fn push(&self, event: &TagEvent) -> Result<()> { + let plan = build_push_plan( + self.driver.as_ref(), + &self.registry, + event, + &self.write_token, + ); + let binary = self.driver.binary(); + + run_engine(binary, &plan.tag_argv, None).await?; + + match &plan.credential { + WriteCredential::DockerConfigEnv { + registry, + username, + token, + } => { + // Write an ephemeral DOCKER_CONFIG whose auths key is byte-identical + // to the tagged registry host:port (H-3), 0600, under the per-project + // dir — deleted when `dir` drops after the push. NEVER merged into + // the user's real ~/.docker/config.json (M-E). + let dir = tempfile::Builder::new() + .prefix("docker-config-") + .tempdir_in(&self.project_dir) + .context("creating ephemeral DOCKER_CONFIG dir")?; + write_docker_config(dir.path(), registry, username, token)?; + run_engine(binary, &plan.push_argv, Some(("DOCKER_CONFIG", dir.path()))).await?; + } + WriteCredential::PodmanCreds { username, token } => { + // podman takes the credential per-invocation on argv (design A10). + let argv = vec![ + "push".to_string(), + "--creds".to_string(), + format!("{username}:{token}"), + plan.target_ref.clone(), + ]; + run_engine(binary, &argv, None).await?; + } + } + Ok(()) + } + + /// The INGEST fallback is not wired up, and says so instead of pretending. + /// + /// It used to run `save -o /ingest.tar ` and return + /// `Ok`. Nothing in the tree ever read that tar - no transfer, no load, no + /// import - so the sync reported success, `notify` then failed with + /// "the registry has no manifest for tag `dev` yet (the push must land + /// before the notify)", and the message pointed at a push that was never + /// attempted. The device never updated and the tar was rewritten on every + /// rebuild forever. + /// + /// Every Docker-Desktop and podman-machine user without the avocado-vm + /// routed lands here, so failing at the point the path is taken - with the + /// remedy - beats a success that unravels one layer down. + async fn ingest(&self, event: &TagEvent) -> Result<()> { + anyhow::bail!( + "container dev cannot sync `{}` on this host yet: the container engine runs inside a \ + VM whose loopback is not the host's, so the registry push path is unreachable, and \ + the INGEST fallback that would replace it is not implemented (it exports a tar \ + nothing transfers). Start the avocado-vm and route it (`avocado vm start`) so the \ + push path becomes reachable, or run the dev loop from a host whose engine is native.", + event.image + ) + } +} + +impl Syncer for EngineSyncer { + fn sync<'a>( + &'a self, + mode: SyncMode, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + match mode { + SyncMode::Push => self.push(event).await, + SyncMode::Ingest => self.ingest(event).await, + } + }) + } +} + +/// Write an ephemeral docker `config.json` with a single `auths` entry keyed to +/// `registry`, mode 0600. +fn write_docker_config( + dir: &std::path::Path, + registry: &str, + username: &str, + token: &str, +) -> Result<()> { + let auth = base64::engine::general_purpose::STANDARD.encode(format!("{username}:{token}")); + let body = serde_json::json!({ "auths": { registry: { "auth": auth } } }); + let path = dir.join("config.json"); + std::fs::write(&path, serde_json::to_vec(&body)?) + .with_context(|| format!("writing ephemeral docker config to {}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("chmod 0600 on ephemeral docker config")?; + } + Ok(()) +} + +/// Run ` ` with an optional single env override, failing on a +/// non-zero exit. +async fn run_engine( + binary: &str, + argv: &[String], + env: Option<(&str, &std::path::Path)>, +) -> Result<()> { + let mut cmd = Command::new(binary); + cmd.args(argv); + // A supersede cancels an in-flight sync by dropping its future, which also + // drops the ephemeral DOCKER_CONFIG tempdir the push authenticates with. Left + // to tokio's default the child outlives that drop and keeps pushing against a + // credential dir that no longer exists — docker then sends no credential and + // the write listener answers 401 "no basic auth credentials". Tie the child's + // lifetime to the future so cancelling a sync actually cancels its push. + cmd.kill_on_drop(true); + if let Some((key, val)) = env { + cmd.env(key, val); + } + let status = cmd + .status() + .await + .with_context(|| format!("running `{binary} {}`", argv.join(" ")))?; + if !status.success() { + anyhow::bail!("`{binary} {}` exited with {status}", argv.join(" ")); + } + Ok(()) +} + +/// Cross-arch guard (task 4.3, design "cross-arch refusal"). +/// +/// A container image built for one CPU architecture cannot run on a device of +/// another, so syncing an amd64 image to an arm64 device is a silent +/// wrong-arch delivery the device engine would fail (or worse, a manifest that +/// pulls but never runs). The guard sits IN the sync path as a [`Syncer`] +/// decorator: it probes the image's platform architecture, compares it against +/// every connected device's reported `hello.arch`, and REFUSES the sync (with +/// actionable buildx guidance) before the wrapped syncer pushes or exports +/// anything. Because a refused sync returns `Err`, [`do_sync_and_notify`] also +/// skips the device notify — so a mismatch never reaches push OR notify. +/// +/// The device architecture comes from the control-WS `hello` frame's `arch` +/// field (task 5.1 records it into a [`DeviceArchBook`]); the guard only reads +/// the snapshot, so it does not depend on the WS implementation. +pub mod arch_guard { + use std::collections::BTreeMap; + use std::future::Future; + use std::pin::Pin; + use std::sync::{Arc, Mutex}; + + use anyhow::{Context, Result}; + use tokio::process::Command; + + use super::super::engine::{EngineDriver, TagEvent}; + use super::{SyncMode, Syncer}; + + /// A CPU architecture canonicalized to the OCI/GOARCH spelling. + /// + /// A device reports `hello.arch` in `uname -m` form (`x86_64`, `aarch64`), + /// while an image's platform architecture is GOARCH (`amd64`, `arm64`). + /// Normalizing both to one spelling lets them compare equal. An unrecognized + /// value is lowercased and compared verbatim, so two identical unknown + /// arches still match rather than spuriously refusing. + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct DeviceArch(String); + + impl DeviceArch { + /// Canonicalize a raw arch string from an image platform or a device + /// `hello.arch`. + pub fn parse(raw: &str) -> Self { + let lowered = raw.trim().to_ascii_lowercase(); + let canon = match lowered.as_str() { + "x86_64" | "amd64" | "x64" => "amd64", + "aarch64" | "arm64" | "arm64v8" => "arm64", + "armv7l" | "armv6l" | "armhf" | "arm" | "arm32v7" => "arm", + "i386" | "i486" | "i586" | "i686" | "386" | "x86" => "386", + "riscv64" => "riscv64", + "ppc64le" => "ppc64le", + "s390x" => "s390x", + _ => lowered.as_str(), + }; + DeviceArch(canon.to_string()) + } + + /// The canonical GOARCH string (`amd64`, `arm64`, …). + pub fn as_str(&self) -> &str { + &self.0 + } + } + + /// A refused cross-arch sync: the image platform does not match a device. + /// + /// The `Display` is the user-facing refusal and carries buildx guidance that + /// names the device's target platform, so a developer can rebuild for the + /// right architecture without guessing the flag. + #[derive(Debug, thiserror::Error)] + #[error( + "refusing to sync image `{image}` (platform `{image_arch}`) to a device reporting arch \ + `{device_arch}`: this would ship a wrong-architecture image the device cannot run. \ + Rebuild for the device platform with buildx, e.g.:\n \ + docker buildx build --platform linux/{device_arch} -t {image} .\n \ + then re-run the sync." + )] + pub struct ArchMismatch { + /// The image reference that was refused. + pub image: String, + /// The image's platform architecture (canonical GOARCH). + pub image_arch: String, + /// The mismatched device's reported architecture (canonical GOARCH). + pub device_arch: String, + } + + /// Refuse the sync unless `image_arch` matches EVERY connected device. + /// + /// A single mismatched device is a refusal — we never ship a wrong-arch + /// image to any device in a fleet. With no connected devices there is + /// nothing to mismatch, so the sync is allowed (it simply reaches no one). + pub fn check_arch( + image: &str, + image_arch: &DeviceArch, + device_arches: &[DeviceArch], + ) -> Result<(), ArchMismatch> { + for dev in device_arches { + if dev != image_arch { + return Err(ArchMismatch { + image: image.to_string(), + image_arch: image_arch.as_str().to_string(), + device_arch: dev.as_str().to_string(), + }); + } + } + Ok(()) + } + + /// Probe an image's platform architecture (task 4.3 seam). + /// + /// The concrete host implementation is [`EngineArchProbe`]; tests substitute + /// a fixed double so the guard's refusal logic is asserted without a real + /// engine. + pub trait ImageArchProbe: Send + Sync { + /// Report the platform architecture of `event`'s image. + fn image_arch<'a>( + &'a self, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>>; + } + + /// A snapshot of the architectures of currently-connected devices, sourced + /// from their `hello.arch` control frames (task 5.1 populates it). + pub trait DeviceArchBook: Send + Sync { + /// The architectures of every device currently known from a `hello`. + fn device_arches(&self) -> Vec; + } + + /// In-memory [`DeviceArchBook`] keyed by device id, populated from `hello` + /// frames. A reconnecting device overwrites its prior entry, so a snapshot + /// never double-counts one device. + #[derive(Default, Clone)] + pub struct HelloArchBook { + by_device: Arc>>, + } + + impl HelloArchBook { + /// A book with no devices recorded yet. + pub fn new() -> Self { + Self::default() + } + + /// Record a device's `hello.arch` (task 5.1 calls this on a hello frame). + /// + /// Prefer [`HelloArchBook::record_session`], which ties the entry to the + /// connection that produced it. A bare insert outlives the device: the + /// book is consulted by `check_arch`, which refuses on ANY mismatch, so a + /// device that is unplugged and replaced by one of another architecture + /// leaves an entry that refuses every later sync for the rest of the + /// session - naming, in its buildx guidance, an architecture no connected + /// device reports. + pub fn record_hello(&self, device_id: &str, arch: &str) { + self.by_device.lock().unwrap().insert( + device_id.to_string(), + LeasedArch { + arch: DeviceArch::parse(arch), + // No connection behind it; only record_session refcounts. + holders: 1, + }, + ); + } + + /// Record `device_id`'s arch for as long as the returned guard lives. + /// + /// The book is trying to answer "what is connected right now", which is + /// something the connection set already knows - so derive it from the + /// connection rather than accumulating it. Dropping the guard removes the + /// entry, and because that runs on every exit path (clean close, error, + /// panic, early return) the book cannot drift from reality the way a + /// remove-on-disconnect call placed at one exit would. + /// + /// Re-recording the same device (a reconnect that overlaps its own + /// previous session) refcounts rather than replacing, so the older + /// session's guard dropping cannot evict the newer session's entry. + pub fn record_session(&self, device_id: &str, arch: &str) -> DeviceArchLease { + let mut by_device = self.by_device.lock().unwrap(); + let entry = by_device + .entry(device_id.to_string()) + .or_insert_with(|| LeasedArch { + arch: DeviceArch::parse(arch), + holders: 0, + }); + entry.arch = DeviceArch::parse(arch); + entry.holders += 1; + DeviceArchLease { + book: self.by_device.clone(), + device_id: device_id.to_string(), + } + } + } + + /// One device's architecture plus how many live connections claim it. + #[derive(Debug, Clone)] + pub(crate) struct LeasedArch { + arch: DeviceArch, + holders: usize, + } + + /// Keeps a device in the [`HelloArchBook`] until dropped. + /// + /// Held by the control server's per-connection task, so the entry's lifetime + /// is exactly the session's. + pub struct DeviceArchLease { + book: Arc>>, + device_id: String, + } + + impl Drop for DeviceArchLease { + fn drop(&mut self) { + let mut by_device = self.book.lock().unwrap(); + if let Some(entry) = by_device.get_mut(&self.device_id) { + entry.holders = entry.holders.saturating_sub(1); + if entry.holders == 0 { + by_device.remove(&self.device_id); + } + } + } + } + + impl DeviceArchBook for HelloArchBook { + fn device_arches(&self) -> Vec { + self.by_device + .lock() + .unwrap() + .values() + .map(|entry| entry.arch.clone()) + .collect() + } + } + + /// The architecture last probed for each image reference. + /// + /// The guard already knows an image's arch - it probes it on every sync - + /// but that knowledge died with the call. `reconcile` runs later, on a + /// device's `hello`, and had no way to ask what architecture the digest it + /// is about to hand out was built for. With an empty device book at push + /// time the guard allows the sync, so a wrong-arch digest could reach the + /// desired state and be shipped to the first device that connected. + /// + /// Recording it here lets the arch outlive the probe, so the check can + /// happen at the moment a device is actually known. Deliberately mirrors + /// [`HelloArchBook`]: same shared-`Arc` clone semantics, same "one map, two + /// halves" wiring. + #[derive(Default, Clone)] + pub struct ImageArchBook { + by_image: Arc>>, + } + + impl ImageArchBook { + /// A book with no images recorded yet. + pub fn new() -> Self { + Self::default() + } + + /// Record the architecture probed for `image`. + pub fn record_image(&self, image: &str, arch: DeviceArch) { + self.by_image + .lock() + .unwrap() + .insert(image.to_string(), arch); + } + + /// The architecture last probed for `image`, if any. + pub fn arch_for(&self, image: &str) -> Option { + self.by_image.lock().unwrap().get(image).cloned() + } + } + + /// Probe the image architecture via ` image inspect --format + /// {{.Architecture}} ` — the engine CLI, consistent with the rest of + /// the driver (no API socket). + pub struct EngineArchProbe { + binary: &'static str, + } + + impl EngineArchProbe { + /// Build a probe driving `driver`'s engine CLI binary. + pub fn new(driver: &dyn EngineDriver) -> Self { + Self { + binary: driver.binary(), + } + } + } + + impl ImageArchProbe for EngineArchProbe { + fn image_arch<'a>( + &'a self, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let output = Command::new(self.binary) + .args([ + "image", + "inspect", + "--format", + "{{.Architecture}}", + &event.image, + ]) + .output() + .await + .with_context(|| { + format!("running `{} image inspect {}`", self.binary, event.image) + })?; + if !output.status.success() { + anyhow::bail!( + "`{} image inspect {}` failed: {}", + self.binary, + event.image, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + let arch = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if arch.is_empty() { + anyhow::bail!( + "`{} image inspect {}` reported an empty architecture", + self.binary, + event.image + ); + } + Ok(DeviceArch::parse(&arch)) + }) + } + } + + /// A [`Syncer`] decorator that refuses a cross-arch sync BEFORE delegating. + /// + /// It probes the image architecture, compares it against the device book, + /// and returns [`ArchMismatch`] on a mismatch — so `inner` (the real + /// PUSH/INGEST syncer) is never invoked and the watcher skips notify. + pub struct ArchGuardSyncer { + inner: Arc, + probe: Arc, + devices: Arc, + images: ImageArchBook, + } + + impl ArchGuardSyncer { + /// Wrap `inner`, guarding it with `probe` (image arch) and `devices` + /// (connected-device arches), recording each probe into `images`. + /// + /// `images` is what makes the guard useful after the fact: an empty + /// device book means there is nobody to disagree with yet, so the sync + /// is allowed, and only the recorded arch lets a later `reconcile` + /// refuse to hand that digest to a device of the wrong architecture. + pub fn new( + inner: Arc, + probe: Arc, + devices: Arc, + images: ImageArchBook, + ) -> Self { + Self { + inner, + probe, + devices, + images, + } + } + } + + impl Syncer for ArchGuardSyncer { + fn sync<'a>( + &'a self, + mode: SyncMode, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let image_arch = self.probe.image_arch(event).await?; + let device_arches = self.devices.device_arches(); + // A mismatch refuses here, before the wrapped syncer pushes or + // exports anything. + check_arch(&event.image, &image_arch, &device_arches)?; + // Record BEFORE delegating, so the arch is available to a later + // reconcile even for the allowed-because-nobody-was-connected + // case - which is precisely the case the record exists for. + // Key on the canonical form, which is what `frame_suits_device` + // looks up. Recording the raw event ref meant a podman user's + // `localhost/my-app:dev` was never found under `my-app:dev`, so + // the cross-arch broadcast filter fell through to its permissive + // arm for every registry-qualified ref. + self.images.record_image( + &super::super::image_ref::canonical(&event.image), + image_arch, + ); + self.inner.sync(mode, event).await + }) + } + } + + #[cfg(test)] + mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::super::{do_sync_and_notify, Notifier}; + + fn ev(image: &str) -> TagEvent { + TagEvent { + image: image.to_string(), + image_id: None, + } + } + + struct FixedProbe(&'static str); + impl ImageArchProbe for FixedProbe { + fn image_arch<'a>( + &'a self, + _event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + let arch = DeviceArch::parse(self.0); + Box::pin(async move { Ok(arch) }) + } + } + + #[derive(Default)] + struct CountingSyncer { + calls: AtomicUsize, + } + impl Syncer for CountingSyncer { + fn sync<'a>( + &'a self, + _mode: SyncMode, + _event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(()) }) + } + } + + #[derive(Default)] + struct CountingNotifier { + calls: AtomicUsize, + } + impl Notifier for CountingNotifier { + fn notify<'a>( + &'a self, + _event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(()) }) + } + } + + // ---- arch normalization ---- + + #[test] + fn parse_canonicalizes_uname_and_goarch_spellings() { + assert_eq!(DeviceArch::parse("x86_64"), DeviceArch::parse("amd64")); + assert_eq!(DeviceArch::parse("aarch64"), DeviceArch::parse("arm64")); + assert_eq!(DeviceArch::parse("armv7l"), DeviceArch::parse("arm")); + assert_eq!(DeviceArch::parse("AMD64"), DeviceArch::parse("amd64")); + assert_ne!(DeviceArch::parse("amd64"), DeviceArch::parse("arm64")); + } + + // ---- pure check_arch logic ---- + + #[test] + fn a_matching_arch_passes_the_check() { + // uname `aarch64` device vs a GOARCH `arm64` image: equal after + // normalization. + assert!(check_arch( + "app:dev", + &DeviceArch::parse("arm64"), + &[DeviceArch::parse("aarch64")] + ) + .is_ok()); + } + + #[test] + fn an_amd64_image_is_refused_on_an_arm64_device() { + let err = check_arch( + "my-app:dev", + &DeviceArch::parse("amd64"), + &[DeviceArch::parse("aarch64")], + ) + .expect_err("an amd64 image must be refused for an arm64 device"); + assert_eq!(err.image_arch, "amd64"); + assert_eq!(err.device_arch, "arm64"); + } + + #[test] + fn any_single_mismatched_device_refuses_the_whole_sync() { + // A fleet with one arm64 and one amd64 device: an amd64 image cannot + // run on the arm64 one, so the whole sync is refused. + let devices = [DeviceArch::parse("arm64"), DeviceArch::parse("amd64")]; + let err = check_arch("app:dev", &DeviceArch::parse("amd64"), &devices) + .expect_err("a mismatch on any device refuses the sync"); + assert_eq!(err.device_arch, "arm64"); + } + + #[test] + fn no_connected_devices_is_not_a_mismatch() { + assert!(check_arch("app:dev", &DeviceArch::parse("amd64"), &[]).is_ok()); + } + + #[test] + fn the_refusal_names_buildx_and_the_device_target_platform() { + let err = check_arch( + "my-app:dev", + &DeviceArch::parse("amd64"), + &[DeviceArch::parse("aarch64")], + ) + .unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("buildx"), + "the refusal must give buildx guidance, not a bare error: {msg}" + ); + assert!( + msg.contains("linux/arm64"), + "the refusal must name the device's target platform: {msg}" + ); + } + + // ---- the guard sits in the sync path (via do_sync_and_notify) ---- + + #[tokio::test] + async fn an_amd64_image_is_refused_before_push_or_notify_on_an_arm64_device() { + let inner = Arc::new(CountingSyncer::default()); + let notifier = CountingNotifier::default(); + let book = HelloArchBook::new(); + book.record_hello("dev-1", "aarch64"); // device reports arm64 + + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), + Arc::new(book), + ImageArchBook::new(), + ); + + do_sync_and_notify(SyncMode::Push, &guard, ¬ifier, &ev("my-app:dev")).await; + + assert_eq!( + inner.calls.load(Ordering::SeqCst), + 0, + "a cross-arch image must be refused before the push runs" + ); + assert_eq!( + notifier.calls.load(Ordering::SeqCst), + 0, + "a refused sync must not notify the device" + ); + } + + #[tokio::test] + async fn the_arch_is_recorded_under_the_key_the_broadcast_filter_reads() { + // The two halves of the guard used different keys. This recorded + // `event.image` verbatim; `ControlServer::frame_suits_device` rebuilt + // its key from the registry-stripped `(image, tag)` of the Sync frame. + // For any registry-qualified ref the lookup missed, `arch_for` + // returned None, and the filter's permissive arm let the frame + // through - an amd64 image to an aarch64 device. + // + // podman writes local refs as `localhost/my-app:dev` and + // `WatchSet::is_watched` is an exact match, so a podman user has to + // configure the qualified ref for the watcher to fire at all. That + // made this every ref on that engine, not an edge case. + let inner = Arc::new(CountingSyncer::default()); + let notifier = CountingNotifier::default(); + let images = ImageArchBook::new(); + let observed = images.clone(); + + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), + // No devices connected: the allowed-because-nobody-was-looking + // case, which is exactly the one the record exists to cover. + Arc::new(HelloArchBook::new()), + images, + ); + + do_sync_and_notify( + SyncMode::Push, + &guard, + ¬ifier, + &ev("localhost/my-app:dev"), + ) + .await; + + assert_eq!( + observed.arch_for("my-app:dev"), + Some(DeviceArch::parse("amd64")), + "the arch must be findable under the canonical key the broadcast \ + filter looks up, not only under the raw event ref" + ); + } + + #[tokio::test] + async fn a_matching_arch_image_proceeds_to_push_and_notify() { + let inner = Arc::new(CountingSyncer::default()); + let notifier = CountingNotifier::default(); + let book = HelloArchBook::new(); + book.record_hello("dev-1", "x86_64"); // amd64 device + + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), + Arc::new(book), + ImageArchBook::new(), + ); + + do_sync_and_notify(SyncMode::Push, &guard, ¬ifier, &ev("my-app:dev")).await; + + assert_eq!( + inner.calls.load(Ordering::SeqCst), + 1, + "a matching-arch image is pushed" + ); + assert_eq!( + notifier.calls.load(Ordering::SeqCst), + 1, + "a matching-arch image notifies the device after the push" + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + use tokio::sync::Notify; + use tokio::time::{timeout, Duration}; + + use crate::utils::container_dev::auth::WRITE_USERNAME; + use crate::utils::container_dev::engine::{DockerDriver, PodmanDriver}; + + fn ev(image: &str) -> TagEvent { + TagEvent { + image: image.to_string(), + image_id: Some(format!("sha256:{image}")), + } + } + + // ---- topology selection: explicit detection, not emergent (D1) ---- + + #[test] + fn native_linux_selects_push() { + let topo = HostTopology { + docker_desktop: false, + vm_routing: false, + }; + assert_eq!(topo.sync_mode(), SyncMode::Push); + } + + #[test] + fn avocado_vm_selects_push_even_on_a_docker_desktop_host() { + // macOS with the avocado-vm routed: docker_desktop is true, but the VM + // push fast path wins. + let topo = HostTopology { + docker_desktop: true, + vm_routing: true, + }; + assert_eq!(topo.sync_mode(), SyncMode::Push); + } + + #[test] + fn docker_desktop_without_vm_selects_ingest() { + // Docker-Desktop / podman-machine with no avocado-vm: PUSH is unreachable, + // so the topology-detected fallback is INGEST — not emergent behavior. + let topo = HostTopology { + docker_desktop: true, + vm_routing: false, + }; + assert_eq!(topo.sync_mode(), SyncMode::Ingest); + } + + // ---- PUSH is delta into the registry; INGEST is a full local export ---- + + #[test] + fn push_plan_retags_onto_the_registry_and_injects_the_write_credential() { + let plan = build_push_plan( + &DockerDriver, + "127.0.0.1:5599", + &ev("my-app:dev"), + &WriteToken::new("wtok"), + ); + assert_eq!(plan.target_ref, "127.0.0.1:5599/my-app:dev"); + assert_eq!( + plan.tag_argv, + vec!["tag", "my-app:dev", "127.0.0.1:5599/my-app:dev"] + ); + assert_eq!(plan.push_argv, vec!["push", "127.0.0.1:5599/my-app:dev"]); + // The delta path pushes to the embedded registry with the host-only write + // token (Basic, via an ephemeral DOCKER_CONFIG keyed to the registry). + match plan.credential { + WriteCredential::DockerConfigEnv { + registry, + username, + token, + } => { + assert_eq!(registry, "127.0.0.1:5599"); + assert_eq!(username, WRITE_USERNAME); + assert_eq!(token, "wtok"); + } + other => panic!("expected an ephemeral DOCKER_CONFIG credential, got {other:?}"), + } + } + + #[test] + fn push_plan_strips_a_podman_localhost_qualifier() { + let plan = build_push_plan( + &PodmanDriver, + "127.0.0.1:5599", + &ev("localhost/my-app:dev"), + &WriteToken::new("wtok"), + ); + // The registry qualifier is stripped so the target is the same repo:tag as + // the docker case, not `127.0.0.1:5599/localhost/my-app:dev`. + assert_eq!(plan.target_ref, "127.0.0.1:5599/my-app:dev"); + } + + #[test] + fn ingest_plan_is_a_full_image_export_not_a_registry_push() { + let plan = build_ingest_plan(&ev("my-app:dev")); + assert_eq!(plan.source_ref, "my-app:dev"); + assert_eq!(plan.export_argv, vec!["save", "my-app:dev"]); + // INGEST must never target the embedded registry — that is the O(full + // image) fallback, distinct from the delta PUSH. + assert!( + !plan + .export_argv + .iter() + .any(|a| a.contains(':') && a.contains('/')), + "INGEST is a local export, it must not push to a registry endpoint: {:?}", + plan.export_argv + ); + assert_eq!(plan.export_argv[0], "save"); + } + + // ---- watcher orchestration: recording doubles for the seams ---- + + #[derive(Default)] + struct Recorder { + /// Ordered log across both seams: `sync-start:`, `sync-done:`, + /// `notify:`. + log: Mutex>, + /// Images whose sync started. + started: Mutex>, + /// Images whose sync ran to completion (i.e. was not cancelled). + completed: Mutex>, + /// Fired after a sync records its start, so a test can send a superseding + /// event only once a push is genuinely in flight. + started_signal: Notify, + /// An image whose sync blocks (models a slow, cancellable push). + slow_image: Mutex>, + } + + impl Recorder { + fn arc() -> Arc { + Arc::new(Self::default()) + } + fn set_slow(&self, image: &str) { + *self.slow_image.lock().unwrap() = Some(image.to_string()); + } + } + + impl Syncer for Recorder { + fn sync<'a>( + &'a self, + mode: SyncMode, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.log + .lock() + .unwrap() + .push(format!("sync-start:{}:{:?}", event.image, mode)); + self.started.lock().unwrap().push(event.image.clone()); + self.started_signal.notify_one(); + let slow = self.slow_image.lock().unwrap().clone(); + if slow.as_deref() == Some(event.image.as_str()) { + // Block long enough that a supersede cancels this future. + sleep(Duration::from_secs(30)).await; + } + self.completed.lock().unwrap().push(event.image.clone()); + self.log + .lock() + .unwrap() + .push(format!("sync-done:{}", event.image)); + Ok(()) + }) + } + } + + impl Notifier for Recorder { + fn notify<'a>( + &'a self, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.log + .lock() + .unwrap() + .push(format!("notify:{}", event.image)); + Ok(()) + }) + } + } + + #[tokio::test] + async fn a_tag_event_pushes_then_notifies() { + let rec = Recorder::arc(); + let (tx, rx) = mpsc::channel(8); + let handle = tokio::spawn(run_watcher( + rx, + SyncMode::Push, + rec.clone() as Arc, + rec.clone() as Arc, + DEBOUNCE, + WatchSet::new(["my-app:dev".to_string()]), + )); + + tx.send(ev("my-app:dev")).await.unwrap(); + // Give the debounce window + a slack margin to settle and run the work. + sleep(DEBOUNCE + Duration::from_millis(200)).await; + drop(tx); + timeout(Duration::from_secs(2), handle) + .await + .expect("watcher exits after the channel closes") + .unwrap(); + + // The sync ran once with the PUSH mode, then the notify followed it. + let log = rec.log.lock().unwrap().clone(); + assert_eq!( + log, + vec![ + "sync-start:my-app:dev:Push".to_string(), + "sync-done:my-app:dev".to_string(), + "notify:my-app:dev".to_string(), + ], + "a rebuild must push (delta) then notify, in that order" + ); + } + + #[tokio::test] + async fn a_second_event_within_the_debounce_window_supersedes_the_first() { + let rec = Recorder::arc(); + let (tx, rx) = mpsc::channel(8); + let handle = tokio::spawn(run_watcher( + rx, + SyncMode::Push, + rec.clone() as Arc, + rec.clone() as Arc, + DEBOUNCE, + WatchSet::new(["v1".to_string(), "v2".to_string()]), + )); + + // Two events well inside the 300 ms window. + tx.send(ev("v1")).await.unwrap(); + sleep(Duration::from_millis(50)).await; + tx.send(ev("v2")).await.unwrap(); + + sleep(DEBOUNCE + Duration::from_millis(200)).await; + drop(tx); + timeout(Duration::from_secs(2), handle) + .await + .unwrap() + .unwrap(); + + // Only the latest event synced; v1 was superseded and never pushed. + let started = rec.started.lock().unwrap().clone(); + assert_eq!( + started, + vec!["v2".to_string()], + "the burst coalesces to the latest tag" + ); + let log = rec.log.lock().unwrap().clone(); + assert_eq!( + log, + vec![ + "sync-start:v2:Push".to_string(), + "sync-done:v2".to_string(), + "notify:v2".to_string(), + ] + ); + } + + #[tokio::test] + async fn a_superseding_event_cancels_an_in_flight_push() { + let rec = Recorder::arc(); + rec.set_slow("v1"); // v1's push blocks until cancelled + let (tx, rx) = mpsc::channel(8); + let handle = tokio::spawn(run_watcher( + rx, + SyncMode::Push, + rec.clone() as Arc, + rec.clone() as Arc, + DEBOUNCE, + WatchSet::new(["v1".to_string(), "v2".to_string()]), + )); + + // v1 settles through the debounce and starts a (blocking) push. + tx.send(ev("v1")).await.unwrap(); + rec.started_signal.notified().await; + + // Now supersede with v2 while v1's push is in flight. + tx.send(ev("v2")).await.unwrap(); + + sleep(DEBOUNCE + Duration::from_millis(200)).await; + drop(tx); + timeout(Duration::from_secs(2), handle) + .await + .unwrap() + .unwrap(); + + let started = rec.started.lock().unwrap().clone(); + let completed = rec.completed.lock().unwrap().clone(); + // Both pushes started, but v1's in-flight push was cancelled by the + // supersede: only v2 completes and notifies. + assert!(started.contains(&"v1".to_string()), "v1's push started"); + assert!(started.contains(&"v2".to_string()), "v2's push started"); + assert_eq!( + completed, + vec!["v2".to_string()], + "the superseded v1 push was cancelled before completion" + ); + let log = rec.log.lock().unwrap().clone(); + assert!( + log.contains(&"notify:v2".to_string()), + "v2 notifies after its push" + ); + assert!( + !log.contains(&"notify:v1".to_string()), + "the cancelled v1 push must not notify" + ); + assert!( + !log.contains(&"sync-done:v1".to_string()), + "the cancelled v1 push must not complete" + ); + } + + #[tokio::test] + async fn an_event_for_an_unwatched_image_is_ignored() { + let rec = Recorder::arc(); + let (tx, rx) = mpsc::channel(8); + let handle = tokio::spawn(run_watcher( + rx, + SyncMode::Push, + rec.clone() as Arc, + rec.clone() as Arc, + DEBOUNCE, + WatchSet::new(["my-app:dev".to_string()]), + )); + + tx.send(ev("some-other-image:latest")).await.unwrap(); + sleep(DEBOUNCE + Duration::from_millis(200)).await; + drop(tx); + timeout(Duration::from_secs(2), handle) + .await + .unwrap() + .unwrap(); + + assert!( + rec.started.lock().unwrap().is_empty(), + "an image absent from `container_dev.images` must never sync" + ); + } + + #[tokio::test] + async fn the_watchers_own_registry_retag_does_not_feed_back_as_a_rebuild() { + // `EngineSyncer::push` runs `docker tag /` before + // every push, and the engine emits a tag event for that retag. Acting on + // it re-enters the sync path, whose own retag emits the next event — an + // unbounded push loop. Each iteration also cancels the previous push + // mid-flight, which orphans it against a deleted DOCKER_CONFIG and yields + // a 401. Measured at 1281 failed pushes from ONE real rebuild. + let rec = Recorder::arc(); + let (tx, rx) = mpsc::channel(8); + let handle = tokio::spawn(run_watcher( + rx, + SyncMode::Push, + rec.clone() as Arc, + rec.clone() as Arc, + DEBOUNCE, + WatchSet::new(["my-app:dev".to_string()]), + )); + + tx.send(ev("10.0.2.2:5601/my-app:dev")).await.unwrap(); + sleep(DEBOUNCE + Duration::from_millis(200)).await; + drop(tx); + timeout(Duration::from_secs(2), handle) + .await + .unwrap() + .unwrap(); + + assert!( + rec.started.lock().unwrap().is_empty(), + "the registry-qualified retag is the watcher's own side effect, not a rebuild" + ); + } + + #[tokio::test] + async fn an_untagged_watched_ref_matches_the_latest_tag() { + // `ref: my-app` (no tag) is a legal config; docker's tag events always + // carry an explicit tag, so the watch set must apply docker's own + // default-tag rule or such a config would silently stop syncing. + let rec = Recorder::arc(); + let (tx, rx) = mpsc::channel(8); + let handle = tokio::spawn(run_watcher( + rx, + SyncMode::Push, + rec.clone() as Arc, + rec.clone() as Arc, + DEBOUNCE, + WatchSet::new(["my-app".to_string()]), + )); + + tx.send(ev("my-app:latest")).await.unwrap(); + sleep(DEBOUNCE + Duration::from_millis(200)).await; + drop(tx); + timeout(Duration::from_secs(2), handle) + .await + .unwrap() + .unwrap(); + + assert_eq!( + rec.started.lock().unwrap().clone(), + vec!["my-app:latest".to_string()], + "an untagged watched ref must match the `:latest` tag event" + ); + } + + #[test] + fn debounce_default_is_300ms() { + assert_eq!(DEBOUNCE, Duration::from_millis(300)); + } + + // ---- cancelling a sync must not orphan the engine subprocess ---- + + #[tokio::test] + async fn cancelling_run_engine_kills_the_child_rather_than_orphaning_it() { + // A supersede cancels an in-flight push by dropping its future. That drop + // also removes the ephemeral DOCKER_CONFIG tempdir the push authenticates + // with, so an engine child that outlives the cancellation keeps running + // against a deleted credential dir and 401s ("no basic auth credentials"). + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("child-survived"); + let argv = vec![ + "-c".to_string(), + format!("sleep 3; touch {}", marker.display()), + ]; + + { + // Same shape as the supersede path: pin, poll so the child is + // genuinely spawned, then cancel by letting the future drop. The + // scope is what drops it — `tokio::pin!` keeps the future in a hidden + // local, so dropping the `Pin` binding alone would cancel nothing. + let work = run_engine("sh", &argv, None); + tokio::pin!(work); + let _ = timeout(Duration::from_millis(300), &mut work).await; + } + + sleep(Duration::from_secs(4)).await; + assert!( + !marker.exists(), + "a cancelled sync must kill its engine child, not leave it running" + ); + } +} diff --git a/src/utils/container_dev/ws.rs b/src/utils/container_dev/ws.rs new file mode 100644 index 00000000..af46dc09 --- /dev/null +++ b/src/utils/container_dev/ws.rs @@ -0,0 +1,1721 @@ +//! Control-only WebSocket channel (design D9; task 5.1). +//! +//! The host and device exchange ONLY control frames over this channel: +//! +//! - host -> device: [`HostFrame::Sync`] `{image, tag, digest}` — the image now +//! available to pull. It carries a digest *reference*, never blob bytes: bulk +//! blob/manifest transfers ride the dedicated bulk HTTPS listener (design D9, +//! tasks 3.7/6.2), NOT this WS. The [`HostFrame`] enum has no blob variant by +//! construction, so a blob transfer cannot be sent as a WS frame. +//! - device -> host: [`DeviceFrame::Hello`] `{device_id, arch, running_digest}`, +//! [`DeviceFrame::Progress`], and [`DeviceFrame::Status`]. +//! +//! Two behaviors are load-bearing (design D5/H2): +//! +//! 1. **Desired-state is RE-DERIVED at `up`, never assumed persistent.** +//! [`DesiredState`] is built solely from the engine's current watched tags +//! ([`DesiredState::derive_from_watched_tags`]); there is no disk/restore +//! constructor. After a host restart the host rebuilds it from the engine's +//! *current* tags, so a digest that changed while the host was down is +//! reflected, not restored from a stale snapshot. +//! 2. **On (re)connect the host reconciles the device's `running_digest`.** A +//! device that reconnects reporting a digest that no longer matches the +//! desired state is driven back to current with a reconcile [`HostFrame::Sync`] +//! ([`DesiredState::reconcile`]). +//! +//! The WS upgrade authenticates through the SAME read/control-token validator +//! seam the bulk listener uses ([`super::auth::read_request_authorized`], task +//! 3.4) — the WS is NOT a second, separately-implemented auth surface (design +//! G-5). A WebSocket upgrade is an HTTP `GET` carrying the same `Authorization` +//! header, so the upgrade callback hands that header straight to the shared +//! validator. +//! +//! This module realizes the watcher's [`super::watcher::Notifier`] seam (task +//! 4.2): [`ControlServer`] broadcasts a [`HostFrame::Sync`] to every connected +//! device when the watcher reports a new tag, and records each device's +//! `hello.arch` into the [`super::watcher::arch_guard::HelloArchBook`] the +//! cross-arch guard (task 4.3) reads. + +use std::collections::BTreeMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result}; +use futures_util::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::net::TcpListener; +use tokio::sync::broadcast; +use tokio_rustls::TlsAcceptor; +use tokio_tungstenite::tungstenite::handshake::server::{ErrorResponse, Request, Response}; +use tokio_tungstenite::tungstenite::http::{header, StatusCode}; +use tokio_tungstenite::tungstenite::Message; + +use super::auth::{read_request_authorized, ReadToken}; +use super::engine::TagEvent; +use super::image_ref::canonical; +use super::store::BlobStore; +use crate::utils::output::{print_warning, OutputLevel}; + +use super::watcher::arch_guard::{DeviceArch, DeviceArchLease, HelloArchBook, ImageArchBook}; +use super::watcher::Notifier; + +/// A host -> device control frame. +/// +/// There is exactly ONE variant, [`HostFrame::Sync`], and it carries only image +/// coordinates plus a content-digest *reference* — never blob bytes. This is the +/// structural guarantee that a bulk transfer can never ride the control WS +/// (design D9): the type has no frame that could carry a blob. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HostFrame { + /// The `{image, tag, digest}` now available for the device to pull over the + /// dedicated bulk listener. `digest` is a `sha256:…` reference, not content. + Sync { + /// Repository component of the watched image (e.g. `my-app`). + image: String, + /// Tag component (e.g. `dev`). + tag: String, + /// Content digest (`sha256:…`) the device should be running. + digest: String, + /// The device systemd unit that consumes this image, from the matching + /// `container_dev.images[].service`. + /// + /// The device needs it to make the sync take effect at all: `docker + /// restart ` re-executes the existing container object, which + /// stays bound to its create-time image id, so a freshly pulled image for + /// the same tag is ignored. Restarting the owning unit re-runs + /// `docker run` and re-resolves the tag. The field was declared in config + /// and never sent, so every device fell back to restarting the container + /// and every sync silently no-opped. + /// + /// Optional on the wire so an older device still parses the frame; it + /// then falls back to its own `AVOCADO_CONTAINER_DEV_SERVICE` and finally + /// to the container restart, exactly as before. + #[serde(default, skip_serializing_if = "Option::is_none")] + service: Option, + }, +} + +/// A device -> host control frame. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum DeviceFrame { + /// Sent on connect and reconnect; carries the digest the device currently + /// runs so the host can reconcile it against the desired state. + Hello(Hello), + /// Progress of an in-flight pull (informational). + Progress(Progress), + /// A device state report (informational). + Status(Status), +} + +/// The device's `hello`: who it is, its CPU arch, and the digest it runs now. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Hello { + /// Stable per-device identity. + pub device_id: String, + /// The device CPU architecture (`uname -m` form, e.g. `aarch64`), recorded + /// into the cross-arch guard's [`HelloArchBook`]. + pub arch: String, + /// The content digest the device is currently running. Empty on a device + /// that has not yet pulled anything. + pub running_digest: String, +} + +/// Progress of an in-flight device pull. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Progress { + /// The image the progress refers to. + pub image: String, + /// Bytes pulled so far. + pub bytes_pulled: u64, +} + +/// A device state report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Status { + /// The reporting device. + pub device_id: String, + /// A short state token (e.g. `running`, `restarting`). + pub state: String, + /// Optional human-readable detail. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +/// Split an image reference (`[registry/]repo[:tag]`) into `(repo, tag)`. +/// +/// Delegates to [`super::image_ref::split`] so this and the watcher cannot drift +/// apart again - they already had, which is how the arch book ended up keyed one +/// way and read the other. +fn split_image_tag(image: &str) -> (String, String) { + super::image_ref::split(image) +} + +/// The host's desired container state: `(image, tag) -> digest`. +/// +/// RE-DERIVED at every `up` from the engine's current watched tags (design D5); +/// there is deliberately NO `Deserialize`/disk-restore path, so the desired +/// state cannot be silently loaded from a stale snapshot across a host restart. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DesiredState { + by_tag: BTreeMap<(String, String), DesiredEntry>, + /// `(image, tag) -> owning systemd unit`, from the project's + /// `container_dev.images[].service`. + /// + /// Kept beside the desired map rather than inside [`DesiredEntry`] because it + /// is static project configuration, not something a push discovers: an entry + /// derived from the engine's watched tags and one recorded after a sync must + /// resolve to the same unit. + services: BTreeMap<(String, String), String>, +} + +/// One desired `(image, tag)` entry: the digest to run, and the architecture it +/// was built for when that is known. +/// +/// The arch rides alongside the digest rather than being looked up later +/// because it is only knowable at push time, when the guard probes the image. +/// `None` means "not probed" - entries derived from the engine's watched tags at +/// `up` have never been through the guard - and an unknown arch is never treated +/// as a mismatch, so this can only refuse deliveries it positively knows are +/// wrong. +#[derive(Debug, Clone, PartialEq, Eq)] +struct DesiredEntry { + digest: String, + arch: Option, +} + +impl DesiredState { + /// Re-derive the desired state from the engine's CURRENT watched tags at + /// `up` (design D5). + /// + /// This is the ONLY way to populate a [`DesiredState`]: the desired mapping + /// is a function of what the engine reports *now*, never a persisted value. + /// Each item is `(image, tag, digest)`. + pub fn derive_from_watched_tags(watched: I) -> Self + where + I: IntoIterator, + { + let by_tag = watched + .into_iter() + .map(|(image, tag, digest)| { + ( + (image, tag), + DesiredEntry { + digest, + // Never probed: these come from the engine's current + // tags, not from a guarded sync. + arch: None, + }, + ) + }) + .collect(); + Self { + by_tag, + services: BTreeMap::new(), + } + } + + /// Record which systemd unit consumes each `(image, tag)`. + /// + /// Called once at `up` from the project's `container_dev.images`. The keys + /// must be derived the same way the push path derives them (through + /// `image_ref::split`), or a lookup silently misses and the device falls back + /// to restarting the container - which is the no-op this exists to end. + pub fn set_services(&mut self, services: I) + where + I: IntoIterator, + { + self.services = services.into_iter().collect(); + } + + /// The unit consuming `(image, tag)`, if the project declared one. + fn service_for(&self, image: &str, tag: &str) -> Option { + self.services + .get(&(image.to_string(), tag.to_string())) + .cloned() + } + + /// Record a fresh `(image, tag) -> digest` after a new sync so a later + /// reconcile compares against the just-pushed digest. + pub fn record_sync(&mut self, image: &str, tag: &str, digest: &str, arch: Option) { + self.by_tag.insert( + (image.to_string(), tag.to_string()), + DesiredEntry { + digest: digest.to_string(), + arch, + }, + ); + } + + /// The desired digest for `(image, tag)`, if watched. + pub fn digest_for(&self, image: &str, tag: &str) -> Option<&str> { + self.by_tag + .get(&(image.to_string(), tag.to_string())) + .map(|entry| entry.digest.as_str()) + } + + /// The desired entries as `(image, tag, digest)` triples. + pub fn entries(&self) -> Vec<(String, String, String)> { + self.by_tag + .iter() + .map(|((image, tag), entry)| (image.clone(), tag.clone(), entry.digest.clone())) + .collect() + } + + /// Reconcile a device's reported `running_digest` against the desired state + /// (design H2). + /// + /// Returns a [`HostFrame::Sync`] for every desired entry whose digest does + /// NOT match what the device runs — driving a device that reconnected with a + /// stale digest back to current. A device already on the desired digest + /// yields no sync. + /// + /// An entry whose recorded architecture disagrees with the device's is never + /// sent. The cross-arch guard cannot cover this on its own: at push time the + /// device book may be empty (the device is still booting, or hours away), so + /// the guard has nobody to disagree with and allows the sync. This is the + /// second half of that check, made at the only moment the device's own arch + /// is known. An entry with no recorded arch is passed through unchanged - the + /// filter refuses only what it positively knows is wrong. + pub fn reconcile(&self, hello: &Hello) -> Vec { + let device_arch = DeviceArch::parse(&hello.arch); + self.by_tag + .iter() + .filter(|(_, entry)| entry.digest != hello.running_digest) + .filter(|((image, _), entry)| match &entry.arch { + Some(image_arch) if *image_arch != device_arch => { + print_warning( + &format!( + "refusing to sync `{image}` (built for {}) to device `{}` \ + (reports {}): rebuild for the device platform", + sanitize_device_text(image_arch.as_str()), + sanitize_device_text(&hello.device_id), + sanitize_device_text(device_arch.as_str()), + ), + OutputLevel::Normal, + ); + false + } + _ => true, + }) + .map(|((image, tag), entry)| HostFrame::Sync { + image: image.clone(), + tag: tag.clone(), + digest: entry.digest.clone(), + service: self.service_for(image, tag), + }) + .collect() + } +} + +/// Print a device `Status` report to the operator. +/// +/// Every field is device-supplied and goes through [`sanitize_device_text`] for +/// the same reason the arch-refusal warning does: `print_warning` is a bare +/// `println!` with an ANSI prefix and no escaping, so a device holding the read +/// token could otherwise forge a success line over its own failure report. +/// +/// `sync_failed` and `needs_rebootstrap` are the two the device raises today; +/// anything else is printed verbatim rather than dropped, so a new device-side +/// state is visible before the host learns to special-case it. +/// The operator-facing text for a device `Status`. +fn device_status_message(status: &Status) -> String { + let device = sanitize_device_text(&status.device_id); + let state = sanitize_device_text(&status.state); + let detail = status + .detail + .as_deref() + .map(sanitize_device_text) + .unwrap_or_default(); + let suffix = if detail.is_empty() { + String::new() + } else { + format!(": {detail}") + }; + match status.state.as_str() { + "needs_rebootstrap" => format!( + "device `{device}` rejected its read/control token{suffix}. Re-run \ + `avocado container dev up` to mint a fresh one." + ), + _ => format!("device `{device}` reports {state}{suffix}"), + } +} + +/// Render device-supplied text safe to print to a terminal. +/// +/// `print_warning` is a bare `println!` with an ANSI prefix and no escaping, and +/// both `device_id` and `arch` come straight off the wire - `DeviceArch::parse` +/// falls through to the raw lowercased input for anything it does not recognize. +/// A device holding the read token (every device does) could put `ESC[2K\r` and a +/// forged green success line in its `device_id` and overwrite the refusal warning +/// on the operator's terminal, so a refused sync would read as a completed one. +/// +/// Keeps printable ASCII and replaces everything else, so the warning stays +/// readable while carrying no control sequence. Truncated because the field is +/// attacker-sized as well as attacker-valued. +fn sanitize_device_text(raw: &str) -> String { + const MAX: usize = 64; + let mut out: String = raw + .chars() + .take(MAX) + .map(|c| { + if c.is_ascii_graphic() || c == ' ' { + c + } else { + '.' + } + }) + .collect(); + if raw.chars().count() > MAX { + out.push('…'); + } + out +} + +/// The control-WS server: authenticates each upgrade through the shared +/// read/control-token seam, reconciles a device's `hello`, and broadcasts +/// host -> device `sync` frames (realizing the watcher's [`Notifier`] seam). +/// +/// Held behind an [`Arc`] so the accept loop, per-connection tasks, and the +/// watcher's notify path all share one instance. +pub struct ControlServer { + /// The per-session Bearer read/control token every WS upgrade is validated + /// against — the SAME token the bulk listener uses (design G-5). + read_token: ReadToken, + /// The desired state, re-derived at `up`; updated on each notify. + desired: Mutex, + /// The cross-arch guard's device-arch book, populated from `hello.arch`. + arch_book: HelloArchBook, + /// Image architectures recorded by the cross-arch guard, read by `notify` so + /// the arch is stored alongside the digest it describes. + image_arches: ImageArchBook, + /// Host -> device fan-out of `sync` frames; each connection subscribes. + tx: broadcast::Sender, + /// The registry store the bulk listener serves, used by `notify` to resolve a + /// tag to the MANIFEST digest the device must pull by. + /// + /// `None` only in unit tests that assert fan-out and reconciliation without a + /// registry; production (`container dev up`) always supplies it. + store: Option>, + /// Where a device `Status` report goes. + /// + /// Injectable purely so the WIRING is testable: a test that calls + /// `device_status_message` directly proves the wording and nothing else, so + /// re-dropping the `Status` arm in `on_device_message` would leave it green - + /// the same defect this PR is fixing elsewhere. Production always passes + /// `print_warning`. + reporter: Reporter, +} + +/// Sink for operator-facing device reports; see [`ControlServer::reporter`]. +type Reporter = Arc; + +impl ControlServer { + /// Build a server over `read_token`, the up-time `desired` state, and the + /// cross-arch guard's two books: `arch_book` (device arches, which this + /// server fills from `hello` frames) and `image_arches` (image arches, which + /// the guard fills and `notify` reads). + pub fn new( + read_token: ReadToken, + desired: DesiredState, + arch_book: HelloArchBook, + image_arches: ImageArchBook, + store: Option>, + ) -> Arc { + let (tx, _rx) = broadcast::channel(64); + Arc::new(Self { + read_token, + desired: Mutex::new(desired), + arch_book, + image_arches, + tx, + store, + reporter: Arc::new(|message| print_warning(message, OutputLevel::Normal)), + }) + } + + /// Replace the device-report sink. Test-only; see [`Self::reporter`]. + #[cfg(test)] + fn with_reporter(self: Arc, reporter: Reporter) -> Arc { + let Ok(mut server) = Arc::try_unwrap(self) else { + panic!("with_reporter must be called while the Arc is still sole-owned"); + }; + server.reporter = reporter; + Arc::new(server) + } + + /// Serve control-WS connections on `listener`, terminating TLS with + /// `acceptor` before any WebSocket byte is read (design D8/D9). + /// + /// This is the production entry point: the device agent connects over + /// `wss://` and pins the per-project session CA, so the control WS enforces + /// the same pinned-CA TLS guarantee the bulk listener does + /// ([`super::registry::BulkListener`]). Each accepted TCP stream is + /// handshaked with the per-project leaf (task 3.6) and, on success, upgraded + /// (with auth) and served on its own task over the resulting + /// [`tokio_rustls::server::TlsStream`]. A TLS handshake failure is a + /// per-connection concern (a client that does not trust the session CA, or a + /// probe): the connection is dropped and the accept loop keeps serving, + /// mirroring [`super::registry`]'s bulk `TlsListener`. + pub async fn serve_tls(self: Arc, listener: TcpListener, acceptor: TlsAcceptor) { + loop { + let stream = match listener.accept().await { + Ok((stream, _peer)) => stream, + Err(_) => { + // Back off and keep serving, matching the bulk listener + // (`super::registry::TlsListener::accept`). Returning here + // ended the control WS for the rest of the session on a + // single transient error - a client that RSTs between SYN + // and accept gives ECONNABORTED, and EMFILE is transient + // too - while `up` kept running and `status` kept reporting + // the session live, so no device could reconnect and + // nothing said why. The sleep is what stops a persistent + // error becoming a busy-spin. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + continue; + } + }; + let acceptor = acceptor.clone(); + let server = Arc::clone(&self); + tokio::spawn(async move { + // Drop a connection whose TLS handshake fails and keep serving; + // do not surface it, do not busy-spin. + let Ok(tls) = acceptor.accept(stream).await else { + return; + }; + let _ = server.handle_connection(tls).await; + }); + } + } + + /// Accept control-WS connections on `listener` over PLAIN TCP. + /// + /// Test-only: production binds the control WS over pinned-CA TLS via + /// [`serve_tls`](Self::serve_tls). This entry exists so the transport-agnostic + /// control logic can be exercised over plain TCP exactly as the auth-module + /// tests do, without a TLS handshake in the loop. It is gated `#[cfg(test)]` + /// so no production path can ever bind the control WS in plaintext. + #[cfg(test)] + pub async fn serve(self: Arc, listener: TcpListener) { + loop { + let Ok((stream, _peer)) = listener.accept().await else { + return; + }; + let server = Arc::clone(&self); + tokio::spawn(async move { + let _ = server.handle_connection(stream).await; + }); + } + } + + /// Upgrade one stream (authenticating via the shared seam) then serve its + /// control frames. + /// + /// Generic over the transport `S` so the SAME connection-handling core drives + /// both the production TLS stream (`TlsStream`) and the plain-TCP + /// stream tests use — the read/control-token validator seam is shared, never + /// duplicated per transport (design G-5). + async fn handle_connection(self: Arc, stream: S) -> Result<()> + where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, + { + let ws = self.accept_authenticated(stream).await?; + self.run_session(ws).await + } + + /// Perform the WebSocket upgrade, rejecting a client that lacks a valid + /// Bearer read/control token. + /// + /// The upgrade callback delegates to [`read_request_authorized`] — the exact + /// function the bulk listener's middleware uses — so the WS cannot diverge + /// from the bulk auth surface (design G-5). A rejected upgrade returns `401` + /// with a bare `Bearer` challenge, matching the read listener (design L-1). + // The upgrade callback's `Result` shape is imposed + // verbatim by tungstenite's `accept_hdr_async` contract, so the large-err + // lint cannot be satisfied by boxing without breaking the trait bound. + #[allow(clippy::result_large_err)] + async fn accept_authenticated( + &self, + stream: S, + ) -> Result> + where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + { + let token = self.read_token.clone(); + let callback = + move |request: &Request, response: Response| -> Result { + if read_request_authorized(request.headers(), &token) { + Ok(response) + } else { + let err = tokio_tungstenite::tungstenite::http::Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header(header::WWW_AUTHENTICATE, "Bearer") + .body(Some("read/control token required".to_string())) + .expect("static 401 response builds"); + Err(err) + } + }; + tokio_tungstenite::accept_hdr_async(stream, callback) + .await + .context("control-WS upgrade") + } + + /// Serve one authenticated connection: reconcile on `hello`, fan out + /// broadcast `sync` frames, and drain informational device frames. + async fn run_session( + self: Arc, + mut ws: tokio_tungstenite::WebSocketStream, + ) -> Result<()> + where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + { + let mut broadcasts = self.tx.subscribe(); + // Holds this device in the arch book for exactly as long as the session + // lasts. Declared here so it drops on EVERY exit from this function - + // clean close, send error, or unwind - rather than at one hand-placed + // removal that a later `return` could route around. + let mut _arch_lease: Option = None; + // This connection's device architecture, learned from its `hello`. The + // broadcast arm below needs it: `reconcile` filters on arch, but a frame + // fanned out by `notify` never goes through `reconcile`, so without this + // the pre-hello hole stays open for any device that connects DURING a + // push - the guard snapshots an empty device book before pushing, and the + // device that arrives mid-push receives the Sync on its broadcast arm. + let mut device_arch: Option = None; + loop { + tokio::select! { + incoming = ws.next() => match incoming { + Some(Ok(msg)) => { + if let Some((frames, lease, arch)) = self.on_device_message(&msg) { + // A reconnecting device re-leases; replacing the old + // guard here drops it, which is correct because it + // belonged to this same session. + if lease.is_some() { + _arch_lease = lease; + } + if arch.is_some() { + device_arch = arch; + } + for frame in frames { + ws.send(encode(&frame)?).await?; + } + } + } + // Connection closed or errored: end the session. + Some(Err(_)) | None => return Ok(()), + }, + host = broadcasts.recv() => match host { + Ok(frame) => { + if self.frame_suits_device(&frame, device_arch.as_ref()) { + ws.send(encode(&frame)?).await?; + } + } + // Lagged past the buffer: skip the missed frames, keep serving. + Err(broadcast::error::RecvError::Lagged(_)) => {} + Err(broadcast::error::RecvError::Closed) => return Ok(()), + }, + } + } + } + + /// Whether `frame` may be sent to a device reporting `device_arch`. + /// + /// The arch check has to happen on the FAN-OUT, not only in `reconcile`. + /// `notify` broadcasts to every subscriber, and a device that connects during + /// a push reconciles against a desired map the push has not written yet - so + /// it gets no reconcile frames, then receives the broadcast one directly. That + /// is the same pre-hello window the recorded image arch exists to close, + /// reached by the other path. + /// + /// Refuses only a positive mismatch: an unrecorded image arch, or a device + /// that has not said hello yet, passes through unchanged. + fn frame_suits_device(&self, frame: &HostFrame, device_arch: Option<&DeviceArch>) -> bool { + let HostFrame::Sync { image, tag, .. } = frame; + let Some(device_arch) = device_arch else { + return true; + }; + let reference = if tag.is_empty() { + image.clone() + } else { + format!("{image}:{tag}") + }; + // Canonical, matching what `ArchGuardSyncer` records. Looking up the + // reference as broadcast found nothing for any registry-qualified ref. + match self.image_arches.arch_for(&canonical(&reference)) { + Some(image_arch) if image_arch != *device_arch => { + print_warning( + &format!( + "not broadcasting `{reference}` (built for {}) to a device reporting {}", + sanitize_device_text(image_arch.as_str()), + sanitize_device_text(device_arch.as_str()), + ), + OutputLevel::Normal, + ); + false + } + _ => true, + } + } + + /// Handle one device -> host frame, returning any host -> device frames to + /// send in response (the reconcile syncs for a `hello`). + /// Returns the frames to send, plus a lease the caller must hold for the + /// rest of the session when this frame put a device in the arch book. + fn on_device_message( + &self, + msg: &Message, + ) -> Option<(Vec, Option, Option)> { + let text = msg.to_text().ok()?; + let frame: DeviceFrame = serde_json::from_str(text).ok()?; + match frame { + DeviceFrame::Hello(hello) => { + // Record the device arch for the cross-arch guard (task 4.3), + // scoped to this connection: the guard refuses on ANY mismatch, + // so an entry that outlived its device would refuse every later + // sync for an architecture nothing connected reports. + let lease = self.arch_book.record_session(&hello.device_id, &hello.arch); + // Reconcile the reported running_digest against the desired state. + let frames = self.desired.lock().unwrap().reconcile(&hello); + let arch = DeviceArch::parse(&hello.arch); + Some((frames, Some(lease), Some(arch))) + } + // A device report needs no host response, but it does need to reach + // the operator. Dropping `Status` silently meant the device could + // report `sync_failed` or `needs_rebootstrap` and nothing surfaced + // anywhere: `Hello` is re-sent only on reconnect, so a healthy link + // showed a device synced at the old digest with no error surface at + // all, and the only evidence lived in the device journal. + DeviceFrame::Status(status) => { + (self.reporter)(&device_status_message(&status)); + None + } + // Progress is informational and high-frequency; printing every one + // would bury the Status lines above it. + DeviceFrame::Progress(_) => None, + } + } +} + +impl Notifier for ControlServer { + /// Notify every connected device that `event`'s image is available: update + /// the desired state with the new digest and broadcast a [`HostFrame::Sync`]. + /// + /// Only a control `sync` frame is ever sent — the bulk pull rides the + /// dedicated listener (design D9), never this WS. + fn notify<'a>( + &'a self, + event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let (image, tag) = split_image_tag(&event.image); + // The device pulls `/@`, so this MUST be the + // registry MANIFEST digest. `event.image_id` is the engine's LOCAL + // image id (a config digest) and names nothing the registry can + // serve: pulling by it fails, so every sync would no-op while the + // control frame looked correct. Resolve the tag against the store the + // bulk listener actually serves. + let digest = match self.store.as_ref() { + Some(store) => store + .resolve_tag(&image, &tag) + .ok() + .flatten() + .ok_or_else(|| { + anyhow::anyhow!( + "refusing to notify `{}`: the registry has no manifest for tag `{}` \ + yet (the push must land before the notify)", + event.image, + tag + ) + })?, + None => event.image_id.clone().unwrap_or_default(), + }; + // An empty digest must never enter the desired state. `reconcile` + // compares it against the device's `running_digest`, which is also + // empty before the device's first pull - so an empty desired digest + // compares EQUAL and the device is silently never told to pull. A + // caller with no digest has nothing to desire; record nothing. + if digest.is_empty() { + return Err(anyhow::anyhow!( + "refusing to notify `{}` with no image digest: the engine did not \ + report an id for it", + event.image + )); + } + // The arch the guard probed for this image, if it went through the + // guard at all. Recorded with the digest so a later reconcile can + // refuse to hand it to a device of another architecture - the guard + // itself cannot, because at push time there may be no device + // connected to compare against. + let arch = self.image_arches.arch_for(&canonical(&event.image)); + let service = { + let mut desired = self.desired.lock().unwrap(); + desired.record_sync(&image, &tag, &digest, arch); + desired.service_for(&image, &tag) + }; + let frame = HostFrame::Sync { + image, + tag, + digest, + service, + }; + // A send with no connected devices is not an error (nobody to notify + // yet); a later `hello` reconciles them. + let _ = self.tx.send(frame); + Ok(()) + }) + } +} + +/// Serialize a [`HostFrame`] into a WebSocket text message. +fn encode(frame: &HostFrame) -> Result { + let json = serde_json::to_string(frame).context("serializing a control frame")?; + Ok(Message::Text(json.into())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::container_dev::image_ref; + use tokio_tungstenite::tungstenite::client::IntoClientRequest; + use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; + + const READ_TOKEN: &str = "read-control-token"; + + /// A `DesiredState` whose services are keyed exactly the way `up` keys them, + /// so the tests exercise the real derivation rather than a hand-built key. + fn desired_with_service(config_ref: &str, service: &str, digest: &str) -> DesiredState { + let (repo, tag) = image_ref::split(config_ref); + let mut desired = DesiredState::derive_from_watched_tags([( + repo.clone(), + tag.clone(), + digest.to_string(), + )]); + desired.set_services([((repo, tag), service.to_string())]); + desired + } + + #[test] + fn reconcile_tells_the_device_which_unit_consumes_the_image() { + // Without this the device restarts the container, which stays bound to + // its create-time image id, so the pulled image never runs and every sync + // no-ops while reporting success. + let desired = desired_with_service("my-app:dev", "app.service", "sha256:new"); + + let frames = desired.reconcile(&hello("")); + + assert_eq!( + frames, + vec![HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: Some("app.service".to_string()), + }] + ); + } + + #[test] + fn the_service_key_survives_a_registry_prefixed_config_ref() { + // THE key-agreement test. `image_ref::split` strips the registry, so a + // config `ref` of `localhost/my-app:dev` must still resolve against the + // stripped `my-app` the frame carries. Keying the service map by the raw + // config string instead looks correct and silently misses here, and a + // missed lookup is indistinguishable from "no service declared" - which + // falls back to the container restart this exists to replace. + let desired = desired_with_service("localhost/my-app:dev", "app.service", "sha256:new"); + + let frames = desired.reconcile(&hello("")); + + assert_eq!(frames.len(), 1, "{frames:?}"); + let HostFrame::Sync { image, service, .. } = &frames[0]; + assert_eq!(image, "my-app", "the frame carries the stripped repo"); + assert_eq!( + service.as_deref(), + Some("app.service"), + "the service must resolve against the stripped repo, not the raw ref" + ); + } + + #[test] + fn an_undeclared_image_carries_no_service() { + // The mirror: a project with no `service:` must send None rather than an + // arbitrary unit, so the device keeps its previous behaviour. Without + // this, a lookup that returned some default would pass the tests above. + let desired = DesiredState::derive_from_watched_tags([( + "other-app".to_string(), + "dev".to_string(), + "sha256:new".to_string(), + )]); + + let frames = desired.reconcile(&hello("")); + + let HostFrame::Sync { service, .. } = &frames[0]; + assert_eq!(service.as_deref(), None); + } + + #[test] + fn a_frame_without_a_service_omits_the_key_on_the_wire() { + // `skip_serializing_if` keeps the frame byte-identical to what older + // devices already parse, so shipping this host does not require every + // device to be upgraded first. + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }; + let json = serde_json::to_string(&frame).unwrap(); + assert!(!json.contains("service"), "{json}"); + } + + #[tokio::test] + async fn a_status_frame_reaches_the_operator_through_on_device_message() { + // The WIRING, not the wording. Re-dropping the `Status` arm in + // `on_device_message` leaves every assertion on `device_status_message` + // green while the host goes back to swallowing the report - which is the + // exact defect being fixed. Nothing but observing the sink catches it. + let seen: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); + let server = ControlServer::new( + ReadToken::new(READ_TOKEN), + DesiredState::default(), + HelloArchBook::new(), + ImageArchBook::new(), + None, + ) + .with_reporter({ + let seen = Arc::clone(&seen); + Arc::new(move |message: &str| seen.lock().unwrap().push(message.to_string())) + }); + + let frame = DeviceFrame::Status(Status { + device_id: "dev-1".to_string(), + state: "sync_failed".to_string(), + detail: Some("my-app:dev @ sha256:new: boom".to_string()), + }); + let text = serde_json::to_string(&frame).unwrap(); + + let response = server.on_device_message(&Message::Text(text.into())); + + assert!( + response.is_none(), + "a report needs no host response, only to be surfaced" + ); + let reports = seen.lock().unwrap().clone(); + assert_eq!(reports.len(), 1, "exactly one report: {reports:?}"); + assert!(reports[0].contains("sync_failed"), "{:?}", reports[0]); + assert!(reports[0].contains("boom"), "{:?}", reports[0]); + } + + #[tokio::test] + async fn a_progress_frame_is_not_reported() { + // The mirror: Progress is high-frequency and would bury the Status lines. + // Without this, a reporter wired to every device frame would pass above. + let seen: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); + let server = ControlServer::new( + ReadToken::new(READ_TOKEN), + DesiredState::default(), + HelloArchBook::new(), + ImageArchBook::new(), + None, + ) + .with_reporter({ + let seen = Arc::clone(&seen); + Arc::new(move |message: &str| seen.lock().unwrap().push(message.to_string())) + }); + + let frame = DeviceFrame::Progress(Progress { + image: "my-app:dev".to_string(), + bytes_pulled: 4096, + }); + let text = serde_json::to_string(&frame).unwrap(); + + server.on_device_message(&Message::Text(text.into())); + + assert!( + seen.lock().unwrap().is_empty(), + "progress must stay silent: {:?}", + seen.lock().unwrap() + ); + } + + #[test] + fn a_device_status_report_is_surfaced_and_sanitized() { + // Dropping Status silently meant a device could report sync_failed and + // nothing appeared anywhere on the host: Hello is re-sent only on + // reconnect, so a healthy link showed the device synced at the old digest + // with the only evidence in the device journal. + let message = device_status_message(&Status { + device_id: "dev-1".to_string(), + state: "sync_failed".to_string(), + detail: Some("my-app:dev @ sha256:new: boom".to_string()), + }); + assert!(message.contains("dev-1"), "{message}"); + assert!(message.contains("sync_failed"), "{message}"); + assert!(message.contains("boom"), "{message}"); + + // Device-supplied text reaches a bare println! with an ANSI prefix, so a + // device holding the read token could otherwise overwrite its own failure + // line with a forged success one. + let forged = device_status_message(&Status { + device_id: "\x1b[2K\rdev-1".to_string(), + state: "sync_failed".to_string(), + detail: None, + }); + assert!( + !forged.contains('\x1b') && !forged.contains('\r'), + "control characters must not survive: {forged:?}" + ); + } + + #[test] + fn a_stale_token_report_names_the_remedy() { + // needs_rebootstrap has no renewal endpoint - the operator must re-run + // `up` - so the generic "reports " line would leave them stuck. + let message = device_status_message(&Status { + device_id: "dev-1".to_string(), + state: "needs_rebootstrap".to_string(), + detail: None, + }); + assert!( + message.contains("avocado container dev up"), + "the only remedy must be named: {message}" + ); + } + + fn hello(running_digest: &str) -> Hello { + Hello { + device_id: "dev-1".to_string(), + arch: "aarch64".to_string(), + running_digest: running_digest.to_string(), + } + } + + // ---- frame protocol: control-only, no blob carrier (design D9) ---- + + #[test] + fn a_sync_frame_round_trips_and_carries_only_a_digest_reference() { + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:abc".to_string(), + service: None, + }; + let json = serde_json::to_string(&frame).unwrap(); + // The wire form is tagged and carries a digest *reference*, never bytes. + assert!(json.contains("\"type\":\"sync\""), "tagged as sync: {json}"); + assert!(json.contains("sha256:abc"), "carries the digest: {json}"); + let back: HostFrame = serde_json::from_str(&json).unwrap(); + assert_eq!(back, frame); + } + + #[test] + fn the_only_host_frame_is_sync_so_no_blob_can_ride_the_ws() { + // Structural guarantee: HostFrame has exactly one variant, Sync, which + // carries image coordinates + a digest reference. There is no variant a + // blob/bulk byte stream could be placed into, so a bulk transfer cannot + // be sent as a WS frame (design D9). This test pins that: if a blob-bytes + // variant were ever added, the exhaustive match below stops compiling. + let frame = HostFrame::Sync { + image: "a".into(), + tag: "b".into(), + digest: "sha256:c".into(), + service: None, + }; + match frame { + HostFrame::Sync { .. } => {} + } + } + + #[test] + fn device_frames_round_trip() { + let frames = vec![ + DeviceFrame::Hello(hello("sha256:run")), + DeviceFrame::Progress(Progress { + image: "my-app:dev".into(), + bytes_pulled: 42, + }), + DeviceFrame::Status(Status { + device_id: "dev-1".into(), + state: "running".into(), + detail: None, + }), + ]; + for frame in frames { + let json = serde_json::to_string(&frame).unwrap(); + let back: DeviceFrame = serde_json::from_str(&json).unwrap(); + assert_eq!(back, frame); + } + } + + // ---- desired-state: re-derived at up, never persisted (design D5) ---- + + #[test] + fn desired_state_is_derived_from_current_watched_tags() { + let desired = DesiredState::derive_from_watched_tags([ + ( + "my-app".to_string(), + "dev".to_string(), + "sha256:aaa".to_string(), + ), + ( + "side".to_string(), + "latest".to_string(), + "sha256:bbb".to_string(), + ), + ]); + assert_eq!(desired.digest_for("my-app", "dev"), Some("sha256:aaa")); + assert_eq!(desired.digest_for("side", "latest"), Some("sha256:bbb")); + assert_eq!(desired.digest_for("absent", "dev"), None); + } + + #[test] + fn a_second_up_rederives_desired_state_from_the_new_current_tags() { + // First `up`: the engine's current watched tag is digest aaa. + let first_up = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:aaa".to_string(), + )]); + assert_eq!(first_up.digest_for("my-app", "dev"), Some("sha256:aaa")); + + // The image is rebuilt while the host is down; the engine's current tag + // is now digest bbb. A fresh `up` RE-DERIVES from the current tags — it + // must reflect bbb, not restore the stale aaa from any persisted state. + let second_up = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:bbb".to_string(), + )]); + assert_eq!( + second_up.digest_for("my-app", "dev"), + Some("sha256:bbb"), + "desired state must be re-derived from current tags, not persisted" + ); + } + + // ---- reconcile: a stale running_digest is driven back to current (H2) ---- + + #[test] + fn a_stale_running_digest_reconciles_to_a_sync() { + let desired = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:new".to_string(), + )]); + // The device reports it is running an older digest. + let frames = desired.reconcile(&hello("sha256:old")); + assert_eq!( + frames, + vec![HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }], + "a stale running_digest must produce a reconcile sync to the desired digest" + ); + } + + #[test] + fn a_device_already_on_the_desired_digest_needs_no_sync() { + let desired = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:current".to_string(), + )]); + assert!( + desired.reconcile(&hello("sha256:current")).is_empty(), + "a device already on the desired digest must not be reconciled" + ); + } + + #[test] + fn split_image_tag_strips_registry_and_defaults_tag() { + assert_eq!( + split_image_tag("my-app:dev"), + ("my-app".into(), "dev".into()) + ); + assert_eq!( + split_image_tag("localhost/my-app:dev"), + ("my-app".into(), "dev".into()) + ); + assert_eq!( + split_image_tag("my-app"), + ("my-app".into(), "latest".into()) + ); + } + + // ---- WS upgrade authenticates via the SHARED read/control validator (G-5) ---- + + /// Spawn a control server over plain TCP; return its `ws://` base URL and the + /// server handle so a test can also drive its notify path. + async fn spawn_server(desired: DesiredState) -> (String, Arc) { + spawn_server_with_images(desired, ImageArchBook::new()).await + } + + /// [`spawn_server`] over a caller-supplied image-arch book, so a test can + /// stage what the guard would have recorded at push time. + async fn spawn_server_with_images( + desired: DesiredState, + images: ImageArchBook, + ) -> (String, Arc) { + let server = ControlServer::new( + ReadToken::new(READ_TOKEN), + desired, + HelloArchBook::new(), + images, + None, + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let serve = Arc::clone(&server); + tokio::spawn(async move { serve.serve(listener).await }); + (format!("ws://{addr}/"), server) + } + + /// A client upgrade request carrying the Bearer read/control token. + fn authed_request(url: &str, token: &str) -> Request { + let mut req = url.into_client_request().unwrap(); + req.headers_mut() + .insert(AUTHORIZATION, format!("Bearer {token}").parse().unwrap()); + req + } + + #[tokio::test] + async fn an_upgrade_without_the_read_control_token_is_rejected() { + let (url, _server) = spawn_server(DesiredState::default()).await; + // No Authorization header at all. + let err = tokio_tungstenite::connect_async(url.into_client_request().unwrap()) + .await + .expect_err("an unauthenticated WS upgrade must be rejected"); + match err { + tokio_tungstenite::tungstenite::Error::Http(resp) => { + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "a tokenless upgrade must be 401" + ); + } + other => panic!("expected an HTTP 401, got {other:?}"), + } + } + + #[tokio::test] + async fn the_write_token_shape_is_rejected_on_the_ws_upgrade() { + // A Basic credential (the write token's transport form) must never + // authorize the control WS — the shared validator only accepts Bearer. + use base64::Engine as _; + let (url, _server) = spawn_server(DesiredState::default()).await; + let mut req = url.into_client_request().unwrap(); + let basic = base64::engine::general_purpose::STANDARD.encode("avocado:write-secret"); + req.headers_mut() + .insert(AUTHORIZATION, format!("Basic {basic}").parse().unwrap()); + let err = tokio_tungstenite::connect_async(req) + .await + .expect_err("a Basic write credential must not authorize the control WS"); + assert!( + matches!(err, tokio_tungstenite::tungstenite::Error::Http(resp) if resp.status() == StatusCode::UNAUTHORIZED), + "the write-token shape must be refused on the WS upgrade with 401" + ); + } + + #[tokio::test] + async fn an_upgrade_with_the_read_control_token_is_accepted() { + let (url, _server) = spawn_server(DesiredState::default()).await; + let (ws, resp) = tokio_tungstenite::connect_async(authed_request(&url, READ_TOKEN)) + .await + .expect("a valid read/control token must be accepted"); + assert_eq!(resp.status(), StatusCode::SWITCHING_PROTOCOLS); + drop(ws); + } + + // ---- end-to-end: a hello with a stale digest triggers a reconcile sync ---- + + #[tokio::test] + async fn a_hello_with_a_stale_running_digest_triggers_a_reconcile_sync() { + let desired = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:new".to_string(), + )]); + let (url, _server) = spawn_server(desired).await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(authed_request(&url, READ_TOKEN)) + .await + .unwrap(); + + // The device announces it is running the OLD digest. + let hello = DeviceFrame::Hello(hello("sha256:old")); + ws.send(Message::Text(serde_json::to_string(&hello).unwrap().into())) + .await + .unwrap(); + + // The host must reconcile it back to the desired digest with a sync. + let msg = ws.next().await.expect("a reconcile sync").unwrap(); + let frame: HostFrame = serde_json::from_str(msg.to_text().unwrap()).unwrap(); + assert_eq!( + frame, + HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }, + "a reconnect with a stale running_digest must reconcile to the desired digest" + ); + } + + #[tokio::test] + async fn a_hello_already_on_the_desired_digest_gets_no_sync() { + let desired = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:current".to_string(), + )]); + let (url, _server) = spawn_server(desired).await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(authed_request(&url, READ_TOKEN)) + .await + .unwrap(); + + ws.send(Message::Text( + serde_json::to_string(&DeviceFrame::Hello(hello("sha256:current"))) + .unwrap() + .into(), + )) + .await + .unwrap(); + + // No reconcile should arrive; a short timeout confirms silence rather + // than a spurious sync. + let quiet = tokio::time::timeout(std::time::Duration::from_millis(300), ws.next()).await; + assert!( + quiet.is_err(), + "a device already on the desired digest must not receive a sync" + ); + } + + // ---- the notify seam broadcasts a control sync, never a blob (D9/4.2) ---- + + #[tokio::test] + async fn notify_broadcasts_a_control_sync_frame_to_a_connected_device() { + let (url, server) = spawn_server(DesiredState::default()).await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(authed_request(&url, READ_TOKEN)) + .await + .unwrap(); + + // Connect and announce a matching hello (empty desired -> no reconcile), + // so the device is subscribed before the notify fires. + ws.send(Message::Text( + serde_json::to_string(&DeviceFrame::Hello(hello(""))) + .unwrap() + .into(), + )) + .await + .unwrap(); + // Let the server register the subscription. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // The watcher reports a new tag over the Notifier seam. + let event = TagEvent { + image: "my-app:dev".to_string(), + image_id: Some("sha256:fresh".to_string()), + }; + server.notify(&event).await.unwrap(); + + let msg = ws.next().await.expect("a broadcast sync").unwrap(); + // It is a text control frame carrying the digest reference — never binary + // blob content. + assert!(msg.is_text(), "a control frame is text, not a binary blob"); + let frame: HostFrame = serde_json::from_str(msg.to_text().unwrap()).unwrap(); + assert_eq!( + frame, + HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:fresh".to_string(), + service: None, + } + ); + } + + #[tokio::test] + async fn notify_records_the_new_digest_into_desired_state() { + let (_url, server) = spawn_server(DesiredState::default()).await; + let event = TagEvent { + image: "my-app:dev".to_string(), + image_id: Some("sha256:fresh".to_string()), + }; + server.notify(&event).await.unwrap(); + // A subsequent reconcile compares against the just-pushed digest. + let stale = server + .desired + .lock() + .unwrap() + .reconcile(&hello("sha256:old")); + assert_eq!(stale.len(), 1, "notify must update the desired digest"); + let current = server + .desired + .lock() + .unwrap() + .reconcile(&hello("sha256:fresh")); + assert!( + current.is_empty(), + "a device on the just-pushed digest needs no reconcile" + ); + } + + // The pre-hello window: the guard allowed this push because no device was + // connected to disagree with, so a wrong-arch digest reached desired state. + // `reconcile` is the second half of the check and must refuse it at the one + // moment the device's own arch is finally known. + // + // Deleting the arch filter in `reconcile` fails this: the Sync frame goes + // out and the device is handed an image it cannot run. + #[tokio::test] + async fn reconcile_refuses_a_wrong_arch_entry_recorded_before_any_device_connected() { + let images = ImageArchBook::new(); + // What the guard records when it probes an amd64 image and finds no + // devices to compare against. + images.record_image("my-app:dev", DeviceArch::parse("amd64")); + + let (_url, server) = spawn_server_with_images(DesiredState::default(), images).await; + let event = TagEvent { + image: "my-app:dev".to_string(), + image_id: Some("sha256:amd64only".to_string()), + }; + server.notify(&event).await.unwrap(); + + // An arm64 device connects afterwards, running nothing yet. + let arm = Hello { + device_id: "dev-arm64".to_string(), + arch: "aarch64".to_string(), + running_digest: String::new(), + }; + let frames = server.desired.lock().unwrap().reconcile(&arm); + assert!( + frames.is_empty(), + "an amd64 image must not be reconciled to an aarch64 device: {frames:?}" + ); + + // The same entry must still reach a device that CAN run it, or the + // filter is just breaking sync. + let x86 = Hello { + device_id: "dev-amd64".to_string(), + arch: "x86_64".to_string(), + running_digest: String::new(), + }; + let frames = server.desired.lock().unwrap().reconcile(&x86); + assert_eq!( + frames.len(), + 1, + "a matching-arch device must still be synced: {frames:?}" + ); + } + + // The regression this pins is a KEY MISMATCH, not a missing filter: the guard + // recorded the arch under the raw event ref while the filter looked it up + // under the registry-stripped one, so `arch_for` answered None and the + // permissive `_ => true` arm passed every frame through. podman qualifies + // local refs as `localhost/…` and `WatchSet::is_watched` is an exact match, so + // a podman user MUST configure the qualified ref for the watcher to fire - + // making this every ref on that engine, and the amd64-to-aarch64 broadcast the + // guard exists to stop. + // + // Recording under the raw ref again fails this and nothing else, which is + // exactly what made the drift survivable for so long. + #[tokio::test] + async fn a_registry_qualified_ref_is_still_arch_filtered_on_the_broadcast() { + let images = ImageArchBook::new(); + // What ArchGuardSyncer records for a podman user's `localhost/my-app:dev`. + images.record_image( + &super::super::image_ref::canonical("localhost/my-app:dev"), + DeviceArch::parse("amd64"), + ); + + let (_url, server) = spawn_server_with_images(DesiredState::default(), images).await; + + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:amd64only".to_string(), + service: None, + }; + assert!( + !server.frame_suits_device(&frame, Some(&DeviceArch::parse("aarch64"))), + "an amd64 image recorded from a registry-qualified ref must not reach an \ + aarch64 device" + ); + assert!( + server.frame_suits_device(&frame, Some(&DeviceArch::parse("x86_64"))), + "the same frame must still reach a device that can run it" + ); + } + + // A device controls both `device_id` and (via the parse fall-through) `arch`, + // and the warning path is a bare println with an ANSI prefix. Control bytes + // must not survive into it: a forged `ESC[2K\r` plus a green success line + // would overwrite the refusal on the operator's terminal, so a refused sync + // would read as a completed one. Fails if the sanitizer stops stripping. + #[test] + fn device_supplied_text_cannot_carry_control_sequences() { + let forged = "dev\x1b[2K\r\x1b[32m[OK] synced successfully"; + let safe = sanitize_device_text(forged); + + assert!(!safe.contains('\x1b'), "ESC must not survive: {safe:?}"); + assert!(!safe.contains('\r'), "CR must not survive: {safe:?}"); + assert!(!safe.contains('\n'), "LF must not survive: {safe:?}"); + assert!( + safe.starts_with("dev"), + "printable text should still be readable: {safe:?}" + ); + + // Attacker-sized as well as attacker-valued. + let long = "a".repeat(500); + let capped = sanitize_device_text(&long); + assert!( + capped.chars().count() <= 65, + "must be truncated, got {} chars", + capped.chars().count() + ); + } + + // The broadcast leg needs the same arch filter `reconcile` has. + // + // `reconcile` only runs on a `hello`. A device that connects DURING a push + // reconciles against a desired map the push has not written yet - so it gets + // no frames - and then receives the pushed Sync directly on its broadcast + // arm. That reaches the same pre-hello hole from the other side. Fails if + // `frame_suits_device` stops filtering. + #[test] + fn a_broadcast_frame_is_withheld_from_a_wrong_arch_device() { + let images = ImageArchBook::new(); + images.record_image("my-app:dev", DeviceArch::parse("amd64")); + let server = ControlServer::new( + ReadToken::new(READ_TOKEN), + DesiredState::default(), + HelloArchBook::new(), + images, + None, + ); + + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:amd64only".to_string(), + service: None, + }; + + assert!( + !server.frame_suits_device(&frame, Some(&DeviceArch::parse("aarch64"))), + "an amd64 image must not be broadcast to an aarch64 device" + ); + assert!( + server.frame_suits_device(&frame, Some(&DeviceArch::parse("x86_64"))), + "a matching device must still receive it" + ); + assert!( + server.frame_suits_device(&frame, None), + "a device that has not said hello yet must not be filtered out" + ); + } + + // An image the guard never probed must still fan out, or an unprobed entry + // would silently stop reaching every device. + #[test] + fn a_broadcast_frame_for_an_unprobed_image_is_sent() { + let server = ControlServer::new( + ReadToken::new(READ_TOKEN), + DesiredState::default(), + HelloArchBook::new(), + ImageArchBook::new(), + None, + ); + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:unprobed".to_string(), + service: None, + }; + assert!(server.frame_suits_device(&frame, Some(&DeviceArch::parse("aarch64")))); + } + + // An entry with no recorded arch is passed through: entries derived from the + // engine's watched tags at `up` never went through the guard, and treating + // "unknown" as "mismatch" would stop reconciling them entirely. + #[tokio::test] + async fn reconcile_passes_through_an_entry_with_no_recorded_arch() { + let desired = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:unprobed".to_string(), + )]); + let (_url, server) = spawn_server(desired).await; + + let frames = server.desired.lock().unwrap().reconcile(&Hello { + device_id: "dev-1".to_string(), + arch: "aarch64".to_string(), + running_digest: String::new(), + }); + assert_eq!( + frames.len(), + 1, + "an unprobed entry must still reconcile: {frames:?}" + ); + } + + // A digest-less event must be refused rather than recorded as "". + // + // The empty string is not an inert placeholder here: `reconcile` filters on + // `digest != hello.running_digest`, and a device that has never pulled + // reports an empty `running_digest` - so an empty desired digest compares + // EQUAL, yields no Sync frame, and the device is silently never told to + // pull. Recording nothing is the only safe response. + #[tokio::test] + async fn notify_refuses_an_event_with_no_image_id() { + let (_url, server) = spawn_server(DesiredState::default()).await; + let event = TagEvent { + image: "my-app:dev".to_string(), + image_id: None, + }; + + let result = server.notify(&event).await; + assert!( + result.is_err(), + "an event with no image id must be refused, not recorded as an empty digest" + ); + + // The decisive assertion: a fresh device (empty running_digest) must not + // be left with nothing to do because of a planted empty entry. + let frames = server.desired.lock().unwrap().reconcile(&hello("")); + assert!( + frames.is_empty(), + "no desired entry should exist at all: {frames:?}" + ); + assert!( + server + .desired + .lock() + .unwrap() + .digest_for("my-app", "dev") + .is_none(), + "the refused event must leave no entry behind" + ); + } + + // ---- production TLS: the control WS runs over the pinned-CA leaf (D8/D9) ---- + + /// Spawn a control server over TLS with a fresh session's leaf-backed server + /// config; return its `wss://` base URL, the minted session (whose CA cert a + /// client pins and whose read/control token it presents), and the handle. + async fn spawn_tls_server( + desired: DesiredState, + ) -> ( + String, + crate::utils::container_dev::tls::DevSession, + Arc, + ) { + let session = crate::utils::container_dev::tls::DevSession::mint("dev-runtime", &[]) + .expect("session mints"); + let server = ControlServer::new( + session.read_token.clone(), + desired, + HelloArchBook::new(), + ImageArchBook::new(), + None, + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let acceptor = TlsAcceptor::from(session.tls.server_config()); + let serve = Arc::clone(&server); + tokio::spawn(async move { serve.serve_tls(listener, acceptor).await }); + (format!("wss://{addr}/"), session, server) + } + + /// A `tokio_tungstenite` TLS connector that trusts ONLY `ca_cert_pem`, so it + /// validates the leaf's `127.0.0.1` IP SAN and rejects any other chain — + /// the same pinned-CA discipline the bulk listener's client uses. + fn pinned_ca_connector(ca_cert_pem: &str) -> tokio_tungstenite::Connector { + use base64::Engine as _; + // Decode the single PEM cert body into DER without an extra dependency. + let body: String = ca_cert_pem + .lines() + .filter(|line| !line.starts_with("-----")) + .collect(); + let der = base64::engine::general_purpose::STANDARD + .decode(body.trim()) + .expect("session CA PEM base64 decodes"); + let mut roots = rustls::RootCertStore::empty(); + roots + .add(rustls::pki_types::CertificateDer::from(der)) + .expect("the session CA cert is a valid trust anchor"); + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + tokio_tungstenite::Connector::Rustls(Arc::new(config)) + } + + #[tokio::test] + async fn a_pinned_ca_tls_upgrade_succeeds_and_reconciles_a_stale_hello() { + let desired = DesiredState::derive_from_watched_tags([( + "my-app".to_string(), + "dev".to_string(), + "sha256:new".to_string(), + )]); + let (url, session, _server) = spawn_tls_server(desired).await; + + // A client that pins ONLY the session CA and presents the Bearer + // read/control token: the wss upgrade must succeed over TLS. + let connector = pinned_ca_connector(session.tls.ca_cert_pem()); + let request = authed_request(&url, session.read_token.secret()); + let (mut ws, resp) = + tokio_tungstenite::connect_async_tls_with_config(request, None, false, Some(connector)) + .await + .expect("a pinned-CA wss upgrade with the read/control token must succeed"); + assert_eq!(resp.status(), StatusCode::SWITCHING_PROTOCOLS); + + // A hello reporting a stale running_digest reconciles to the desired one. + ws.send(Message::Text( + serde_json::to_string(&DeviceFrame::Hello(hello("sha256:old"))) + .unwrap() + .into(), + )) + .await + .unwrap(); + + let msg = ws.next().await.expect("a reconcile sync over TLS").unwrap(); + let frame: HostFrame = serde_json::from_str(msg.to_text().unwrap()).unwrap(); + assert_eq!( + frame, + HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }, + "a stale hello over the pinned-CA TLS control WS must reconcile to the desired digest" + ); + } + + #[tokio::test] + async fn a_client_that_does_not_trust_the_session_ca_fails_the_tls_handshake() { + let (url, _session, _server) = spawn_tls_server(DesiredState::default()).await; + + // Pin a DIFFERENT session's CA: it did not sign the server leaf, so the + // TLS handshake must fail before any WebSocket upgrade is attempted. + let other = crate::utils::container_dev::tls::DevSession::mint("other-runtime", &[]) + .expect("a second session mints"); + let connector = pinned_ca_connector(other.tls.ca_cert_pem()); + let result = tokio_tungstenite::connect_async_tls_with_config( + url.into_client_request().unwrap(), + None, + false, + Some(connector), + ) + .await; + assert!( + result.is_err(), + "a client that does not trust the session CA must fail the TLS handshake" + ); + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 221a8edd..19785746 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,6 +1,7 @@ pub mod config; pub mod config_edit; pub mod container; +pub mod container_dev; #[cfg(target_os = "macos")] pub mod disk_writer; pub mod ext_fetch; diff --git a/src/utils/remote.rs b/src/utils/remote.rs index 61007ef1..53f0e391 100644 --- a/src/utils/remote.rs +++ b/src/utils/remote.rs @@ -353,6 +353,74 @@ impl SshClient { }) } + /// Run a command on the remote host, feeding `stdin_data` to its stdin. + /// + /// For delivering file content to a device without depending on a decoder + /// being present there. Embedding a payload in the command means either + /// solving shell quoting for arbitrary bytes or base64-encoding it and + /// decoding on the far side - and `base64` is absent from a minimal target + /// such as Avocado OS, where `printf %s '' | base64 -d` fails with + /// `sh: base64: not found`. A payload on stdin needs nothing but the remote + /// shell, and it also keeps secrets out of the remote process list. + pub async fn run_command_with_stdin(&self, command: &str, stdin_data: &[u8]) -> Result { + use tokio::io::AsyncWriteExt as _; + + if self.verbose { + print_info( + &format!( + "Running remote command (stdin {} bytes): {command}", + stdin_data.len() + ), + OutputLevel::Verbose, + ); + } + + let mut args = self.base_ssh_args(); + args.extend([self.remote.ssh_target(), command.to_string()]); + + let mut child = AsyncCommand::new("ssh") + .args(&args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("Failed to run command on remote: {command}"))?; + + // Write and close stdin before waiting: the remote `cat` will not see EOF + // until the pipe closes, so holding it open past the write deadlocks both + // sides. Taking the handle drops it at the end of this block. + { + let mut stdin = child + .stdin + .take() + .context("ssh stdin was not piped as requested")?; + stdin + .write_all(stdin_data) + .await + .with_context(|| format!("Failed to write stdin for remote command: {command}"))?; + stdin + .shutdown() + .await + .with_context(|| format!("Failed to close stdin for remote command: {command}"))?; + } + + let output = child + .wait_with_output() + .await + .with_context(|| format!("Failed to run command on remote: {command}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!( + "Remote command failed: {}\nError: {}", + command, + stderr.trim() + ); + } + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } + /// Run a command on the remote host, inheriting stdin/stdout/stderr /// /// This method properly forwards Ctrl+C and other signals to the remote process diff --git a/src/utils/runtime.rs b/src/utils/runtime.rs index 019b0fd0..9e5d2d9a 100644 --- a/src/utils/runtime.rs +++ b/src/utils/runtime.rs @@ -247,6 +247,7 @@ mod tests { rootfs: None, initramfs: None, var: None, + container_dev: None, }, ) } diff --git a/tests/container_dev_arch.rs b/tests/container_dev_arch.rs new file mode 100644 index 00000000..0d328f8d --- /dev/null +++ b/tests/container_dev_arch.rs @@ -0,0 +1,453 @@ +//! Cross-arch refusal integration test (task 8.3). +//! +//! Falsifier this file guards (must be false): a mismatched-arch image is +//! delivered to the device. +//! +//! The cross-arch guard (task 4.3) lives at +//! `avocado_cli::utils::container_dev::watcher::arch_guard`. A container image +//! built for one CPU architecture cannot run on a device of another, so the +//! guard probes an image's platform architecture, compares it against every +//! connected device's reported `hello.arch`, and REFUSES the sync before the +//! wrapped syncer ships anything. This file asserts that refusal at the +//! integration level against the REAL guard types — `ArchGuardSyncer`, +//! `HelloArchBook` (the live device-arch book fed by `record_hello`), +//! `DeviceArch`, `check_arch`, and `ArchMismatch`. The only doubles are the two +//! seams the guard was designed to accept: an [`ImageArchProbe`] (image arch, +//! standing in for a real `image inspect`) and an inner [`Syncer`] (the thing +//! that would actually ship). A refusal is proven concretely: the returned +//! error is an [`ArchMismatch`] AND the inner syncer's ship count stays 0, so a +//! wrong-arch image never reaches the device. + +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use anyhow::Result; +use base64::Engine as _; + +use avocado_cli::utils::container_dev::engine::TagEvent; +use avocado_cli::utils::container_dev::watcher::arch_guard::{ + check_arch, ArchGuardSyncer, ArchMismatch, DeviceArch, DeviceArchBook, HelloArchBook, + ImageArchBook, ImageArchProbe, +}; +use avocado_cli::utils::container_dev::watcher::{SyncMode, Syncer}; + +fn ev(image: &str) -> TagEvent { + TagEvent { + image: image.to_string(), + image_id: Some(format!("sha256:{image}")), + } +} + +/// A probe reporting a fixed image architecture, so the guard's refusal logic is +/// exercised without a real engine `image inspect`. +struct FixedProbe(&'static str); + +impl ImageArchProbe for FixedProbe { + fn image_arch<'a>( + &'a self, + _event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + let arch = DeviceArch::parse(self.0); + Box::pin(async move { Ok(arch) }) + } +} + +/// The thing that would actually ship the image. It records every sync call so a +/// refusal is provable as "the ship never happened" (count stays 0), and a pass +/// is provable as "the ship ran exactly once". +#[derive(Default)] +struct ShipRecorder { + ships: AtomicUsize, +} + +impl ShipRecorder { + fn ship_count(&self) -> usize { + self.ships.load(Ordering::SeqCst) + } +} + +impl Syncer for ShipRecorder { + fn sync<'a>( + &'a self, + _mode: SyncMode, + _event: &'a TagEvent, + ) -> Pin> + Send + 'a>> { + self.ships.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(()) }) + } +} + +// ---- assertion 1: a mismatched-arch image is refused, not shipped ---- + +#[tokio::test] +async fn a_mismatched_arch_image_is_refused_and_never_ships() { + let inner = Arc::new(ShipRecorder::default()); + let book = HelloArchBook::new(); + book.record_hello("dev-1", "aarch64"); // device reports arm64 + + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), // image built for x86_64 + Arc::new(book) as Arc, + ImageArchBook::new(), + ); + + let err = guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect_err("an amd64 image must be refused for an arm64 device"); + + let mismatch = err + .downcast_ref::() + .expect("the refusal must be an ArchMismatch, not some unrelated error"); + assert_eq!(mismatch.image_arch, "amd64"); + assert_eq!(mismatch.device_arch, "arm64"); + + assert_eq!( + inner.ship_count(), + 0, + "a refused cross-arch sync must never ship the wrong-arch image to the device" + ); +} + +// ---- assertion 2: a matching-arch image IS shipped (positive control) ---- + +#[tokio::test] +async fn a_matching_arch_image_is_shipped() { + let inner = Arc::new(ShipRecorder::default()); + let book = HelloArchBook::new(); + book.record_hello("dev-1", "x86_64"); // device reports amd64 + + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), // image built for x86_64: matches + Arc::new(book) as Arc, + ImageArchBook::new(), + ); + + guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect("a matching-arch image must be allowed through the guard"); + + assert_eq!( + inner.ship_count(), + 1, + "a matching-arch image must be shipped exactly once (the guard discriminates, \ + it does not refuse everything)" + ); +} + +// ---- assertion 3: fleet model is fleet-wide — ANY mismatched device refuses ---- +// +// The implemented guard is NOT per-device: `check_arch` refuses the whole sync +// if the image mismatches ANY connected device (watcher.rs `check_arch`). These +// cases assert that real design, not a per-device ship-to-the-matching-one model. + +#[tokio::test] +async fn any_single_mismatched_device_in_a_fleet_refuses_the_whole_sync() { + let inner = Arc::new(ShipRecorder::default()); + let book = HelloArchBook::new(); + book.record_hello("dev-amd64", "x86_64"); // matches the amd64 image + book.record_hello("dev-arm64", "aarch64"); // does NOT match + + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), + Arc::new(book) as Arc, + ImageArchBook::new(), + ); + + let err = guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect_err("an amd64 image must be refused because the arm64 device cannot run it"); + + let mismatch = err + .downcast_ref::() + .expect("the refusal must be an ArchMismatch"); + assert_eq!(mismatch.device_arch, "arm64"); + + assert_eq!( + inner.ship_count(), + 0, + "a fleet-wide refusal must ship to NO device, not even the matching amd64 one" + ); +} + +#[tokio::test] +async fn a_homogeneous_matching_fleet_is_shipped() { + let inner = Arc::new(ShipRecorder::default()); + let book = HelloArchBook::new(); + book.record_hello("dev-a", "aarch64"); // arm64 + book.record_hello("dev-b", "arm64"); // arm64 (uname vs GOARCH spelling) + + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("arm64")), // image matches every device + Arc::new(book) as Arc, + ImageArchBook::new(), + ); + + guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect("an image matching every device in the fleet must be allowed"); + + assert_eq!( + inner.ship_count(), + 1, + "an all-matching fleet must ship exactly once" + ); +} + +// ---- the pure guard function `check_arch` discriminates match from mismatch ---- + +#[test] +fn check_arch_refuses_a_mismatch_and_names_the_arches() { + let err = check_arch( + "my-app:dev", + &DeviceArch::parse("amd64"), + &[DeviceArch::parse("aarch64")], + ) + .expect_err("an amd64 image must be refused for an arm64 device"); + assert_eq!(err.image, "my-app:dev"); + assert_eq!(err.image_arch, "amd64"); + assert_eq!(err.device_arch, "arm64"); + + // The refusal is actionable: it names buildx and the device target platform, + // so a bare `Err(())` sentinel would fail this. + let msg = err.to_string(); + assert!( + msg.contains("buildx"), + "refusal must give buildx guidance: {msg}" + ); + assert!( + msg.contains("linux/arm64"), + "refusal must name the device target platform: {msg}" + ); +} + +#[test] +fn check_arch_allows_a_uname_vs_goarch_match() { + // A device reporting uname `aarch64` and an image with GOARCH `arm64` are the + // same architecture; the guard must NOT spuriously refuse them. + check_arch( + "app:dev", + &DeviceArch::parse("arm64"), + &[DeviceArch::parse("aarch64")], + ) + .expect("a uname/GOARCH-equivalent arch must pass the guard"); +} + +// ---- assertion 5: a device that disconnects stops constraining the guard ---- + +/// Within one `up`, a developer tests against an aarch64 board, unplugs it, and +/// attaches an amd64 one. The arch book must not still be refusing amd64 syncs +/// on behalf of a board that is gone - `check_arch` refuses on ANY mismatch, so +/// a stale entry blocks the rest of the session and its buildx guidance names an +/// architecture nothing connected reports. +/// +/// Asserted through the guard rather than by inspecting the book, because the +/// property that matters is "the sync goes through", not "the map is empty". +#[tokio::test] +async fn a_disconnected_device_no_longer_blocks_a_sync() { + let book = HelloArchBook::new(); + let inner = Arc::new(ShipRecorder::default()); + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), // the image the developer now builds + Arc::new(book.clone()) as Arc, + ImageArchBook::new(), + ); + + // The arm64 board is connected: an amd64 sync is correctly refused. + let arm_session = book.record_session("dev-arm64", "aarch64"); + guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect_err("an amd64 image must be refused while an arm64 board is attached"); + assert_eq!(inner.ship_count(), 0); + + // The board is unplugged - its session ends. + drop(arm_session); + + guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect("the departed board must not keep refusing syncs"); + assert_eq!( + inner.ship_count(), + 1, + "the sync must actually ship once nothing disagrees with it" + ); +} + +/// Two overlapping connections from one device must not evict each other: the +/// older session ending cannot remove an entry the newer one still needs, or a +/// reconnect would silently disarm the guard. +#[tokio::test] +async fn overlapping_sessions_for_one_device_refcount() { + let book = HelloArchBook::new(); + let inner = Arc::new(ShipRecorder::default()); + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), + Arc::new(book.clone()) as Arc, + ImageArchBook::new(), + ); + + let first = book.record_session("dev-arm64", "aarch64"); + let second = book.record_session("dev-arm64", "aarch64"); + + drop(first); + guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect_err("the device is still connected on its second session"); + + drop(second); + guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect("with every session closed the guard must stop refusing"); +} + +/// Trust the session CA the way the device agent does, so the control-WS +/// upgrade is exercised over the real pinned-CA TLS rather than plaintext. +fn pinned_ca_connector(ca_cert_pem: &str) -> tokio_tungstenite::Connector { + let body: String = ca_cert_pem + .lines() + .filter(|line| !line.starts_with("-----")) + .collect(); + let der = base64::engine::general_purpose::STANDARD + .decode(body.trim()) + .expect("session CA PEM base64 decodes"); + let mut roots = rustls::RootCertStore::empty(); + roots + .add(rustls::pki_types::CertificateDer::from(der)) + .expect("the session CA cert is a valid trust anchor"); + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + tokio_tungstenite::Connector::Rustls(Arc::new(config)) +} + +/// The guard only works because the book the `ControlServer` writes and the book +/// the `ArchGuardSyncer` reads are the SAME map. Every other test in this file +/// hands the guard a book it populated by hand, so all of them would still pass +/// with the two sides disconnected - which is exactly how the guard sat +/// unreachable before it was wired into `up`. +/// +/// This drives the real path: a device sends a `Hello` over the control WS, the +/// server records its arch, and the guard - holding only a clone of the book it +/// was constructed with - refuses a mismatched image it never saw recorded. +/// Making `HelloArchBook::clone` a deep clone fails here. +/// +/// Unwiring the guard in `up` does NOT - this test builds the `ControlServer` and +/// `ArchGuardSyncer` itself and clones the book by hand, so it verifies the +/// sharing semantics the wiring depends on, not the wiring. Nothing in the suite +/// exercises `DevUpCommand`, so changing `up` to hand the guard a fresh book +/// would leave every test green. That gap is real and unclosed. +#[tokio::test] +async fn a_hello_recorded_by_the_control_server_is_visible_to_the_guard() { + use avocado_cli::utils::container_dev::tls::DevSession; + use avocado_cli::utils::container_dev::ws::{ControlServer, DesiredState, DeviceFrame, Hello}; + use futures_util::SinkExt as _; + use tokio::net::TcpListener; + use tokio_rustls::TlsAcceptor; + use tokio_tungstenite::tungstenite::client::IntoClientRequest; + use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; + use tokio_tungstenite::tungstenite::Message; + + let session = DevSession::mint("dev-runtime", &[]).expect("session mints"); + + // One book, cloned into both halves — exactly what `up` does. + let book = HelloArchBook::new(); + let server = ControlServer::new( + session.read_token.clone(), + DesiredState::default(), + book.clone(), + ImageArchBook::new(), + None, + ); + let inner = Arc::new(ShipRecorder::default()); + // A third handle on the same book, used only to observe when the server has + // processed the hello - so the poll below does not itself drive syncs. + let observer = book.clone(); + let guard = ArchGuardSyncer::new( + inner.clone() as Arc, + Arc::new(FixedProbe("amd64")), // image built for x86_64 + Arc::new(book) as Arc, + ImageArchBook::new(), + ); + + // Nothing recorded yet: the guard has no device to disagree with. + guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect("an empty book has nobody to mismatch"); + assert_eq!(inner.ship_count(), 1); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let acceptor = TlsAcceptor::from(session.tls.server_config()); + tokio::spawn(async move { server.serve_tls(listener, acceptor).await }); + + let mut request = format!("wss://127.0.0.1:{}/", addr.port()) + .into_client_request() + .expect("ws request builds"); + request.headers_mut().insert( + AUTHORIZATION, + format!("Bearer {}", session.read_token.secret()) + .parse() + .unwrap(), + ); + let connector = pinned_ca_connector(session.tls.ca_cert_pem()); + let (mut ws, _resp) = + tokio_tungstenite::connect_async_tls_with_config(request, None, false, Some(connector)) + .await + .expect("authenticated control-WS upgrade succeeds"); + + // An aarch64 device announces itself. Only the SERVER touches the book here. + let hello = DeviceFrame::Hello(Hello { + device_id: "dev-arm64".to_string(), + arch: "aarch64".to_string(), + running_digest: String::new(), + }); + ws.send(Message::text(serde_json::to_string(&hello).unwrap())) + .await + .expect("hello sends"); + + // Wait for the server to record it, bounded so a regression fails rather + // than hangs. Observing the book directly keeps the wait from driving syncs + // of its own, so the ship count below means exactly one thing. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while observer.device_arches().is_empty() { + assert!( + std::time::Instant::now() < deadline, + "the control server's hello never reached the guard's book: the two halves \ + are not sharing one map" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + + let before = inner.ship_count(); + let refused = guard + .sync(SyncMode::Push, &ev("my-app:dev")) + .await + .expect_err("an amd64 image must be refused once an arm64 device has said hello"); + + let mismatch = refused + .downcast_ref::() + .expect("the refusal must be an ArchMismatch"); + assert_eq!(mismatch.device_arch, "arm64"); + assert_eq!(mismatch.image_arch, "amd64"); + assert_eq!( + inner.ship_count(), + before, + "the refused sync must not have shipped anything" + ); +} diff --git a/tests/container_dev_e2e.rs b/tests/container_dev_e2e.rs new file mode 100644 index 00000000..187e00f1 --- /dev/null +++ b/tests/container_dev_e2e.rs @@ -0,0 +1,373 @@ +//! End-to-end round-trip for Container Dev Mode (task 8.1), driven at the +//! interface level against the REAL listeners a device talks to. +//! +//! Two falsifiable properties of the sync round-trip are asserted: +//! +//! 1. **Delta pull.** After a one-line change confined to the final layer, a +//! device that already holds the previous image pulls ONLY the changed layer +//! over the dedicated bulk listener — the shared config and base layers are +//! byte-identical by digest and are never re-transferred. Falsifier: the +//! whole image is re-pulled on a one-line change. +//! 2. **Restart trigger.** The host tells a connected device to move to the new +//! digest over the control WS: a device that reconnects reporting the stale +//! running digest receives a `sync` frame carrying the new digest — the +//! signal that drives the device to pull-and-restart the container. +//! Falsifier: no sync is delivered, so the container is never restarted. +//! +//! The push side uses the plain-HTTP write listener (Basic write token); the +//! pull and control sides use the session's pinned-CA TLS leaf, matching +//! production. Push and pull share ONE per-project store, so a blob pushed on +//! the write leg is pullable on the bulk leg — the actual round-trip. + +use std::collections::HashSet; +use std::net::SocketAddr; +use std::sync::Arc; + +use avocado_cli::utils::container_dev::auth::WRITE_USERNAME; +use avocado_cli::utils::container_dev::registry::{write_router, BulkListener}; +use avocado_cli::utils::container_dev::store::BlobStore; +use avocado_cli::utils::container_dev::tls::DevSession; +use avocado_cli::utils::container_dev::watcher::arch_guard::{HelloArchBook, ImageArchBook}; +use avocado_cli::utils::container_dev::ws::ControlServer; +use avocado_cli::utils::container_dev::ws::{DesiredState, DeviceFrame, Hello, HostFrame}; + +use base64::Engine as _; +use futures_util::{SinkExt as _, StreamExt as _}; +use sha2::{Digest as _, Sha256}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use tokio_rustls::TlsAcceptor; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; +use tokio_tungstenite::tungstenite::Message; + +const RUNTIME: &str = "dev-runtime"; +const NAME: &str = "my-app"; +const TAG: &str = "dev"; + +/// Compute the OCI digest (`sha256:`) of `bytes`. +fn digest_of(bytes: &[u8]) -> String { + let hex: String = Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("sha256:{hex}") +} + +/// A single-platform image manifest referencing `config` and `layers` by digest. +fn manifest_for(config: &[u8], layers: &[&[u8]]) -> Vec { + let layer_entries: Vec<_> = layers + .iter() + .map(|l| { + serde_json::json!({ + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": digest_of(l), + "size": l.len(), + }) + }) + .collect(); + serde_json::to_vec(&serde_json::json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": digest_of(config), + "size": config.len(), + }, + "layers": layer_entries, + })) + .unwrap() +} + +/// The push (write) + pull (bulk) round-trip harness over ONE shared store. +struct Harness { + write_base: String, + bulk_base: String, + session: DevSession, + _bulk: BulkListener, + _dir: TempDir, +} + +/// Stand up the write listener (plain HTTP, Basic-gated) and the bulk read +/// listener (TLS, Bearer-gated) over a single shared per-project store. +async fn harness() -> Harness { + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + + let write_app = write_router(Arc::clone(&store), session.write_token.clone()); + let write_tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let write_addr = write_tcp.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(write_tcp, write_app).await.unwrap(); + }); + + let bulk = BulkListener::bind( + SocketAddr::from(([127, 0, 0, 1], 0)), + Arc::clone(&store), + session.read_token.clone(), + session.tls.server_config(), + ) + .await + .expect("bulk listener binds"); + + Harness { + write_base: format!("http://{write_addr}"), + bulk_base: format!("https://127.0.0.1:{}", bulk.local_addr().port()), + session, + _bulk: bulk, + _dir: dir, + } +} + +/// A reqwest client trusting ONLY the session CA (validates the leaf's SANs). +fn tls_client(session: &DevSession) -> reqwest::Client { + let ca = reqwest::Certificate::from_pem(session.tls.ca_cert_pem().as_bytes()) + .expect("session CA cert parses"); + reqwest::Client::builder() + .add_root_certificate(ca) + .build() + .expect("TLS client builds") +} + +/// Push a blob monolithically to the write listener with the Basic write token. +async fn push_blob(h: &Harness, bytes: &[u8]) { + let digest = digest_of(bytes); + let resp = reqwest::Client::new() + .post(format!( + "{}/v2/{NAME}/blobs/uploads/?digest={digest}", + h.write_base + )) + .basic_auth(WRITE_USERNAME, Some(h.session.write_token.secret())) + .body(bytes.to_vec()) + .send() + .await + .expect("blob push completes"); + assert_eq!(resp.status().as_u16(), 201, "a blob push must be created"); +} + +/// Push a manifest under `TAG` with the Basic write token; returns its digest. +async fn push_manifest(h: &Harness, manifest: &[u8]) -> String { + let resp = reqwest::Client::new() + .put(format!("{}/v2/{NAME}/manifests/{TAG}", h.write_base)) + .basic_auth(WRITE_USERNAME, Some(h.session.write_token.secret())) + .body(manifest.to_vec()) + .send() + .await + .expect("manifest push completes"); + assert_eq!( + resp.status().as_u16(), + 201, + "a manifest push must be created" + ); + digest_of(manifest) +} + +/// The digests a manifest references (config + every layer), in wire order. +fn referenced_digests(manifest: &[u8]) -> Vec { + let v: serde_json::Value = serde_json::from_slice(manifest).unwrap(); + let mut out = vec![v["config"]["digest"].as_str().unwrap().to_string()]; + for layer in v["layers"].as_array().unwrap() { + out.push(layer["digest"].as_str().unwrap().to_string()); + } + out +} + +/// Simulate a device pull over the bulk listener: fetch the manifest, then GET +/// only the referenced blobs NOT already in `local`. Records each fetched blob +/// into `local` and returns the total bytes of blob bodies actually fetched. +async fn device_pull(h: &Harness, local: &mut HashSet) -> u64 { + let client = tls_client(&h.session); + let manifest = client + .get(format!("{}/v2/{NAME}/manifests/{TAG}", h.bulk_base)) + .bearer_auth(h.session.read_token.secret()) + .send() + .await + .expect("manifest pull completes"); + assert_eq!( + manifest.status().as_u16(), + 200, + "the manifest must be pullable" + ); + let manifest_bytes = manifest.bytes().await.unwrap(); + + let mut fetched_bytes = 0u64; + for digest in referenced_digests(&manifest_bytes) { + if local.contains(&digest) { + continue; // already on the device — a delta pull skips it + } + let blob = client + .get(format!("{}/v2/{NAME}/blobs/{digest}", h.bulk_base)) + .bearer_auth(h.session.read_token.secret()) + .send() + .await + .expect("blob pull completes"); + assert_eq!( + blob.status().as_u16(), + 200, + "a referenced blob must be pullable" + ); + let body = blob.bytes().await.unwrap(); + assert_eq!( + digest_of(&body), + digest, + "the pulled blob must match its digest" + ); + fetched_bytes += body.len() as u64; + local.insert(digest); + } + fetched_bytes +} + +// ---- 1. a one-line change pulls only the changed layer over the bulk listener ---- + +#[tokio::test] +async fn a_one_line_change_pulls_only_the_changed_layer() { + let h = harness().await; + + // A shared config and base layer, plus a top layer that differs between the + // two builds — "a one-line change confined to the final layer". + let config = b"image-config-json".to_vec(); + let base_layer = vec![0xABu8; 512 * 1024]; // 512 KiB shared base + let top_v1 = b"top layer, revision 1".to_vec(); + let top_v2 = b"top layer, revision 2 (one line changed)".to_vec(); + + // Push v1 and pull it: the device now holds config + base + top_v1. + push_blob(&h, &config).await; + push_blob(&h, &base_layer).await; + push_blob(&h, &top_v1).await; + let v1_manifest = manifest_for(&config, &[&base_layer, &top_v1]); + let v1_digest = push_manifest(&h, &v1_manifest).await; + + let mut device_blobs = HashSet::new(); + let v1_bytes = device_pull(&h, &mut device_blobs).await; + assert_eq!( + v1_bytes, + (config.len() + base_layer.len() + top_v1.len()) as u64, + "the first pull fetches the whole image (config + base + top)" + ); + + // One-line change: only the top layer differs. Push v2 (shared blobs dedup + // in the content-addressed store) and re-tag. + push_blob(&h, &top_v2).await; + let v2_manifest = manifest_for(&config, &[&base_layer, &top_v2]); + let v2_digest = push_manifest(&h, &v2_manifest).await; + assert_ne!( + v1_digest, v2_digest, + "a changed image must have a new manifest digest" + ); + + // The device pulls again. It must fetch ONLY the changed top layer: the + // shared config and base layer are byte-identical by digest and already + // local, so a delta pull never re-transfers them. + let v2_bytes = device_pull(&h, &mut device_blobs).await; + assert_eq!( + v2_bytes, + top_v2.len() as u64, + "the second pull must transfer ONLY the changed layer, not the whole image \ + (got {v2_bytes} bytes, expected {})", + top_v2.len() + ); + // The shared base layer (the bulk of the image) was NOT re-pulled. + assert!( + v2_bytes < base_layer.len() as u64, + "a one-line change must not re-transfer the shared base layer" + ); +} + +// ---- 2. the device is told to restart with the new digest over the control WS ---- + +#[tokio::test] +async fn a_stale_device_is_synced_to_the_new_digest_over_the_control_ws() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + + let v1_digest = digest_of(b"running-image-v1"); + let v2_digest = digest_of(b"running-image-v2"); + + // Desired state after the change: the tag points at v2 (re-derived at `up` + // from the engine's current watched tags, design D5). + let desired = DesiredState::derive_from_watched_tags([( + NAME.to_string(), + TAG.to_string(), + v2_digest.clone(), + )]); + let server = ControlServer::new( + session.read_token.clone(), + desired, + HelloArchBook::new(), + ImageArchBook::new(), + None, + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let acceptor = TlsAcceptor::from(session.tls.server_config()); + tokio::spawn(async move { server.serve_tls(listener, acceptor).await }); + + // The device dials the control WS with the Bearer read/control token, pinning + // the session CA (production discipline). + let mut request = format!("wss://127.0.0.1:{}/", addr.port()) + .into_client_request() + .expect("ws request builds"); + request.headers_mut().insert( + AUTHORIZATION, + format!("Bearer {}", session.read_token.secret()) + .parse() + .unwrap(), + ); + let connector = pinned_ca_connector(session.tls.ca_cert_pem()); + let (mut ws, _resp) = + tokio_tungstenite::connect_async_tls_with_config(request, None, false, Some(connector)) + .await + .expect("authenticated control-WS upgrade succeeds"); + + // The device reports the STALE digest it is currently running. + let hello = DeviceFrame::Hello(Hello { + device_id: "dev-1".to_string(), + arch: "x86_64".to_string(), + running_digest: v1_digest.clone(), + }); + ws.send(Message::text(serde_json::to_string(&hello).unwrap())) + .await + .expect("hello sends"); + + // The host reconciles and pushes a sync to the NEW digest — the trigger that + // drives the device to pull-and-restart the container. + let msg = tokio::time::timeout(std::time::Duration::from_secs(5), ws.next()) + .await + .expect("a sync frame arrives before the timeout") + .expect("the ws stream yields a frame") + .expect("the frame is not an error"); + let frame: HostFrame = serde_json::from_str(msg.to_text().unwrap()).unwrap(); + assert_eq!( + frame, + HostFrame::Sync { + image: NAME.to_string(), + tag: TAG.to_string(), + digest: v2_digest.clone(), + // This harness builds its DesiredState directly, without a config, + // so no service is declared for the image. + service: None, + }, + "a device reporting the stale digest must be told to move to the new digest" + ); +} + +/// A `tokio_tungstenite` TLS connector trusting ONLY `ca_cert_pem` — the pinned-CA +/// discipline the production control-WS client uses. +fn pinned_ca_connector(ca_cert_pem: &str) -> tokio_tungstenite::Connector { + let body: String = ca_cert_pem + .lines() + .filter(|line| !line.starts_with("-----")) + .collect(); + let der = base64::engine::general_purpose::STANDARD + .decode(body.trim()) + .expect("session CA PEM base64 decodes"); + let mut roots = rustls::RootCertStore::empty(); + roots + .add(rustls::pki_types::CertificateDer::from(der)) + .expect("the session CA cert is a valid trust anchor"); + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + tokio_tungstenite::Connector::Rustls(Arc::new(config)) +} diff --git a/tests/container_dev_security.rs b/tests/container_dev_security.rs new file mode 100644 index 00000000..f5b8634e --- /dev/null +++ b/tests/container_dev_security.rs @@ -0,0 +1,368 @@ +//! Security assertions for the Container Dev Mode embedded registry and control +//! WebSocket, driven at the interface level against the REAL listeners +//! (task 8.2). +//! +//! Every case here spins up a live listener from one [`DevSession`] and drives +//! it with a pinned-CA client, then asserts the exact `401` a removed gate would +//! turn into a success. The falsifiers this file guards (ALL must be false): +//! +//! - an unauthenticated read/WS is served; +//! - an unauthenticated write succeeds on either interface; +//! - the Bearer read/control token authorizes a write (H-A compromised device); +//! - a wrong-password Basic credential authorizes a write (G-3); +//! - the Basic write token is honored on a read route (M-2). +//! +//! The write listener is served over plain HTTP (its gate is the Basic write +//! token, which the tests exercise directly); the bulk read listener and the +//! control WS run over the session's pinned-CA TLS leaf, matching production. + +use std::net::SocketAddr; +use std::sync::Arc; + +use avocado_cli::utils::container_dev::auth::WRITE_USERNAME; +use avocado_cli::utils::container_dev::registry::{write_router, BulkListener}; +use avocado_cli::utils::container_dev::store::BlobStore; +use avocado_cli::utils::container_dev::tls::DevSession; +use avocado_cli::utils::container_dev::watcher::arch_guard::{HelloArchBook, ImageArchBook}; +use avocado_cli::utils::container_dev::ws::{ControlServer, DesiredState}; + +use base64::Engine as _; +use sha2::{Digest as _, Sha256}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use tokio_rustls::TlsAcceptor; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; + +const RUNTIME: &str = "dev-runtime"; + +/// Compute the OCI digest (`sha256:`) of `bytes`. +fn digest_of(bytes: &[u8]) -> String { + let hex: String = Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("sha256:{hex}") +} + +/// A minimal single-platform image manifest to push at a write route. +fn manifest_bytes() -> Vec { + serde_json::to_vec(&serde_json::json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "size": 7, + }, + "layers": [], + })) + .unwrap() +} + +/// Bind the dedicated bulk READ listener (Bearer-gated, TLS) over a fresh +/// per-project store seeded with `blob`, using `session`'s read token and leaf. +/// +/// Returns the loopback `https://` base URL, the live listener handle (kept +/// alive by the caller), the seeded blob digest, and the temp-dir guard. +async fn spawn_bulk(session: &DevSession, blob: &[u8]) -> (String, BulkListener, String, TempDir) { + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "proj").expect("store opens")); + let digest = digest_of(blob); + store.write_blob(&digest, blob).unwrap(); + + let listener = BulkListener::bind( + SocketAddr::from(([127, 0, 0, 1], 0)), + store, + session.read_token.clone(), + session.tls.server_config(), + ) + .await + .expect("bulk listener binds"); + let base = format!("https://127.0.0.1:{}", listener.local_addr().port()); + (base, listener, digest, dir) +} + +/// Start the WRITE listener (Basic-gated) over a fresh per-project store, using +/// `session`'s write token. Served over plain HTTP; the gate under test is the +/// Basic credential, not the transport. Returns its base URL and the temp-dir +/// guard. +async fn spawn_write(session: &DevSession) -> (String, TempDir) { + let dir = TempDir::new().unwrap(); + let store = Arc::new(BlobStore::at(dir.path(), "wproj").expect("store opens")); + let app = write_router(store, session.write_token.clone()); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = tcp.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(tcp, app).await.unwrap(); + }); + (format!("http://{addr}"), dir) +} + +/// Start the control WS server over the session's pinned-CA TLS leaf; return its +/// `wss://` base URL. The gate under test is the Bearer read/control token on +/// the upgrade. +async fn spawn_ws_tls(session: &DevSession) -> String { + let server = ControlServer::new( + session.read_token.clone(), + DesiredState::default(), + HelloArchBook::new(), + ImageArchBook::new(), + None, + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let acceptor = TlsAcceptor::from(session.tls.server_config()); + tokio::spawn(async move { server.serve_tls(listener, acceptor).await }); + format!("wss://{addr}/") +} + +/// A reqwest client trusting ONLY the session CA, so it validates the leaf's +/// `127.0.0.1` IP SAN and rejects any other chain (never native roots). +fn tls_client(session: &DevSession) -> reqwest::Client { + let ca = reqwest::Certificate::from_pem(session.tls.ca_cert_pem().as_bytes()) + .expect("session CA cert parses"); + reqwest::Client::builder() + .add_root_certificate(ca) + .build() + .expect("TLS client builds") +} + +/// A `tokio_tungstenite` TLS connector trusting ONLY `ca_cert_pem` — the same +/// pinned-CA discipline the production control-WS client uses. +fn pinned_ca_connector(ca_cert_pem: &str) -> tokio_tungstenite::Connector { + // Decode the single PEM cert body into DER without an extra dependency. + let body: String = ca_cert_pem + .lines() + .filter(|line| !line.starts_with("-----")) + .collect(); + let der = base64::engine::general_purpose::STANDARD + .decode(body.trim()) + .expect("session CA PEM base64 decodes"); + let mut roots = rustls::RootCertStore::empty(); + roots + .add(rustls::pki_types::CertificateDer::from(der)) + .expect("the session CA cert is a valid trust anchor"); + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + tokio_tungstenite::Connector::Rustls(Arc::new(config)) +} + +// ---- 1. an unauthenticated read on the bulk listener is rejected ---- + +#[tokio::test] +async fn unauthenticated_read_is_rejected_with_401() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let (base, _listener, digest, _dir) = spawn_bulk(&session, b"a-container-layer").await; + let client = tls_client(&session); + + // The API version ping with no Authorization header. + let ping = client + .get(format!("{base}/v2/")) + .send() + .await + .expect("anonymous ping completes"); + assert_eq!( + ping.status().as_u16(), + 401, + "an unauthenticated read on the bulk listener must be refused" + ); + + // A blob path with no Authorization header must also be refused before any + // bytes are served. + let blob = client + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .send() + .await + .expect("anonymous blob pull completes"); + assert_eq!( + blob.status().as_u16(), + 401, + "an unauthenticated blob pull must be refused" + ); +} + +// ---- 2. an unauthenticated WS upgrade is rejected ---- + +#[tokio::test] +async fn unauthenticated_ws_upgrade_is_rejected_with_401() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let url = spawn_ws_tls(&session).await; + let connector = pinned_ca_connector(session.tls.ca_cert_pem()); + + // No Authorization header on the upgrade request at all. + let request = url.into_client_request().expect("ws request builds"); + let err = + tokio_tungstenite::connect_async_tls_with_config(request, None, false, Some(connector)) + .await + .expect_err("an unauthenticated WS upgrade must be rejected"); + match err { + tokio_tungstenite::tungstenite::Error::Http(resp) => assert_eq!( + resp.status().as_u16(), + 401, + "a tokenless control-WS upgrade must be 401" + ), + other => panic!("expected an HTTP 401 on the WS upgrade, got {other:?}"), + } +} + +// ---- 3. an unauthenticated write is refused on BOTH interfaces ---- + +#[tokio::test] +async fn unauthenticated_write_is_refused_on_both_interfaces() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let body = manifest_bytes(); + + // (a) the write listener: manifest PUT, blob-upload POST, and the gated + // `GET /v2/` ping all refuse an anonymous request. + let (write_base, _wdir) = spawn_write(&session).await; + let anon = reqwest::Client::new(); + + let put = anon + .put(format!("{write_base}/v2/my-app/manifests/dev")) + .body(body.clone()) + .send() + .await + .expect("anonymous manifest PUT completes"); + assert_eq!( + put.status().as_u16(), + 401, + "an anonymous manifest write must be refused" + ); + + let post = anon + .post(format!("{write_base}/v2/my-app/blobs/uploads/")) + .send() + .await + .expect("anonymous upload POST completes"); + assert_eq!( + post.status().as_u16(), + 401, + "an anonymous blob-upload open must be refused" + ); + + let ping = anon + .get(format!("{write_base}/v2/")) + .send() + .await + .expect("anonymous write-listener ping completes"); + assert_eq!( + ping.status().as_u16(), + 401, + "the write listener's gated ping must refuse an anonymous request" + ); + + // (b) the bulk READ listener exposes NO write route: an anonymous write verb + // is refused by the read gate before any routing, so it never reaches a + // write handler (there is none on this listener). + let (bulk_base, _listener, _digest, _bdir) = spawn_bulk(&session, b"seed").await; + let bulk_put = tls_client(&session) + .put(format!("{bulk_base}/v2/my-app/manifests/dev")) + .body(body) + .send() + .await + .expect("anonymous PUT to the bulk listener completes"); + assert_eq!( + bulk_put.status().as_u16(), + 401, + "a write verb on the bulk read listener must not be served/authorized" + ); +} + +// ---- 4. the Bearer read/control token is refused on EVERY write route (H-A) ---- + +#[tokio::test] +async fn bearer_read_control_token_is_refused_on_every_write_route() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let (write_base, _wdir) = spawn_write(&session).await; + let read = session.read_token.secret(); + let client = reqwest::Client::new(); + + // A compromised device holds ONLY the Bearer read/control token. Presenting + // it on any write route must be refused — the write listener requires Basic. + + // manifest PUT + let put = client + .put(format!("{write_base}/v2/my-app/manifests/dev")) + .bearer_auth(read) + .body(manifest_bytes()) + .send() + .await + .expect("bearer manifest PUT completes"); + assert_eq!( + put.status().as_u16(), + 401, + "the Bearer read/control token must not authorize a manifest write" + ); + + // blob-upload POST + let post = client + .post(format!("{write_base}/v2/my-app/blobs/uploads/")) + .bearer_auth(read) + .send() + .await + .expect("bearer upload POST completes"); + assert_eq!( + post.status().as_u16(), + 401, + "the Bearer read/control token must not authorize a blob-upload open" + ); + + // dedup HEAD probe + let missing = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + let head = client + .head(format!("{write_base}/v2/my-app/blobs/{missing}")) + .bearer_auth(read) + .send() + .await + .expect("bearer dedup HEAD completes"); + assert_eq!( + head.status().as_u16(), + 401, + "the Bearer read/control token must not authorize a dedup probe" + ); +} + +// ---- 5. a wrong-password Basic credential is refused on a write route (G-3) ---- + +#[tokio::test] +async fn wrong_password_basic_credential_is_refused_on_a_write_route() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let (write_base, _wdir) = spawn_write(&session).await; + + // The correct username but a password that is not the session write token. + let resp = reqwest::Client::new() + .put(format!("{write_base}/v2/my-app/manifests/dev")) + .basic_auth(WRITE_USERNAME, Some("not-the-write-token")) + .body(manifest_bytes()) + .send() + .await + .expect("wrong-password write completes"); + assert_eq!( + resp.status().as_u16(), + 401, + "a Basic credential with the wrong password must be refused on a write route" + ); +} + +// ---- 6. the Basic write token is refused on a read route (M-2) ---- + +#[tokio::test] +async fn basic_write_token_is_refused_on_a_read_route() { + let session = DevSession::mint(RUNTIME, &[]).expect("session mints"); + let (base, _listener, digest, _dir) = spawn_bulk(&session, b"layer-bytes").await; + + // The host-only write token presented in its Basic transport form on the + // bulk READ listener must be refused: read routes accept only Bearer. + let resp = tls_client(&session) + .get(format!("{base}/v2/my-app/blobs/{digest}")) + .basic_auth(WRITE_USERNAME, Some(session.write_token.secret())) + .send() + .await + .expect("basic-on-read pull completes"); + assert_eq!( + resp.status().as_u16(), + 401, + "the Basic write token must not be honored on a read route" + ); +}