diff --git a/.github/workflows/test-ova.yml b/.github/workflows/test-ova.yml index ee99979..5f90291 100644 --- a/.github/workflows/test-ova.yml +++ b/.github/workflows/test-ova.yml @@ -22,13 +22,13 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - name: Install bats and jq + - name: Install bats, jq and zstd run: | sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends bats jq + sudo apt-get install -y --no-install-recommends bats jq zstd - name: Run logic and config tests - run: bats ova/tests/generate-env.bats ova/tests/generate-compose.bats ova/tests/firewall.bats + run: bats ova/tests/generate-env.bats ova/tests/generate-compose.bats ova/tests/firewall.bats ova/tests/dg-ctl.bats integration: name: Real bring-up @@ -37,10 +37,10 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - name: Install bats and jq + - name: Install bats, jq and zstd run: | sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends bats jq + sudo apt-get install -y --no-install-recommends bats jq zstd - name: Login to GitHub container registry uses: docker/login-action@v4 diff --git a/ova/defguard.pkr.hcl b/ova/defguard.pkr.hcl index 46fb548..7c5e7bd 100644 --- a/ova/defguard.pkr.hcl +++ b/ova/defguard.pkr.hcl @@ -92,6 +92,21 @@ build { destination = "/tmp/defguard-init.service" } + provisioner "file" { + source = "files/dg-ctl" + destination = "/tmp/dg-ctl" + } + + provisioner "file" { + source = "files/install-dg-ctl.sh" + destination = "/tmp/install-dg-ctl.sh" + } + + provisioner "file" { + source = "manifest.json" + destination = "/tmp/manifest.json" + } + provisioner "file" { source = "files/defguard-firewall.sh" destination = "/tmp/defguard-firewall.sh" @@ -126,6 +141,13 @@ build { "echo 'DEFGUARD_GATEWAY_TAG=${var.gateway_tag}' | sudo tee -a /opt/stacks/defguard/init/.image-tags > /dev/null", "sudo mv /tmp/99-defguard.cfg /etc/cloud/cloud.cfg.d/99-defguard.cfg", "sudo mv /tmp/defguard-init.service /etc/systemd/system/defguard-init.service", + "sudo mkdir -p /opt/defguard/backups", + "sudo install -m 750 -o root -g root /tmp/dg-ctl /opt/defguard/dg-ctl", + "sudo install -m 750 -o root -g root /tmp/install-dg-ctl.sh /opt/defguard/install-dg-ctl.sh", + "sudo ln -sf /opt/defguard/dg-ctl /usr/local/bin/dg-ctl", + "jq -n --arg v \"$(jq -r .ova_version /tmp/manifest.json)\" --arg ref \"$(jq -r .template_ref /tmp/manifest.json)\" --arg core '${var.core_tag}' --arg proxy '${var.proxy_tag}' --arg gateway '${var.gateway_tag}' '{ova_version: $v, template_ref: $ref, tags: {core: $core, proxy: $proxy, gateway: $gateway}}' | sudo tee /opt/defguard/state.json > /dev/null", + "sudo chmod 600 /opt/defguard/state.json", + "rm -f /tmp/manifest.json", "sudo mv /tmp/defguard-firewall.sh /opt/stacks/defguard/defguard-firewall.sh", "sudo chmod +x /opt/stacks/defguard/defguard-firewall.sh", "sudo mv /tmp/defguard-firewall.service /etc/systemd/system/defguard-firewall.service", diff --git a/ova/files/dg-ctl b/ova/files/dg-ctl new file mode 100644 index 0000000..828ab0a --- /dev/null +++ b/ova/files/dg-ctl @@ -0,0 +1,653 @@ +#!/bin/bash +# defguard OVA maintenance CLI, installed as /opt/defguard/dg-ctl. +set -euo pipefail + +DG_CTL_VERSION="2.1.0" + +STACK_DIR="${DEFGUARD_STACK_DIR:-/opt/stacks/defguard}" +OVA_DIR="${DEFGUARD_OVA_DIR:-/opt/defguard}" +BACKUPS_DIR="$OVA_DIR/backups" +STATE_FILE="$OVA_DIR/state.json" + +COMPOSE_FILE="$STACK_DIR/docker-compose.yml" +ENV_FILE="$STACK_DIR/.env" +INIT_DIR="$STACK_DIR/init" +APPLIED_PROFILES_FILE="$INIT_DIR/.applied-profiles" + +OVA_REPO="${DEFGUARD_OVA_REPO:-DefGuard/deployment}" +MANIFEST_URL="${DEFGUARD_OVA_MANIFEST_URL:-https://raw.githubusercontent.com/$OVA_REPO/ova-upgrade-process/ova/manifest.json}" +SOURCE_DIR="${DEFGUARD_OVA_SOURCE_DIR:-}" + +KEEP_BACKUPS="${KEEP_BACKUPS:-3}" +HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-180}" +# Window used to catch a service that starts and then crash-loops. +HEALTH_SETTLE="${HEALTH_SETTLE:-15}" + +WORK_DIR="" +cleanup() { [ -n "$WORK_DIR" ] && rm -rf "$WORK_DIR"; return 0; } +trap cleanup EXIT + +log() { echo "[dg-ctl] $*"; } +warn() { echo "[dg-ctl] WARNING: $*" >&2; } +die() { echo "[dg-ctl] ERROR: $*" >&2; exit 1; } + +compose() { + docker compose -f "$COMPOSE_FILE" --project-directory "$STACK_DIR" "$@" +} + +need_root() { + [ "${DEFGUARD_OVA_ALLOW_NONROOT:-0}" = "1" ] && return 0 + [ "$(id -u)" = "0" ] || die "must run as root (try: sudo $0 $*)" +} + +need_stack() { + [ -f "$COMPOSE_FILE" ] || die "$COMPOSE_FILE not found; this does not look like a defguard OVA host" + [ -f "$ENV_FILE" ] || die "$ENV_FILE not found" + docker compose version >/dev/null 2>&1 || die "docker compose v2 is not available" + command -v jq >/dev/null 2>&1 || die "jq is required" +} + +env_value() { + [ -f "$ENV_FILE" ] || return 0 + sed -n "s/^$1=//p" "$ENV_FILE" | tail -n1 +} + +fetch() { + local src="$1" dest="$2" + case "$src" in + file://*) cp "${src#file://}" "$dest" ;; + /*) cp "$src" "$dest" ;; + *) curl -fsSL --retry 3 -o "$dest" "$src" ;; + esac +} + +fetch_ova_file() { + local ref="$1" name="$2" dest="$3" + if [ -n "$SOURCE_DIR" ]; then + cp "$SOURCE_DIR/ova/files/$name" "$dest" + else + fetch "https://raw.githubusercontent.com/$OVA_REPO/$ref/ova/files/$name" "$dest" + fi +} + +read_manifest() { + local tmp + tmp="$(mktemp)" + fetch "$MANIFEST_URL" "$tmp" || { rm -f "$tmp"; die "could not fetch manifest from $MANIFEST_URL"; } + jq -e . "$tmp" || { rm -f "$tmp"; die "manifest at $MANIFEST_URL is not valid JSON"; } + rm -f "$tmp" +} + +state_value() { + [ -f "$STATE_FILE" ] || { echo "unknown"; return 0; } + jq -r --arg k "$1" '.[$k] // "unknown"' "$STATE_FILE" 2>/dev/null || echo "unknown" +} + +# "unknown" / "2.0-unknown" (state.json seeded before this field existed) sorts +# as the oldest possible version, so every migration is considered pending. +normalize_version() { + case "$1" in + ""|unknown|*-unknown) echo "0.0.0" ;; + *) echo "$1" ;; + esac +} + +version_le() { + [ "$1" = "$(printf '%s\n%s\n' "$1" "$2" | sort -V | head -n1)" ] +} + +# resolve_pending_migrations FROM TO MANIFEST - prints, one per line, the +# migration versions in MANIFEST's "migrations" array with FROM < v <= TO. +resolve_pending_migrations() { + local from to manifest v + from="$(normalize_version "$1")" + to="$(normalize_version "$2")" + manifest="$3" + while IFS= read -r v; do + [ -n "$v" ] || continue + if ! version_le "$v" "$from" && version_le "$v" "$to"; then + echo "$v" + fi + done < <(jq -r '.migrations[]? // empty' <<<"$manifest" | sort -V) +} + +# Prints the profiles this stack was generated with, one per line. The +# generated compose file has no profiles: key left in it, so this has to come +# from what generate-compose.sh recorded. +resolve_applied_profiles() { + if [ -s "$APPLIED_PROFILES_FILE" ]; then + sed '/^$/d' "$APPLIED_PROFILES_FILE" + return 0 + fi + + # OVAs built before .applied-profiles existed: the generator's header comment. + local header + header="$(sed -n 's/^# Selected profiles: //p' "$COMPOSE_FILE" | head -n1)" + if [ -n "$header" ]; then + tr '[:space:]' '\n' <<<"$header" | sed '/^$/d' + return 0 + fi + + # Last resort: infer from the service list. + local services profiles=() + services="$(compose config --services 2>/dev/null | tr '\n' ' ')" || true + case " $services " in *" core "*) profiles+=(core) ;; esac + case " $services " in *" edge "*) profiles+=(edge) ;; esac + case " $services " in *" gateway "*) profiles+=(gateway) ;; esac + case " $services " in *" dockge "*) profiles+=(dockge) ;; esac + [ "${#profiles[@]}" -gt 0 ] \ + || die "cannot determine which profiles this stack runs; create $APPLIED_PROFILES_FILE (one profile per line) and retry" + printf '%s\n' "${profiles[@]}" +} + +has_profile() { + local want="$1" p + shift + for p in "$@"; do [ "$p" = "$want" ] && return 0; done + return 1 +} + +backup_id_now() { + local label="${1:-}" + local id + id="$(date -u +%Y%m%d-%H%M%SZ)" + [ -n "$label" ] && id="$id-$label" + echo "$id" +} + +do_backup() { + local label="${1:-}" id dir profiles digests checksum + id="$(backup_id_now "$label")" + dir="$BACKUPS_DIR/$id" + # Same-second backups (e.g. two upgrades run back to back) would otherwise + # collide on the timestamp-only id; disambiguate instead of failing. + if [ -e "$dir" ]; then + local n=2 + while [ -e "$BACKUPS_DIR/$id-$n" ]; do n=$((n + 1)); done + id="$id-$n" + dir="$BACKUPS_DIR/$id" + fi + mkdir -p "$BACKUPS_DIR" + + local free_kb need_kb + free_kb="$(df -Pk "$OVA_DIR" | awk 'NR==2 {print $4}')" + need_kb="$(du -sk "$STACK_DIR/.volumes" 2>/dev/null | awk '{print $1}')" + need_kb="${need_kb:-0}" + if [ "$free_kb" -lt "$need_kb" ]; then + die "not enough free space in $OVA_DIR: ${free_kb}K free, ${need_kb}K of volumes to archive" + fi + + mkdir -p "$dir" + profiles="$(resolve_applied_profiles)" + + # Digests first: they are only readable while the containers/images are known + # to compose, and they cost nothing. + digests="$(compose images --format json 2>/dev/null || echo '[]')" + echo "$digests" > "$dir/image-digests" + + log "stopping stack for a consistent backup" >&2 + compose stop >&2 + + log "archiving volumes" >&2 + if [ -d "$STACK_DIR/.volumes" ]; then + tar --zstd -cf "$dir/volumes.tar.zst" -C "$STACK_DIR" .volumes + else + warn ".volumes does not exist; archiving an empty volume set" + tar --zstd -cf "$dir/volumes.tar.zst" -C "$STACK_DIR" --files-from /dev/null + fi + checksum="$(sha256sum "$dir/volumes.tar.zst" | awk '{print $1}')" + + cp "$ENV_FILE" "$dir/env" + chmod 600 "$dir/env" + cp "$COMPOSE_FILE" "$dir/docker-compose.yml" + printf '%s\n' "$profiles" > "$dir/applied-profiles" + + jq -n \ + --arg id "$id" \ + --arg label "$label" \ + --arg created "$(date -u +%FT%TZ)" \ + --arg host "$(hostname)" \ + --arg ova_version "$(state_value ova_version)" \ + --arg core "$(env_value DEFGUARD_CORE_TAG)" \ + --arg proxy "$(env_value DEFGUARD_PROXY_TAG)" \ + --arg gateway "$(env_value DEFGUARD_GATEWAY_TAG)" \ + --arg checksum "$checksum" \ + '{id: $id, label: $label, created: $created, host: $host, ova_version: $ova_version, + tags: {core: $core, proxy: $proxy, gateway: $gateway}, + volumes_sha256: $checksum}' > "$dir/meta.json" + + log "backup written to $dir" >&2 + echo "$id" +} + +prune_backups() { + local keep="$KEEP_BACKUPS" all old + [ "$keep" -gt 0 ] 2>/dev/null || return 0 + # Backup ids are generated timestamps, so ls is safe here. + # shellcheck disable=SC2012 + mapfile -t all < <(ls -1 "$BACKUPS_DIR" 2>/dev/null | LC_ALL=C sort) + [ "${#all[@]}" -gt "$keep" ] || return 0 + old=("${all[@]:0:${#all[@]}-keep}") + local b + for b in "${old[@]}"; do + log "pruning old backup $b (KEEP_BACKUPS=$keep)" + rm -rf "${BACKUPS_DIR:?}/${b:?}" + done +} + +do_list_backups() { + [ -d "$BACKUPS_DIR" ] || { log "no backups yet"; return 0; } + local b any=0 + printf '%-28s %-22s %-12s %s\n' ID CREATED OVA_VERSION SIZE + # shellcheck disable=SC2012 + for b in $(ls -1 "$BACKUPS_DIR" 2>/dev/null | LC_ALL=C sort); do + any=1 + printf '%-28s %-22s %-12s %s\n' \ + "$b" \ + "$(jq -r '.created // "?"' "$BACKUPS_DIR/$b/meta.json" 2>/dev/null || echo '?')" \ + "$(jq -r '.ova_version // "?"' "$BACKUPS_DIR/$b/meta.json" 2>/dev/null || echo '?')" \ + "$(du -sh "$BACKUPS_DIR/$b" | awk '{print $1}')" + done + [ "$any" = 1 ] || log "no backups yet" +} + +do_rollback() { + local id="" skip_tests=0 dir + while [ $# -gt 0 ]; do + case "$1" in + --skip-tests) skip_tests=1; shift ;; + -*) die "unknown option for rollback: $1" ;; + *) id="$1"; shift ;; + esac + done + [ -n "$id" ] || die "usage: dg-ctl rollback [--skip-tests] (see: dg-ctl list-backups)" + dir="$BACKUPS_DIR/$id" + [ -d "$dir" ] || die "no such backup: $id" + for f in volumes.tar.zst env docker-compose.yml meta.json; do + [ -f "$dir/$f" ] || die "backup $id is incomplete: missing $f" + done + + # Verify before deleting anything. + local want have + want="$(jq -r '.volumes_sha256 // ""' "$dir/meta.json")" + have="$(sha256sum "$dir/volumes.tar.zst" | awk '{print $1}')" + [ -n "$want" ] || die "backup $id has no checksum recorded; refusing to restore" + [ "$want" = "$have" ] || die "backup $id is corrupt: volumes.tar.zst checksum mismatch" + + log "rolling back to $id" + compose stop || true + compose down --remove-orphans || true + + rm -rf "$STACK_DIR/.volumes" + tar --zstd -xf "$dir/volumes.tar.zst" -C "$STACK_DIR" + install -m 600 "$dir/env" "$ENV_FILE" + cp "$dir/docker-compose.yml" "$COMPOSE_FILE" + [ -f "$dir/applied-profiles" ] && { mkdir -p "$INIT_DIR"; cp "$dir/applied-profiles" "$APPLIED_PROFILES_FILE"; } + + compose up -d --remove-orphans + + if [ "$skip_tests" = 1 ]; then + log "rollback to $id complete; run '$0 test' when the stack has settled" + elif do_test; then + log "rollback to $id complete and healthy" + else + die "rollback to $id restored the stack but health checks failed; inspect with: journalctl -t defguard-core" + fi +} + +tcp_open() { + local host="$1" port="$2" + timeout 3 bash -c "exec 3<>/dev/tcp/$host/$port" 2>/dev/null +} + +check() { + local name="$1" + shift + if "$@"; then + echo " PASS $name" + return 0 + fi + echo " FAIL $name" + return 1 +} + +retry_until() { + local deadline=$(( $(date +%s) + HEALTH_TIMEOUT )) + while :; do + "$@" && return 0 + [ "$(date +%s)" -ge "$deadline" ] && return 1 + sleep 3 + done +} + +service_running() { + local svc="$1" state + state="$(compose ps --format json "$svc" 2>/dev/null \ + | jq -rs 'flatten | .[0].State // "missing"' 2>/dev/null || echo missing)" + [ "$state" = "running" ] +} + +restart_count() { + local svc="$1" cid + cid="$(compose ps -q "$svc" 2>/dev/null | head -n1)" + [ -n "$cid" ] || { echo -1; return 0; } + docker inspect -f '{{.RestartCount}}' "$cid" 2>/dev/null || echo -1 +} + +core_health() { curl -fsS http://localhost:8000/api/v1/health >/dev/null 2>&1; } + +db_ready() { + local user + user="$(env_value POSTGRES_USER)" + compose exec -T db pg_isready -U "${user:-defguard}" >/dev/null 2>&1 +} + +# 50051 is only published in segmented deployments. +edge_grpc_published() { compose port edge 50051 >/dev/null 2>&1; } + +no_fatal_logs() { + local tag="$1" since="$2" + command -v journalctl >/dev/null 2>&1 || return 0 + ! journalctl -t "$tag" --since "$since" --no-pager 2>/dev/null \ + | grep -Eq 'FATAL|panicked at|thread .* panicked' +} + +# do_test [since] - since is a journalctl timestamp bounding the log scan. +do_test() { + local since="${1:--5 min}" + local -a profiles services + mapfile -t profiles < <(resolve_applied_profiles) + mapfile -t services < <(compose config --services) + + log "health checks (profiles: ${profiles[*]})" + local failed=0 svc + + local -A before=() + for svc in "${services[@]}"; do + check "$svc is running" retry_until service_running "$svc" || failed=1 + before[$svc]="$(restart_count "$svc")" + done + + if has_profile core "${profiles[@]}"; then + check "core /api/v1/health responds" retry_until core_health || failed=1 + check "db accepts connections" retry_until db_ready || failed=1 + fi + if has_profile edge "${profiles[@]}"; then + check "edge :8080 accepts connections" retry_until tcp_open localhost 8080 || failed=1 + check "edge :443 accepts connections" retry_until tcp_open localhost 443 || failed=1 + if edge_grpc_published; then + check "edge :50051 accepts connections" retry_until tcp_open localhost 50051 || failed=1 + fi + fi + if has_profile gateway "${profiles[@]}"; then + check "gateway health port :55003 accepts connections" retry_until tcp_open localhost 55003 || failed=1 + fi + + # Catch crash loops: a container can report running and die seconds later. + log "watching ${HEALTH_SETTLE}s for restarts" + sleep "$HEALTH_SETTLE" + for svc in "${services[@]}"; do + local now + now="$(restart_count "$svc")" + check "$svc did not restart" [ "$now" = "${before[$svc]}" ] || failed=1 + done + + for svc in "${services[@]}"; do + check "$svc logged no fatal errors" no_fatal_logs "defguard-$svc" "$since" || failed=1 + done + + [ "$failed" = 0 ] || return 1 + log "all health checks passed" +} + +set_env_tag() { + local key="$1" value="$2" + if grep -q "^$key=" "$ENV_FILE"; then + sed -i.bak "s|^$key=.*|$key=$value|" "$ENV_FILE" && rm -f "$ENV_FILE.bak" + else + printf '%s=%s\n' "$key" "$value" >> "$ENV_FILE" + fi +} + +do_upgrade() { + local ref="" core_tag="" proxy_tag="" gateway_tag="" + local assume_yes=0 skip_backup=0 skip_tests=0 + + while [ $# -gt 0 ]; do + case "$1" in + --ref) ref="${2:?--ref needs a value}"; shift 2 ;; + --core-tag) core_tag="${2:?--core-tag needs a value}"; shift 2 ;; + --proxy-tag) proxy_tag="${2:?--proxy-tag needs a value}"; shift 2 ;; + --gateway-tag) gateway_tag="${2:?--gateway-tag needs a value}"; shift 2 ;; + -y|--yes) assume_yes=1; shift ;; + --no-backup) skip_backup=1; shift ;; + --skip-tests) skip_tests=1; shift ;; + *) die "unknown option for upgrade: $1" ;; + esac + done + + command -v zstd >/dev/null 2>&1 || die "zstd is required for backups (apt-get install zstd)" + + local manifest ova_version + manifest="$(read_manifest)" + ova_version="$(jq -r '.ova_version // "unknown"' <<<"$manifest")" + : "${ref:=$(jq -r '.template_ref // "main"' <<<"$manifest")}" + : "${core_tag:=$(jq -r '.core_tag // empty' <<<"$manifest")}" + : "${proxy_tag:=$(jq -r '.proxy_tag // empty' <<<"$manifest")}" + : "${gateway_tag:=$(jq -r '.gateway_tag // empty' <<<"$manifest")}" + [ -n "$core_tag" ] && [ -n "$proxy_tag" ] && [ -n "$gateway_tag" ] \ + || die "manifest is missing image tags and none were given on the command line" + + local -a profiles + mapfile -t profiles < <(resolve_applied_profiles) + + cat < $core_tag + proxy: $(env_value DEFGUARD_PROXY_TAG) -> $proxy_tag + gateway: $(env_value DEFGUARD_GATEWAY_TAG) -> $gateway_tag + version: $(state_value ova_version) -> $ova_version +EOF + if [ "$assume_yes" != 1 ]; then + read -r -p "[dg-ctl] proceed? [y/N] " answer + case "$answer" in y|Y|yes|YES) ;; *) die "aborted" ;; esac + fi + + WORK_DIR="$(mktemp -d)" + local work="$WORK_DIR" + local new_init="$work/init" + mkdir -p "$new_init" + + log "fetching compose template and generator at $ref" + local f + for f in docker-compose.template.yaml lib.sh generate-compose.sh; do + fetch_ova_file "$ref" "$f" "$new_init/$f" || die "could not fetch ova/files/$f at ref $ref" + done + + local from_version + from_version="$(state_value ova_version)" + local -a pending_migrations + mapfile -t pending_migrations < <(resolve_pending_migrations "$from_version" "$ova_version" "$manifest") + if [ "${#pending_migrations[@]}" -gt 0 ]; then + log "fetching migrations: ${pending_migrations[*]}" + mkdir -p "$work/migrations" + local m + for m in "${pending_migrations[@]}"; do + fetch_ova_file "$ref" "migrations/$m.sh" "$work/migrations/$m.sh" \ + || die "could not fetch migration $m at ref $ref" + bash -n "$work/migrations/$m.sh" || die "migration $m does not parse; nothing was changed" + done + fi + + # Validate before touching the stack: a broken template must not cost downtime. + local -a profile_args=() + for f in "${profiles[@]}"; do profile_args+=(--profile "$f"); done + docker compose -f "$new_init/docker-compose.template.yaml" \ + --project-directory "$STACK_DIR" --env-file "$ENV_FILE" "${profile_args[@]}" \ + config -q >/dev/null 2>&1 \ + || die "fetched template at ref $ref does not parse; nothing was changed" + + local backup_id="none" + if [ "$skip_backup" = 1 ]; then + warn "--no-backup given: no rollback point will exist" + else + backup_id="$(do_backup upgrade)" + fi + + local started_at + started_at="$(date -u +'%Y-%m-%d %H:%M:%S')" + + if [ "${#pending_migrations[@]}" -gt 0 ]; then + local m + for m in "${pending_migrations[@]}"; do + log "running migration $m" + if ! STACK_DIR="$STACK_DIR" OVA_DIR="$OVA_DIR" ENV_FILE="$ENV_FILE" \ + FROM_VERSION="$from_version" TO_VERSION="$ova_version" \ + bash "$work/migrations/$m.sh"; then + die "migration $m failed; stack left as-is. Restore data with: $0 rollback $backup_id" + fi + done + fi + + log "updating image tags in $ENV_FILE" + set_env_tag DEFGUARD_CORE_TAG "$core_tag" + set_env_tag DEFGUARD_PROXY_TAG "$proxy_tag" + set_env_tag DEFGUARD_GATEWAY_TAG "$gateway_tag" + chmod 600 "$ENV_FILE" + + log "regenerating $COMPOSE_FILE" + cp "$COMPOSE_FILE" "$work/previous-docker-compose.yml" + # generate-compose.sh is a no-op when the file exists, and reads its profile + # selection from the flag files it then deletes - so recreate that input. + rm -f "$COMPOSE_FILE" + local -a stack_profiles=() + for f in "${profiles[@]}"; do + [ "$f" = "dockge" ] && continue + stack_profiles+=("$f") + done + printf '%s\n' "${stack_profiles[@]}" > "$STACK_DIR/active-profiles" + if has_profile dockge "${profiles[@]}"; then + touch "$STACK_DIR/enable-docker-management" + fi + + if ! DEFGUARD_STACK_DIR="$STACK_DIR" DEFGUARD_INIT_DIR="$new_init" \ + bash "$new_init/generate-compose.sh"; then + cp "$work/previous-docker-compose.yml" "$COMPOSE_FILE" + rm -f "$STACK_DIR/active-profiles" "$STACK_DIR/enable-docker-management" + die "compose regeneration failed; previous docker-compose.yml restored. Restore data with: $0 rollback $backup_id" + fi + # The regenerated file is the new source of truth for the profile set. + mkdir -p "$INIT_DIR" + printf '%s\n' "${profiles[@]}" > "$APPLIED_PROFILES_FILE" + + log "pulling images" + compose pull + log "starting stack" + compose up -d --remove-orphans + + jq -n \ + --arg ova_version "$ova_version" \ + --arg ref "$ref" \ + --arg core "$core_tag" --arg proxy "$proxy_tag" --arg gateway "$gateway_tag" \ + --arg upgraded_at "$(date -u +%FT%TZ)" \ + --arg backup "$backup_id" \ + --arg cli "$DG_CTL_VERSION" \ + --argjson migrations "$(printf '%s\n' "${pending_migrations[@]:-}" | jq -R 'select(length > 0)' | jq -s .)" \ + '{ova_version: $ova_version, template_ref: $ref, cli_version: $cli, + tags: {core: $core, proxy: $proxy, gateway: $gateway}, + upgraded_at: $upgraded_at, backup_id: $backup, migrations_applied: $migrations}' > "$STATE_FILE" + + if [ "$skip_tests" = 1 ]; then + log "--skip-tests given; run '$0 test' when the stack has settled" + elif ! do_test "$started_at"; then + echo >&2 + warn "the upgraded stack did not pass health checks" + if [ "$backup_id" != "none" ]; then + echo "[dg-ctl] roll back with: $0 rollback $backup_id" >&2 + fi + exit 1 + fi + + prune_backups + log "upgrade to $ova_version complete" +} + +do_version() { + echo "dg-ctl cli: $DG_CTL_VERSION" + echo "ova version: $(state_value ova_version)" + echo "template ref: $(state_value template_ref)" + echo "core tag: $(env_value DEFGUARD_CORE_TAG)" + echo "proxy tag: $(env_value DEFGUARD_PROXY_TAG)" + echo "gateway tag: $(env_value DEFGUARD_GATEWAY_TAG)" + local applied + applied="$(jq -r '.migrations_applied // [] | join(", ")' "$STATE_FILE" 2>/dev/null)" || applied="" + [ -n "$applied" ] && echo "migrations: $applied" + local manifest + if manifest="$(read_manifest 2>/dev/null)"; then + echo "available: $(jq -r '.ova_version // "unknown"' <<<"$manifest") (core $(jq -r '.core_tag' <<<"$manifest"), proxy $(jq -r '.proxy_tag' <<<"$manifest"), gateway $(jq -r '.gateway_tag' <<<"$manifest"))" + else + echo "available: could not reach $MANIFEST_URL" + fi +} + +do_self_update() { + local manifest ref tmp + manifest="$(read_manifest)" + ref="$(jq -r '.template_ref // "main"' <<<"$manifest")" + tmp="$(mktemp)" + fetch_ova_file "$ref" "dg-ctl" "$tmp" || { rm -f "$tmp"; die "could not fetch dg-ctl at ref $ref"; } + bash -n "$tmp" || { rm -f "$tmp"; die "fetched dg-ctl does not parse; keeping the current one"; } + install -m 750 -o root -g root "$tmp" "$OVA_DIR/dg-ctl" + rm -f "$tmp" + log "CLI updated from ref $ref: $("$OVA_DIR/dg-ctl" --version 2>/dev/null || true)" +} + +usage() { + cat < [options] + + upgrade [--ref REF] [--core-tag T] [--proxy-tag T] [--gateway-tag T] + [--yes] [--no-backup] [--skip-tests] + back up, run any pending structural migrations, refresh + the stack from the release manifest, then health-test it + backup [--label L] cold full backup of volumes, .env and compose file + list-backups show local backups + rollback [--skip-tests] + restore a backup and bring that stack back up + test run the health checks against the running stack + version show installed and available versions + self-update replace this CLI with the published one + help this text + +environment: + DEFGUARD_OVA_MANIFEST_URL release manifest (default: $OVA_REPO@ova-upgrade-process) + DEFGUARD_OVA_SOURCE_DIR take ova/files/* from a local checkout + KEEP_BACKUPS backups to keep after an upgrade (default 3) +EOF +} + +main() { + local cmd="${1:-help}" + [ $# -gt 0 ] && shift || true + case "$cmd" in + upgrade) need_root "$cmd"; need_stack; do_upgrade "$@" ;; + backup) + need_root "$cmd"; need_stack + local label="" + [ "${1:-}" = "--label" ] && { label="${2:?--label needs a value}"; shift 2; } + mkdir -p "$BACKUPS_DIR" + do_backup "$label" + ;; + list-backups) do_list_backups ;; + rollback) need_root "$cmd"; need_stack; do_rollback "$@" ;; + test) need_stack; do_test "$@" ;; + version|--version|-v) do_version ;; + self-update) need_root "$cmd"; do_self_update ;; + help|--help|-h) usage ;; + *) usage >&2; die "unknown command: $cmd" ;; + esac +} + +main "$@" diff --git a/ova/files/docker-setup.sh b/ova/files/docker-setup.sh index 40eb776..cb64c9c 100644 --- a/ova/files/docker-setup.sh +++ b/ova/files/docker-setup.sh @@ -5,7 +5,8 @@ apt-get update apt-get full-upgrade -y # open-vm-tools: graceful shutdown, guest IP reporting, and time sync under VMware. # jq: strips profiles from generate-compose.sh's flattened docker-compose.yml -apt-get install -y ca-certificates curl open-vm-tools jq +# zstd: compresses stack backups taken by /opt/defguard/dg-ctl +apt-get install -y ca-certificates curl open-vm-tools jq zstd install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc diff --git a/ova/files/generate-compose.sh b/ova/files/generate-compose.sh index 189f8da..30ebf16 100755 --- a/ova/files/generate-compose.sh +++ b/ova/files/generate-compose.sh @@ -55,13 +55,18 @@ trap 'rm -f "$TMP_COMPOSE_FILE"' EXIT | docker compose -f - -p defguard --project-directory "$STACK_DIR" config } > "$TMP_COMPOSE_FILE" -sed -i \ +sed -i.bak \ -e "s/__DEFGUARD_CORE_TAG__/\${DEFGUARD_CORE_TAG}/" \ -e "s/__DEFGUARD_PROXY_TAG__/\${DEFGUARD_PROXY_TAG}/" \ -e "s/__DEFGUARD_GATEWAY_TAG__/\${DEFGUARD_GATEWAY_TAG}/" \ - "$TMP_COMPOSE_FILE" + "$TMP_COMPOSE_FILE" && rm -f "$TMP_COMPOSE_FILE.bak" mv "$TMP_COMPOSE_FILE" "$COMPOSE_FILE" +# The generated file carries no profiles: keys, so record the selection for +# later regeneration (see /opt/defguard/dg-ctl upgrade). +mkdir -p "$INIT_DIR" +printf '%s\n' "${_profiles[@]}" > "$INIT_DIR/.applied-profiles" + rm -f "$PROFILES_FILE" "$ENABLE_DOCKER_MGMT_FILE" echo "DefGuard: generated $COMPOSE_FILE for profiles: ${_profiles[*]}" diff --git a/ova/files/install-dg-ctl.sh b/ova/files/install-dg-ctl.sh new file mode 100644 index 0000000..fb11850 --- /dev/null +++ b/ova/files/install-dg-ctl.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Installs the defguard OVA maintenance CLI onto an already-deployed OVA VM, +# which shipped without it. Idempotent - re-running just refreshes the CLI. +# +# curl -fsSL https://raw.githubusercontent.com/DefGuard/deployment/main/ova/files/install-dg-ctl.sh | sudo bash +set -euo pipefail + +STACK_DIR="${DEFGUARD_STACK_DIR:-/opt/stacks/defguard}" +OVA_DIR="${DEFGUARD_OVA_DIR:-/opt/defguard}" +OVA_REPO="${DEFGUARD_OVA_REPO:-DefGuard/deployment}" +MANIFEST_URL="${DEFGUARD_OVA_MANIFEST_URL:-https://raw.githubusercontent.com/$OVA_REPO/ova-upgrade-process/ova/manifest.json}" +SOURCE_DIR="${DEFGUARD_OVA_SOURCE_DIR:-}" + +log() { echo "[dg-ctl-install] $*"; } +die() { echo "[dg-ctl-install] ERROR: $*" >&2; exit 1; } + +[ "$(id -u)" = "0" ] || die "run as root (pipe into 'sudo bash')" +[ -f "$STACK_DIR/docker-compose.yml" ] \ + || die "$STACK_DIR/docker-compose.yml not found; this does not look like a defguard OVA host" + +# jq and zstd are baked into OVA 2.1 images but not into earlier ones. +missing=() +command -v jq >/dev/null 2>&1 || missing+=(jq) +command -v zstd >/dev/null 2>&1 || missing+=(zstd) +command -v curl >/dev/null 2>&1 || missing+=(curl) +if [ "${#missing[@]}" -gt 0 ]; then + log "installing missing dependencies: ${missing[*]}" + apt-get update -qq + apt-get install -y --no-install-recommends "${missing[@]}" +fi + +fetch() { + local src="$1" dest="$2" + case "$src" in + file://*) cp "${src#file://}" "$dest" ;; + /*) cp "$src" "$dest" ;; + *) curl -fsSL --retry 3 -o "$dest" "$src" ;; + esac +} + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +fetch "$MANIFEST_URL" "$work/manifest.json" || die "could not fetch manifest from $MANIFEST_URL" +REF="$(jq -r '.template_ref // "main"' "$work/manifest.json")" + +if [ -n "$SOURCE_DIR" ]; then + cp "$SOURCE_DIR/ova/files/dg-ctl" "$work/dg-ctl" +else + fetch "https://raw.githubusercontent.com/$OVA_REPO/$REF/ova/files/dg-ctl" "$work/dg-ctl" \ + || die "could not fetch dg-ctl at ref $REF" +fi +bash -n "$work/dg-ctl" || die "downloaded dg-ctl does not parse; aborting" + +mkdir -p "$OVA_DIR/backups" +install -m 750 -o root -g root "$work/dg-ctl" "$OVA_DIR/dg-ctl" +ln -sf "$OVA_DIR/dg-ctl" /usr/local/bin/dg-ctl + +# Seed state from what is actually deployed. ova_version is unknown for images +# built before the manifest existed; the tags in .env are the truth. +if [ ! -f "$OVA_DIR/state.json" ]; then + env_value() { sed -n "s/^$1=//p" "$STACK_DIR/.env" | tail -n1; } + jq -n \ + --arg core "$(env_value DEFGUARD_CORE_TAG)" \ + --arg proxy "$(env_value DEFGUARD_PROXY_TAG)" \ + --arg gateway "$(env_value DEFGUARD_GATEWAY_TAG)" \ + '{ova_version: "2.0-unknown", template_ref: "unknown", + tags: {core: $core, proxy: $proxy, gateway: $gateway}}' > "$OVA_DIR/state.json" + chmod 600 "$OVA_DIR/state.json" +fi + +log "installed $OVA_DIR/dg-ctl (ref $REF)" +log "next: sudo $OVA_DIR/dg-ctl upgrade" diff --git a/ova/files/migrations/README.md b/ova/files/migrations/README.md new file mode 100644 index 0000000..b88ed90 --- /dev/null +++ b/ova/files/migrations/README.md @@ -0,0 +1,29 @@ +# OVA structural migrations + +`dg-ctl upgrade` runs these when it needs to change the layout of +`/opt/stacks/defguard` or `/opt/defguard` in a way the compose template +regeneration alone can't express (moving/renaming a volume directory, seeding +a new required `.env` var, dropping a new file under `init/`, etc). + +## Adding one + +1. Add a script here named after the OVA version that introduces the change, + e.g. `2.2.0.sh`. +2. List that version in `migrations` in `ova/manifest.json`, in ascending + order. +3. The script receives, as environment variables: + - `STACK_DIR` - the stack directory (`/opt/stacks/defguard`) + - `OVA_DIR` - the OVA CLI directory (`/opt/defguard`) + - `ENV_FILE` - `$STACK_DIR/.env` + - `FROM_VERSION` / `TO_VERSION` - the OVA versions being upgraded between +4. Exit non-zero to abort the upgrade. The pre-upgrade backup has already been + taken by this point, and the compose file/tags have not yet been touched, + so `dg-ctl rollback ` is always safe to run after a failed migration. +5. Scripts must be idempotent: `dg-ctl upgrade` can be re-run after a partial + failure, and `resolve_pending_migrations` decides what's pending from the + *target* OVA version, not from what actually ran last time. + +`dg-ctl` only runs migrations with `FROM_VERSION < version <= TO_VERSION` +(string `FROM_VERSION`/`TO_VERSION` of `unknown` or `*-unknown`, from OVAs +that predate `state.json`, are treated as version `0.0.0` - every migration +runs). diff --git a/ova/manifest.json b/ova/manifest.json new file mode 100644 index 0000000..8d4bb4f --- /dev/null +++ b/ova/manifest.json @@ -0,0 +1,8 @@ +{ + "ova_version": "2.1.0", + "core_tag": "2", + "proxy_tag": "2", + "gateway_tag": "2", + "template_ref": "ova-upgrade-process", + "migrations": [] +} diff --git a/ova/test/test-deployment-modes.sh b/ova/test/test-deployment-modes.sh index 66ba006..db60f5e 100755 --- a/ova/test/test-deployment-modes.sh +++ b/ova/test/test-deployment-modes.sh @@ -125,6 +125,52 @@ verify_mode() { || { log "$mode: /opt/stacks/defguard contains '$actual', expected only docker-compose.yml, .env, .volumes, defguard-firewall.sh, init"; return 1; } reprovision_mode "$mode" "$ip" || return 1 + upgrade_mode "$mode" "$ip" || return 1 + + return 0 +} + +upgrade_mode() { + local mode="$1" ip="$2" names probe + [ "${SKIP_UPGRADE_TEST:-0}" = "1" ] && { log "$mode: skipping upgrade test"; return 0; } + + log "$mode: exercising /opt/defguard/dg-ctl upgrade" + + local has_db=0 + case " ${EXPECT[$mode]} " in *" db "*) has_db=1 ;; esac + if [ "$has_db" = 1 ]; then + vm_ssh "$ip" "sudo docker compose -f /opt/stacks/defguard/docker-compose.yml exec -T db \ + psql -qtAX -U defguard -d defguard -c \"CREATE TABLE IF NOT EXISTS ova_upgrade_probe (v text); \ + INSERT INTO ova_upgrade_probe VALUES ('survived');\"" \ + || { log "$mode: could not seed the upgrade probe row"; return 1; } + fi + + vm_ssh "$ip" "sudo ${UPGRADE_ENV:-} /opt/defguard/dg-ctl upgrade --yes" \ + || { log "$mode: dg-ctl upgrade failed"; return 1; } + + vm_ssh "$ip" "sudo test -f /opt/defguard/state.json" \ + || { log "$mode: upgrade did not record /opt/defguard/state.json"; return 1; } + vm_ssh "$ip" "sudo /opt/defguard/dg-ctl list-backups | grep -q Z" \ + || { log "$mode: upgrade left no backup behind"; return 1; } + + names="$(wait_services "$ip" "${EXPECT[$mode]}")" \ + || { log "$mode: upgrade: expected services did not come back; running: $(tr '\n' ' ' <<<"$names")"; return 1; } + + local svc + for svc in ${FORBID[$mode]}; do + has_service "$names" "$svc" \ + && { log "$mode: upgrade: unexpected service '$svc' is running"; return 1; } + done + + if [ "$has_db" = 1 ]; then + probe="$(vm_ssh "$ip" "sudo docker compose -f /opt/stacks/defguard/docker-compose.yml exec -T db \ + psql -qtAX -U defguard -d defguard -c 'SELECT v FROM ova_upgrade_probe;'" | tr -d '[:space:]')" + [ "$probe" = "survived" ] \ + || { log "$mode: upgrade lost database state (probe returned '$probe')"; return 1; } + fi + + vm_ssh "$ip" "sudo /opt/defguard/dg-ctl test" \ + || { log "$mode: post-upgrade health checks failed"; return 1; } return 0 } diff --git a/ova/tests/dg-ctl.bats b/ova/tests/dg-ctl.bats new file mode 100644 index 0000000..901bcfe --- /dev/null +++ b/ova/tests/dg-ctl.bats @@ -0,0 +1,269 @@ +#!/usr/bin/env bats +# Exercises /opt/defguard/dg-ctl against a temp stack: real `docker compose config` +# (offline), everything daemon-facing faked by tests/stub/docker-stub. + +load helpers + +CLI="$FILES_DIR/dg-ctl" + +setup() { + command -v docker >/dev/null 2>&1 || skip "docker not installed" + docker compose version >/dev/null 2>&1 || skip "docker compose v2 not available" + command -v jq >/dev/null 2>&1 || skip "jq not installed" + command -v zstd >/dev/null 2>&1 || skip "zstd not installed" + command -v sha256sum >/dev/null 2>&1 || skip "sha256sum not available" + make_stack + make_ova_home + stub_docker + # Health checks must not sit in retry loops in unit tests. + export HEALTH_TIMEOUT=1 HEALTH_SETTLE=0 +} + +teardown() { + teardown_stub_docker + teardown_ova_home + teardown_stack + teardown_migration_source +} + +@test "backup captures volumes, env, compose file and a matching checksum" { + seed_stack core + run bash "$CLI" backup --label unit + [ "$status" -eq 0 ] + id="${lines[-1]}" + dir="$OVA_HOME/backups/$id" + + for f in volumes.tar.zst env docker-compose.yml applied-profiles image-digests meta.json; do + [ -f "$dir/$f" ] + done + [ "$(jq -r '.volumes_sha256' "$dir/meta.json")" = "$(sha256sum "$dir/volumes.tar.zst" | awk '{print $1}')" ] + [ "$(jq -r '.tags.core' "$dir/meta.json")" = "2" ] + [ "$(cat "$dir/applied-profiles")" = "core" ] + [ "$(file_mode "$dir/env")" = "600" ] +} + +@test "list-backups shows a created backup" { + seed_stack core + id="$(bash "$CLI" backup 2>/dev/null | tail -n1)" + run bash "$CLI" list-backups + [ "$status" -eq 0 ] + [[ "$output" == *"$id"* ]] +} + +@test "backups beyond KEEP_BACKUPS are pruned on upgrade" { + seed_stack core + write_manifest + for n in 1 2 3 4; do mkdir -p "$OVA_HOME/backups/2000010$n-000000Z"; done + KEEP_BACKUPS=2 run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + # 2 kept: the freshly taken one sorts last, so only it plus one placeholder. + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 2 ] +} + +@test "rollback restores volumes and refuses a corrupt archive" { + seed_stack core + id="$(bash "$CLI" backup 2>/dev/null | tail -n1)" + + echo "damaged-after-backup" > "$STACK_DIR/.volumes/db/marker" + run bash "$CLI" rollback --skip-tests "$id" + [ "$status" -eq 0 ] + [ "$(cat "$STACK_DIR/.volumes/db/marker")" = "seeded-db-file" ] + + printf 'corrupt' >> "$OVA_HOME/backups/$id/volumes.tar.zst" + echo "damaged-again" > "$STACK_DIR/.volumes/db/marker" + run bash "$CLI" rollback "$id" + [ "$status" -ne 0 ] + [[ "$output" == *"checksum mismatch"* ]] + # Refusal happens before anything is deleted. + [ "$(cat "$STACK_DIR/.volumes/db/marker")" = "damaged-again" ] +} + +@test "rollback rejects an unknown backup id" { + seed_stack core + run bash "$CLI" rollback 19700101-000000Z + [ "$status" -ne 0 ] + [[ "$output" == *"no such backup"* ]] +} + +@test "profile resolution prefers init/.applied-profiles" { + seed_stack core + printf 'core\ngateway\n' > "$INIT_DIR/.applied-profiles" + run bash "$CLI" test + [[ "$output" == *"profiles: core gateway"* ]] +} + +@test "profile resolution falls back to the generated header comment" { + seed_stack edge + rm -f "$INIT_DIR/.applied-profiles" + run bash "$CLI" test + [[ "$output" == *"profiles: edge"* ]] +} + +@test "profile resolution falls back to the service list" { + seed_stack core + rm -f "$INIT_DIR/.applied-profiles" + grep -v '^# Selected profiles:' "$STACK_DIR/docker-compose.yml" > "$STACK_DIR/stripped.yml" + mv "$STACK_DIR/stripped.yml" "$STACK_DIR/docker-compose.yml" + run bash "$CLI" test + [[ "$output" == *"profiles: core"* ]] +} + +@test "upgrade rewrites only the tag lines in .env" { + seed_stack core + write_manifest 9.9.9 9.1 9.2 9.3 + db_password_before="$(sed -n 's/^DEFGUARD_DB_PASSWORD=//p' "$STACK_DIR/.env")" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(sed -n 's/^DEFGUARD_CORE_TAG=//p' "$STACK_DIR/.env")" = "9.1" ] + [ "$(sed -n 's/^DEFGUARD_PROXY_TAG=//p' "$STACK_DIR/.env")" = "9.2" ] + [ "$(sed -n 's/^DEFGUARD_GATEWAY_TAG=//p' "$STACK_DIR/.env")" = "9.3" ] + [ "$(sed -n 's/^DEFGUARD_DB_PASSWORD=//p' "$STACK_DIR/.env")" = "$db_password_before" ] + [ "$(file_mode "$STACK_DIR/.env")" = "600" ] +} + +@test "upgrade regenerates the compose file for the same profile set" { + seed_stack core + write_manifest + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + unset COMPOSE_PROFILES + [ "$(docker compose -f "$STACK_DIR/docker-compose.yml" config --services | sort | xargs)" = "core db" ] + [ ! -e "$STACK_DIR/active-profiles" ] + [ "$(cat "$INIT_DIR/.applied-profiles")" = "core" ] +} + +@test "upgrade command line tags override the manifest" { + seed_stack core + write_manifest 9.9.9 9 9 9 + run bash "$CLI" upgrade --yes --skip-tests --core-tag 3.3.3 + [ "$status" -eq 0 ] + [ "$(sed -n 's/^DEFGUARD_CORE_TAG=//p' "$STACK_DIR/.env")" = "3.3.3" ] + [ "$(sed -n 's/^DEFGUARD_PROXY_TAG=//p' "$STACK_DIR/.env")" = "9" ] +} + +@test "upgrade records state.json" { + seed_stack core + write_manifest 2.1.7 2.1 2.1 2.1 + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(jq -r '.ova_version' "$OVA_HOME/state.json")" = "2.1.7" ] + [ "$(jq -r '.template_ref' "$OVA_HOME/state.json")" = "test-ref" ] + [ "$(jq -r '.tags.gateway' "$OVA_HOME/state.json")" = "2.1" ] + [ "$(jq -r '.backup_id' "$OVA_HOME/state.json")" != "none" ] +} + +@test "upgrade aborts without touching the stack when the fetched template is broken" { + seed_stack core + write_manifest + broken="$(mktemp -d)" + mkdir -p "$broken/ova/files" + cp "$FILES_DIR/lib.sh" "$FILES_DIR/generate-compose.sh" "$broken/ova/files/" + echo "this: is: not: a: compose: file" > "$broken/ova/files/docker-compose.template.yaml" + export DEFGUARD_OVA_SOURCE_DIR="$broken" + before="$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"does not parse"* ]] + [ "$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" = "$before" ] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] + rm -rf "$broken" +} + +@test "upgrade with an unreachable manifest changes nothing" { + seed_stack core + export DEFGUARD_OVA_MANIFEST_URL="file://$STACK_DIR/does-not-exist.json" + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] +} + +@test "a failing health check exits non-zero and prints the rollback command" { + seed_stack gateway + write_manifest + DOCKER_STUB_STATE=exited run bash "$CLI" upgrade --yes + [ "$status" -ne 0 ] + [[ "$output" == *"FAIL"* ]] + [[ "$output" == *"rollback"* ]] + # The upgrade itself happened; only the verdict failed. + [ -f "$OVA_HOME/state.json" ] +} + +@test "--no-backup takes no backup" { + seed_stack core + write_manifest + run bash "$CLI" upgrade --yes --skip-tests --no-backup + [ "$status" -eq 0 ] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] + [ "$(jq -r '.backup_id' "$OVA_HOME/state.json")" = "none" ] +} + +@test "upgrade runs a pending migration and records it in state.json" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'echo "migrated $FROM_VERSION -> $TO_VERSION" > "$STACK_DIR/migration-ran"' + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [[ "$output" == *"running migration 5.0.0"* ]] + [ -f "$STACK_DIR/migration-ran" ] + [[ "$(cat "$STACK_DIR/migration-ran")" == "migrated unknown -> 5.0.0" ]] + [ "$(jq -r '.migrations_applied | join(",")' "$OVA_HOME/state.json")" = "5.0.0" ] +} + +@test "upgrade does not re-run a migration already covered by the current version" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'echo ran >> "$STACK_DIR/migration-ran"' + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(wc -l < "$STACK_DIR/migration-ran")" -eq 1 ] + teardown_migration_source + + write_manifest 6.0.0 6 6 6 '["5.0.0","6.0.0"]' + # Only 6.0.0.sh is staged; if 5.0.0 were (wrongly) re-fetched, the upgrade + # would fail outright since no such file exists here. + stage_migration 6.0.0 'echo ran >> "$STACK_DIR/migration-ran"' + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(wc -l < "$STACK_DIR/migration-ran")" -eq 2 ] + [ "$(jq -r '.migrations_applied | join(",")' "$OVA_HOME/state.json")" = "6.0.0" ] +} + +@test "a failing migration aborts the upgrade before tags or compose change" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'exit 1' + before="$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"migration 5.0.0 failed"* ]] + [[ "$output" == *"rollback"* ]] + [ "$(sed -n 's/^DEFGUARD_CORE_TAG=//p' "$STACK_DIR/.env")" = "2" ] # untouched, seeded before the manifest's "5" + [ "$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" = "$before" ] + # a backup was taken before the migration ran, so rollback is possible + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 1 ] +} + +@test "version reports installed and available versions" { + seed_stack core + write_manifest 4.5.6 4 4 4 + run bash "$CLI" version + [ "$status" -eq 0 ] + [[ "$output" == *"core tag: 2"* ]] + [[ "$output" == *"4.5.6"* ]] +} + +@test "an unknown command fails with usage" { + run bash "$CLI" definitely-not-a-command + [ "$status" -ne 0 ] + [[ "$output" == *"usage: dg-ctl"* ]] +} + +@test "commands refuse to run outside a defguard stack" { + run bash "$CLI" test + [ "$status" -ne 0 ] + [[ "$output" == *"does not look like a defguard OVA host"* ]] +} diff --git a/ova/tests/helpers.bash b/ova/tests/helpers.bash index b9d4322..3340e8d 100644 --- a/ova/tests/helpers.bash +++ b/ova/tests/helpers.bash @@ -29,6 +29,85 @@ DEFGUARD_GATEWAY_TAG=${3:-test-gateway} EOF } +file_mode() { + stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1" +} + +make_ova_home() { + OVA_HOME="$(mktemp -d)" + mkdir -p "$OVA_HOME/backups" + export DEFGUARD_OVA_DIR="$OVA_HOME" + export DEFGUARD_OVA_ALLOW_NONROOT=1 +} + +teardown_ova_home() { + [ -n "${OVA_HOME:-}" ] && rm -rf "$OVA_HOME" + return 0 +} + +stub_docker() { + STUB_BIN="$(mktemp -d)" + DOCKER_REAL="$(command -v docker)" + export DOCKER_REAL + export DOCKER_STUB_LOG="$STUB_BIN/calls.log" + cp "$STUB_DIR/docker-stub" "$STUB_BIN/docker" + chmod +x "$STUB_BIN/docker" + PATH="$STUB_BIN:$PATH" + export PATH +} + +teardown_stub_docker() { + [ -n "${STUB_BIN:-}" ] && rm -rf "$STUB_BIN" + return 0 +} + +seed_stack() { + if [ "$#" -gt 0 ]; then + printf '%s\n' "$@" > "$STACK_DIR/active-profiles" + fi + write_image_tags "${CORE_TAG:-2}" "${PROXY_TAG:-2}" "${GATEWAY_TAG:-2}" + bash "$FILES_DIR/generate-env.sh" >/dev/null + bash "$FILES_DIR/generate-compose.sh" >/dev/null + mkdir -p "$STACK_DIR/.volumes/db" "$STACK_DIR/.volumes/certs" + echo "seeded-db-file" > "$STACK_DIR/.volumes/db/marker" +} + +# A file:// manifest, so upgrades can be exercised without network access. +# $5, if given, is a JSON array literal for the "migrations" field. +write_manifest() { + cat > "$STACK_DIR/manifest.json" <.sh so +# do_upgrade's fetch_ova_file picks it up, and points DEFGUARD_OVA_SOURCE_DIR +# at the copy. Caller must write_manifest with matching "migrations" first (or +# after - only DEFGUARD_OVA_SOURCE_DIR needs to be set last). +stage_migration() { + local version="$1" body="$2" + MIGRATION_SOURCE_DIR="$(mktemp -d)" + mkdir -p "$MIGRATION_SOURCE_DIR/ova/files/migrations" + cp "$FILES_DIR/docker-compose.template.yaml" "$FILES_DIR/lib.sh" "$FILES_DIR/generate-compose.sh" \ + "$MIGRATION_SOURCE_DIR/ova/files/" + printf '%s\n' "$body" > "$MIGRATION_SOURCE_DIR/ova/files/migrations/$version.sh" + export DEFGUARD_OVA_SOURCE_DIR="$MIGRATION_SOURCE_DIR" +} + +teardown_migration_source() { + [ -n "${MIGRATION_SOURCE_DIR:-}" ] && rm -rf "$MIGRATION_SOURCE_DIR" + return 0 +} + # Minimal .env so compose interpolation of the *_TAG variables succeeds. write_env() { cat > "$STACK_DIR/.env" </dev/null 2>&1 || skip "zstd not installed" + echo "core" > "$STACK_DIR/active-profiles" + bring_up + wait_for_health + + psql() { docker compose -f "$STACK_DIR/docker-compose.yml" exec -T db psql -qtAX -U defguard -d defguard "$@"; } + psql -c "CREATE TABLE ova_upgrade_probe (v text); INSERT INTO ova_upgrade_probe VALUES ('survived');" + + OVA_HOME="$(mktemp -d)" + cat > "$OVA_HOME/manifest.json" < `compose ps` reports services not running +# DOCKER_STUB_RESTARTS=2 -> `inspect` reports that many restarts +# DOCKER_STUB_PULL_FAIL=1 -> `compose pull` exits non-zero +set -uo pipefail + +: "${DOCKER_REAL:?DOCKER_REAL must point at the real docker binary}" +[ -n "${DOCKER_STUB_LOG:-}" ] && printf '%s\n' "$*" >> "$DOCKER_STUB_LOG" + +passthrough() { exec "$DOCKER_REAL" "$@"; } + +verb="" +skip_next=0 +seen_compose=0 +for a in "$@"; do + if [ "$skip_next" = 1 ]; then skip_next=0; continue; fi + case "$a" in + compose) seen_compose=1; continue ;; + -f|--project-directory|--env-file|--profile|-p) skip_next=1; continue ;; + -*) continue ;; + *) verb="$a"; break ;; + esac +done + +if [ "$seen_compose" != 1 ]; then + case "${1:-}" in + inspect) + printf '%s\n' "${DOCKER_STUB_RESTARTS:-0}" + exit 0 + ;; + *) passthrough "$@" ;; + esac +fi + +case "$verb" in + version|config) passthrough "$@" ;; + pull) exit "${DOCKER_STUB_PULL_FAIL:-0}" ;; + up|stop|down|start|restart) exit 0 ;; + images) echo '[]'; exit 0 ;; + port) exit 1 ;; + exec) exit "${DOCKER_STUB_EXEC_FAIL:-0}" ;; + ps) + # -q asks for container ids; --format json asks for state. + if [[ " $* " == *" -q "* ]]; then + echo "stubcontainerid" + else + printf '{"Service":"stub","State":"%s"}\n' "${DOCKER_STUB_STATE:-running}" + fi + exit 0 + ;; + *) exit 0 ;; +esac