From 6a2608c00a8dff41242c9e3ede5ddeb2e04da54a Mon Sep 17 00:00:00 2001 From: DIZAMU <117295462+DIZAMU@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:50:31 +0100 Subject: [PATCH 1/3] feat(deploy): deploy the cluster from a pinned GHCR release with Argo CD --- deploy/argocd/application.yaml | 22 +++ deploy/argocd/install/kustomization.yaml | 28 +++ deploy/argocd/project.yaml | 42 +++++ deploy/k8s-dev/kustomization.yaml | 32 ++++ deploy/k8s/api.yaml | 9 +- deploy/k8s/api/Dockerfile | 11 +- deploy/k8s/dbt.yaml | 7 +- deploy/k8s/kustomization.yaml | 17 +- scripts/build-api-image.sh | 29 +++ scripts/k8s-gitops-prove.sh | 213 +++++++++++++++++++++++ scripts/k8s-memory.sh | 48 +++++ scripts/k8s-up.sh | 181 ++++++++++++++++--- 12 files changed, 604 insertions(+), 35 deletions(-) create mode 100644 deploy/argocd/application.yaml create mode 100644 deploy/argocd/install/kustomization.yaml create mode 100644 deploy/argocd/project.yaml create mode 100644 deploy/k8s-dev/kustomization.yaml create mode 100755 scripts/build-api-image.sh create mode 100755 scripts/k8s-gitops-prove.sh create mode 100755 scripts/k8s-memory.sh diff --git a/deploy/argocd/application.yaml b/deploy/argocd/application.yaml new file mode 100644 index 0000000..8783f23 --- /dev/null +++ b/deploy/argocd/application.yaml @@ -0,0 +1,22 @@ +# RootCause's Application (spec section 14): Argo CD syncs deploy/k8s from +# main into the cluster. Automated sync with prune (what leaves git leaves +# the cluster) and self-heal (a change made by hand is reverted). +# scripts/k8s-up.sh applies it; with --dev it turns automated sync off +# instead, and scripts/k8s-gitops-prove.sh fails while it is off. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: rootcause + namespace: argocd +spec: + project: rootcause + source: + repoURL: git@github.com:DIZAMU/RootCause.git + targetRevision: main + path: deploy/k8s + destination: + server: https://kubernetes.default.svc + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/deploy/argocd/install/kustomization.yaml b/deploy/argocd/install/kustomization.yaml new file mode 100644 index 0000000..9920a00 --- /dev/null +++ b/deploy/argocd/install/kustomization.yaml @@ -0,0 +1,28 @@ +# Argo CD core (spec section 8.2): the application controller, the repo +# server and redis, with no UI, no SSO and no notifications. scripts/k8s-up.sh +# downloads core-install.yaml for the version below, checks its SHA-256, puts +# it next to this file in a temporary directory and applies the result. The +# manifest itself is not committed. +# +# Images are pinned by digest. The ApplicationSet controller is removed: +# RootCause has one Application (deploy/argocd/application.yaml). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: argocd +resources: + - core-install.yaml +images: + - name: quay.io/argoproj/argocd + digest: sha256:8e5cc5bb008df8a09dba439462e58c94c55b7735888d3ebde0c25fc05e4e5335 + - name: public.ecr.aws/docker/library/redis + digest: sha256:08ad0b1d280850169a790dba1393ff7a90aef951fc19632cf4d3ce4f78e679ba +patches: + - target: + kind: Deployment + name: argocd-applicationset-controller + patch: | + $patch: delete + apiVersion: apps/v1 + kind: Deployment + metadata: + name: argocd-applicationset-controller diff --git a/deploy/argocd/project.yaml b/deploy/argocd/project.yaml new file mode 100644 index 0000000..1f182cf --- /dev/null +++ b/deploy/argocd/project.yaml @@ -0,0 +1,42 @@ +# The Argo CD project RootCause's Application belongs to (spec section 13.2). +# It limits what a bad commit on main can make Argo CD do: one source repo, +# the two RootCause namespaces on this cluster only, no cluster-wide objects +# at all (the namespaces come from scripts/k8s-up.sh), and only the kinds +# deploy/k8s uses. Argo CD's core install has no default project, so an +# Application outside this one syncs nothing. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: rootcause + namespace: argocd +spec: + description: RootCause in the local cluster + sourceRepos: + - git@github.com:DIZAMU/RootCause.git + destinations: + - server: https://kubernetes.default.svc + namespace: rootcause-system + - server: https://kubernetes.default.svc + namespace: sandboxes + clusterResourceWhitelist: [] + namespaceResourceWhitelist: + - group: "" + kind: Service + - group: "" + kind: ServiceAccount + - group: "" + kind: ResourceQuota + - group: "" + kind: LimitRange + - group: apps + kind: Deployment + - group: apps + kind: StatefulSet + - group: batch + kind: CronJob + - group: networking.k8s.io + kind: NetworkPolicy + - group: rbac.authorization.k8s.io + kind: Role + - group: rbac.authorization.k8s.io + kind: RoleBinding diff --git a/deploy/k8s-dev/kustomization.yaml b/deploy/k8s-dev/kustomization.yaml new file mode 100644 index 0000000..b7ed024 --- /dev/null +++ b/deploy/k8s-dev/kustomization.yaml @@ -0,0 +1,32 @@ +# The cluster with the API image built from the working tree +# (scripts/k8s-up.sh --dev): rootcause/api:local, imported into the nodes +# with k3d, so unreleased code can be tried. k8s-up.sh turns Argo CD's +# automated sync off first, or it would put the release back; a run without +# --dev turns it on again. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../k8s +images: + - name: ghcr.io/dizamu/rootcause-api + newName: rootcause/api + newTag: local +patches: + - target: + kind: Deployment + name: rootcause-api + patch: | + - op: replace + path: /spec/template/spec/containers/0/imagePullPolicy + value: Never + - op: remove + path: /spec/template/spec/imagePullSecrets + - target: + kind: CronJob + name: dbt + patch: | + - op: replace + path: /spec/jobTemplate/spec/template/spec/initContainers/0/imagePullPolicy + value: Never + - op: remove + path: /spec/jobTemplate/spec/template/spec/imagePullSecrets diff --git a/deploy/k8s/api.yaml b/deploy/k8s/api.yaml index 71168d7..730f674 100644 --- a/deploy/k8s/api.yaml +++ b/deploy/k8s/api.yaml @@ -39,6 +39,10 @@ spec: rootcause.instance: main spec: serviceAccountName: rootcause-api + # The image is private on GHCR. scripts/k8s-up.sh creates this Secret + # from the owner's token; it is never in git. + imagePullSecrets: + - name: ghcr-pull # The kubernetes runtime needs the service account's token; the # ServiceAccount itself does not mount it anywhere else. automountServiceAccountToken: true @@ -52,8 +56,9 @@ spec: type: RuntimeDefault containers: - name: api - image: rootcause/api:local - imagePullPolicy: Never + # kustomization.yaml pins the release by digest. + image: rootcause/api + imagePullPolicy: IfNotPresent args: [serve, --listen-all, --port, "7300"] env: - name: RC_RUNTIME diff --git a/deploy/k8s/api/Dockerfile b/deploy/k8s/api/Dockerfile index de21017..06ddb02 100644 --- a/deploy/k8s/api/Dockerfile +++ b/deploy/k8s/api/Dockerfile @@ -1,11 +1,14 @@ # The API and dashboard image for the cluster (spec section 8.2). The context -# is a directory scripts/k8s-up.sh assembles: the rootcause binary, built with -# -tags webdist so the dashboard is embedded, and the exercises folder, whose -# scripts the engine copies into sandboxes. Nothing else: no shell tools are -# needed, and the process runs as an unprivileged user. +# is a directory scripts/build-api-image.sh assembles: the rootcause binary, +# built with -tags webdist so the dashboard is embedded, and the exercises +# folder, whose scripts the engine copies into sandboxes. Nothing else: no +# shell tools are needed, and the process runs as an unprivileged user. FROM debian:13-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad COPY rootcause /opt/rootcause/rootcause COPY exercises /opt/rootcause/exercises +# Links the GHCR package to the repository, so release.yml's GITHUB_TOKEN may +# push to it. +LABEL org.opencontainers.image.source=https://github.com/DIZAMU/RootCause USER 10001:10001 ENV RC_ROOT=/opt/rootcause HOME=/tmp ENTRYPOINT ["/opt/rootcause/rootcause"] diff --git a/deploy/k8s/dbt.yaml b/deploy/k8s/dbt.yaml index 88a8684..c625707 100644 --- a/deploy/k8s/dbt.yaml +++ b/deploy/k8s/dbt.yaml @@ -27,6 +27,8 @@ spec: app: rootcause-dbt spec: restartPolicy: Never + imagePullSecrets: + - name: ghcr-pull automountServiceAccountToken: false enableServiceLinks: false hostUsers: false @@ -38,8 +40,9 @@ spec: type: RuntimeDefault initContainers: - name: project - image: rootcause/api:local - imagePullPolicy: Never + # The API image, pinned in kustomization.yaml. + image: rootcause/api + imagePullPolicy: IfNotPresent args: [dbt-project, /project] resources: limits: diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml index 435b317..32e0869 100644 --- a/deploy/k8s/kustomization.yaml +++ b/deploy/k8s/kustomization.yaml @@ -1,14 +1,23 @@ -# Everything in the cluster except the rootcause-db Secret and the -# api-to-apiserver NetworkPolicy, which scripts/k8s-up.sh creates (the first -# holds the generated password, the second the API server's address). +# Everything Argo CD syncs from main into the cluster (spec sections 8.2 and +# 14). Not in here, because scripts/k8s-up.sh creates them: the namespaces +# (namespaces.yaml; Argo CD may not create cluster-wide objects), the +# rootcause-db and ghcr-pull Secrets (they hold secrets) and the +# api-to-apiserver NetworkPolicy (it holds the API server's address). # k3d.yaml is the cluster itself, not a resource. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - namespaces.yaml - rbac.yaml - sandboxes.yaml - system-network.yaml - postgres.yaml - api.yaml - dbt.yaml +# The release the cluster runs, by digest only. release.yml prints the digest +# of each vX.Y.Z it pushes; the owner puts it here in a pull request, and +# ci.yml checks that it is a pushed release. The zero digest means no release +# has been deployed yet: scripts/k8s-up.sh then needs --dev. +images: + - name: rootcause/api + newName: ghcr.io/dizamu/rootcause-api + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 diff --git a/scripts/build-api-image.sh b/scripts/build-api-image.sh new file mode 100755 index 0000000..e4c4a30 --- /dev/null +++ b/scripts/build-api-image.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Builds the API image (deploy/k8s/api/Dockerfile) under the tag given: the +# rootcause binary with the dashboard embedded (-tags webdist), plus the +# exercises folder. scripts/k8s-up.sh (local images) and release.yml (GHCR) +# both build it here, so the cluster runs what the release pushes. +# +# Usage: scripts/build-api-image.sh [version] +# version goes into the image's org.opencontainers.image.version label +# (default "dev"). +set -euo pipefail + +[[ $# -ge 1 && $# -le 2 ]] || { echo "usage: build-api-image.sh [version]" >&2; exit 2; } +tag="$1" +version="${2:-dev}" + +root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$root" + +ctx="$(mktemp -d)" +trap 'rm -rf "$ctx"' EXIT + +scripts/build-web.sh >/dev/null +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags webdist -o "$ctx/rootcause" ./cmd/rootcause +cp -r exercises "$ctx/exercises" +docker build -q --platform linux/amd64 \ + --label "org.opencontainers.image.version=$version" \ + --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \ + -t "$tag" -f deploy/k8s/api/Dockerfile "$ctx" >/dev/null +echo "built $tag" diff --git a/scripts/k8s-gitops-prove.sh b/scripts/k8s-gitops-prove.sh new file mode 100755 index 0000000..ef9b68e --- /dev/null +++ b/scripts/k8s-gitops-prove.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# Proves the Phase 8 done-when criteria on the running cluster (spec section +# 18), after scripts/k8s-up.sh without --dev: +# +# 1. Argo CD's rootcause Application syncs deploy/k8s from main with +# automated sync, prune and self-heal on, and is Synced and Healthy at +# the commit main points at on GitHub. A forgotten --dev fails here. +# 2. The API pod runs the release pinned in deploy/k8s/kustomization.yaml, +# by digest, and so does the dbt CronJob. With RC_RELEASE_DIGEST set +# (from release.yml's job summary), that digest must match too. +# 3. A change made by hand is reverted: the sandboxes ResourceQuota is +# patched, and Argo CD must put it back. +# 4. The rootcause project refuses an Application that points at another +# namespace or another repository, and the Application tracks nothing +# outside rootcause-system and sandboxes. +# 5. The objects k8s-up.sh creates outside git (rootcause-db, ghcr-pull, +# api-to-apiserver, the namespaces) are still there: prune leaves them. +# +# The test Applications are removed at the end. Exit 0 only if every check +# passes. +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$root" +context=k3d-rootcause +kc() { kubectl --context "$context" "$@"; } +failures=0 +pass() { echo "PASS $*"; } +fail() { echo "FAIL $*"; failures=$((failures + 1)); } +# same : pass if they are equal. +same() { + if [[ "$1" == "$2" ]]; then pass "$3"; else fail "$3: got '$1', want '$2'"; fi +} +# exists : pass if the object is there. +exists() { + if kc get "$1" "$2" -n "$3" >/dev/null 2>&1; then pass "$1 $3/$2 kept"; else fail "$1 $3/$2 missing"; fi +} +repo_url=git@github.com:DIZAMU/RootCause.git +image=ghcr.io/dizamu/rootcause-api + +# quota_saved is the sandboxes ResourceQuota's pod limit before the drift +# test. If the test ends without Argo CD putting it back (a timeout, or the +# script is stopped), cleanup restores it, so the proof never leaves a weaker +# limit behind. +quota_saved="" +restore_quota() { + [[ -n "$quota_saved" ]] || return 0 + local now + now="$(kc get resourcequota sandboxes -n sandboxes -o jsonpath='{.spec.hard.pods}' 2>/dev/null || true)" + if [[ "$now" != "$quota_saved" ]]; then + if kc patch resourcequota sandboxes -n sandboxes --type merge \ + -p "{\"spec\":{\"hard\":{\"pods\":\"$quota_saved\"}}}" >/dev/null 2>&1; then + echo "note ResourceQuota pods restored to $quota_saved by the script, not by Argo CD" + else + echo "FAIL could not restore ResourceQuota pods to $quota_saved; it is $now" + fi + fi + quota_saved="" +} + +cleanup() { + restore_quota + kc delete application -n argocd rootcause-escape-ns rootcause-escape-repo \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true +} +trap cleanup EXIT + +app() { kc get application rootcause -n argocd -o jsonpath="$1"; } + +echo "== 1. the Application" +if ! kc get application rootcause -n argocd >/dev/null 2>&1; then + fail "no Application rootcause in argocd (run scripts/k8s-up.sh without --dev)" + echo + echo "k8s-gitops-prove: $failures check(s) failed" + exit 1 +fi +src="$(app '{.spec.project} {.spec.source.repoURL} {.spec.source.targetRevision} {.spec.source.path}')" +if [[ "$src" == "rootcause $repo_url main deploy/k8s" ]]; then + pass "project rootcause, $repo_url, main, deploy/k8s" +else + fail "Application source is '$src'" +fi +policy="$(app '{.spec.syncPolicy.automated.prune} {.spec.syncPolicy.automated.selfHeal}')" +if [[ "$policy" == "true true" ]]; then + pass "automated sync with prune and self-heal" +else + fail "automated sync is off or incomplete (prune selfHeal: '$policy'); a run with --dev turns it off" +fi +main_sha="$(git ls-remote "$repo_url" refs/heads/main | cut -f1)" +[[ -n "$main_sha" ]] || main_sha="$(git ls-remote origin refs/heads/main | cut -f1)" +state="" +for _ in $(seq 60); do + state="$(app '{.status.sync.status} {.status.health.status} {.status.sync.revision}')" + [[ "$state" == "Synced Healthy $main_sha" ]] && break + sleep 5 +done +if [[ "$state" == "Synced Healthy $main_sha" ]]; then + pass "Synced and Healthy at main ($main_sha)" +else + fail "state '$state', main is $main_sha" +fi + +echo "== 2. the release" +revision="$(app '{.status.sync.revision}')" +pinned="" +if git cat-file -e "$revision" 2>/dev/null; then + pinned="$(git show "$revision:deploy/k8s/kustomization.yaml" | + awk '$1 == "newName:" && $2 == "'"$image"'" { found = 1; next } found && $1 == "digest:" { print $2; exit }')" +else + fail "commit $revision is not in this clone (git fetch origin, then re-run)" +fi +if [[ "$pinned" =~ ^sha256:[0-9a-f]{64}$ && "$pinned" != sha256:0000000000000000000000000000000000000000000000000000000000000000 ]]; then + pass "kustomization at $revision pins $image@$pinned" +else + fail "kustomization at $revision pins '$pinned'" +fi +deployed="$(kc get deployment rootcause-api -n rootcause-system -o jsonpath='{.spec.template.spec.containers[0].image}')" +cron="$(kc get cronjob dbt -n rootcause-system -o jsonpath='{.spec.jobTemplate.spec.template.spec.initContainers[0].image}')" +same "$deployed" "$image@$pinned" "Deployment image" +same "$cron" "$image@$pinned" "dbt CronJob image" +running="$(kc get pods -n rootcause-system -l app=rootcause-api,rootcause.instance=main \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[0].imageID}{"\n"}{end}')" +if [[ "$(wc -l <<<"$running")" -eq 1 && "${running##*@}" == "$pinned" ]]; then + pass "the API pod runs $pinned" +else + fail "the API pod(s) run: $running" +fi +if [[ -n "${RC_RELEASE_DIGEST:-}" ]]; then + same "$pinned" "$RC_RELEASE_DIGEST" "pinned digest equals RC_RELEASE_DIGEST" +else + echo "note RC_RELEASE_DIGEST not set; the release summary's digest was not compared" +fi + +echo "== 3. drift is reverted" +want="$(kc get resourcequota sandboxes -n sandboxes -o jsonpath='{.spec.hard.pods}')" +quota_saved="$want" +kc patch resourcequota sandboxes -n sandboxes --type merge -p '{"spec":{"hard":{"pods":"40"}}}' >/dev/null +got="" +for _ in $(seq 60); do + got="$(kc get resourcequota sandboxes -n sandboxes -o jsonpath='{.spec.hard.pods}')" + [[ "$got" == "$want" ]] && break + sleep 2 +done +if [[ "$want" == 4 && "$got" == "$want" ]]; then + quota_saved="" + pass "ResourceQuota pods patched to 40, Argo CD put back $got" +else + fail "ResourceQuota pods is $got two minutes after the patch, git says 4" + restore_quota +fi + +echo "== 4. the project's limits" +# escape : an Application in project rootcause that +# Argo CD must refuse with an InvalidSpecError and never sync. +escape() { + kc apply -f - >/dev/null </dev/null +} +# kube-system holds nothing of RootCause's; the Application lists it as the +# default namespace, which the project does not allow. +escape rootcause-escape-ns "$repo_url" kube-system +escape rootcause-escape-repo https://github.com/argoproj/argocd-example-apps.git rootcause-system +outside="$(app '{range .status.resources[*]}{.namespace}{"\n"}{end}' | sort -u | grep -vxE 'rootcause-system|sandboxes' || true)" +if [[ -z "$outside" ]]; then + pass "every tracked resource is in rootcause-system or sandboxes" +else + fail "tracked resources outside the project's namespaces: $outside" +fi + +echo "== 5. objects outside git" +for obj in "secret rootcause-db rootcause-system" "secret ghcr-pull rootcause-system" \ + "networkpolicy api-to-apiserver rootcause-system" "secret repo-rootcause argocd"; do + read -r kind name ns <<<"$obj" + exists "$kind" "$name" "$ns" +done +for ns in rootcause-system sandboxes; do + if kc get namespace "$ns" >/dev/null 2>&1; then pass "namespace $ns kept"; else fail "namespace $ns missing"; fi +done + +echo +if [[ "$failures" -gt 0 ]]; then + echo "k8s-gitops-prove: $failures check(s) failed" + exit 1 +fi +echo "k8s-gitops-prove: every check passed" diff --git a/scripts/k8s-memory.sh b/scripts/k8s-memory.sh new file mode 100755 index 0000000..766c8f6 --- /dev/null +++ b/scripts/k8s-memory.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Prints the memory the k3d cluster uses (spec section 17, OQ-4): each node +# container's usage from docker, then the working set of every container in +# the cluster, read from each node's container runtime (the cluster runs no +# metrics-server). Ends with the WSL2 total from free -m. +# +# Usage: scripts/k8s-memory.sh [namespace] (only that namespace's pods) +set -euo pipefail + +cluster=rootcause +only="${1:-}" + +nodes="$(docker ps --format '{{.Names}}' --filter "label=k3d.cluster=$cluster" | grep -E -- '-(server|agent)-[0-9]+$' | sort)" +[[ -n "$nodes" ]] || { echo "k8s-memory: cluster $cluster is not running" >&2; exit 1; } + +echo "== nodes (docker)" +# shellcheck disable=SC2086 # one argument per node +docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' $nodes + +echo "== containers (working set)" +for n in $nodes; do + docker exec "$n" crictl stats -o json +done | python3 -c ' +import json, sys +only = sys.argv[1] +dec = json.JSONDecoder() +text = sys.stdin.read() +rows, total, i = [], 0, 0 +while i < len(text): + if text[i].isspace(): + i += 1 + continue + doc, i = dec.raw_decode(text, i) + for s in doc.get("stats", []): + labels = s["attributes"]["labels"] + ns = labels.get("io.kubernetes.pod.namespace", "") + if only and ns != only: + continue + mib = int((s.get("memory") or {}).get("workingSetBytes", {}).get("value", 0)) / 2**20 + total += mib + rows.append((ns, labels.get("io.kubernetes.pod.name", ""), s["attributes"]["metadata"]["name"], mib)) +for ns, pod, c, mib in sorted(rows): + print(f"{mib:7.0f} MiB {ns}/{pod}/{c}") +print(f"{total:7.0f} MiB total") +' "$only" + +echo "== WSL2 (free -m)" +free -m diff --git a/scripts/k8s-up.sh b/scripts/k8s-up.sh index 7c82bce..1087fb4 100755 --- a/scripts/k8s-up.sh +++ b/scripts/k8s-up.sh @@ -2,16 +2,32 @@ # Brings up the v2 cluster (spec section 8.2) and deploys RootCause into it. # Safe to re-run: every step creates what is missing and updates the rest. # +# Default (GitOps, spec section 14): Argo CD syncs deploy/k8s from main on +# GitHub, so the cluster runs the release pinned there, never the working +# tree. # 1. The k3d cluster from deploy/k8s/k3d.yaml (started if it exists). -# 2. Images: the API image (rootcause built with the dashboard, plus the -# exercises) and every exercise image, built with docker and imported -# into the cluster's nodes. Nothing is pushed anywhere. -# 3. The rootcause-db Secret. Its password is generated once and kept in -# deploy/k8s/.env, which git ignores (spec section 13.2). It never -# appears in a manifest, on a command line or in this script's output. -# 4. The manifests (kubectl apply -k deploy/k8s), and the NetworkPolicy -# that lets the API reach the Kubernetes API server's address. -# 5. A restart of the API, so it runs the image just built. +# 2. The exercise images, built with docker and imported into the nodes +# (sandbox pods never pull). +# 3. The namespaces, and the Secrets that are never in git: rootcause-db +# (a password generated once into git-ignored deploy/k8s/.env), +# ghcr-pull (the owner's GHCR token) and Argo CD's repository +# credential (the owner's read-only deploy key). No secret appears in a +# manifest, on a command line or in this script's output. +# 4. Argo CD core, from its pinned and checksum-verified install manifest, +# then deploy/argocd (the project and the Application). +# 5. The NetworkPolicy that lets the API reach the Kubernetes API server's +# address, and a wait until Argo CD reports Synced and Healthy. +# +# --dev: the API image is built from the working tree (rootcause/api:local) +# and applied with deploy/k8s-dev. Argo CD's automated sync, if Argo CD is +# installed, is turned off first, or it would put the release back. A later +# run without --dev turns it on again. +# +# The owner's credentials (paths can be changed with these variables): +# RC_GHCR_TOKEN_FILE ~/.config/rootcause/ghcr-token +# a classic token with read:packages only +# RC_ARGOCD_DEPLOY_KEY ~/.config/rootcause/argocd-deploy-key +# the private half of a read-only deploy key # # Needs docker, k3d, kubectl (scripts/install-k8s-tools.sh), Go and Node. set -euo pipefail @@ -21,11 +37,50 @@ cd "$root" cluster=rootcause context="k3d-$cluster" envfile="$root/deploy/k8s/.env" +ghcr_token_file="${RC_GHCR_TOKEN_FILE:-$HOME/.config/rootcause/ghcr-token}" +deploy_key_file="${RC_ARGOCD_DEPLOY_KEY:-$HOME/.config/rootcause/argocd-deploy-key}" +ghcr_user=DIZAMU +repo_url=git@github.com:DIZAMU/RootCause.git + +# Argo CD core (spec section 8.2). To update: change the version, download +# manifests/core-install.yaml for it, put its sha256sum here, and pin the +# images it names by digest in deploy/argocd/install/kustomization.yaml. +ARGOCD_VERSION=v3.5.6 +ARGOCD_SHA256=d3d9d3555ad77eb4b6d27b1ec0d2af7978a91537e84d54631f983c227f0a3d31 + +mode=gitops +case "${1:-}" in +"") ;; +--dev) mode=dev ;; +*) echo "usage: k8s-up.sh [--dev]" >&2; exit 2 ;; +esac for tool in docker k3d kubectl go; do command -v "$tool" >/dev/null || { echo "k8s-up: $tool not found on PATH" >&2; exit 1; } done +# need_secret_file : the file must exist and be private. +need_secret_file() { + if [[ ! -s "$1" ]]; then + echo "k8s-up: $2 not found at $1 (see README, Kubernetes)" >&2 + exit 1 + fi + if [[ -n "$(find "$1" -perm /077)" ]]; then + echo "k8s-up: $1 is readable by others; run chmod 600 $1" >&2 + exit 1 + fi +} + +if [[ "$mode" == gitops ]]; then + if grep -q 'digest: sha256:0\{64\}$' deploy/k8s/kustomization.yaml; then + echo "k8s-up: deploy/k8s/kustomization.yaml pins no release yet (zero digest)." >&2 + echo "k8s-up: push a release tag and merge its digest first, or run k8s-up.sh --dev" >&2 + exit 1 + fi + need_secret_file "$ghcr_token_file" "the GHCR read:packages token" + need_secret_file "$deploy_key_file" "the Argo CD deploy key" +fi + echo "== cluster" if k3d cluster get "$cluster" >/dev/null 2>&1; then k3d cluster start "$cluster" --wait >/dev/null @@ -38,13 +93,27 @@ kc() { kubectl --context "$context" "$@"; } tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT +# app_exists: Argo CD is installed and has the rootcause Application. +app_exists() { + kc get crd applications.argoproj.io >/dev/null 2>&1 && + kc get application rootcause -n argocd >/dev/null 2>&1 +} + +# sync_state prints the Application's sync, health and policy. +sync_state() { + if ! app_exists; then + echo "argo cd: no rootcause Application (not managed by Argo CD)" + return + fi + kc get application rootcause -n argocd -o jsonpath='argo cd: sync {.status.sync.status} at {.status.sync.revision}, health {.status.health.status}, automated {.spec.syncPolicy.automated}{"\n"}' +} + echo "== images" -scripts/build-web.sh >/dev/null -mkdir "$tmp/api" -CGO_ENABLED=0 go build -tags webdist -o "$tmp/api/rootcause" ./cmd/rootcause -cp -r exercises "$tmp/api/exercises" -docker build -q -t rootcause/api:local -f deploy/k8s/api/Dockerfile "$tmp/api" >/dev/null -images=(rootcause/api:local) +images=() +if [[ "$mode" == dev ]]; then + scripts/build-api-image.sh rootcause/api:local + images+=(rootcause/api:local) +fi for dir in exercises/*/; do id="$(basename "$dir")" version="$(awk '$1 == "version:" { print $2; exit }' "$dir/exercise.yaml")" @@ -55,7 +124,8 @@ done k3d image import --cluster "$cluster" "${images[@]}" >/dev/null echo "imported ${#images[@]} images" -echo "== secret" +echo "== namespaces and secrets" +kc apply -f deploy/k8s/namespaces.yaml >/dev/null if [[ ! -f "$envfile" ]]; then ( umask 077 @@ -71,15 +141,52 @@ password="$(sed -n 's/^POSTGRES_PASSWORD=//p' "$envfile")" printf 'postgres://rootcause:%s@postgres:5432/rootcause?sslmode=disable' "$password" >"$tmp/database-url" ) unset password -kc apply -f deploy/k8s/namespaces.yaml >/dev/null kc create secret generic rootcause-db -n rootcause-system \ --from-file=password="$tmp/password" --from-file=database-url="$tmp/database-url" \ --dry-run=client -o yaml | kc apply -f - >/dev/null rm -f "$tmp/password" "$tmp/database-url" echo "secret rootcause-db applied" -echo "== manifests" -kc apply -k deploy/k8s +if [[ "$mode" == gitops ]]; then + # The pull secret is built in a private file, so the token never reaches + # a command line. + ( + umask 077 + auth="$(printf '%s:%s' "$ghcr_user" "$(tr -d '[:space:]' <"$ghcr_token_file")" | base64 -w0)" + printf '{"auths":{"ghcr.io":{"auth":"%s"}}}' "$auth" >"$tmp/dockerconfig.json" + ) + kc create secret generic ghcr-pull -n rootcause-system \ + --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson="$tmp/dockerconfig.json" \ + --dry-run=client -o yaml | kc apply -f - >/dev/null + rm -f "$tmp/dockerconfig.json" + echo "secret ghcr-pull applied" + + echo "== argo cd $ARGOCD_VERSION" + mkdir "$tmp/argocd" + curl -fsSL --retry 3 -o "$tmp/argocd/core-install.yaml" \ + "https://raw.githubusercontent.com/argoproj/argo-cd/$ARGOCD_VERSION/manifests/core-install.yaml" + if ! echo "$ARGOCD_SHA256 $tmp/argocd/core-install.yaml" | sha256sum --check --status; then + echo "k8s-up: checksum mismatch for Argo CD $ARGOCD_VERSION core-install.yaml" >&2 + exit 1 + fi + cp deploy/argocd/install/kustomization.yaml "$tmp/argocd/" + kc create namespace argocd --dry-run=client -o yaml | kc apply -f - >/dev/null + # Server-side: the CRDs are too large for the last-applied annotation. + kc apply --server-side --force-conflicts -k "$tmp/argocd" >/dev/null + kc rollout status statefulset/argocd-application-controller -n argocd --timeout=600s + kc rollout status deployment/argocd-repo-server -n argocd --timeout=600s + kc rollout status deployment/argocd-redis -n argocd --timeout=600s + + kc create secret generic repo-rootcause -n argocd \ + --from-literal=type=git --from-literal=url="$repo_url" \ + --from-file=sshPrivateKey="$deploy_key_file" \ + --dry-run=client -o yaml | + kc label --local -f - argocd.argoproj.io/secret-type=repository -o yaml | + kc apply -f - >/dev/null + echo "secret repo-rootcause applied" +fi + +echo "== api-to-apiserver" apiserver="$(kc get endpointslice -n default -l kubernetes.io/service-name=kubernetes \ -o jsonpath='{.items[0].endpoints[0].addresses[0]} {.items[0].ports[0].port}')" read -r apiserver_ip apiserver_port <<<"$apiserver" @@ -104,8 +211,36 @@ spec: protocol: TCP YAML -echo "== rollout" -kc rollout status statefulset/postgres -n rootcause-system --timeout=180s -kc rollout restart deployment/rootcause-api -n rootcause-system >/dev/null -kc rollout status deployment/rootcause-api -n rootcause-system --timeout=180s +if [[ "$mode" == dev ]]; then + echo "== manifests (dev)" + if app_exists; then + kc patch application rootcause -n argocd --type json \ + -p '[{"op": "remove", "path": "/spec/syncPolicy/automated"}]' >/dev/null 2>&1 || true + echo "argo cd automated sync turned off" + fi + kc apply -k deploy/k8s-dev + kc rollout status statefulset/postgres -n rootcause-system --timeout=180s + kc rollout restart deployment/rootcause-api -n rootcause-system >/dev/null + kc rollout status deployment/rootcause-api -n rootcause-system --timeout=180s +else + echo "== argo cd application" + kc apply -f deploy/argocd/project.yaml -f deploy/argocd/application.yaml + # Ask for a refresh now instead of at the next poll. + kc annotate application rootcause -n argocd argocd.argoproj.io/refresh=normal --overwrite >/dev/null + state="" + for _ in $(seq 120); do + state="$(kc get application rootcause -n argocd -o jsonpath='{.status.sync.status}/{.status.health.status}')" + [[ "$state" == Synced/Healthy ]] && break + sleep 5 + done + if [[ "$state" != Synced/Healthy ]]; then + sync_state + kc get application rootcause -n argocd -o jsonpath='{range .status.conditions[*]}{.type}: {.message}{"\n"}{end}' >&2 + echo "k8s-up: Argo CD did not reach Synced/Healthy within 10 minutes" >&2 + exit 1 + fi +fi + +echo "== state" +sync_state echo "Dashboard: http://localhost:7300" From 4f86f0574d40c9ab571f26f1b35684d75913ec34 Mon Sep 17 00:00:00 2001 From: DIZAMU <117295462+DIZAMU@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:50:31 +0100 Subject: [PATCH 2/3] ci: add the release workflow and the pinned-release check --- .github/workflows/ci.yml | 21 ++++++ .github/workflows/exercises.yml | 5 +- .github/workflows/release.yml | 95 +++++++++++++++++++++++++ scripts/ci-release-image.sh | 119 ++++++++++++++++++++++++++++++++ 4 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100755 scripts/ci-release-image.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 449b364..1e4665f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,3 +107,24 @@ jobs: --require TestBuildRecoversFromKilledEngine --require TestPlaySvc01 -tags webdist ./... + + # The release deploy/k8s pins must be a pushed vX.Y.Z (spec section 14). + # GITHUB_TOKEN is the only credential, with packages: read on this job only. + # A pull request from a fork gets no package access, so the job does not + # run there. + release-image: + name: ci-release-image + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: pinned digest is a release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: scripts/ci-release-image.sh check-kustomization diff --git a/.github/workflows/exercises.yml b/.github/workflows/exercises.yml index 5681327..889ef1f 100644 --- a/.github/workflows/exercises.yml +++ b/.github/workflows/exercises.yml @@ -1,7 +1,8 @@ # Exercise validation (spec section 14): for each changed exercise and seeds # 1, 2 and 3, build the image; break then check must fail; break, solution, # check must pass. Pull requests and pushes to main validate only the changed -# exercises; tags and manual runs validate all of them. +# exercises; tags and manual runs validate all of them. Tags reach this +# workflow through release.yml, which calls it before it pushes an image. # # The workflow always runs and finds the changed exercises itself, instead of # using a paths filter, so it can become a required check: a filtered @@ -12,7 +13,7 @@ on: pull_request: push: branches: [main] - tags: ["v*"] + workflow_call: workflow_dispatch: permissions: {} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..adb3037 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,95 @@ +# Release (spec section 14): on a vX.Y.Z tag on main, validate every exercise, +# then build the API image and push it to ghcr.io/dizamu/rootcause-api under +# that tag only. The job summary has the digest; the owner pins it in +# deploy/k8s/kustomization.yaml in a pull request, and Argo CD deploys it +# once that is merged. Nothing here commits or pushes to the repository. +# +# GITHUB_TOKEN is the only credential. packages: write is granted to the push +# job alone. +name: release + +on: + push: + tags: ["v*"] + +permissions: {} + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + check: + name: release-check + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # origin/main, for the on-main check + persist-credentials: false + - name: tag is vX.Y.Z, on main, and new + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + run: scripts/ci-release-image.sh check-tag "$TAG" "$GITHUB_SHA" + + validate: + needs: check + permissions: + contents: read + uses: ./.github/workflows/exercises.yml + + push: + name: release-push + needs: [check, validate] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: web/.nvmrc + cache: npm + cache-dependency-path: web/package-lock.json + - name: build the API image + env: + TAG: ${{ github.ref_name }} + run: scripts/build-api-image.sh "ghcr.io/dizamu/rootcause-api:$TAG" "$TAG" + - name: push to GHCR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + run: | + image="ghcr.io/dizamu/rootcause-api:$TAG" + printf '%s' "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + docker push "$image" + docker logout ghcr.io + digest="$(docker inspect --format '{{index .RepoDigests 0}}' "$image")" + digest="${digest#*@}" + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "no digest for $image" >&2; exit 1; } + { + echo "## ghcr.io/dizamu/rootcause-api:$TAG" + echo + echo "Digest: \`$digest\`" + echo + echo "To deploy it, set this in deploy/k8s/kustomization.yaml in a pull request:" + echo + echo '```yaml' + echo "images:" + echo " - name: rootcause/api" + echo " newName: ghcr.io/dizamu/rootcause-api" + echo " digest: $digest" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/scripts/ci-release-image.sh b/scripts/ci-release-image.sh new file mode 100755 index 0000000..88d585a --- /dev/null +++ b/scripts/ci-release-image.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# CI helpers for the API image on GHCR (spec section 14). release.yml and +# ci.yml call them; they read GITHUB_TOKEN and GITHUB_ACTOR from the +# workflow and never print the token. +# +# check-tag the tag is vX.Y.Z, its commit is on +# origin/main, and the image tag is not on GHCR yet +# check-kustomization the digest pinned in deploy/k8s/kustomization.yaml +# is the zero placeholder, or the digest of a vX.Y.Z +# tag on GHCR +set -euo pipefail + +registry=ghcr.io +repo=dizamu/rootcause-api +zero=sha256:0000000000000000000000000000000000000000000000000000000000000000 +accept="application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json" + +root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$root" + +die() { echo "ci-release-image: $*" >&2; exit 1; } + +# bearer prints a pull token for the repository, from GITHUB_TOKEN. GHCR +# answers 401 or 403 both when the token may not read the package and when +# the package does not exist yet (before the first release); bearer then +# prints nothing. Callers decide what that means. +bearer() { + [[ -n "${GITHUB_TOKEN:-}" ]] || die "GITHUB_TOKEN is not set" + local body code + body="$(mktemp)" + code="$(curl -sS -o "$body" -w '%{http_code}' -u "${GITHUB_ACTOR:-x}:$GITHUB_TOKEN" \ + "https://$registry/token?service=$registry&scope=repository:$repo:pull")" || { rm -f "$body"; die "GHCR token request failed"; } + case "$code" in + 200) python3 -c 'import json, sys; print(json.load(sys.stdin).get("token", ""))' <"$body" ;; + 401 | 403) ;; + *) rm -f "$body"; die "GHCR token request answered $code" ;; + esac + rm -f "$body" +} + +# manifest_digest prints the digest GHCR reports for a +# tag or digest, or nothing if it does not exist or may not be read. +manifest_digest() { + local code headers + headers="$(mktemp)" + code="$(curl -sS -o /dev/null -D "$headers" -w '%{http_code}' -I \ + -H "Authorization: Bearer $1" -H "Accept: $accept" \ + "https://$registry/v2/$repo/manifests/$2")" + case "$code" in + 200) tr -d '\r' <"$headers" | awk -F': ' 'tolower($1) == "docker-content-digest" { print $2 }' ;; + 401 | 403 | 404) ;; + *) rm -f "$headers"; die "GHCR answered $code for $repo:$2" ;; + esac + rm -f "$headers" +} + +# release_tags prints the repository's vX.Y.Z tags. +release_tags() { + local out + out="$(curl -fsS -H "Authorization: Bearer $1" "https://$registry/v2/$repo/tags/list?n=1000")" || return 0 + python3 -c 'import json, sys; print("\n".join(json.load(sys.stdin).get("tags") or []))' <<<"$out" | + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' || true +} + +check_tag() { + local tag="$1" sha="$2" + [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "tag $tag is not vX.Y.Z" + git rev-parse --verify -q origin/main >/dev/null || die "origin/main is not fetched (checkout needs fetch-depth: 0)" + git merge-base --is-ancestor "$sha" origin/main || die "tag $tag points at $sha, which is not on main" + local token + token="$(bearer)" + # No token is the first release: the package does not exist yet. If + # instead the token may not read it, the push fails as well. + if [[ -n "$token" && -n "$(manifest_digest "$token" "$tag")" ]]; then + die "$registry/$repo:$tag already exists; release tags are never pushed twice" + fi + echo "tag $tag: vX.Y.Z, on main, not on GHCR yet" +} + +check_kustomization() { + local digest + digest="$(awk '$1 == "newName:" && $2 == "'"$registry/$repo"'" { found = 1; next } + found && $1 == "digest:" { print $2; exit }' deploy/k8s/kustomization.yaml)" + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || die "no sha256 digest for $registry/$repo in deploy/k8s/kustomization.yaml" + if grep -qE '^\s*newTag:' deploy/k8s/kustomization.yaml; then + die "deploy/k8s/kustomization.yaml sets newTag; the release is pinned by digest only" + fi + if [[ "$digest" == "$zero" ]]; then + echo "kustomization: zero placeholder, no release deployed yet" + return + fi + local token tag + token="$(bearer)" + [[ -n "$token" ]] || die "$digest is pinned, but $registry/$repo cannot be read (no release yet, or no read access)" + [[ -n "$(manifest_digest "$token" "$digest")" ]] || die "$registry/$repo@$digest does not exist" + while read -r tag; do + [[ -n "$tag" ]] || continue + if [[ "$(manifest_digest "$token" "$tag")" == "$digest" ]]; then + echo "kustomization: $digest is release $tag" + return + fi + done < <(release_tags "$token") + die "$registry/$repo@$digest exists but is no vX.Y.Z release" +} + +case "${1:-}" in +check-tag) + [[ $# -eq 3 ]] || die "usage: check-tag " + check_tag "$2" "$3" + ;; +check-kustomization) + [[ $# -eq 1 ]] || die "usage: check-kustomization" + check_kustomization + ;; +*) + echo "usage: ci-release-image.sh check-tag | check-kustomization" >&2 + exit 2 + ;; +esac From 2d03b3b8958ff89aedcf231bd53073ee10b2fe29 Mon Sep 17 00:00:00 2001 From: DIZAMU <117295462+DIZAMU@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:50:31 +0100 Subject: [PATCH 3/3] docs: document GitOps, the release flow and the Argo CD measurements --- README.md | 39 ++++++++++++++++++++++++++++++++------- docs/SPEC.md | 34 +++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 08477be..8830f32 100644 --- a/README.md +++ b/README.md @@ -30,16 +30,40 @@ Exercises that boot systemd (such as svc-02) need a host with cgroup v2. WSL2 us ### Kubernetes (v2) -The same sessions can run in a local k3d cluster (spec section 8.2): the API and dashboard, Postgres and dbt in the namespace `rootcause-system`, one pod per session in `sandboxes`, and the terminal in the browser. +The same sessions can run in a local k3d cluster (spec section 8.2): the API and dashboard, Postgres and dbt in the namespace `rootcause-system`, one pod per session in `sandboxes`, and the terminal in the browser. Argo CD in `argocd` deploys the cluster from `main` (see GitOps below). 1. Run `scripts/install-k8s-tools.sh`. It puts pinned, checksum-verified `k3d` and `kubectl` in `~/.local/bin`, without sudo. -2. Run `scripts/k8s-up.sh`. It creates the cluster (1 server, 2 agents), builds and imports the images, generates the database password into git-ignored `deploy/k8s/.env`, and deploys everything. Re-run it after a change. +2. Run `scripts/k8s-up.sh --dev` to try the working tree: it creates the cluster (1 server, 2 agents), builds the API image and the exercise images and imports them, generates the database password into git-ignored `deploy/k8s/.env`, and applies `deploy/k8s-dev`. Re-run it after a change. If Argo CD is installed, `--dev` turns its automated sync off first. 3. Open http://localhost:7300. Start an exercise and work in the terminal on the exercise page. 4. Run `scripts/k8s-prove.sh` to check the isolation: the API's permissions, the network policies, and one full session through the dashboard. That session plays on a separate database, `rootcause_cluster_test`, through a second API pod, and leaves the real one untouched. -5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob. +5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob. `scripts/k8s-memory.sh` prints the memory each node and container uses. `RC_RUNTIME=kubernetes` points the CLI at the cluster, for example `RC_RUNTIME=kubernetes scripts/validate-exercise.sh perm-01-shared-folder 1`. svc-02 boots systemd, which does not run in a pod, so it is refused there (spec section 13.3). +### GitOps (v2) + +Without `--dev`, `scripts/k8s-up.sh` installs Argo CD core (pinned, checksum-verified) and the Application in `deploy/argocd`, which syncs `deploy/k8s` from `main` with automated sync, prune and self-heal. The cluster then runs the API image pinned by digest in `deploy/k8s/kustomization.yaml`, pulled from the private package `ghcr.io/dizamu/rootcause-api`. Exercise images are still built and imported by `k8s-up.sh`; sandbox pods never pull. + +A change reaches the cluster like this, with no kubectl by hand: + +1. Merge the change to `main`. +2. Tag it and push the tag. `git tag -s` signs it with the same SSH key as the commits; check before you push: + ```bash + git checkout main && git pull --ff-only + git tag -s v0.8.0 -m v0.8.0 + git tag -v v0.8.0 + git push origin v0.8.0 + ``` + `release.yml` checks that the tag is `vX.Y.Z` on `main` and new, validates every exercise, builds the API image and pushes it under that tag only. Its job summary shows the digest. +3. In a pull request, set that digest in `deploy/k8s/kustomization.yaml` (`images:`, digest only). `ci-release-image` checks that it is a pushed release. When it is merged, Argo CD rolls it out. + +Check the result with `scripts/k8s-gitops-prove.sh` (set `RC_RELEASE_DIGEST` to the digest from the job summary to compare it too): the Application is Synced and Healthy at `main` with self-heal on, the API runs the pinned digest, a change made by hand is reverted, and the project refuses other namespaces and repositories. It fails while a `--dev` run has turned automated sync off. + +Two credentials stay outside the repo, in `~/.config/rootcause` (chmod 600). `k8s-up.sh` turns them into Secrets and never prints them: + +- `argocd-deploy-key`: the private half of a read-only deploy key for this repository. Make it with `ssh-keygen -t ed25519 -N '' -C rootcause-argocd -f ~/.config/rootcause/argocd-deploy-key`, then add the `.pub` file under Settings, Deploy keys, with write access off. The key has no passphrase, so it is unencrypted on disk. That is acceptable for a read-only key to one private repository, and the Secret in the cluster holds it in plain text anyway. +- `ghcr-token`: a personal access token (classic) with only the `read:packages` scope and a 90-day expiry, made under Settings, Developer settings, Personal access tokens, Tokens (classic). GitHub Packages accepts only classic tokens. A classic `read:packages` token can read every package your account can see, not just this one. When it expires, make a new one, save it to the same file and run `k8s-up.sh` again. + ## Tests Unit tests need nothing: `go test ./...`. Integration tests need a database whose name ends in `_test`, and Docker with userns-remap: @@ -59,14 +83,15 @@ go test -count=1 ./... ## CI -GitHub Actions runs four workflows (spec section 14). Every action is pinned by commit SHA, and every job gets read-only access to the repository and nothing else. +GitHub Actions runs five workflows (spec section 14). Every action is pinned by commit SHA, and every job gets read-only access to the repository and nothing else, except `packages: read` on `ci-release-image` and `release-check`, and `packages: write` on `release-push`. | Workflow | Check name | Does | |---|---|---| -| `ci.yml` | `ci-checks`, `ci-test` | gofmt, go vet, golangci-lint, shellcheck, i18n parity, the dashboard build, and every Go test with Postgres and Docker. Fails if any test is skipped | -| `exercises.yml` | `exercises` | Validates the changed exercises with seeds 1, 2 and 3. All of them on tags `v*` and on manual runs | +| `ci.yml` | `ci-checks`, `ci-test`, `ci-release-image` | gofmt, go vet, golangci-lint, shellcheck, i18n parity, the dashboard build, and every Go test with Postgres and Docker. Fails if any test is skipped. `ci-release-image` checks that the digest in `deploy/k8s/kustomization.yaml` is a pushed `vX.Y.Z` release, or the zero placeholder | +| `exercises.yml` | `exercises` | Validates the changed exercises with seeds 1, 2 and 3. All of them on manual runs, and when `release.yml` calls it | | `dbt.yml` | `dbt` | dbt build against a Postgres service container, when the dbt project or its inputs changed | | `commit-policy.yml` | `commit-policy` | Rejects Co-authored-by trailers, tool attribution lines, author emails other than the owner's, and commits GitHub does not show as Verified | +| `release.yml` | `release-check`, `exercises`, `release-push` | On a tag `v*`: the tag must be `vX.Y.Z` on `main` and not pushed before; every exercise is validated; then the API image is pushed to GHCR under that tag. It never commits | Run the same commit check locally with `scripts/check-commit-policy.sh origin/main HEAD`. @@ -78,7 +103,7 @@ On the free plan, branch protection does not apply to private repositories, so a - Restrict deletions, and block force pushes. - Require a pull request before merging, with merge commits only (squash and rebase drop the SSH signatures). - Require signed commits. -- Require status checks to pass, with the branch up to date: `ci-checks`, `ci-test`, `exercises`, `dbt`, `commit-policy`. Every workflow starts on every PR and decides inside the job what to run, so none of these checks stays pending. +- Require status checks to pass, with the branch up to date: `ci-checks`, `ci-test`, `ci-release-image`, `exercises`, `dbt`, `commit-policy`. Every workflow starts on every PR and decides inside the job what to run, so none of these checks stays pending. Known gap to close when the ruleset goes on: the workflows, `scripts/check-commit-policy.sh` and `.githooks/commit-msg` run from the PR's own commits, so a PR could weaken the checks that judge it. Before the repo goes public, make `commit-policy.yml` check out the base branch's copy of the script and hook, and require review of changes under `.github/`, `scripts/` and `.githooks/`. diff --git a/docs/SPEC.md b/docs/SPEC.md index 5fb447c..ca063a5 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -246,15 +246,17 @@ WSL2 ├── namespace rootcause-system │ ├── api + dashboard (Deployment) │ ├── postgres (StatefulSet) - │ ├── dbt (CronJob every 10 minutes; a Job on demand by script) - │ └── argocd (Phase 8) - └── namespace sandboxes - └── one pod per session: the exercise image, nothing added + │ └── dbt (CronJob every 10 minutes; a Job on demand by script) + ├── namespace sandboxes + │ └── one pod per session: the exercise image, nothing added + └── namespace argocd + └── Argo CD core: application controller, repo server, redis ``` - Reached from Windows through k3d port mapping to `127.0.0.1:7300` on the host only. Tailscale only if remote access is needed. - The browser terminal is xterm.js in the dashboard, on a WebSocket to the API. The API runs `bash` in the session pod through the Kubernetes exec API (the v1 `docker exec` model), so nothing in a sandbox listens on the network and sandboxes stay closed to ingress. Phase 7 chose this over ttyd in the pod, which would have needed a listening port and a NetworkPolicy opening every sandbox to `rootcause-system`, plus an extra binary in every exercise image. -- `scripts/k8s-up.sh` builds the images, imports them into the cluster's nodes (`k3d image import`), creates the database Secret and applies `deploy/k8s`. `scripts/k8s-prove.sh` proves the Phase 7 criteria. The CLI uses the cluster with `RC_RUNTIME=kubernetes`; `rootcause selftest` then validates exercises in sandbox pods. +- `scripts/k8s-up.sh` creates the cluster, builds the exercise images and imports them into the cluster's nodes (`k3d image import`; sandbox pods never pull), creates the namespaces and the Secrets, installs Argo CD and applies `deploy/argocd`. Argo CD then syncs `deploy/k8s` from `main`, and the API runs the release pinned there by digest from GHCR (section 14). `scripts/k8s-up.sh --dev` instead builds the API image from the working tree and applies `deploy/k8s-dev`, after turning Argo CD's automated sync off. `scripts/k8s-prove.sh` proves the Phase 7 criteria and `scripts/k8s-gitops-prove.sh` the Phase 8 ones. The CLI uses the cluster with `RC_RUNTIME=kubernetes`; `rootcause selftest` then validates exercises in sandbox pods. +- Argo CD is the core install (no UI, no SSO, no notifications, no ApplicationSet controller) at a pinned version, with its install manifest checked against a pinned SHA-256 and its images pinned by digest. Its one Application belongs to the AppProject `rootcause` (section 13.2). ### 8.3 Runtime abstraction @@ -391,7 +393,10 @@ Implementation: - `rootcause-system` is default-deny too. The API accepts 7300 and reaches Postgres, cluster DNS and the Kubernetes API server's address; dbt reaches Postgres and DNS; Postgres accepts the API and dbt. - Session pods: their own user namespace (`hostUsers: false`, so pod root is not node root), no service account token mounted, no service links, seccomp `RuntimeDefault`, no hostPath volumes, all capabilities dropped and the exercise's allowlist added back, no privilege escalation, memory and CPU limits, in-memory `emptyDir` for `tmpfs`, and no cluster DNS. The pause container is PID 1 and reaps stopped processes. The process limit is the kubelet's `pod-max-pids` (256), since Kubernetes has no per-pod setting. - Pod Security Admission enforces `restricted` in `rootcause-system` and `baseline` in `sandboxes`. -- Secrets never enter the repo. Local secrets live in git-ignored `.env` files. Cluster secrets are created outside git: `scripts/k8s-up.sh` generates the Postgres password once into git-ignored `deploy/k8s/.env` and creates the Secret from files, so the password never appears in a manifest, on a command line or in output. Sealed secrets are for Phase 8 to decide, with GitOps. +- Secrets never enter the repo. Local secrets live in git-ignored `.env` files. Cluster secrets are created outside git by `scripts/k8s-up.sh`, from files, so no secret appears in a manifest, on a command line or in output: `rootcause-db` (the Postgres password, generated once into git-ignored `deploy/k8s/.env`), `ghcr-pull` (the owner's GHCR token) and `repo-rootcause` in `argocd` (the owner's read-only deploy key). The owner keeps the token and the key in `~/.config/rootcause`, mode 600. Phase 8 decided against sealed secrets: there are three secrets, all created by one script on one laptop. +- What is not in git is created by `scripts/k8s-up.sh` and never tracked by Argo CD, so prune leaves it: the namespaces, the three Secrets, and the NetworkPolicy to the API server's address (which is known only once the cluster runs). +- The AppProject `rootcause` limits what a bad commit on `main` can make Argo CD do: this repository as the only source, `rootcause-system` and `sandboxes` on this cluster as the only destinations, no cluster-wide resources at all, and only the namespaced kinds `deploy/k8s` uses. The core install has no `default` project. `scripts/k8s-gitops-prove.sh` checks that an Application pointing at another namespace or another repository is refused. +- Argo CD's application controller can write anywhere in the cluster; the AppProject limits what it is asked to do, not what it can do. It is a trust boundary next to the API (Phase 9). ### 13.3 Known limits (v1) @@ -404,6 +409,8 @@ Implementation: Known limits (v2, Kubernetes): - k3d runs its node containers privileged and in the host's user namespace, so node root is host root, whatever Docker's userns-remap says. Sandbox pods are isolated from it by `hostUsers: false`; system pods run as non-root users in their own user namespaces as well. Phase 9 reviews this gap (OQ-3). +- Pod Security Admission `baseline` on `sandboxes` does not require `hostUsers: false`, so a compromised API or Argo CD could create a pod in the host's user namespace. Phase 9 adds a ValidatingAdmissionPolicy on `sandboxes` for both service accounts. +- The API image's `exercises/` and the exercise images imported by `scripts/k8s-up.sh` come from different trees: the release and the working tree. A changed exercise bumps its `version`, and so its image tag, so a missing image fails the session start with `ErrImageNeverPull` instead of running a stale image. - A check runner may connect to any sandbox on the internal network, not only its own. With two sessions, one session's runner could reach the other session's sandbox. Per-session NetworkPolicies would need the API to manage NetworkPolicies, which section 13.2 does not allow. - `init: systemd` exercises (svc-02) do not run on Kubernetes: in a pod with its own user namespace the cgroup filesystem stays owned by host root, so systemd cannot boot. The CLI, the dashboard and the selftest refuse such an exercise there with a clear error. There is no restart before a check either (section 8.3). - The cluster's dbt CronJob runs `dbt build` and fails on dbt's own exit code; the node-count check of section 9.2 applies to builds the binary starts. @@ -416,12 +423,14 @@ Known limits (v2, Kubernetes): | `exercises.yml` | Changes under `exercises/` | For each changed exercise and 3 seeds: build the image; break then check must fail; break, solution, check must pass | | `dbt.yml` | Changes under `dbt/` | dbt build against a Postgres service container with seed data: a scripted series of sessions played through the engine | | `commit-policy.yml` | Every PR | Rejects Co-authored-by trailers, tool attribution lines, author emails other than the owner's, and unsigned commits | -| `release.yml` (v2) | Tags on main | Builds and pushes images to GHCR, updates image tags in `deploy/k8s`. Argo CD syncs | +| `release.yml` (v2) | Tags `vX.Y.Z` on main | Checks that the tag is `vX.Y.Z`, on `main` and not pushed before, validates every exercise (it calls `exercises.yml`), then builds the API image and pushes it to `ghcr.io/dizamu/rootcause-api` under that tag only. The job summary shows the digest. It never commits: the owner pins the digest in `deploy/k8s/kustomization.yaml` in a pull request, `ci.yml` checks it, and Argo CD syncs it once merged | - Private repo on the free plan means limited Actions minutes. Only changed exercises are validated per PR. Full validation runs on tags. - `exercises.yml` and `dbt.yml` start on every PR and push to main, and decide inside the job whether anything changed, so they can be required checks. - Every action is pinned by commit SHA. Workflows have no permissions by default, and each job gets read access to the repository only. -- GHCR images are private while the repo is private, so the cluster needs an image pull secret. +- GHCR images are private while the repo is private, so the cluster needs an image pull secret. GitHub Packages accepts only classic personal access tokens; the pull secret uses one with `read:packages` only and a 90-day expiry. +- Only the API image goes to GHCR. Exercise images stay local (`k3d image import`): all nine images together are about 210 MB compressed, and the free plan's 500 MB of private package storage would fill within about two releases. +- `packages: write` is granted only to `release.yml`'s push job, `packages: read` only to its tag check and to `ci.yml`'s `ci-release-image` job (not on pull requests from forks). GITHUB_TOKEN is the only credential. - Images target linux/amd64 only, until something needs ARM. ## 15. Git and authorship @@ -455,7 +464,7 @@ Known limits (v2, Kubernetes): - Docker Engine installed directly in WSL2, not Docker Desktop. - Toolchain: Go (latest stable), Node LTS (build time only, for the dashboard). dbt runs in a container. - v1 resource budget: at most 2 concurrent sandboxes, default 256 MiB and 0.5 CPU each. -- v2 resource budget: k3d with 1 server and 2 agents. If memory is tight, measure Argo CD first (OQ-4). +- v2 resource budget: k3d with 1 server and 2 agents, Argo CD core, at most 2 sandboxes. Measured in Phase 8: about 2.5 GB of the 8 GB WSL2 cap at peak (OQ-4). `scripts/k8s-memory.sh` prints the current use per node and container. ## 18. Phases and acceptance criteria @@ -517,6 +526,7 @@ Done when: a session runs fully in the cluster from the browser, a sandbox pod c ### Phase 8: GitOps - `release.yml`, GHCR images, Argo CD syncing `deploy/k8s`. +- `scripts/k8s-gitops-prove.sh`: the Application Synced and Healthy at `main` with automated sync, prune and self-heal on; the running API image equals the pinned release digest; a change made by hand is reverted; the project refuses other destinations and sources. Done when: a merged change reaches the cluster without manual kubectl. @@ -543,7 +553,7 @@ The owner decides when the repo goes public. Ready means: | OQ-1 | Service exercises usually rely on systemd, which often needs extra container privileges | Resolved in Phase 3: a) systemd as PID 1. It boots under the v1 security flags with no capability outside the allowlist, given in-memory `/run` and `/run/lock`. A unit that is started but not enabled is gone after `docker restart`; an enabled one is running again. journald and tmp.mount are masked (they need SETPCAP). Exercises opt in with `init: systemd` | Phase 3 | | OQ-2 | Docker manages resolv.conf inside containers | Resolved in Phase 3: `/etc/resolv.conf` is a bind mount. In-place writes persist for the session, but tools that replace the file (`sed -i`, some editors) fail with "Device or resource busy", and `docker restart` regenerates it. So net-01's fault lives in lookup order (`/etc/nsswitch.conf`) or the local DNS server config, and its fix never needs a resolv.conf edit | Phase 3 | | OQ-3 | The learner is root inside the sandbox | v1 resolved: userns-remap (`/etc/docker/daemon.json` with `{"userns-remap": "default"}`). Chosen over rootless Docker because it keeps the existing daemon and needs one config change. Verified with `docker info` showing `name=userns`; container root maps to host UID 100000. The CLI refuses to run sessions without userns or rootless mode. v2: since Phase 7 every sandbox pod has its own user namespace (`hostUsers: false`, verified on this kernel: pod root maps to a high unprivileged host UID). Open for Phase 9: the k3d nodes themselves run privileged as host root, and gVisor stays an option | Phase 9 (v2) | -| OQ-4 | Argo CD memory use on a 16 GB laptop next to the cluster | Argo CD, or Flux if too heavy. Phase 7 measured WSL2 memory in use (`free -m`, without page cache): 0.9 GB without the cluster, 1.7 GB with the bare cluster (1 server, 2 agents), 2.1 GB at peak with Postgres, the API, a session and a selftest with its runner | Phase 8 | +| OQ-4 | Argo CD memory use on a 16 GB laptop next to the cluster | Resolved in Phase 8: Argo CD core v3.5.6, since it stays far below the agreed limit of 0.75 GB added (otherwise Flux). Phase 7 measured WSL2 memory in use (`free -m`, without page cache): 0.9 GB without the cluster, 1.7 GB with the bare cluster (1 server, 2 agents), 2.1 GB at peak with Postgres, the API, a session and a selftest with its runner. Argo CD's working set per pod (`scripts/k8s-memory.sh`): idle with no Application 226 MiB (application controller 149, repo server 56, redis 21); peak during a sync 369 MiB; steady with Applications synced 362 to 384 MiB (controller 262 to 270, repo server 71 to 103, redis 19 to 21). WSL2 in use with Argo CD: 2.27 GB before and 2.50 GB at peak during `scripts/k8s-prove.sh` (two sessions, a runner, a second API pod), so Argo CD adds about 0.4 GB | Phase 8 | | OQ-5 | Public live demo | Recording only, or Oracle Cloud free tier (ARM, idle reclaim risk) | After v1 | | OQ-6 | Elo ratings | Keep 2 up / 2 down, or switch | When the catalog passes about 30 exercises | @@ -573,7 +583,9 @@ RootCause/ ├── dbt/ dbt project; embed.go embeds it in the binary ├── deploy/ │ ├── compose/ v1: Postgres (dbt runs with docker run, see 9.2) -│ └── k8s/ v2: k3d cluster config, manifests (kustomize), api/Dockerfile +│ ├── k8s/ v2: k3d cluster config, manifests (kustomize) Argo CD syncs, api/Dockerfile +│ ├── k8s-dev/ v2: overlay with the API image built from the working tree +│ └── argocd/ v2: Argo CD install kustomization, the AppProject and the Application ├── locales/ en.json, no.json ├── scripts/ validate-exercise.sh, build-web.sh, CI helpers, k8s-*.sh ├── .githooks/ commit-msg