Phase 8

.github/workflows/ci.yml

@@ -107,3 +107,24 @@ jobs:
           --require TestBuildRecoversFromKilledEngine
           --require TestPlaySvc01
           -tags webdist ./...
+
+  # The release deploy/k8s pins must be a pushed vX.Y.Z (spec section 14).
+  # GITHUB_TOKEN is the only credential, with packages: read on this job only.
+  # A pull request from a fork gets no package access, so the job does not
+  # run there.
+  release-image:
+    name: ci-release-image
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: pinned digest is a release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: scripts/ci-release-image.sh check-kustomization

.github/workflows/exercises.yml

@@ -1,7 +1,8 @@
 # Exercise validation (spec section 14): for each changed exercise and seeds
 # 1, 2 and 3, build the image; break then check must fail; break, solution,
 # check must pass. Pull requests and pushes to main validate only the changed
-# exercises; tags and manual runs validate all of them.
+# exercises; tags and manual runs validate all of them. Tags reach this
+# workflow through release.yml, which calls it before it pushes an image.
 #
 # The workflow always runs and finds the changed exercises itself, instead of
 # using a paths filter, so it can become a required check: a filtered
@@ -12,7 +13,7 @@ on:
   pull_request:
   push:
     branches: [main]
-    tags: ["v*"]
+  workflow_call:
   workflow_dispatch:
 
 permissions: {}

.github/workflows/release.yml

@@ -0,0 +1,95 @@
+# Release (spec section 14): on a vX.Y.Z tag on main, validate every exercise,
+# then build the API image and push it to ghcr.io/dizamu/rootcause-api under
+# that tag only. The job summary has the digest; the owner pins it in
+# deploy/k8s/kustomization.yaml in a pull request, and Argo CD deploys it
+# once that is merged. Nothing here commits or pushes to the repository.
+#
+# GITHUB_TOKEN is the only credential. packages: write is granted to the push
+# job alone.
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions: {}
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  check:
+    name: release-check
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0 # origin/main, for the on-main check
+          persist-credentials: false
+      - name: tag is vX.Y.Z, on main, and new
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ github.ref_name }}
+        run: scripts/ci-release-image.sh check-tag "$TAG" "$GITHUB_SHA"
+
+  validate:
+    needs: check
+    permissions:
+      contents: read
+    uses: ./.github/workflows/exercises.yml
+
+  push:
+    name: release-push
+    needs: [check, validate]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version-file: web/.nvmrc
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+      - name: build the API image
+        env:
+          TAG: ${{ github.ref_name }}
+        run: scripts/build-api-image.sh "ghcr.io/dizamu/rootcause-api:$TAG" "$TAG"
+      - name: push to GHCR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          image="ghcr.io/dizamu/rootcause-api:$TAG"
+          printf '%s' "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+          docker push "$image"
+          docker logout ghcr.io
+          digest="$(docker inspect --format '{{index .RepoDigests 0}}' "$image")"
+          digest="${digest#*@}"
+          [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "no digest for $image" >&2; exit 1; }
+          {
+            echo "## ghcr.io/dizamu/rootcause-api:$TAG"
+            echo
+            echo "Digest: \`$digest\`"
+            echo
+            echo "To deploy it, set this in deploy/k8s/kustomization.yaml in a pull request:"
+            echo
+            echo '```yaml'
+            echo "images:"
+            echo "  - name: rootcause/api"
+            echo "    newName: ghcr.io/dizamu/rootcause-api"
+            echo "    digest: $digest"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"

README.md

@@ -30,16 +30,40 @@ Exercises that boot systemd (such as svc-02) need a host with cgroup v2. WSL2 us
 
 ### Kubernetes (v2)
 
-The same sessions can run in a local k3d cluster (spec section 8.2): the API and dashboard, Postgres and dbt in the namespace `rootcause-system`, one pod per session in `sandboxes`, and the terminal in the browser.
+The same sessions can run in a local k3d cluster (spec section 8.2): the API and dashboard, Postgres and dbt in the namespace `rootcause-system`, one pod per session in `sandboxes`, and the terminal in the browser. Argo CD in `argocd` deploys the cluster from `main` (see GitOps below).
 
 1. Run `scripts/install-k8s-tools.sh`. It puts pinned, checksum-verified `k3d` and `kubectl` in `~/.local/bin`, without sudo.
-2. Run `scripts/k8s-up.sh`. It creates the cluster (1 server, 2 agents), builds and imports the images, generates the database password into git-ignored `deploy/k8s/.env`, and deploys everything. Re-run it after a change.
+2. Run `scripts/k8s-up.sh --dev` to try the working tree: it creates the cluster (1 server, 2 agents), builds the API image and the exercise images and imports them, generates the database password into git-ignored `deploy/k8s/.env`, and applies `deploy/k8s-dev`. Re-run it after a change. If Argo CD is installed, `--dev` turns its automated sync off first.
 3. Open http://localhost:7300. Start an exercise and work in the terminal on the exercise page.
 4. Run `scripts/k8s-prove.sh` to check the isolation: the API's permissions, the network policies, and one full session through the dashboard. That session plays on a separate database, `rootcause_cluster_test`, through a second API pod, and leaves the real one untouched.
-5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob.
+5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob. `scripts/k8s-memory.sh` prints the memory each node and container uses.
 
 `RC_RUNTIME=kubernetes` points the CLI at the cluster, for example `RC_RUNTIME=kubernetes scripts/validate-exercise.sh perm-01-shared-folder 1`. svc-02 boots systemd, which does not run in a pod, so it is refused there (spec section 13.3).
 
+### GitOps (v2)
+
+Without `--dev`, `scripts/k8s-up.sh` installs Argo CD core (pinned, checksum-verified) and the Application in `deploy/argocd`, which syncs `deploy/k8s` from `main` with automated sync, prune and self-heal. The cluster then runs the API image pinned by digest in `deploy/k8s/kustomization.yaml`, pulled from the private package `ghcr.io/dizamu/rootcause-api`. Exercise images are still built and imported by `k8s-up.sh`; sandbox pods never pull.
+
+A change reaches the cluster like this, with no kubectl by hand:
+
+1. Merge the change to `main`.
+2. Tag it and push the tag. `git tag -s` signs it with the same SSH key as the commits; check before you push:
+   ```bash
+   git checkout main && git pull --ff-only
+   git tag -s v0.8.0 -m v0.8.0
+   git tag -v v0.8.0
+   git push origin v0.8.0
+   ```
+   `release.yml` checks that the tag is `vX.Y.Z` on `main` and new, validates every exercise, builds the API image and pushes it under that tag only. Its job summary shows the digest.
+3. In a pull request, set that digest in `deploy/k8s/kustomization.yaml` (`images:`, digest only). `ci-release-image` checks that it is a pushed release. When it is merged, Argo CD rolls it out.
+
+Check the result with `scripts/k8s-gitops-prove.sh` (set `RC_RELEASE_DIGEST` to the digest from the job summary to compare it too): the Application is Synced and Healthy at `main` with self-heal on, the API runs the pinned digest, a change made by hand is reverted, and the project refuses other namespaces and repositories. It fails while a `--dev` run has turned automated sync off.
+
+Two credentials stay outside the repo, in `~/.config/rootcause` (chmod 600). `k8s-up.sh` turns them into Secrets and never prints them:
+
+- `argocd-deploy-key`: the private half of a read-only deploy key for this repository. Make it with `ssh-keygen -t ed25519 -N '' -C rootcause-argocd -f ~/.config/rootcause/argocd-deploy-key`, then add the `.pub` file under Settings, Deploy keys, with write access off. The key has no passphrase, so it is unencrypted on disk. That is acceptable for a read-only key to one private repository, and the Secret in the cluster holds it in plain text anyway.
+- `ghcr-token`: a personal access token (classic) with only the `read:packages` scope and a 90-day expiry, made under Settings, Developer settings, Personal access tokens, Tokens (classic). GitHub Packages accepts only classic tokens. A classic `read:packages` token can read every package your account can see, not just this one. When it expires, make a new one, save it to the same file and run `k8s-up.sh` again.
+
 ## Tests
 
 Unit tests need nothing: `go test ./...`. Integration tests need a database whose name ends in `_test`, and Docker with userns-remap:
@@ -59,14 +83,15 @@ go test -count=1 ./...
 
 ## CI
 
-GitHub Actions runs four workflows (spec section 14). Every action is pinned by commit SHA, and every job gets read-only access to the repository and nothing else.
+GitHub Actions runs five workflows (spec section 14). Every action is pinned by commit SHA, and every job gets read-only access to the repository and nothing else, except `packages: read` on `ci-release-image` and `release-check`, and `packages: write` on `release-push`.
 
 | Workflow | Check name | Does |
 |---|---|---|
-| `ci.yml` | `ci-checks`, `ci-test` | gofmt, go vet, golangci-lint, shellcheck, i18n parity, the dashboard build, and every Go test with Postgres and Docker. Fails if any test is skipped |
-| `exercises.yml` | `exercises` | Validates the changed exercises with seeds 1, 2 and 3. All of them on tags `v*` and on manual runs |
+| `ci.yml` | `ci-checks`, `ci-test`, `ci-release-image` | gofmt, go vet, golangci-lint, shellcheck, i18n parity, the dashboard build, and every Go test with Postgres and Docker. Fails if any test is skipped. `ci-release-image` checks that the digest in `deploy/k8s/kustomization.yaml` is a pushed `vX.Y.Z` release, or the zero placeholder |
+| `exercises.yml` | `exercises` | Validates the changed exercises with seeds 1, 2 and 3. All of them on manual runs, and when `release.yml` calls it |
 | `dbt.yml` | `dbt` | dbt build against a Postgres service container, when the dbt project or its inputs changed |
 | `commit-policy.yml` | `commit-policy` | Rejects Co-authored-by trailers, tool attribution lines, author emails other than the owner's, and commits GitHub does not show as Verified |
+| `release.yml` | `release-check`, `exercises`, `release-push` | On a tag `v*`: the tag must be `vX.Y.Z` on `main` and not pushed before; every exercise is validated; then the API image is pushed to GHCR under that tag. It never commits |
 
 Run the same commit check locally with `scripts/check-commit-policy.sh origin/main HEAD`.
 
@@ -78,7 +103,7 @@ On the free plan, branch protection does not apply to private repositories, so a
 - Restrict deletions, and block force pushes.
 - Require a pull request before merging, with merge commits only (squash and rebase drop the SSH signatures).
 - Require signed commits.
-- Require status checks to pass, with the branch up to date: `ci-checks`, `ci-test`, `exercises`, `dbt`, `commit-policy`. Every workflow starts on every PR and decides inside the job what to run, so none of these checks stays pending.
+- Require status checks to pass, with the branch up to date: `ci-checks`, `ci-test`, `ci-release-image`, `exercises`, `dbt`, `commit-policy`. Every workflow starts on every PR and decides inside the job what to run, so none of these checks stays pending.
 
 Known gap to close when the ruleset goes on: the workflows, `scripts/check-commit-policy.sh` and `.githooks/commit-msg` run from the PR's own commits, so a PR could weaken the checks that judge it. Before the repo goes public, make `commit-policy.yml` check out the base branch's copy of the script and hook, and require review of changes under `.github/`, `scripts/` and `.githooks/`.
 

deploy/argocd/application.yaml

@@ -0,0 +1,22 @@
+# RootCause's Application (spec section 14): Argo CD syncs deploy/k8s from
+# main into the cluster. Automated sync with prune (what leaves git leaves
+# the cluster) and self-heal (a change made by hand is reverted).
+# scripts/k8s-up.sh applies it; with --dev it turns automated sync off
+# instead, and scripts/k8s-gitops-prove.sh fails while it is off.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: rootcause
+  namespace: argocd
+spec:
+  project: rootcause
+  source:
+    repoURL: git@github.com:DIZAMU/RootCause.git
+    targetRevision: main
+    path: deploy/k8s
+  destination:
+    server: https://kubernetes.default.svc
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

deploy/argocd/install/kustomization.yaml

@@ -0,0 +1,28 @@
+# Argo CD core (spec section 8.2): the application controller, the repo
+# server and redis, with no UI, no SSO and no notifications. scripts/k8s-up.sh
+# downloads core-install.yaml for the version below, checks its SHA-256, puts
+# it next to this file in a temporary directory and applies the result. The
+# manifest itself is not committed.
+#
+# Images are pinned by digest. The ApplicationSet controller is removed:
+# RootCause has one Application (deploy/argocd/application.yaml).
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: argocd
+resources:
+  - core-install.yaml
+images:
+  - name: quay.io/argoproj/argocd
+    digest: sha256:8e5cc5bb008df8a09dba439462e58c94c55b7735888d3ebde0c25fc05e4e5335
+  - name: public.ecr.aws/docker/library/redis
+    digest: sha256:08ad0b1d280850169a790dba1393ff7a90aef951fc19632cf4d3ce4f78e679ba
+patches:
+  - target:
+      kind: Deployment
+      name: argocd-applicationset-controller
+    patch: |
+      $patch: delete
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: argocd-applicationset-controller

deploy/argocd/project.yaml

@@ -0,0 +1,42 @@
+# The Argo CD project RootCause's Application belongs to (spec section 13.2).
+# It limits what a bad commit on main can make Argo CD do: one source repo,
+# the two RootCause namespaces on this cluster only, no cluster-wide objects
+# at all (the namespaces come from scripts/k8s-up.sh), and only the kinds
+# deploy/k8s uses. Argo CD's core install has no default project, so an
+# Application outside this one syncs nothing.
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: rootcause
+  namespace: argocd
+spec:
+  description: RootCause in the local cluster
+  sourceRepos:
+    - git@github.com:DIZAMU/RootCause.git
+  destinations:
+    - server: https://kubernetes.default.svc
+      namespace: rootcause-system
+    - server: https://kubernetes.default.svc
+      namespace: sandboxes
+  clusterResourceWhitelist: []
+  namespaceResourceWhitelist:
+    - group: ""
+      kind: Service
+    - group: ""
+      kind: ServiceAccount
+    - group: ""
+      kind: ResourceQuota
+    - group: ""
+      kind: LimitRange
+    - group: apps
+      kind: Deployment
+    - group: apps
+      kind: StatefulSet
+    - group: batch
+      kind: CronJob
+    - group: networking.k8s.io
+      kind: NetworkPolicy
+    - group: rbac.authorization.k8s.io
+      kind: Role
+    - group: rbac.authorization.k8s.io
+      kind: RoleBinding

deploy/k8s-dev/kustomization.yaml

@@ -0,0 +1,32 @@
+# The cluster with the API image built from the working tree
+# (scripts/k8s-up.sh --dev): rootcause/api:local, imported into the nodes
+# with k3d, so unreleased code can be tried. k8s-up.sh turns Argo CD's
+# automated sync off first, or it would put the release back; a run without
+# --dev turns it on again.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../k8s
+images:
+  - name: ghcr.io/dizamu/rootcause-api
+    newName: rootcause/api
+    newTag: local
+patches:
+  - target:
+      kind: Deployment
+      name: rootcause-api
+    patch: |
+      - op: replace
+        path: /spec/template/spec/containers/0/imagePullPolicy
+        value: Never
+      - op: remove
+        path: /spec/template/spec/imagePullSecrets
+  - target:
+      kind: CronJob
+      name: dbt
+    patch: |
+      - op: replace
+        path: /spec/jobTemplate/spec/template/spec/initContainers/0/imagePullPolicy
+        value: Never
+      - op: remove
+        path: /spec/jobTemplate/spec/template/spec/imagePullSecrets

deploy/k8s/api.yaml

@@ -39,6 +39,10 @@ spec:
         rootcause.instance: main
     spec:
       serviceAccountName: rootcause-api
+      # The image is private on GHCR. scripts/k8s-up.sh creates this Secret
+      # from the owner's token; it is never in git.
+      imagePullSecrets:
+        - name: ghcr-pull
       # The kubernetes runtime needs the service account's token; the
       # ServiceAccount itself does not mount it anywhere else.
       automountServiceAccountToken: true
@@ -52,8 +56,9 @@ spec:
           type: RuntimeDefault
       containers:
         - name: api
-          image: rootcause/api:local
-          imagePullPolicy: Never
+          # kustomization.yaml pins the release by digest.
+          image: rootcause/api
+          imagePullPolicy: IfNotPresent
           args: [serve, --listen-all, --port, "7300"]
           env:
             - name: RC_RUNTIME

deploy/k8s/api/Dockerfile

@@ -1,11 +1,14 @@
 # The API and dashboard image for the cluster (spec section 8.2). The context
-# is a directory scripts/k8s-up.sh assembles: the rootcause binary, built with
-# -tags webdist so the dashboard is embedded, and the exercises folder, whose
-# scripts the engine copies into sandboxes. Nothing else: no shell tools are
-# needed, and the process runs as an unprivileged user.
+# is a directory scripts/build-api-image.sh assembles: the rootcause binary,
+# built with -tags webdist so the dashboard is embedded, and the exercises
+# folder, whose scripts the engine copies into sandboxes. Nothing else: no
+# shell tools are needed, and the process runs as an unprivileged user.
 FROM debian:13-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad
 COPY rootcause /opt/rootcause/rootcause
 COPY exercises /opt/rootcause/exercises
+# Links the GHCR package to the repository, so release.yml's GITHUB_TOKEN may
+# push to it.
+LABEL org.opencontainers.image.source=https://github.com/DIZAMU/RootCause
 USER 10001:10001
 ENV RC_ROOT=/opt/rootcause HOME=/tmp
 ENTRYPOINT ["/opt/rootcause/rootcause"]