From 9e2c37a512dcfd46b55d3a8da627cdb63d011263 Mon Sep 17 00:00:00 2001 From: DIZAMU <117295462+DIZAMU@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:36:52 +0100 Subject: [PATCH] chore: prepare the repository for the public v1.0.0 release --- .../scripts}/ci-changed-exercises.sh | 10 +- .../scripts}/ci-check-admission.sh | 6 +- .../scripts}/ci-check-credential-args.sh | 10 +- .../scripts}/ci-enable-userns.sh | 6 +- {scripts => .github/scripts}/ci-go-test.sh | 6 +- .../scripts}/ci-release-image.sh | 4 +- .../scripts}/test-commit-msg.sh | 2 +- .github/workflows/ci.yml | 20 +- .github/workflows/commit-policy.yml | 2 +- .github/workflows/dbt.yml | 2 +- .github/workflows/exercises.yml | 12 +- .github/workflows/release.yml | 4 +- .gitignore | 2 +- README.md | 226 ++++--- SECURITY.md | 20 + cmd/rootcause/main.go | 4 +- dbt/dbt_project.yml | 2 +- dbt/embed.go | 2 +- dbt/models/marts/mart_exercise_stats.sql | 2 +- dbt/models/marts/mart_review_queue.sql | 2 +- dbt/models/marts/mart_topic_levels.sql | 2 +- dbt/models/staging/sources.yml | 2 +- deploy/argocd/application.yaml | 2 +- deploy/argocd/install/kustomization.yaml | 2 +- deploy/argocd/project.yaml | 4 +- deploy/compose/docker-compose.yml | 2 +- deploy/k8s/admission.yaml | 2 +- deploy/k8s/api.yaml | 2 +- deploy/k8s/api/Dockerfile | 2 +- deploy/k8s/dbt.yaml | 4 +- deploy/k8s/k3d.yaml | 4 +- deploy/k8s/kustomization.yaml | 2 +- deploy/k8s/namespaces.yaml | 2 +- deploy/k8s/postgres.yaml | 4 +- deploy/k8s/rbac.yaml | 2 +- deploy/k8s/sandboxes.yaml | 6 +- docs/ARCHITECTURE.md | 501 ++++++++++++++ docs/SPEC.md | 630 ------------------ .../0001-go-engine-and-runtime-interface.md | 22 +- .../adr/0002-dashboard-embedded-local-only.md | 14 +- docs/adr/0003-postgres-compose-migrations.md | 14 +- docs/adr/0004-dbt-in-a-container.md | 16 +- docs/adr/0005-bash-exercise-contract.md | 16 +- docs/adr/0006-grading-and-adaptation.md | 18 +- docs/adr/0007-seeded-variants.md | 14 +- docs/adr/0008-languages-and-tickets.md | 12 +- docs/adr/0009-single-local-profile.md | 13 +- docs/adr/0010-docker-sandboxes-userns.md | 22 + docs/adr/0010-v1-docker-sandboxes-userns.md | 22 - docs/adr/0011-k3d-user-namespaces.md | 24 + docs/adr/0011-v2-k3d-user-namespaces.md | 24 - docs/adr/0012-sandbox-isolation-model.md | 32 +- docs/adr/0013-web-terminal-exec.md | 16 +- docs/adr/0014-ci-github-actions.md | 20 +- docs/adr/0015-gitops-argo-cd.md | 22 +- docs/adr/0016-argo-cd-trust-boundary.md | 14 +- docs/adr/0017-ghcr-images-and-tokens.md | 19 + docs/adr/0017-ghcr-private-images.md | 20 - docs/adr/0018-commits-and-authorship.md | 18 +- docs/adr/0019-known-limits.md | 19 +- docs/adr/0020-public-api-image.md | 19 + docs/adr/README.md | 25 +- docs/images/inbox.png | Bin 0 -> 236726 bytes docs/images/session.png | Bin 0 -> 253079 bytes docs/images/stats.png | Bin 0 -> 159435 bytes exercises/disk-01-stopped-saving/i18n/en.yaml | 4 +- exercises/disk-01-stopped-saving/i18n/no.yaml | 4 +- exercises/log-01-crash-on-start/i18n/en.yaml | 2 +- exercises/log-01-crash-on-start/i18n/no.yaml | 2 +- exercises/net-01-no-names/i18n/en.yaml | 4 +- exercises/net-01-no-names/i18n/no.yaml | 4 +- exercises/net-02-outside-only/i18n/en.yaml | 2 +- exercises/net-02-outside-only/i18n/no.yaml | 2 +- exercises/perm-01-shared-folder/i18n/en.yaml | 4 +- exercises/perm-01-shared-folder/i18n/no.yaml | 4 +- exercises/perm-02-backup-denied/i18n/en.yaml | 12 +- exercises/perm-02-backup-denied/i18n/no.yaml | 10 +- exercises/svc-01-intranet-config/i18n/en.yaml | 2 +- exercises/svc-01-intranet-config/i18n/no.yaml | 2 +- internal/cli/app.go | 2 +- internal/cli/commands.go | 6 +- internal/cli/errors.go | 2 +- internal/cli/errors_test.go | 2 +- internal/cli/next.go | 8 +- internal/cli/runtime.go | 4 +- internal/cli/serve.go | 6 +- internal/dbt/dbt.go | 8 +- internal/exercise/load.go | 10 +- internal/exercise/types.go | 12 +- internal/exercise/variant.go | 2 +- internal/grading/grading.go | 6 +- internal/i18n/i18n.go | 2 +- internal/i18n/i18n_test.go | 2 +- internal/play/play.go | 8 +- internal/runtime/docker/docker.go | 4 +- internal/runtime/docker/docker_test.go | 2 +- internal/runtime/kubernetes/kubernetes.go | 4 +- internal/runtime/runtime.go | 4 +- internal/selector/integration_test.go | 2 +- internal/selector/load.go | 4 +- internal/selector/selector.go | 10 +- internal/server/api.go | 2 +- internal/server/cluster_test.go | 2 +- internal/server/server.go | 6 +- internal/server/server_test.go | 2 +- internal/server/views.go | 2 +- internal/session/engine.go | 20 +- internal/session/reaper.go | 2 +- internal/session/selftest.go | 4 +- internal/store/analytics.go | 4 +- internal/store/event.go | 2 +- internal/store/exercise.go | 2 +- internal/store/migrate.go | 2 +- internal/store/migrations/0001_init.sql | 2 +- internal/store/session.go | 4 +- internal/store/session_test.go | 2 +- internal/store/store.go | 2 +- locales/en.json | 68 +- locales/locales.go | 2 +- locales/no.json | 68 +- scripts/check-commit-policy.sh | 4 +- scripts/k8s-gitops-prove.sh | 4 +- scripts/k8s-memory.sh | 2 +- scripts/k8s-prove.sh | 4 +- scripts/k8s-up.sh | 6 +- scripts/k8s-watch-sync.sh | 2 +- scripts/ui-check.sh | 2 +- web/embed.go | 2 +- web/public/assets/fonts/README.md | 8 +- web/src/shell/Sidebar.tsx | 2 +- web/src/shell/keyboard.tsx | 2 +- web/src/tokens.css | 4 +- web/src/views/Exercises.tsx | 2 +- web/src/views/History.tsx | 2 +- web/src/views/Inbox.tsx | 2 +- web/src/views/Stats.tsx | 2 +- 136 files changed, 1156 insertions(+), 1234 deletions(-) rename {scripts => .github/scripts}/ci-changed-exercises.sh (75%) rename {scripts => .github/scripts}/ci-check-admission.sh (94%) rename {scripts => .github/scripts}/ci-check-credential-args.sh (86%) rename {scripts => .github/scripts}/ci-enable-userns.sh (83%) rename {scripts => .github/scripts}/ci-go-test.sh (91%) rename {scripts => .github/scripts}/ci-release-image.sh (97%) rename {scripts => .github/scripts}/test-commit-msg.sh (96%) create mode 100644 SECURITY.md create mode 100644 docs/ARCHITECTURE.md delete mode 100644 docs/SPEC.md create mode 100644 docs/adr/0010-docker-sandboxes-userns.md delete mode 100644 docs/adr/0010-v1-docker-sandboxes-userns.md create mode 100644 docs/adr/0011-k3d-user-namespaces.md delete mode 100644 docs/adr/0011-v2-k3d-user-namespaces.md create mode 100644 docs/adr/0017-ghcr-images-and-tokens.md delete mode 100644 docs/adr/0017-ghcr-private-images.md create mode 100644 docs/adr/0020-public-api-image.md create mode 100644 docs/images/inbox.png create mode 100644 docs/images/session.png create mode 100644 docs/images/stats.png diff --git a/scripts/ci-changed-exercises.sh b/.github/scripts/ci-changed-exercises.sh similarity index 75% rename from scripts/ci-changed-exercises.sh rename to .github/scripts/ci-changed-exercises.sh index 2d4c090..f6d9b01 100755 --- a/scripts/ci-changed-exercises.sh +++ b/.github/scripts/ci-changed-exercises.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # ci-changed-exercises.sh prints the ids of the exercises CI must validate, -# one per line (spec section 14): +# one per line (ARCHITECTURE.md section 14): # # all every exercise # the exercises with a file changed between the two commits. @@ -9,11 +9,11 @@ # # Exercises deleted in the change are left out. # -# Usage: scripts/ci-changed-exercises.sh all -# scripts/ci-changed-exercises.sh +# Usage: .github/scripts/ci-changed-exercises.sh all +# .github/scripts/ci-changed-exercises.sh set -euo pipefail -root="$(cd "$(dirname "$0")/.." && pwd)" +root="$(cd "$(dirname "$0")/../.." && pwd)" cd "$root" all() { @@ -29,7 +29,7 @@ case "$#" in ;; 2) changed="$(git diff --name-only "$1" "$2")" - if grep -qxE 'scripts/ci-changed-exercises\.sh|scripts/validate-exercise\.sh|\.github/workflows/exercises\.yml' <<<"$changed"; then + if grep -qxE '\.github/scripts/ci-changed-exercises\.sh|scripts/validate-exercise\.sh|\.github/workflows/exercises\.yml' <<<"$changed"; then all exit 0 fi diff --git a/scripts/ci-check-admission.sh b/.github/scripts/ci-check-admission.sh similarity index 94% rename from scripts/ci-check-admission.sh rename to .github/scripts/ci-check-admission.sh index fcaa07f..85da615 100755 --- a/scripts/ci-check-admission.sh +++ b/.github/scripts/ci-check-admission.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Checks that deploy/k8s, as Argo CD renders it, holds the admission policy -# for sandbox pods in full (spec section 13.2): every rule's message from a +# for sandbox pods in full (ARCHITECTURE.md section 13.2): every rule's message from a # to l, failurePolicy Fail, and a binding with validationActions exactly # [Deny], scoped to sandboxes, with no matchConditions that could exempt a # requester. ci.yml runs it with the pinned kubectl from @@ -11,10 +11,10 @@ # contacting a server (the label it adds stays in this pipe), and Python's # standard json module reads the result. # -# Usage: scripts/ci-check-admission.sh [kubectl] (default: kubectl on PATH) +# Usage: .github/scripts/ci-check-admission.sh [kubectl] (default: kubectl on PATH) set -euo pipefail -root="$(cd "$(dirname "$0")/.." && pwd)" +root="$(cd "$(dirname "$0")/../.." && pwd)" cd "$root" kubectl="${1:-kubectl}" diff --git a/scripts/ci-check-credential-args.sh b/.github/scripts/ci-check-credential-args.sh similarity index 86% rename from scripts/ci-check-credential-args.sh rename to .github/scripts/ci-check-credential-args.sh index a7d9e70..879e459 100755 --- a/scripts/ci-check-credential-args.sh +++ b/.github/scripts/ci-check-credential-args.sh @@ -9,11 +9,11 @@ # # Pass such values on stdin instead (curl --config -). # -# Usage: scripts/ci-check-credential-args.sh [file...] (default: the other scripts/*.sh) -# scripts/ci-check-credential-args.sh --self-test +# Usage: .github/scripts/ci-check-credential-args.sh [file...] (default: the other scripts/*.sh) +# .github/scripts/ci-check-credential-args.sh --self-test set -euo pipefail -root="$(cd "$(dirname "$0")/.." && pwd)" +root="$(cd "$(dirname "$0")/../.." && pwd)" cd "$root" pattern='(^|[[:space:]])(-u|--user|-H|--header)[[:space:]]+"([^"]*\$\(|[^"]*\$\{?[A-Za-z_]*([Tt][Oo][Kk][Ee][Nn]|[Pp][Aa][Ss][Ss][Ww][Oo][Rr][Dd])|[Aa]uthorization:)' @@ -48,8 +48,8 @@ files=("$@") if [[ ${#files[@]} -eq 0 ]]; then # Every script but this one, whose comments and self-test hold the # patterns on purpose. - for f in scripts/*.sh; do - [[ "$f" == scripts/ci-check-credential-args.sh ]] || files+=("$f") + for f in scripts/*.sh .github/scripts/*.sh; do + [[ "$f" == .github/scripts/ci-check-credential-args.sh ]] || files+=("$f") done fi if hits="$(grep -nHE -- "$pattern" "${files[@]}")"; then diff --git a/scripts/ci-enable-userns.sh b/.github/scripts/ci-enable-userns.sh similarity index 83% rename from scripts/ci-enable-userns.sh rename to .github/scripts/ci-enable-userns.sh index 1478fae..6faa743 100755 --- a/scripts/ci-enable-userns.sh +++ b/.github/scripts/ci-enable-userns.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # ci-enable-userns.sh turns on Docker's user namespace remapping on a CI -# runner, the isolation the engine requires for sandboxes (spec section 13.1, -# OQ-3). It merges "userns-remap": "default" into /etc/docker/daemon.json, +# runner, the isolation the engine requires for sandboxes (ARCHITECTURE.md section +# 13.1). It merges "userns-remap": "default" into /etc/docker/daemon.json, # restarts Docker, and waits until `docker info` reports name=userns. # # Restarting Docker with userns-remap switches its storage root, so start # containers (Postgres, sandboxes) only after this script. # -# Usage (as root): scripts/ci-enable-userns.sh +# Usage (as root): .github/scripts/ci-enable-userns.sh set -euo pipefail conf=/etc/docker/daemon.json diff --git a/scripts/ci-go-test.sh b/.github/scripts/ci-go-test.sh similarity index 91% rename from scripts/ci-go-test.sh rename to .github/scripts/ci-go-test.sh index c548fba..32d7cfc 100755 --- a/scripts/ci-go-test.sh +++ b/.github/scripts/ci-go-test.sh @@ -9,8 +9,8 @@ # Integration tests skip only when their database or Docker was never # configured, so in CI a skip means a broken setup, not a pass. # -# Usage: scripts/ci-go-test.sh [--require TestName]... [go test args]... -# Example: scripts/ci-go-test.sh --require TestScriptedSelection -tags webdist ./... +# Usage: .github/scripts/ci-go-test.sh [--require TestName]... [go test args]... +# Example: .github/scripts/ci-go-test.sh --require TestScriptedSelection -tags webdist ./... set -euo pipefail # Skipped unless asked for on purpose: RC_DBT_STRESS=1, or RC_CLUSTER_URL @@ -25,7 +25,7 @@ while [ "${1:-}" = "--require" ]; do done [ "$#" -gt 0 ] || { echo "usage: ci-go-test.sh [--require TestName]... [go test args]..." >&2; exit 2; } -root="$(cd "$(dirname "$0")/.." && pwd)" +root="$(cd "$(dirname "$0")/../.." && pwd)" log="$(mktemp)" trap 'rm -f "$log"' EXIT diff --git a/scripts/ci-release-image.sh b/.github/scripts/ci-release-image.sh similarity index 97% rename from scripts/ci-release-image.sh rename to .github/scripts/ci-release-image.sh index b15f45d..adb65d7 100755 --- a/scripts/ci-release-image.sh +++ b/.github/scripts/ci-release-image.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# CI helpers for the API image on GHCR (spec section 14). release.yml and +# CI helpers for the API image on GHCR (ARCHITECTURE.md section 14). release.yml and # ci.yml call them; they read GITHUB_TOKEN and GITHUB_ACTOR from the # workflow and never print the token. # @@ -15,7 +15,7 @@ repo=dizamu/rootcause-api zero=sha256:0000000000000000000000000000000000000000000000000000000000000000 accept="application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json" -root="$(cd "$(dirname "$0")/.." && pwd)" +root="$(cd "$(dirname "$0")/../.." && pwd)" cd "$root" die() { echo "ci-release-image: $*" >&2; exit 1; } diff --git a/scripts/test-commit-msg.sh b/.github/scripts/test-commit-msg.sh similarity index 96% rename from scripts/test-commit-msg.sh rename to .github/scripts/test-commit-msg.sh index 80fc9c0..3f02eca 100755 --- a/scripts/test-commit-msg.sh +++ b/.github/scripts/test-commit-msg.sh @@ -3,7 +3,7 @@ # and messages it must reject. Exit 0 when every case behaves as expected. set -uo pipefail -root="$(cd "$(dirname "$0")/.." && pwd)" +root="$(cd "$(dirname "$0")/../.." && pwd)" hook="$root/.githooks/commit-msg" tmp="$(mktemp)" trap 'rm -f "$tmp"' EXIT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5d5647..3b8aa10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -# CI for every pull request and every push to main (spec section 14). +# CI for every pull request and every push to main (ARCHITECTURE.md section 14). # A push to a PR branch already runs through pull_request, so pushes to other # branches are left out to save Actions minutes. name: ci @@ -57,17 +57,17 @@ jobs: - name: shellcheck run: | shellcheck --version - shellcheck scripts/*.sh .githooks/commit-msg exercises/*/*.sh + shellcheck scripts/*.sh .github/scripts/*.sh .githooks/commit-msg exercises/*/*.sh - name: admission policy shape run: | scripts/install-k8s-tools.sh "$RUNNER_TEMP/k8s-bin" - scripts/ci-check-admission.sh "$RUNNER_TEMP/k8s-bin/kubectl" + .github/scripts/ci-check-admission.sh "$RUNNER_TEMP/k8s-bin/kubectl" - name: no credentials on command lines run: | - scripts/ci-check-credential-args.sh --self-test - scripts/ci-check-credential-args.sh + .github/scripts/ci-check-credential-args.sh --self-test + .github/scripts/ci-check-credential-args.sh - name: commit-msg hook cases - run: bash scripts/test-commit-msg.sh + run: bash .github/scripts/test-commit-msg.sh - name: i18n key parity run: | go test -count=1 -run 'TestLocaleParity' ./internal/i18n/ @@ -97,7 +97,7 @@ jobs: run: scripts/build-web.sh # Before any container starts: the restart switches Docker's storage. - name: enable Docker userns-remap - run: sudo scripts/ci-enable-userns.sh + run: sudo .github/scripts/ci-enable-userns.sh - name: start Postgres with rootcause init run: | pw="$(openssl rand -hex 24)" @@ -110,13 +110,13 @@ jobs: env: RC_REQUIRE_DOCKER: "1" run: >- - scripts/ci-go-test.sh + .github/scripts/ci-go-test.sh --require TestScriptedSelection --require TestBuildRecoversFromKilledEngine --require TestPlaySvc01 -tags webdist ./... - # The release deploy/k8s pins must be a pushed vX.Y.Z (spec section 14). + # The release deploy/k8s pins must be a pushed vX.Y.Z (ARCHITECTURE.md section 14). # GITHUB_TOKEN is the only credential, with packages: read on this job only. # A pull request from a fork gets no package access, so the job does not # run there. @@ -135,4 +135,4 @@ jobs: - name: pinned digest is a release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: scripts/ci-release-image.sh check-kustomization + run: .github/scripts/ci-release-image.sh check-kustomization diff --git a/.github/workflows/commit-policy.yml b/.github/workflows/commit-policy.yml index b1f66c5..2450a8f 100644 --- a/.github/workflows/commit-policy.yml +++ b/.github/workflows/commit-policy.yml @@ -1,4 +1,4 @@ -# Commit policy (spec sections 14 and 15): every commit in a pull request has +# Commit policy (ARCHITECTURE.md sections 14 and 15): every commit in a pull request has # no Co-authored-by trailer or tool attribution line, the owner's author and # committer email, and a signature GitHub verifies. name: commit-policy diff --git a/.github/workflows/dbt.yml b/.github/workflows/dbt.yml index bd795aa..149e2b7 100644 --- a/.github/workflows/dbt.yml +++ b/.github/workflows/dbt.yml @@ -86,7 +86,7 @@ jobs: RC_DBT_HOST: postgres RC_REQUIRE_DOCKER: "1" run: >- - scripts/ci-go-test.sh + .github/scripts/ci-go-test.sh --require TestScriptedSelection --require TestBuildRecoversFromKilledEngine --require TestBuildFailureKeepsLog diff --git a/.github/workflows/exercises.yml b/.github/workflows/exercises.yml index 889ef1f..f8069d2 100644 --- a/.github/workflows/exercises.yml +++ b/.github/workflows/exercises.yml @@ -1,4 +1,4 @@ -# Exercise validation (spec section 14): for each changed exercise and seeds +# Exercise validation (ARCHITECTURE.md section 14): for each changed exercise and seeds # 1, 2 and 3, build the image; break then check must fail; break, solution, # check must pass. Pull requests and pushes to main validate only the changed # exercises; tags and manual runs validate all of them. Tags reach this @@ -46,13 +46,13 @@ jobs: PUSH_AFTER: ${{ github.sha }} run: | if [ "$EVENT" = workflow_dispatch ] || [ "$REF_TYPE" = tag ]; then - ids="$(scripts/ci-changed-exercises.sh all)" + ids="$(.github/scripts/ci-changed-exercises.sh all)" elif [ "$EVENT" = pull_request ]; then - ids="$(scripts/ci-changed-exercises.sh "$PR_BASE" "$PR_HEAD")" + ids="$(.github/scripts/ci-changed-exercises.sh "$PR_BASE" "$PR_HEAD")" elif [ -z "$PUSH_BEFORE" ] || [ "$PUSH_BEFORE" = 0000000000000000000000000000000000000000 ]; then - ids="$(scripts/ci-changed-exercises.sh all)" + ids="$(.github/scripts/ci-changed-exercises.sh all)" else - ids="$(scripts/ci-changed-exercises.sh "$PUSH_BEFORE" "$PUSH_AFTER")" + ids="$(.github/scripts/ci-changed-exercises.sh "$PUSH_BEFORE" "$PUSH_AFTER")" fi echo "exercises to validate: ${ids:-none}" # One line, space-separated; empty when nothing changed. @@ -64,7 +64,7 @@ jobs: go-version-file: go.mod - name: enable Docker userns-remap if: steps.list.outputs.ids != '' - run: sudo scripts/ci-enable-userns.sh + run: sudo .github/scripts/ci-enable-userns.sh - name: validate, seeds 1 2 3 if: steps.list.outputs.ids != '' env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index adb3037..252f604 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# Release (spec section 14): on a vX.Y.Z tag on main, validate every exercise, +# Release (ARCHITECTURE.md section 14): on a vX.Y.Z tag on main, validate every exercise, # then build the API image and push it to ghcr.io/dizamu/rootcause-api under # that tag only. The job summary has the digest; the owner pins it in # deploy/k8s/kustomization.yaml in a pull request, and Argo CD deploys it @@ -35,7 +35,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ github.ref_name }} - run: scripts/ci-release-image.sh check-tag "$TAG" "$GITHUB_SHA" + run: .github/scripts/ci-release-image.sh check-tag "$TAG" "$GITHUB_SHA" validate: needs: check diff --git a/.gitignore b/.gitignore index 490fec0..abf99d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# Secrets: local env files are never committed (spec section 13.2). +# Secrets: local env files are never committed (ARCHITECTURE.md section 13.2). .env # Compiled binary and build output. /rootcause diff --git a/README.md b/README.md index 3f673a1..4970e4c 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,90 @@ # RootCause -Hands-on IT troubleshooting practice. Each exercise is a broken Linux machine with a planted fault, delivered as a helpdesk ticket. The learner finds and fixes the fault in a terminal, an automated check grades the result, and the next exercise adapts to the learner's weak spots. +Practise fixing real, broken Linux servers. -**Status:** in development. +RootCause hands you a helpdesk ticket and a Linux machine with a planted fault. You find the cause and fix it in a terminal, an automated check grades the result, and the next ticket adapts to your weak spots. Everything runs locally, in English or Norwegian. -See [`docs/SPEC.md`](docs/SPEC.md) for the full specification. + + +![The inbox, with the recommended ticket pinned first](docs/images/inbox.png) + +## What it does + +- **Real machines.** Every exercise is a disposable container, or a Kubernetes pod, with a real fault in permissions, services, logs, networking or disk. +- **Graded by the end state.** A check tests what you left behind, never the commands you typed. Any valid fix passes, and the usual shortcuts fail. +- **Hints with a cost.** Three hint tiers go from where to look to nearly the answer. A solve earns 3, 2 or 1 stars depending on the hints used. Time is recorded but never scored. +- **Adaptive.** Each topic has a level from 1 to 3. RootCause picks your weakest topic next and brings back exercises you struggled with, as a new variant. +- **Randomized variants.** File names, ports, users and senders change from a seed, so a repeat is never the exact same machine. +- **Statistics.** A dbt pipeline turns every session into progress, a review queue and difficulty calibration. +- **A CLI and a dashboard.** Use either or both, and they follow each other. + +![A solved session with the walkthrough](docs/images/session.png) + +## Quick start + +RootCause runs on Linux (tested on Ubuntu 24.04) and on Windows 11 with WSL2. You need Docker Engine with the Compose plugin, Go 1.27 and Node 24. Node is only needed to build the dashboard. + +Each exercise runs in a container where you are root, so root in the container must not be root on your machine. Docker has to use user namespace remapping or rootless mode, and RootCause refuses to start a session otherwise. + +1. Turn on user namespace remapping. Create `/etc/docker/daemon.json` with `{"userns-remap": "default"}` and run `sudo systemctl restart docker`. Docker then uses a separate storage area, so images are pulled again. Check that `docker info --format '{{json .SecurityOptions}}'` shows `name=userns`. +2. Clone the repository and set a database password. + ```bash + git clone https://github.com/DIZAMU/RootCause.git && cd RootCause + cp deploy/compose/.env.example deploy/compose/.env # then set POSTGRES_PASSWORD + ``` +3. Build the dashboard and the binary. + ```bash + scripts/build-web.sh + go build -tags webdist -o rootcause ./cmd/rootcause + ``` +4. Start the database and create your profile with `./rootcause init`. +5. Run `./rootcause serve` and open http://localhost:7300. Pick a ticket, then run `./rootcause shell` in a terminal next to the browser to work on the machine. Stop the server with Ctrl-C. + +The first start of each exercise builds its image, which takes a minute. The first statistics build pulls the dbt image. + +You can also play from the CLI alone. `./rootcause next` starts the recommended ticket, and `./rootcause` on its own lists every command. + +The exercise that boots systemd (svc-02) needs cgroup v2, which is the default on current distributions and on WSL2. `docker info --format '{{.CgroupVersion}}'` must print `2`. + +### Using the dashboard + +- The inbox shows the recommended ticket first, with why it was picked. +- Ctrl+K opens the command palette, Ctrl+Enter runs the check (in the terminal too), and `?` shows every shortcut. +- A browser's first visit opens a short guided intro, and the "?" button shows it again. +- "Reset all data" in the help dialog, or `./rootcause reset --all`, deletes your history and starts you fresh. It cannot be undone. + +![Topic levels, solves per day and statistics per exercise](docs/images/stats.png) + +## Kubernetes mode + +The same sessions can run in a local k3d cluster. The API, dashboard, Postgres and dbt run in one namespace, each session gets a locked-down pod in another, and the terminal is right in the browser. + +1. Run `scripts/install-k8s-tools.sh`. It puts pinned, checksum-verified `k3d` and `kubectl` in `~/.local/bin`, without sudo. +2. Run `scripts/k8s-up.sh --dev`. It creates the cluster (1 server, 2 agents), builds every image from your checkout, imports them and deploys RootCause. It needs no credentials. +3. Open http://localhost:7300 and work in the terminal on the session page. +4. Run `scripts/k8s-prove.sh` to check the isolation. It tests the API's permissions, the network policies, the admission policy, one full session, and that every published port is bound to 127.0.0.1. +5. Run `scripts/k8s-down.sh` to stop the cluster. Its data stays. + +Without `--dev`, Argo CD deploys the cluster from this repository's `main`, running the released image pinned by digest. That is the author's own GitOps setup, and it expects a deploy key and a token in `~/.config/rootcause`. [ARCHITECTURE.md](docs/ARCHITECTURE.md) explains how it works. ## How it works -RootCause has two ways to run sessions. Both use the same engine, database and dashboard; only the runtime that creates sandboxes changes (`RC_RUNTIME`). The boxes marked as trust boundaries are where one side must not be able to reach into the other. The decisions behind each part are in [`docs/adr`](docs/adr/README.md). +Both modes share the same engine, database and dashboard, and only the runtime that creates sandboxes changes. The boxes marked as trust boundaries are where one side must not be able to reach into the other. -### v1: local, with Docker +### Docker mode ```mermaid flowchart LR - browser["Browser on Windows"] - subgraph wsl["WSL2 host (trust boundary: the owner's machine)"] + browser["Browser"] + subgraph host["Linux or WSL2 host (trust boundary, your machine)"] cli["rootcause CLI"] - serve["rootcause serve
127.0.0.1:7300
Host check + token"] - engine["Engine: sessions, grading, selector"] + serve["rootcause serve
127.0.0.1:7300
Host check and token"] + engine["Engine
sessions, grading, selector"] pg[("PostgreSQL
Docker Compose")] dbt["dbt container
no capabilities, read-only"] subgraph docker["Docker Engine with userns-remap"] - subgraph sb["Sandbox container (trust boundary: learner is root here)"] - machine["Exercise image
cap-drop ALL + allowlist
no network or internal only"] + subgraph sb["Sandbox container (trust boundary, learner is root here)"] + machine["Exercise image
cap-drop ALL and an allowlist
no network, or internal only"] end end end @@ -35,28 +96,28 @@ flowchart LR engine -->|docker CLI| sb ``` -### v2: the local k3d cluster, deployed by Argo CD +### Kubernetes mode ```mermaid flowchart LR - browser["Browser on Windows"] - subgraph github["GitHub (private)"] + browser["Browser"] + subgraph github["GitHub"] repo["main branch
deploy/k8s"] ghcr["GHCR
rootcause-api vX.Y.Z"] release["release.yml
on a vX.Y.Z tag"] end - subgraph host["WSL2 host"] - subgraph k3d["k3d cluster (trust boundary: nodes run as host root)"] - subgraph argocd["argocd (trust boundary: writes anywhere it is asked)"] + subgraph host["Linux or WSL2 host"] + subgraph k3d["k3d cluster (trust boundary, nodes run as host root)"] + subgraph argocd["argocd (trust boundary, writes anywhere it is asked)"] argo["Argo CD core
project rootcause"] end - subgraph system["rootcause-system: PSA restricted, default-deny"] - api["API + dashboard
Role: pods in sandboxes only"] + subgraph system["rootcause-system
PSA restricted, default-deny"] + api["API and dashboard
Role limited to pods in sandboxes"] pgk[("PostgreSQL")] cron["dbt CronJob"] end - subgraph sandboxes["sandboxes: PSA baseline, admission policy, default-deny"] - pod["Session pod (trust boundary: own user namespace)
hostUsers false, no token, emptyDir only"] + subgraph sandboxes["sandboxes
PSA baseline, admission policy, default-deny"] + pod["Session pod (trust boundary, own user namespace)
hostUsers false, no token, emptyDir only"] runner["Check runner pod"] end end @@ -70,80 +131,39 @@ flowchart LR argo -->|read-only deploy key| repo argo -->|sync, self-heal| system argo -->|sync, self-heal| sandboxes - ghcr -->|pull, read-only token| api + ghcr -->|pull| api ``` - A sandbox can reach neither the internet nor `rootcause-system`, and only a check runner may connect to it. - The API's service account may only create, read, exec into and delete pods in `sandboxes`. The admission policy refuses any sandbox pod that is not locked down, whoever creates it. - Everything the cluster publishes listens on the host's 127.0.0.1 only. -## Local setup - -RootCause runs in WSL2 or Linux with Docker Engine and Go. Each exercise runs in a container where you are root, so root in the container must not be root on your machine. Docker must use user namespace remapping or rootless mode. The CLI refuses to start sessions otherwise. - -1. Create `/etc/docker/daemon.json` with `{"userns-remap": "default"}`, then run `sudo systemctl restart docker`. Docker then uses a separate storage area, so images you pulled before are pulled again. -2. Check that `docker info --format '{{json .SecurityOptions}}'` shows `name=userns`. -3. Copy `deploy/compose/.env.example` to `deploy/compose/.env` and set a password. -4. Run `go run ./cmd/rootcause init`, then `go run ./cmd/rootcause list`. -5. Run `go run ./cmd/rootcause next` to start the recommended exercise. Statistics are built with dbt in a container (the first run pulls the image). +The full design, from the exercise format to the security model, is in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). The reasons behind each choice are in the [decision records](docs/adr/README.md). -### Dashboard +## Exercises -The dashboard is a React app in `web/`, built into the binary. Building it needs Node 24 (see `web/.nvmrc`), and only at build time. - -1. Run `scripts/build-web.sh`. It runs `npm ci` from the committed lockfile, a type check, and `vite build` into `web/dist`. -2. Build the binary with the dashboard embedded: `go build -tags webdist -o rootcause ./cmd/rootcause`. -3. Run `./rootcause serve` and open http://localhost:7300 in a browser on Windows. Stop it with Ctrl-C. - -The server listens on 127.0.0.1 only. WSL2 forwards `localhost` from Windows to it, so no `.wslconfig` change is needed. Work in the sandbox with `./rootcause shell` in a WSL terminal next to the browser; in the cluster the terminal is part of the page. - -The home page is an inbox of tickets with the recommended one on top. A browser's first visit opens a short guided intro; the "?" button at the bottom of the sidebar shows it again, with all the shortcuts. Press Ctrl+K anywhere for the command palette, Ctrl+Enter to run the check (in the terminal too), and `?` for help. In the terminal, Tab completes; press Esc, then Tab, to move on with the keyboard. - -To start over as a new learner, use "Reset all data" in the help dialog, or `rootcause reset --all` in the CLI (type RESET to confirm, or pass `--yes` in a script). It deletes your history, sessions, levels, statistics and settings, ends a session in progress and removes its sandbox, and cannot be undone. It acts only on the database the dashboard or CLI uses. - -`scripts/ui-check.sh ` checks the dashboard in Windows' Chrome on this laptop: the first-run intro, the collapsed sidebar, centred pages on wide screens, the reset dialog, accessibility (axe-core, downloaded once into `~/.cache/rootcause` and checked against a pinned SHA-256), layout shift, the keyboard rules, the Tab order, terminal latency and the CSP, with screenshots in both languages. Point it at a server on a test database: it runs a real check. With `RC_UI_WIPE=1` it ends by resetting all data through the dashboard, which empties that database. The dashboard follows the CLI: hints, checks and results from either side show up in both. A binary built without `-tags webdist` still runs `serve`, but shows a page that says how to build the dashboard. - -Exercises that boot systemd (such as svc-02) need a host with cgroup v2. WSL2 uses cgroup v2 by default. Check with `docker info --format '{{.CgroupVersion}}'`, which must print `2`. - -### Kubernetes (v2) - -The same sessions can run in a local k3d cluster (spec section 8.2): the API and dashboard, Postgres and dbt in the namespace `rootcause-system`, one pod per session in `sandboxes`, and the terminal in the browser. Argo CD in `argocd` deploys the cluster from `main` (see GitOps below). - -1. Run `scripts/install-k8s-tools.sh`. It puts pinned, checksum-verified `k3d` and `kubectl` in `~/.local/bin`, without sudo. -2. Run `scripts/k8s-up.sh --dev` to try the working tree: it creates the cluster (1 server, 2 agents), builds the API image and the exercise images and imports them, generates the database password into git-ignored `deploy/k8s/.env`, and applies `deploy/k8s-dev`. Re-run it after a change. If Argo CD is installed, `--dev` turns its automated sync off first. -3. Open http://localhost:7300. Start an exercise and work in the terminal on the exercise page. -4. Run `scripts/k8s-prove.sh` to check the isolation: the API's permissions, the network policies, one full session through the dashboard, that every published port is bound to 127.0.0.1, and the sandbox admission policy (each forbidden pod change refused, as the API and as Argo CD). That session plays on a separate database, `rootcause_cluster_test`, through a second API pod, and leaves the real one untouched. -5. Run `scripts/k8s-down.sh` to stop the cluster and free memory; its data stays. `scripts/k8s-dbt.sh` rebuilds the statistics at once instead of waiting for the 10-minute CronJob. `scripts/k8s-memory.sh` prints the memory each node and container uses. - -`RC_RUNTIME=kubernetes` points the CLI at the cluster, for example `RC_RUNTIME=kubernetes scripts/validate-exercise.sh perm-01-shared-folder 1`. svc-02 boots systemd, which does not run in a pod, so it is refused there (spec section 13.3). - -### GitOps (v2) - -Without `--dev`, `scripts/k8s-up.sh` installs Argo CD core (pinned, checksum-verified) and the Application in `deploy/argocd`, which syncs `deploy/k8s` from `main` with automated sync, prune and self-heal. The cluster then runs the API image pinned by digest in `deploy/k8s/kustomization.yaml`, pulled from the private package `ghcr.io/dizamu/rootcause-api`. Exercise images are still built and imported by `k8s-up.sh`; sandbox pods never pull. - -A change reaches the cluster like this, with no kubectl by hand: - -1. Merge the change to `main`. -2. Tag it and push the tag. `git tag -s` signs it with the same SSH key as the commits; check before you push: - ```bash - git checkout main && git pull --ff-only - git tag -s v0.8.0 -m v0.8.0 - git tag -v v0.8.0 - git push origin v0.8.0 - ``` - `release.yml` checks that the tag is `vX.Y.Z` on `main` and new, validates every exercise, builds the API image and pushes it under that tag only. Its job summary shows the digest. -3. In a pull request, set that digest in `deploy/k8s/kustomization.yaml` (`images:`, digest only). `ci-release-image` checks that it is a pushed release. When it is merged, Argo CD rolls it out. +| Ticket | Topic | Difficulty | +|---|---|---| +| We can't save to the shared folder | Permissions | 1 | +| The nightly backup fails | Permissions | 2 | +| The intranet is down | Services | 1 | +| Vaktliste is gone after every reboot | Services | 2 | +| Lagerapp will not start | Logs | 1 | +| The server cannot find our internal machines | Networking | 2 | +| kundeportal works on the server, but not for us | Networking | 1 | +| The meter app stopped saving | Disk | 1 | -Check the result with `scripts/k8s-gitops-prove.sh` (set `RC_RELEASE_DIGEST` to the digest from the job summary to compare it too): the Application is Synced and Healthy at `main` with self-heal on, the API runs the pinned digest, a change made by hand is reverted, and the project refuses other namespaces and repositories. It fails while a `--dev` run has turned automated sync off. +Each exercise is a folder in [`exercises/`](exercises) with a Dockerfile, the scripts that plant and check the fault, a reference solution that only CI uses, and the texts in both languages. -Two credentials stay outside the repo, in `~/.config/rootcause` (chmod 600). `k8s-up.sh` turns them into Secrets and never prints them: +## Tests -- `argocd-deploy-key`: the private half of a read-only deploy key for this repository. Make it with `ssh-keygen -t ed25519 -N '' -C rootcause-argocd -f ~/.config/rootcause/argocd-deploy-key`, then add the `.pub` file under Settings, Deploy keys, with write access off. The key has no passphrase, so it is unencrypted on disk. That is acceptable for a read-only key to one private repository, and the Secret in the cluster holds it in plain text anyway. -- `ghcr-token`: a personal access token (classic) with only the `read:packages` scope and a 90-day expiry, made under Settings, Developer settings, Personal access tokens, Tokens (classic). GitHub Packages accepts only classic tokens. A classic `read:packages` token can read every package your account can see, not just this one. When it expires, make a new one, save it to the same file and run `k8s-up.sh` again. +Unit tests need nothing beyond Go. -## Tests +```bash +go test ./... +``` -Unit tests need nothing: `go test ./...`. Integration tests need a database whose name ends in `_test`, and Docker with userns-remap: +Integration tests need a database whose name ends in `_test`, and Docker with userns-remap. They never touch any other database. ```bash docker exec rootcause-postgres-1 createdb -U rootcause rootcause_test @@ -151,40 +171,30 @@ export RC_TEST_DATABASE_URL="postgres://rootcause:@127.0.0.1:5432/root go test -count=1 ./... ``` -- A test skips only when `RC_TEST_DATABASE_URL` is unset, or when Docker is missing. If the variable is set but the database is unreachable or not a `_test` database, the test fails. -- `RC_REQUIRE_DOCKER=1` makes a missing Docker fail the tests too. CI sets it. -- Tests that run dbt create and drop their own database (for example `rootcause_selector_test`), because a dbt build reads every row in its database. -- `RC_DBT_STRESS=1 go test -run TestBuildStress ./internal/dbt/` kills dbt builds at random points and checks that the next build succeeds. `RC_DBT_STRESS_ROUNDS` sets the rounds (default 10). -- A failed dbt build saves its full `dbt.log` under `~/.cache/rootcause/dbt-logs/` and prints the path. The newest 20 are kept. -- `TestClusterSession` plays a session against the deployed cluster and runs only with `RC_CLUSTER_URL` set; `scripts/k8s-prove.sh` sets it. -- `TestSandboxAdmission` checks the sandbox admission policy against the pods the runtime builds, with server-side dry runs as the API and as Argo CD, and runs only with `RC_CLUSTER_ADMISSION` set; `scripts/k8s-prove.sh` sets it too. +- A test skips only when `RC_TEST_DATABASE_URL` is unset or Docker is missing. If the variable is set but the database is unreachable or not a `_test` database, the test fails. CI fails on any skipped test. +- `scripts/validate-exercise.sh ` proves one exercise end to end. Break then check must fail, and break, solution, check must pass. +- `scripts/ui-check.sh` checks the dashboard in a real browser, covering accessibility, the keyboard rules, layout shift and both languages. ## CI -GitHub Actions runs five workflows (spec section 14). Every action is pinned by commit SHA, and every job gets read-only access to the repository and nothing else, except `packages: read` on `ci-release-image` and `release-check`, and `packages: write` on `release-push`. - -| Workflow | Check name | Does | -|---|---|---| -| `ci.yml` | `ci-checks`, `ci-test`, `ci-release-image` | gofmt, go vet, golangci-lint, shellcheck, i18n parity, the dashboard build, and every Go test with Postgres and Docker. Fails if any test is skipped. Also fails if a script puts a credential on a command line, or if the rendered sandbox admission policy loses a rule, its Fail policy or its Deny-only binding. `ci-release-image` checks that the digest in `deploy/k8s/kustomization.yaml` is a pushed `vX.Y.Z` release, or the zero placeholder | -| `exercises.yml` | `exercises` | Validates the changed exercises with seeds 1, 2 and 3. All of them on manual runs, and when `release.yml` calls it | -| `dbt.yml` | `dbt` | dbt build against a Postgres service container, when the dbt project or its inputs changed | -| `commit-policy.yml` | `commit-policy` | Rejects Co-authored-by trailers, tool attribution lines, author emails other than the owner's, and commits GitHub does not show as Verified | -| `release.yml` | `release-check`, `exercises`, `release-push` | On a tag `v*`: the tag must be `vX.Y.Z` on `main` and not pushed before; every exercise is validated; then the API image is pushed to GHCR under that tag. It never commits | +GitHub Actions runs five workflows. Every action is pinned by commit SHA, and every job gets read-only access unless it needs more. -Run the same commit check locally with `scripts/check-commit-policy.sh origin/main HEAD`. +| Workflow | Does | +|---|---| +| `ci.yml` | gofmt, go vet, golangci-lint, shellcheck, i18n parity, the dashboard build, and every Go test with Postgres and Docker. It also checks that no script puts a credential on a command line and that the sandbox admission policy keeps every rule | +| `exercises.yml` | Validates every changed exercise with 3 seeds | +| `dbt.yml` | Runs the dbt build and its tests against Postgres | +| `commit-policy.yml` | Checks that every commit is signed and follows the commit rules | +| `release.yml` | On a `vX.Y.Z` tag, validates every exercise and pushes the API image to GHCR | -### When the repo goes public +## About -On the free plan, branch protection does not apply to private repositories, so a red check does not block the merge button. The rule until then: never merge a PR with a red check. When the repo goes public, add a ruleset under Settings, Rules, Rulesets: +RootCause is my first real project. I made a small thing called GeoDex before, but this is the first time I took an idea all the way to a finished product, and I built it purely for fun. The learning curve was steep, and I am honestly amazed at how it turned out. -- Target: the default branch (`main`). Enforcement: Active. No bypass list. -- Restrict deletions, and block force pushes. -- Require a pull request before merging, with merge commits only (squash and rebase drop the SSH signatures). -- Require signed commits. -- Require status checks to pass, with the branch up to date: `ci-checks`, `ci-test`, `ci-release-image`, `exercises`, `dbt`, `commit-policy`. Every workflow starts on every PR and decides inside the job what to run, so none of these checks stays pending. +I used AI (Claude) throughout, to understand the fundamentals, to implement, and to work through the architecture and logic. -Known gap to close when the ruleset goes on: the workflows, `scripts/check-commit-policy.sh` and `.githooks/commit-msg` run from the PR's own commits, so a PR could weaken the checks that judge it. Before the repo goes public, make `commit-policy.yml` check out the base branch's copy of the script and hook, and require review of changes under `.github/`, `scripts/` and `.githooks/`. +I would love to hear what you think. If something breaks, feels off or could be better, please [open an issue](https://github.com/DIZAMU/RootCause/issues). ## License -PolyForm Noncommercial 1.0.0. See [`LICENSE`](LICENSE). +[PolyForm Noncommercial 1.0.0](LICENSE). You may use, change and share RootCause for any noncommercial purpose. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..41283cb --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security + +RootCause gives the learner root inside a sandbox on purpose, so the sandbox isolation matters. The design and its known limits are described in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) (section 13) and in [ADR 0019](docs/adr/0019-known-limits.md). + +## Reporting a vulnerability + +Please report a vulnerability privately through GitHub. Open the **Security** tab of this repository and choose **Report a vulnerability**. Do not open a public issue for it. + +Helpful things to include are what you found, how to reproduce it, and which mode it affects (Docker mode or Kubernetes mode). I will reply as soon as I can. + +## In scope + +- A way out of a sandbox, to the host or to another sandbox +- A way for a sandbox to reach the internet, `rootcause-system` or the Kubernetes API +- A way for another website to call the local API or read its token +- A secret that ends up in the repository, in a log or on a command line + +## Out of scope + +The limits already listed in ADR 0019, such as a learner faking a check from inside their own sandbox, or k3d's privileged nodes. diff --git a/cmd/rootcause/main.go b/cmd/rootcause/main.go index b64697d..7ed1f4c 100644 --- a/cmd/rootcause/main.go +++ b/cmd/rootcause/main.go @@ -1,4 +1,4 @@ -// Command rootcause is the RootCause CLI (spec section 10). +// Command rootcause is the RootCause CLI (ARCHITECTURE.md section 10). // // The play commands are init, next, start, shell, ticket, hint, check, reset, // giveup, status, list, stats, serve (the dashboard) and lang. The developer @@ -138,7 +138,7 @@ func cmdSelfTest(ctx context.Context, args []string) int { } // cmdDBTProject writes the embedded dbt project into an existing directory. -// The cluster's dbt CronJob runs it as an init container (spec section 8.2). +// The cluster's dbt CronJob runs it as an init container (ARCHITECTURE.md section 8.2). func cmdDBTProject(args []string) int { if len(args) != 1 { fmt.Fprintln(os.Stderr, "dbt-project: expected exactly one directory") diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index be66195..a29bb90 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -1,4 +1,4 @@ -# RootCause analytics (spec section 9.2). dbt reads the engine's source tables +# RootCause analytics (ARCHITECTURE.md section 9.2). dbt reads the engine's source tables # in schema public and writes every model to schema analytics, so a build never # touches the engine's own tables. name: rootcause diff --git a/dbt/embed.go b/dbt/embed.go index ac7915b..799ae93 100644 --- a/dbt/embed.go +++ b/dbt/embed.go @@ -1,4 +1,4 @@ -// Package dbtproject embeds the dbt project (spec section 9.2) in the binary. +// Package dbtproject embeds the dbt project (ARCHITECTURE.md section 9.2) in the binary. // internal/dbt extracts it into a fresh directory for each build, so a build // never reads the working tree, where a file caught mid-save could silently // drop models or tests from the run. diff --git a/dbt/models/marts/mart_exercise_stats.sql b/dbt/models/marts/mart_exercise_stats.sql index 48eeb7b..fb0a966 100644 --- a/dbt/models/marts/mart_exercise_stats.sql +++ b/dbt/models/marts/mart_exercise_stats.sql @@ -1,5 +1,5 @@ -- Per exercise: solve rate, hint use and duration over all counted sessions, --- and the calibration flag (spec section 7.3). The flag is set only once an +-- and the calibration flag (ARCHITECTURE.md section 7.3). The flag is set only once an -- exercise has at least 10 counted sessions: too_hard when difficulty 1 is -- solved under 50% of the time, too_easy when difficulty 3 is solved over -- 90% of the time. diff --git a/dbt/models/marts/mart_review_queue.sql b/dbt/models/marts/mart_review_queue.sql index 2bc995f..a8c199a 100644 --- a/dbt/models/marts/mart_review_queue.sql +++ b/dbt/models/marts/mart_review_queue.sql @@ -1,4 +1,4 @@ --- Exercises that come back for review (spec section 7.2, rule 1): per learner +-- Exercises that come back for review (ARCHITECTURE.md section 7.2, rule 1): per learner -- and exercise, the latest counted session was a fail or a 1-star solve. It is -- due 72 hours after that session ended; the selector compares due_at with -- the current time, so a queue built earlier is still correct later. A later diff --git a/dbt/models/marts/mart_topic_levels.sql b/dbt/models/marts/mart_topic_levels.sql index 0b9cfc3..5419454 100644 --- a/dbt/models/marts/mart_topic_levels.sql +++ b/dbt/models/marts/mart_topic_levels.sql @@ -1,4 +1,4 @@ --- The current level per learner and topic (spec section 7.1): the "to" value +-- The current level per learner and topic (ARCHITECTURE.md section 7.1): the "to" value -- of the topic's latest level_changed event, or 1 if it never changed. Also -- when the topic was last practiced (any finished session that was not -- voided), for the selector's tie-break. diff --git a/dbt/models/staging/sources.yml b/dbt/models/staging/sources.yml index 159b2d4..fea5256 100644 --- a/dbt/models/staging/sources.yml +++ b/dbt/models/staging/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: engine - description: Source tables written by the engine (spec section 9.1). + description: Source tables written by the engine (ARCHITECTURE.md section 9.1). schema: public tables: - name: session diff --git a/deploy/argocd/application.yaml b/deploy/argocd/application.yaml index ad01add..2f4a561 100644 --- a/deploy/argocd/application.yaml +++ b/deploy/argocd/application.yaml @@ -1,4 +1,4 @@ -# RootCause's Application (spec section 14): Argo CD syncs deploy/k8s from +# RootCause's Application (ARCHITECTURE.md section 14): Argo CD syncs deploy/k8s from # main into the cluster. Automated sync with prune (what leaves git leaves # the cluster) and self-heal (a change made by hand is reverted). # scripts/k8s-up.sh applies it; with --dev it turns automated sync off diff --git a/deploy/argocd/install/kustomization.yaml b/deploy/argocd/install/kustomization.yaml index 9920a00..abb3dd3 100644 --- a/deploy/argocd/install/kustomization.yaml +++ b/deploy/argocd/install/kustomization.yaml @@ -1,4 +1,4 @@ -# Argo CD core (spec section 8.2): the application controller, the repo +# Argo CD core (ARCHITECTURE.md section 8.2): the application controller, the repo # server and redis, with no UI, no SSO and no notifications. scripts/k8s-up.sh # downloads core-install.yaml for the version below, checks its SHA-256, puts # it next to this file in a temporary directory and applies the result. The diff --git a/deploy/argocd/project.yaml b/deploy/argocd/project.yaml index e4e5676..57648e6 100644 --- a/deploy/argocd/project.yaml +++ b/deploy/argocd/project.yaml @@ -1,9 +1,9 @@ -# The Argo CD project RootCause's Application belongs to (spec section 13.2). +# The Argo CD project RootCause's Application belongs to (ARCHITECTURE.md section 13.2). # It limits what a bad commit on main can make Argo CD do: one source repo, # the two RootCause namespaces on this cluster only, and only the kinds # deploy/k8s uses. The one cluster-wide kind allowed is the admission policy # for sandbox pods and its binding, so self-heal restores the policy if it is -# deleted or weakened (spec section 13.2); the namespaces still come from +# deleted or weakened (ARCHITECTURE.md section 13.2); the namespaces still come from # scripts/k8s-up.sh. Argo CD's core install has no default project, so an # Application outside this one syncs nothing. apiVersion: argoproj.io/v1alpha1 diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index c69fa04..d4a6a99 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -1,4 +1,4 @@ -# v1 data services (spec section 8.1). Postgres is bound to 127.0.0.1 only, so +# v1 data services (ARCHITECTURE.md section 8.1). Postgres is bound to 127.0.0.1 only, so # it is never reachable from outside this machine. The password comes from a # git-ignored .env file (see .env.example); it is never committed. name: rootcause diff --git a/deploy/k8s/admission.yaml b/deploy/k8s/admission.yaml index a1a9b8e..96c17fd 100644 --- a/deploy/k8s/admission.yaml +++ b/deploy/k8s/admission.yaml @@ -1,4 +1,4 @@ -# Admission rules for every pod in the sandboxes namespace (spec section 13.2). +# Admission rules for every pod in the sandboxes namespace (ARCHITECTURE.md section 13.2). # The runtime builds sandbox pods this way already (internal/runtime/kubernetes, # podFor); this policy makes the API server refuse any pod that is not built # this way, whoever asks: the API, Argo CD or an administrator. There are no diff --git a/deploy/k8s/api.yaml b/deploy/k8s/api.yaml index 730f674..22536fb 100644 --- a/deploy/k8s/api.yaml +++ b/deploy/k8s/api.yaml @@ -1,4 +1,4 @@ -# The API and dashboard (spec section 8.2): rootcause serve with the +# The API and dashboard (ARCHITECTURE.md section 8.2): rootcause serve with the # kubernetes runtime. It runs as the rootcause-api service account, which may # only manage pods in sandboxes (rbac.yaml). The dashboard is reached through # the NodePort, which k3d publishes on the host's 127.0.0.1:7300 only diff --git a/deploy/k8s/api/Dockerfile b/deploy/k8s/api/Dockerfile index 06ddb02..21fe136 100644 --- a/deploy/k8s/api/Dockerfile +++ b/deploy/k8s/api/Dockerfile @@ -1,4 +1,4 @@ -# The API and dashboard image for the cluster (spec section 8.2). The context +# The API and dashboard image for the cluster (ARCHITECTURE.md section 8.2). The context # is a directory scripts/build-api-image.sh assembles: the rootcause binary, # built with -tags webdist so the dashboard is embedded, and the exercises # folder, whose scripts the engine copies into sandboxes. Nothing else: no diff --git a/deploy/k8s/dbt.yaml b/deploy/k8s/dbt.yaml index c625707..489f49a 100644 --- a/deploy/k8s/dbt.yaml +++ b/deploy/k8s/dbt.yaml @@ -1,5 +1,5 @@ -# dbt for the cluster (spec sections 8.2 and 9.3): a CronJob builds the marts -# every 10 minutes. The API has no rights to start Jobs (spec section 13.2), +# dbt for the cluster (ARCHITECTURE.md sections 8.2 and 9.3): a CronJob builds the marts +# every 10 minutes. The API has no rights to start Jobs (ARCHITECTURE.md section 13.2), # so the dashboard shows when the marts were last built instead. # scripts/k8s-dbt.sh starts a build at once. # diff --git a/deploy/k8s/k3d.yaml b/deploy/k8s/k3d.yaml index d709b1c..d6e7017 100644 --- a/deploy/k8s/k3d.yaml +++ b/deploy/k8s/k3d.yaml @@ -1,4 +1,4 @@ -# The local v2 cluster (spec section 8.2): 1 server and 2 agents. Created by +# The local v2 cluster (ARCHITECTURE.md section 8.2): 1 server and 2 agents. Created by # scripts/k8s-up.sh with `k3d cluster create --config`. # # The dashboard is published on the host's 127.0.0.1:7300 only, through the @@ -31,7 +31,7 @@ options: nodeFilters: - server:* # Kubernetes has no per-pod process limit; the kubelet applies this one - # to every pod (spec section 13.1's limit of 256). + # to every pod (ARCHITECTURE.md section 13.1's limit of 256). - arg: --kubelet-arg=pod-max-pids=256 nodeFilters: - server:* diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml index e101d6e..3c3422c 100644 --- a/deploy/k8s/kustomization.yaml +++ b/deploy/k8s/kustomization.yaml @@ -1,4 +1,4 @@ -# Everything Argo CD syncs from main into the cluster (spec sections 8.2 and +# Everything Argo CD syncs from main into the cluster (ARCHITECTURE.md sections 8.2 and # 14). Not in here, because scripts/k8s-up.sh creates them: the namespaces # (namespaces.yaml; Argo CD may not create cluster-wide objects), the # rootcause-db and ghcr-pull Secrets (they hold secrets) and the diff --git a/deploy/k8s/namespaces.yaml b/deploy/k8s/namespaces.yaml index 322335d..e4febd3 100644 --- a/deploy/k8s/namespaces.yaml +++ b/deploy/k8s/namespaces.yaml @@ -1,4 +1,4 @@ -# The two namespaces of spec section 8.2. Pod Security Admission enforces the +# The two namespaces of ARCHITECTURE.md section 8.2. Pod Security Admission enforces the # restricted profile on the system workloads and baseline on sandboxes: the # learner is root in a sandbox pod (in its own user namespace), which the # restricted profile forbids. diff --git a/deploy/k8s/postgres.yaml b/deploy/k8s/postgres.yaml index 4626666..2860f5c 100644 --- a/deploy/k8s/postgres.yaml +++ b/deploy/k8s/postgres.yaml @@ -1,6 +1,6 @@ -# Postgres for the cluster (spec section 8.2), on a local-path volume. The +# Postgres for the cluster (ARCHITECTURE.md section 8.2), on a local-path volume. The # password comes from the rootcause-db Secret, which scripts/k8s-up.sh -# creates outside git (spec section 13.2). +# creates outside git (ARCHITECTURE.md section 13.2). apiVersion: v1 kind: Service metadata: diff --git a/deploy/k8s/rbac.yaml b/deploy/k8s/rbac.yaml index 6445b24..a98e416 100644 --- a/deploy/k8s/rbac.yaml +++ b/deploy/k8s/rbac.yaml @@ -1,4 +1,4 @@ -# The API's identity (spec section 13.2): it may create, read, exec into and +# The API's identity (ARCHITECTURE.md section 13.2): it may create, read, exec into and # delete pods in sandboxes. Nothing else, and nowhere else: a Role, not a # ClusterRole. scripts/k8s-prove.sh checks this with kubectl auth can-i. apiVersion: v1 diff --git a/deploy/k8s/sandboxes.yaml b/deploy/k8s/sandboxes.yaml index ba33c2f..86fee9b 100644 --- a/deploy/k8s/sandboxes.yaml +++ b/deploy/k8s/sandboxes.yaml @@ -1,11 +1,11 @@ -# Limits and network isolation for the sandboxes namespace (spec section +# Limits and network isolation for the sandboxes namespace (ARCHITECTURE.md section # 13.2). The runtime in internal/runtime/kubernetes labels every pod with # rootcause.network ("none" or "internal"); the engine labels check runners # rootcause.runner=true. # # Nothing in or out by default: no internet, no cluster DNS, no # rootcause-system, no other sandbox. The one exception is a check runner, -# which may connect to sandboxes on the internal network (spec section 13.3 +# which may connect to sandboxes on the internal network (ARCHITECTURE.md section 13.3 # names the limit: with two sessions, a runner could reach the other one). apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -52,7 +52,7 @@ spec: matchLabels: rootcause.runner: "true" --- -# At most two sessions and a check runner at a time (spec section 8.2's +# At most two sessions and a check runner at a time (ARCHITECTURE.md section 8.2's # memory budget on an 8 GB WSL2 host). apiVersion: v1 kind: ResourceQuota diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..14a2b87 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,501 @@ +# RootCause architecture + +This document describes how RootCause works, from the exercise format to the security model. The reasons behind the main choices are in the [decision records](adr/README.md). Section numbers are stable, because comments in the code refer to them. + +The product is called RootCause. The CLI command and the binary are lowercase, `rootcause`, and the Go module path is `github.com/DIZAMU/RootCause`. + +RootCause runs in two ways that share the same engine, database and dashboard. + +| Mode | Sandboxes | Terminal | +|---|---|---| +| Docker mode | Containers on the local Docker Engine | The learner's own, through `rootcause shell` | +| Kubernetes mode | Pods in a local k3d cluster, deployed by Argo CD | In the browser, on the session page | + +`RC_RUNTIME` selects the mode. It is `docker` by default, or `kubernetes`. + +## 1. Purpose + +### 1.1 Problem + +New IT and operations staff often learn troubleshooting on production systems, or not at all. Courses teach commands, but few let you practise fixing a broken machine safely and repeatedly, in both English and Norwegian. + +### 1.2 Goals + +- A practice tool that hands you a broken Linux machine as a helpdesk ticket and grades the fix. +- One working system that shows containers, Kubernetes, CI/CD, a data pipeline and security design together. +- Free to use for anyone, for any noncommercial purpose. + +### 1.3 Non-goals + +- Public hosting or several users +- Accounts or login +- Windows or Active Directory exercises +- A mobile layout +- Commercial use + +## 2. Key choices + +| Area | Choice | +|---|---| +| Engine | Go, for the CLI, the local server, grading and sandbox control | +| Dashboard | React and TypeScript (Vite), built and embedded in the Go binary | +| Database | PostgreSQL, run with Docker Compose in Docker mode | +| Transformations | dbt Core with the Postgres adapter, run in a container | +| Fault and check scripts | Bash | +| Sandboxes | Docker Engine, or Kubernetes with k3d | +| CI/CD | GitHub Actions, with GitOps through Argo CD in Kubernetes mode | +| Interface | A CLI and a local web dashboard | +| Accounts | None, one local profile | +| Languages | English and Norwegian | +| Exercise framing | Helpdesk tickets | +| Scoring | 1 to 3 stars, based on the hints used | +| Time | Recorded, never scored | +| Adaptation | Levels per topic with a 2 up, 2 down rule | +| Variants | Randomized from a seed, in several exercises | +| Commits | Conventional Commits, SSH-signed | + +## 3. Concepts + +| Term | Meaning | +|---|---| +| Exercise | A folder that defines a healthy machine, a fault, a check, a reference solution and the texts | +| Variant | One concrete version of an exercise, chosen by a seed. The fault type and difficulty stay the same, the details change (file names, ports, users, the ticket's sender) | +| Session | One attempt at one variant in one sandbox, from the start until it is solved, given up or expired | +| Sandbox | The isolated container or pod a session runs in. It is thrown away afterwards | +| Check | The automated grader. It tests the end state, never the commands typed | +| Hint tiers | Hint 1 says where to look, hint 2 which tool to use, and hint 3 nearly gives the answer | +| Walkthrough | The explained solution, shown after a solve or after giving up | +| Topic level | The learner's level in a topic, from 1 to 3 | + +## 4. Exercise format + +### 4.1 Folder layout + +``` +exercises/svc-01-intranet-config/ +├── exercise.yaml metadata and runtime settings +├── Dockerfile builds the base machine (no exercise scripts inside) +├── setup.sh optional, builds the healthy state at session start +├── break.sh plants the fault at session start +├── check.sh grades the end state +├── solution.sh the reference fix, used only by CI +└── i18n/ + ├── en.yaml ticket, hints, walkthrough, check feedback + └── no.yaml +``` + +Other files may sit beside these when the Dockerfile copies them into the image, such as an app the machine runs (`app/`). Exercise scripts are never copied into the image. + +### 4.2 exercise.yaml + +```yaml +id: svc-01-intranet-config +version: 1 +topic: services # permissions | services | logs | networking | disk +difficulty: 1 # 1 to 3 +estimated_minutes: 10 +network: none # none | internal +init: none # none | systemd (optional, default none) +resources: + memory: 256Mi + cpus: 0.5 +tmpfs: # optional size-limited in-memory volumes + - path: /srv/data + size: 32Mi +session_ttl_minutes: 60 +check: + runs_from: sandbox # sandbox | runner + before: none # none | restart +capabilities: [SETUID, SETGID, CHOWN] # added back after --cap-drop ALL, only from the loader's allowlist +variants: + site_name: [intranet, wiki, portal] + port: [8080, 8081, 8088] +``` + +- `init: systemd` boots systemd as PID 1, for exercises about services at boot. The image runs systemd as its entrypoint, and the sandbox gets in-memory `/run` and `/run/lock` instead of Docker's init, since systemd reaps processes itself. A new sandbox counts as ready only when `systemctl is-system-running` reports `running`. After the check's restart, `degraded` also counts, so the check grades what the learner left. +- `tmpfs` volumes count toward `resources.memory`, and their total must stay below it. +- `runs_from: runner` needs `network: internal` and cannot be combined with `init: systemd`. +- Every sandbox runs within `resources`. The default is 256 MiB and half a CPU, and at most two sandboxes run at once. + +### 4.3 Script contracts + +**setup.sh** (optional) +- Builds the healthy state from the variant values, for example by rendering a config and starting a service. +- Receives the same environment variables as break.sh. It is copied in, run as root and deleted before break.sh runs. +- It exists so the healthy state is never baked into the image, where the learner could read it or re-run it as a shortcut. + +**break.sh** +- Receives `RC_SEED` and every variant value as an `RC_VAR_` environment variable. +- Is deterministic, so the same seed always plants the same fault. +- Is copied into the sandbox at session start, run as root and then deleted, leaving no trace of itself. +- May print derived values as one line of JSON, such as `{"line": 14}`. These are stored with the session and fill placeholders in the hints and the walkthrough. + +**check.sh** +- Exits 0 for a pass and 1 for a fail. Exit 2 means the check could not run, which is an infrastructure error and never counts against the learner. +- Prints one line of JSON, such as `{"passed": false, "feedback": "site_unreachable"}`. The feedback key maps to a text that describes the symptom, never the fix. +- Accepts every valid fix and rejects shortcuts. Each check has at least one assertion against shortcuts. For example, the config must be valid and the original site content must be served, so replacing the config with a default one fails. +- Is copied in at check time and deleted afterwards, so the criteria cannot be read during the session. +- With `runs_from: runner` it runs in a separate runner container instead. That is a fresh copy of the exercise image on the internal network, with no added capabilities, destroyed after the check. `RC_SANDBOX_HOST` holds the sandbox's host name. + +**solution.sh** +- The reference fix. It is never copied into a sandbox image or a session, and only CI uses it. + +### 4.4 Texts (i18n/en.yaml) + +```yaml +ticket: + title: "The intranet is down" + from: "{sender_name}, {department}" + body: | + Hi, since this morning nobody in {department} can open {site_name}. + Someone changed something yesterday. Can you take a look? +hints: + - "Check whether the web server accepts its own configuration." + - "nginx has a built-in config test. Run nginx -t" + - "Line {line} of the config file has a syntax error." +walkthrough: | + Explains the cause, how to find it, the fix, and why the fix works. +feedback: + site_unreachable: "The site still doesn't respond." +``` + +Placeholders are filled from variant values and derived values. Both language files must define the same keys, and CI enforces it. + +## 5. Exercise catalog + +| ID | Topic | Ticket | Fault (hidden) | Check passes when | Shortcut that fails | +|---|---|---|---|---|---| +| perm-01-shared-folder | permissions | Økonomi can't save to the shared folder | Group write missing, or the wrong group owns the share | A group member can create and edit a file | Making the share world-writable | +| perm-02-backup-denied | permissions | The nightly backup fails with "permission denied" | The script lacks the execute bit, or the backup user cannot write to the target | The backup runs as the backup user and produces an archive | A world-writable script, or running as root | +| svc-01-intranet-config | services | The intranet went down after a config change | A syntax error in the nginx config | The config test passes and the original page is served | A default page or an empty config | +| svc-02-dies-on-reboot | services | The app works until the server reboots | The service is not set to start at boot | The app runs after a sandbox restart | Starting it by hand, since the check restarts first | +| log-01-crash-on-start | logs | The app crashes on start | A wrong path or a missing value in the app's config, visible in its log | The app runs and its health endpoint returns OK | The health endpoint must report the correct config | +| net-01-no-names | networking | The server reaches IP addresses but not names | Name resolution misconfigured (resolver address or lookup order) | Internal host names resolve to the expected address | Hard-coding the answer in the hosts file | +| net-02-outside-only | networking | The site works on the server but not from outside | The service listens only on the loopback address | A request from outside the sandbox succeeds | The check runs from the runner, not inside | +| disk-01-stopped-saving | disk | The server stopped saving data | The data volume is full because of a large hidden file | Free space is above the threshold and the app writes again | The required data files must still exist | + +- Sandboxes have no internet access, so net-01 runs its own small DNS server (dnsmasq) inside the sandbox, with an internal zone. +- disk-01 mounts a size-limited tmpfs as its data volume. +- Every exercise has randomized variants. + +## 6. Grading and fairness + +| Result | Stars | +|---|---| +| Solved with no hints | 3 | +| Solved with hint 1 or 2 | 2 | +| Solved after hint 3 | 1 | +| Gave up | 0, and the walkthrough is shown | + +- Time is recorded for the statistics, never used in scores or levels. +- Checks are unlimited, and a failed check costs nothing. +- A reset gives a fresh copy of the same variant with no penalty. Revealed hints stay revealed. +- Giving up becomes possible once hint 3 is revealed. +- Infrastructure errors (check exit 2, a crashed sandbox) void the session, so it does not count. +- An expired session counts as a fail only if at least one check was run. + +## 7. Adaptation + +### 7.1 Topic levels + +The engine applies these at once. +- Every topic starts at level 1, and levels go from 1 to 3. +- Two solves in a row in a topic, each with at least 2 stars at the current level, move it up. +- Two fails in a row in a topic (given up, or expired after a check) move it down. +- Each change is logged as a `level_changed` event. + +### 7.2 Next exercise + +The selector reads the dbt marts and applies these rules in order. +1. Reviews come first. An exercise comes back when its latest counted session (solved, given up, or expired after a check) was a fail or a 1-star solve that ended at least 72 hours ago. The oldest due exercise goes first, and a later solve with 2 or more stars takes it out of the queue. It comes back as a new variant, using the first of up to 20 random seeds whose values differ from the last session's. +2. Otherwise the weakest topic is picked. That is the lowest level, with ties broken by the topic practised least recently (never practised first) and then by name. +3. Within the topic, the difficulty closest to the topic level wins, unseen exercises first, then the one attempted least recently. +4. The same exercise never comes twice in a row. This rule wins over rule 1. + +### 7.3 Calibration + +dbt computes the solve rate, hint use and duration per exercise. Once an exercise has at least 10 sessions, it is flagged when difficulty 1 is solved less than half the time, or difficulty 3 more than 9 times in 10. With one learner the flags are for information only. + +## 8. Architecture + +### 8.1 Docker mode + +``` +Linux or WSL2 +├── rootcause (Go binary) +│ ├── CLI +│ ├── local server, API and embedded dashboard, 127.0.0.1 only +│ └── Docker runtime (the docker CLI) +└── Docker Engine + ├── postgres (Compose) + ├── dbt (a container, run on demand) + └── sandbox containers, one per session + +Browser → http://localhost:7300 +``` + +In Docker mode the terminal is the learner's own. `rootcause shell` opens a shell inside the sandbox. + +### 8.2 Kubernetes mode + +``` +Linux or WSL2 +└── k3d cluster, 1 server and 2 agents + ├── namespace rootcause-system + │ ├── api and dashboard (Deployment) + │ ├── postgres (StatefulSet) + │ └── dbt (a CronJob every 10 minutes, or a Job on demand by script) + ├── namespace sandboxes + │ └── one pod per session, the exercise image with nothing added + └── namespace argocd + └── Argo CD core, with the application controller, repo server and redis +``` + +- The cluster is reached through k3d's port mapping to `127.0.0.1:7300` on the host only. +- The browser terminal is xterm.js in the dashboard, on a WebSocket to the API. The API runs `bash` in the session pod through the Kubernetes exec API, the same model as `docker exec`. Nothing in a sandbox listens on the network, so sandboxes stay closed to incoming traffic (ADR 0013). +- `scripts/k8s-up.sh` creates the cluster, builds the exercise images and imports them into the nodes with `k3d image import`, so sandbox pods never pull. It then creates the namespaces and the Secrets, installs Argo CD and applies `deploy/argocd`. Argo CD syncs `deploy/k8s` from `main`, and the API runs the release pinned there by digest (section 14). +- `scripts/k8s-up.sh --dev` builds the API image from the working tree instead and applies `deploy/k8s-dev`, after turning Argo CD's automated sync off. It needs no credentials. +- `scripts/k8s-prove.sh` proves the isolation, and `scripts/k8s-gitops-prove.sh` proves the GitOps setup. The CLI uses the cluster with `RC_RUNTIME=kubernetes`, and `rootcause selftest` then validates exercises in sandbox pods. +- Argo CD is the core install at a pinned version, without the UI, SSO, notifications or the ApplicationSet controller. Its install manifest is checked against a pinned SHA-256 and its images are pinned by digest. Its one Application belongs to the AppProject `rootcause` (section 13.2). +- The whole cluster with Argo CD and two sessions uses about 2.5 GB of memory at peak. `scripts/k8s-memory.sh` prints the current use per node and container. + +### 8.3 Runtime abstraction + +All sandbox control goes through one interface, so Kubernetes mode swaps the implementation, not the callers. + +```go +type Runtime interface { + Create(ctx context.Context, spec SandboxSpec) (Sandbox, error) + Exec(ctx context.Context, id string, cmd []string, stdin io.Reader) (ExecResult, error) + CopyIn(ctx context.Context, id string, path string, data []byte) error + Restart(ctx context.Context, id string) error + Destroy(ctx context.Context, id string) error + List(ctx context.Context) ([]Sandbox, error) +} +``` + +The implementations are `internal/runtime/docker` and `internal/runtime/kubernetes` (client-go). Optional abilities sit beside the interface. These are `Sheller` (`rootcause shell` in Docker mode), `Terminal` (the web terminal in Kubernetes mode), `Diagnoser` and `SystemdChecker`. + +The Kubernetes runtime makes each sandbox a pod in `sandboxes`. `Create` returns once the container runs, as `docker run -d` does. `Exec` and `CopyIn` go through the exec API, and `CopyIn` streams the file to `cat`. `Restart` is not supported, because a restarted container starts again from its image. The only exercise that restarts before its check boots systemd, which the runtime refuses first. Pods on the internal network get a hosts entry for every other internal pod, so a check runner reaches its sandbox by name, as on Docker's internal network. + +### 8.4 Session lifecycle + +1. The selector picks an exercise and a seed. +2. The runtime creates the sandbox from the exercise image, with its resource limits and network mode. +3. setup.sh (if present) and then break.sh are copied in, run with the seed and variant values, and deleted. The values are passed only to these commands, never to the sandbox environment. Derived values are stored with the session. +4. The ticket is shown in the CLI and the dashboard. +5. The learner works. Hints, checks and resets are logged as events. +6. On a check, the sandbox restarts if the exercise asks for it, check.sh runs in the sandbox or from the runner, and the result is logged. +7. On a solve or after giving up, the walkthrough is shown, the stars and the level rule are applied, the sandbox is destroyed and a dbt build starts. +8. A reaper destroys sandboxes that are past their time limit. + +## 9. Data + +### 9.1 Source tables + +The engine writes these. + +| Table | Holds | +|---|---| +| `learner` | One row | +| `exercise` | Synced from exercise.yaml on startup (id, version, topic, difficulty) | +| `session` | id, learner, exercise id and version, seed, variant values, derived values, status, sandbox id, started_at, ended_at, expires_at | +| `event` | id, occurred_at, session_id, type, payload (jsonb) | + +The event types are `session_started`, `hint_revealed`, `check_run`, `reset`, `gave_up`, `solved`, `expired`, `voided` and `level_changed`. + +Migrations are versioned SQL files in `internal/store/migrations`, applied on startup. + +### 9.2 dbt models + +| Layer | Model | Purpose | +|---|---|---| +| staging | stg_events, stg_sessions, stg_exercises | Typed, cleaned source data | +| intermediate | int_session_outcomes | One row per finished session, with outcome, stars, hints, checks and duration | +| marts | mart_topic_levels | The current level per learner and topic, from level_changed events, and when the topic was last practised | +| marts | mart_review_queue | Review candidates per learner (rule 1 of section 7.2), with the time each is due | +| marts | mart_exercise_stats | Solve rate, hint use, duration and the calibration flag | +| marts | mart_progress | Per learner, topic and day, the sessions, solves, average stars and the level at the end of the day | +| marts | mart_build_info | One row with the time of the latest build. The dashboard shows it, so statistics that lag behind are visibly older | + +dbt reads the source tables in the schema `public` and writes every model to the schema `analytics`, so a build never touches the engine's tables. The outcomes in `int_session_outcomes` are `solved`, `gave_up`, `expired_fail` (expired after a check), `expired_skip` (expired with no check) and `voided`. Only the first three count in the statistics and the review queue. The engine stays the authority for levels (section 7.1). `mart_topic_levels` replays its `level_changed` events, and a test asserts that the two agree. + +The tests cover unique and not-null keys, accepted values for event types, statuses and outcomes, and the relationships between sessions and events. Singular tests assert one level row per learner and topic, and one review row per learner and exercise. + +The project lives in `dbt/` and is embedded in the binary. `internal/dbt` extracts it into a fresh temporary directory for each build, so a build never reads the working tree, and runs `dbt build` in the pinned `dbt-postgres` image with `docker run`. The container joins the Compose network and reaches Postgres by service name. It runs with no capabilities, a read-only root filesystem and the project mounted read-only, with targets and logs in `/tmp`. The connection comes from the engine's database URL, and the password reaches the container only through its environment. dbt runs only against the Compose Postgres, never over the host network. For tests and CI only, `RC_DBT_NETWORK` and `RC_DBT_HOST` name another Postgres container's network and host, such as a CI service container. + +Every dbt container is labelled with its database. Before a build, containers left by an earlier build of the same database are removed. A build passes only when dbt exits 0 and runs exactly as many nodes as the project defines, so a run that silently skips models or tests fails. A failed build keeps its full `dbt.log` in the user cache directory and prints the path. + +### 9.3 When dbt runs + +- In Docker mode dbt runs after each solve and each give up, after the result is shown and with a 120-second limit. It also runs on `rootcause stats --refresh`, and in `rootcause next` when the marts are stale. Expiry by the reaper does not run dbt, and the next `rootcause next` catches up. In the dashboard, a solve or give up returns at once and dbt runs in the background, as does a rebuild when the home view finds the marts stale. Starting the recommended exercise from the dashboard rebuilds stale marts first, like `rootcause next`. A Postgres advisory lock lets only one build run at a time, across the CLI and the dashboard. +- In Kubernetes mode a CronJob runs every 10 minutes (`concurrencyPolicy: Forbid`). The API's service account may not start Jobs (section 13.2), so the dashboard shows when the marts were last built instead. `scripts/k8s-dbt.sh` runs a build at once. An init container writes the project embedded in the binary (`rootcause dbt-project`), so the cluster builds exactly what the binary ships. The API runs with `RC_DBT_MODE=cronjob` and never starts dbt itself. +- The marts are stale when they are missing, or when the learner's latest finished session is newer than the latest one in `int_session_outcomes`. `rootcause next` then rebuilds them once. If they are still stale or dbt fails, the selector falls back to topic levels alone, using rules 2 to 4 of section 7.2 with the engine's levels and the source tables, without the review queue. A failed build never changes a session's result. + +## 10. CLI + +| Command | Does | +|---|---| +| `rootcause init` | Starts Postgres, runs the migrations and creates the local profile | +| `rootcause next` | Starts the recommended next exercise | +| `rootcause start ` | Starts a specific exercise | +| `rootcause shell` | Opens a shell in the active sandbox | +| `rootcause ticket` | Shows the active ticket again | +| `rootcause hint` | Reveals the next hint | +| `rootcause check` | Grades the active session | +| `rootcause reset` | Gives a fresh copy of the same variant | +| `rootcause reset --all [--yes]` | Deletes all of the learner's data, like "Reset all data" in the dashboard. It asks for the confirmation word (RESET, or NULLSTILL in Norwegian) unless `--yes` is given | +| `rootcause giveup` | Ends the session and shows the walkthrough (after hint 3) | +| `rootcause status` | Shows the topic levels, the active session and the review queue | +| `rootcause list` | Lists all exercises with their best stars | +| `rootcause stats [--refresh]` | Shows statistics per exercise from the marts, with calibration flags. `--refresh` rebuilds the marts first | +| `rootcause serve [--port N] [--listen-all]` | Starts the local dashboard on 127.0.0.1, port 7300 by default, until Ctrl-C. It reaps expired sessions every minute while it runs. `--listen-all` is only for the cluster and is refused unless `RC_RUNTIME=kubernetes` | +| `rootcause lang en\|no` | Sets the interface language | + +## 11. Dashboard + +The dashboard is a calm workbench. It uses flat panels divided by hairlines, nearly square corners, Inter for text and JetBrains Mono for ids, commands and the terminal, and a teal accent. It has a light and a dark theme and follows the system unless switched. + +- **Sidebar.** Inbox, Exercises, History, Stats and the session in progress, the five topic level rings, the language and theme switches, and a "?" button for help. It collapses (remembered per browser) to a 64 px rail with only the mark, the expand toggle, the page icons, one language button, one theme button and "?". The width animates only after the learner toggles it, never on load, and changes at once under reduced motion. On a phone it becomes a top bar. +- **Help.** The "?" button or key shows the keyboard shortcuts and the switch for single-key shortcuts, "Show the intro again", and a Data section with "Reset all data". +- **Intro.** A browser's first visit opens a four-step guided intro in a dialog. It covers the inbox and why a ticket is picked, the session, hints and stars, and levels and stats. In Docker mode the session step points to `rootcause shell`, and in Kubernetes mode to the terminal on the page. Back, Next and Skip move through it, Esc skips, and the browser remembers when it is finished or skipped. +- **Reset all data.** A dialog says what goes (history, sessions, topic levels, statistics and settings) and that it cannot be undone. Its button works only once the learner has typed the confirmation word (RESET, or NULLSTILL in Norwegian). The reset ends a session in progress and destroys its sandbox, and deletes the sessions, events, statistics and profile language in one transaction. Then it rebuilds the statistics where the server builds them. The browser's stored preferences are cleared and the app loads fresh, with the intro. The exercise catalog and the migrations stay. +- **Inbox.** The home page shows the tickets as a helpdesk inbox, one row each with title, sender, topic, difficulty, a status tag (next, due, new, tried, or solved with the best stars) and the exercise id. The recommended ticket is pinned first, with why it was picked, the last result and what would change that. A session in progress is pinned above everything. +- **Session start.** The engine's real steps (build the image, start the machine, set up, plant the fault) tick off as they happen, on the row that was clicked. Only the exercise id and the step reach the browser. +- **Session.** The ticket and hints are on the left and the work is on the right. The check bar has Run check, Show the next hint with the stars it leaves, Reset, an optional timer, and Give up behind a menu, with each destructive action confirmed in place. Below it are the last check's result, the terminal (Kubernetes mode) or the `rootcause shell` command (Docker mode), and a check log. A solve plays the app's one expressive animation, under 1.2 seconds, which can be skipped and never takes focus from the terminal. +- **Exercises.** The catalog by topic, with difficulty, best stars and when each was last played. +- **History.** Finished sessions with outcome, stars, hints, checks and time, filtered by topic and outcome. +- **Stats.** The topic rings with progress toward the next level, solves per day as a column chart with a data table, and per exercise the sessions, solve rate, hints, time and calibration flag. Where the server builds the marts itself, there is a "rebuild now" button. +- **Wide screens.** The page column is centred, up to 1440 px (the inbox up to 1280 px), with prose kept to a readable width. +- **Keyboard.** Ctrl+K opens a command palette and Ctrl+Enter runs the check, from the terminal too. `g` then `i`, `e`, `h` or `s` goes to a page, `[` folds the sidebar and `?` opens help. The single-key shortcuts can be switched off and never fire while typing in a field, a dialog or the terminal. The terminal keeps Tab for completion, and Esc then Tab leaves it. +- **Languages.** The EN/NO switch sets the profile language, the same one `rootcause lang` sets. +- **Accessibility.** WCAG 2.2 AA is the floor. Every task works by keyboard alone, reduced motion is respected, and layouts reflow at 320 px and at 200% zoom. + +How it is built. + +- The app (Vite, React, TypeScript, plain CSS, hash routing) lives in `web/`. `scripts/build-web.sh` builds it into `web/dist` with `npm ci` from the committed lockfile, and `go build -tags webdist` embeds it. Without the tag, `/` serves a translated page that explains how to build it. +- `internal/server` is a JSON API over the same engine, selector and store as the CLI. Both front ends call the shared steps in `internal/play`, such as the next pick and seed, the exercise texts and the statistics rebuild. Exercise texts are filled in on the server in the learner's language. UI strings come from `locales/*.json` through `/api/i18n/{lang}`, so there is one copy of every string. +- The read endpoints are `GET /api/overview`, `/api/exercises`, `/api/history`, `/api/stats`, `/api/session` (204 if there is none), `/api/sessions/{id}` and `/api/i18n/{lang}`. The actions are `POST /api/start` (`{"exercise": ""}` starts the recommended one), `/api/hint`, `/api/check`, `/api/reset`, `/api/giveup`, `/api/stats/rebuild` and `/api/data/reset`, plus `PUT /api/lang` and `GET /api/terminal` (a WebSocket, in Kubernetes mode). `/api/stats/rebuild` runs one rebuild at a time and answers 409 while one runs, or 404 where a CronJob builds the marts. `/api/data/reset` needs `{"confirm": "RESET"}` and answers 409 while another action, such as a session start, runs. Actions name the session the page shows and are refused if it is no longer the active one. +- Before a session exists, the inbox and the catalog fill each ticket's header with the first value of each variant. A session shows its own variant. +- The dashboard follows the CLI by polling, the open session every 2 seconds and the overview every few seconds and on focus. The two front ends share only the database. +- The session view never contains the seed, the variant values or the derived values. Hints appear once revealed, and the walkthrough once the session is solved or given up. +- Everything the dashboard needs is in the binary, including its fonts (SIL OFL, with the licences next to the files) and its icons. It loads nothing from elsewhere. The terminal's code loads only on a session page. +- `scripts/ui-check.sh` checks the dashboard in a real browser. It covers the first-run intro, the collapsed rail, centred pages on wide screens, the reset dialog, axe-core accessibility rules, layout shift, the keyboard rules, the Tab order, terminal latency, the CSP, and screenshots in both languages. It is not part of CI. + +## 12. Languages + +- UI strings live in `locales/en.json` and `locales/no.json`. No user-facing text is hard-coded in Go or React. +- Exercise texts live in `exercises//i18n/{en,no}.yaml`. +- CI fails if a key exists in one language and not the other. +- Code, comments, commit messages and docs are in English. +- Counts use a `_one` and `_other` pair of keys, the two plural forms English and Norwegian both have. +- Interface text is plain and warm. Buttons start with a verb, praise is only for what was done, and there is at most one exclamation mark (on a solve without hints). There are no streaks, no guilt and no urgency. Layouts leave room for Norwegian, which runs longer. + +## 13. Security model + +### 13.1 Docker mode + +- Sandboxes never get the Docker socket, the host network or privileged mode. +- `--cap-drop ALL`, then only the capabilities an exercise needs. +- Memory, CPU and process limits on every sandbox. +- Network `none`, or an internal network with no route out. +- The learner is root inside the sandbox, so container root must not map to host root. Docker must use user namespace remapping or rootless mode (ADR 0010), and the CLI refuses to start a session otherwise. +- The local server binds to 127.0.0.1 only. +- It refuses any request whose Host header is not `127.0.0.1:PORT` or `localhost:PORT`, which blocks DNS rebinding. +- Every `/api` request must carry the header `X-RootCause-Token`, a random token made per server process and written only into the served `index.html`. Another website cannot read that page, so it cannot call the API. The server sends no CORS headers, and every response forbids framing. +- The web terminal's WebSocket cannot carry headers, so the browser offers the token as a subprotocol (`rootcause.token.`) and the server answers with `rootcause.terminal` only. The request's `Origin` must be the dashboard itself, and the Host check applies as above. +- The Content-Security-Policy allows scripts from the dashboard only. Styles may be inline, because xterm.js writes its layout into `