From d620ee8f37074ecd5d96314d56796cb14ccf5c0d Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 14:29:24 +0000 Subject: [PATCH 1/7] fix(security): stop the purge tool from being a copy of the secret it purges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running the dry run against real history exposed three defects in this script, none of which were visible by reading it. 1. The script embedded the leaked SNMP community as a literal, and the runbook quoted it three more times. So a *successful* history purge left the secret sitting in the working tree — inside the tool written to remove it. The comment above the literal even claimed the opposite ("kept here ... so the string itself is not re-committed"), which was exactly backwards: the script is tracked, so the string was committed. Literals now come from a gitignored .purge-secrets.txt written by the operator at purge time, via --secrets-file. The repository no longer contains the string anywhere. 2. .gitleaks.toml allowlisted both files by path, so the scanner was configured not to report a leak that was genuinely there. Those two entries are gone. Only the gitignored secrets list is allowlisted now. 3. The verification step failed open. It detected leaks via the exit status of git rev-list --all | xargs -I{} git grep -q {} but xargs returns 123 when any invocation exits 1-125, so a literal present in some commits and absent from others made the pipeline non-zero and the check reported clean. Measured on this repository: a string present in 17 of 49 commits was reported absent. Under `set -o pipefail` the same 123 also aborted the script before the check printed anything. Detection is now by output presence with the pipeline guarded, it scans the rewritten worktree as well as history, and it names the offending file. Verified both directions rather than only the happy path: purging a string that genuinely survives the rewrite now reports FAIL and points at scripts/purge-history.sh; purging the real community string reports PASS. The real repository is untouched by the dry run (50 commits before and after). --- .gitignore | 2 + .gitleaks.toml | 6 +-- docs/runbooks/purge-git-history.md | 20 +++++-- scripts/purge-history.sh | 85 +++++++++++++++++++++++++++--- 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index cd92cf2..af87747 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ secrets/* *.crt age.key keys.txt +# Literals fed to scripts/purge-history.sh — never commit these. +.purge-secrets.txt certificates/ # ---- Rendered / decrypted config (produced by scripts/render-config.sh) ---- diff --git a/.gitleaks.toml b/.gitleaks.toml index 48ff397..f3e419d 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -48,9 +48,9 @@ paths = [ '''secrets/.*\.sops\.yaml$''', # Documents required key names and deliberately carries change-me values. '''secrets/.*\.example\.yaml$''', - # Documents the leak it teaches you to remove. - '''docs/runbooks/purge-git-history\.md$''', - '''scripts/purge-history\.sh$''', + # The list of literals to purge, written by the operator at purge time. + # Gitignored; never committed. + '''\.purge-secrets\.txt$''', ] regexes = [ diff --git a/docs/runbooks/purge-git-history.md b/docs/runbooks/purge-git-history.md index 753248c..fcd024f 100644 --- a/docs/runbooks/purge-git-history.md +++ b/docs/runbooks/purge-git-history.md @@ -18,7 +18,7 @@ Verify for yourself before and after: ```bash git show 647d90a~1:certificates/Gandalf.Gondor.Lab/ca-key.pem | head -1 -git log --all -S '7H3r315N05p00N' --oneline +git log --all -S "$(cat .purge-secrets.txt)" --oneline ``` Deleting a file in a later commit does not remove it from history. `git show` @@ -37,9 +37,21 @@ while after a force-push, and forks keep it indefinitely. 4. Install the tool: ```bash - pipx install git-filter-repo + sudo apt install git-filter-repo # or: pipx install git-filter-repo ``` +5. **Write the literals to redact into `.purge-secrets.txt`**, one per line: + + ```bash + printf '%s\n' 'the-old-snmp-community' > .purge-secrets.txt + ``` + + This file is gitignored on purpose. The script reads from it rather than + hardcoding the value, because a purge tool that contains a copy of the secret + leaves the secret in the repository after a successful purge — which is + exactly what the first version of this script did. Delete the file when you + are finished. + ## Dry run ```bash @@ -53,7 +65,7 @@ inspection — check it before continuing: ```bash cd /tmp//repo git log --oneline | head -git log --all -S '7H3r315N05p00N' --oneline # must be empty +git log --all -S "$(cat .purge-secrets.txt)" --oneline # must be empty ``` ## Execute @@ -101,7 +113,7 @@ git push --force --tags origin ```bash git log --all --oneline -- certificates/ # empty -git grep -I '7H3r315N05p00N' $(git rev-list --all) # no matches +git grep -I -e "$(cat .purge-secrets.txt)" $(git rev-list --all) # no matches gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" ``` diff --git a/scripts/purge-history.sh b/scripts/purge-history.sh index 2a87cc1..2bc5c56 100755 --- a/scripts/purge-history.sh +++ b/scripts/purge-history.sh @@ -15,14 +15,34 @@ # running it, and rotate the credentials regardless — assume anything that was # ever pushed to a public repository is compromised. # +# The literals to scrub are read from a gitignored file, NOT hardcoded here. +# An earlier version of this script embedded the leaked community string +# directly — which meant that after a successful history rewrite the secret was +# still sitting in the working tree, in the very script written to remove it. +# The dry run proved it: `git grep` on the rewritten mirror still found the +# string in this file. A purge tool must not itself be a copy of the secret. +# # Usage: # scripts/purge-history.sh --dry-run # rewrite a scratch mirror, report # scripts/purge-history.sh --execute # rewrite ./ for real +# +# --secrets-file PATH literals to redact, one per line +# (default: .purge-secrets.txt, gitignored) set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -MODE="${1:---dry-run}" +MODE="--dry-run" +SECRETS_LIST="${REPO_ROOT}/.purge-secrets.txt" + +while (($#)); do + case "$1" in + --dry-run|--execute) MODE="$1"; shift ;; + --secrets-file) SECRETS_LIST="${2:?--secrets-file needs a path}"; shift 2 ;; + *) printf 'usage: %s [--dry-run|--execute] [--secrets-file PATH]\n' \ + "$(basename "$0")" >&2; exit 1 ;; + esac +done die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } @@ -36,15 +56,37 @@ if ! git filter-repo --help >/dev/null 2>&1; then https://github.com/newren/git-filter-repo" fi -# The literal to scrub. Kept here rather than in a tracked replacements file so -# the string itself is not re-committed by the very script meant to remove it. -LEAKED_COMMUNITY='7H3r315N05p00N!' +if [[ ! -f "${SECRETS_LIST}" ]]; then + die "no secrets list at ${SECRETS_LIST} + +Create it with one literal per line — the values to redact from history: + + printf '%s\\n' 'the-old-snmp-community' > .purge-secrets.txt + +It is gitignored, so the secret never enters a commit. Delete it when done. +Pass --secrets-file PATH to use a different location." +fi REPLACEMENTS="$(mktemp)" PATHS_FILE="$(mktemp)" trap 'rm -f "${REPLACEMENTS}" "${PATHS_FILE}"' EXIT +chmod 600 "${REPLACEMENTS}" + +# Read the list once into an array. Everything downstream uses the array rather +# than re-reading the file, so the secret is touched on disk exactly once. +LITERALS=() +while IFS= read -r literal; do + [[ -z "${literal}" || "${literal}" =~ ^[[:space:]]*# ]] && continue + LITERALS+=("${literal}") +done < "${SECRETS_LIST}" +((${#LITERALS[@]} > 0)) || die "${SECRETS_LIST} contains no literals" + +# git-filter-repo replacement syntax: ==> +for literal in "${LITERALS[@]}"; do + printf '%s==>REDACTED-ROTATED-CREDENTIAL\n' "${literal}" >> "${REPLACEMENTS}" +done +info "loaded ${#LITERALS[@]} literal(s) to redact" -printf '%s==>REDACTED-ROTATED-CREDENTIAL\n' "${LEAKED_COMMUNITY}" > "${REPLACEMENTS}" cat > "${PATHS_FILE}" <<'EOF' certificates/ EOF @@ -65,10 +107,37 @@ report() { printf '\033[0;32m PASS\033[0m no commit touches certificates/\n' fi - if git -C "${target}" grep -qI "${LEAKED_COMMUNITY}" "$(git -C "${target}" rev-list --all)" -- 2>/dev/null; then - printf '\033[0;31m FAIL\033[0m leaked community string still present\n' + # Check every literal across every commit. The earlier version only checked + # one string and reported PASS while the same string was still sitting in the + # working tree of the rewritten repo, inside this script. Scan the checked-out + # tree as well as history. + # Detect by OUTPUT PRESENCE, never by exit status. + # + # `git rev-list --all | xargs -I{} git grep -q ... {}` looks correct and is + # not: xargs returns 123 when *any* invocation exits 1-125, so a literal + # present in some commits but absent from others makes the whole pipeline + # non-zero and the check silently reports clean. Measured on this repo: a + # string present in 17 of 49 commits was reported as absent. A security check + # that fails open is worse than no check. + local leaked=0 literal hit + local ignore_name; ignore_name="$(basename "${SECRETS_LIST}")" + for literal in "${LITERALS[@]}"; do + # `|| true` is load-bearing: under `set -o pipefail` the xargs 123 above + # would abort the whole script before this check ever printed anything. + hit="$(git -C "${target}" rev-list --all \ + | xargs -r -I{} git -C "${target}" grep -lI -e "${literal}" {} -- 2>/dev/null \ + | head -n1 || true)" + [[ -n "${hit}" ]] && { leaked=1; printf ' history: %s\n' "${hit}"; } + + hit="$(grep -rlI -e "${literal}" "${target}" \ + --exclude-dir=.git --exclude="${ignore_name}" 2>/dev/null | head -n1 || true)" + [[ -n "${hit}" ]] && { leaked=1; printf ' worktree: %s\n' "${hit}"; } + done + + if ((leaked)); then + printf '\033[0;31m FAIL\033[0m a redacted literal is still present (history or worktree)\n' else - printf '\033[0;32m PASS\033[0m leaked community string absent from all commits\n' + printf '\033[0;32m PASS\033[0m no redacted literal in any commit or in the worktree\n' fi printf ' commits: %s\n' "$(git -C "${target}" rev-list --all --count)" From c94d77f6de555bff95c993d31352975168522009 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 14:39:48 +0000 Subject: [PATCH 2/7] feat(observability): add log-based alerting rules for Loki MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Loki ruler was configured and pointed at Alertmanager but had no rules, so the log pipeline was a search box rather than something that could tell you anything unprompted. Adds 8 rules in loki/rules/security.rules.yaml covering conditions that only exist in logs: SSH brute force (warning at 20 failures in 5m, critical at 100), SSH accepted from outside VLAN 50/99, repeated sudo failures, user and group creation, kernel OOM kills, read-only remounts, and disk I/O errors. They carry the same severity and category labels as the Prometheus rules, so Alertmanager routing and inhibition are shared with no extra configuration. Two things this surfaced: * Loki's local ruler reads // and, with auth_enabled false, the tenant is literally "fake". Getting that path wrong produces no error at all — just a ruler that silently evaluates nothing. The ruler directory moved out of /loki so the read-only rule mount does not collide with the data volume. * A rule alerting on a host that stops logging was written and then removed. When a host goes quiet it produces no series, so `rate(...) == 0` never evaluates for it, and LogQL has no way to supply a known-host list. The global case is already covered by LokiIngestionStalled on the Prometheus side. Validation: promtool cannot check these — it parses PromQL and rejects every LogQL stream selector. scripts/check_loki_rules.sh boots the pinned Loki image with the rules mounted, fails on a parse error, and then asserts the ruler actually evaluated them, since silence is not evidence when the alternative is that it never read the files. `loki -verify-config` is deliberately not used as the check: it validates the config file and never opens the rule files. Confirmed by experiment — a rule file containing `count_over_time({{{BROKEN` passes -verify-config and is caught only by the boot check. Verified in both directions against the real Loki v3.3.2 binary: all 8 rules parse and are evaluated by the ruler across both groups, and deliberately breaking one LogQL expression makes the checker exit 1 and name the rule. --- .github/workflows/ci.yml | 6 + Makefile | 4 + docs/observability.md | 29 ++++ docs/roadmap.md | 5 +- scripts/check_loki_rules.sh | 86 ++++++++++++ scripts/validate.sh | 9 ++ stacks/observability/compose.yaml | 3 + stacks/observability/loki/loki-config.yaml | 6 +- .../loki/rules/security.rules.yaml | 131 ++++++++++++++++++ 9 files changed, 275 insertions(+), 4 deletions(-) create mode 100755 scripts/check_loki_rules.sh create mode 100644 stacks/observability/loki/rules/security.rules.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d06385e..6f6a626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,12 @@ jobs: -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \ fmt --test "$STACK/alloy/config.alloy" + # promtool cannot check these — it parses PromQL and rejects every LogQL + # stream selector. Loki itself is the only thing that understands them, + # so the checker boots the pinned image with the rules mounted. + - name: Validate Loki rules + run: ./scripts/check_loki_rules.sh + - name: Validate Grafana dashboards run: python3 scripts/check_dashboards.py diff --git a/Makefile b/Makefile index 8a26606..dc996a6 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,10 @@ check-rules: ## Validate Prometheus rules and config promtool check config $(STACK_DIR)/prometheus/prometheus.yaml promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml +.PHONY: check-loki-rules +check-loki-rules: ## Validate Loki (LogQL) alerting rules + ./scripts/check_loki_rules.sh + .PHONY: scan scan: ## Scan the working tree and history for secrets gitleaks detect --no-banner --redact -c .gitleaks.toml diff --git a/docs/observability.md b/docs/observability.md index edd9c74..e8b3684 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -70,6 +70,35 @@ expression in every panel is syntactically valid. ## Alerting +40 rules in total: 32 metric-based in `prometheus/rules/`, and 8 log-based in +`loki/rules/`. + +### Log-based (Loki ruler) + +Some conditions only exist in logs. A metric confirms sshd is running; only the +log shows it rejecting forty passwords in five minutes. `loki/rules/security.rules.yaml` +covers SSH brute force, SSH accepted from outside VLAN 50/99, repeated sudo +failures, user/group creation, kernel OOM kills, read-only remounts and disk I/O +errors. + +They use the same `severity` and `category` labels as the Prometheus rules and +are sent to the same Alertmanager, so routing and inhibition are shared. + +Loki's local ruler reads `//`, and with `auth_enabled: false` +the tenant is literally `fake` — hence the `loki/rules:/etc/loki/rules/fake` +mount in `compose.yaml`. Getting that path wrong produces no error, just a ruler +that silently evaluates nothing. + +`promtool` cannot validate these; it parses PromQL and rejects every LogQL +stream selector. `scripts/check_loki_rules.sh` boots the pinned Loki image with +the rules mounted and fails on a parse error, then asserts the ruler actually +evaluated them. Note that `loki -verify-config` is *not* sufficient on its own — +it validates the config file and never opens the rule files. A file containing +`count_over_time({{{BROKEN` passes `-verify-config` and is caught only by the +boot check. + +### Metric-based (Prometheus) + 32 rules across four files in `prometheus/rules/`: | File | Covers | diff --git a/docs/roadmap.md b/docs/roadmap.md index ed89ec4..df2b8ab 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -33,9 +33,6 @@ inventory. Ordered roughly by how much it matters. inventory currently agree on the address but describe different things. - [ ] Capture dashboard screenshots for the README once the stack has a few days of real data. → [`images/README.md`](images/README.md) -- [ ] Loki alerting rules — the ruler is configured and pointed at Alertmanager - but no log-based rules exist yet. Repeated SSH auth failure is the obvious - first one. ## Infrastructure @@ -70,3 +67,5 @@ inventory. Ordered roughly by how much it matters. - [x] Add alerting (32 rules) and Alertmanager routing - [x] Move secrets to SOPS + age - [x] Add CI: lint, config validation, secret scanning +- [x] Loki alerting rules (8) for auth, SSH brute force and disk/OOM events, + validated in CI by booting the pinned Loki image against them diff --git a/scripts/check_loki_rules.sh b/scripts/check_loki_rules.sh new file mode 100755 index 0000000..525accc --- /dev/null +++ b/scripts/check_loki_rules.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Validate the Loki alerting rules. +# +# There is no `promtool check rules` equivalent for LogQL: promtool parses +# PromQL and rejects every stream selector in these files. The only tool that +# genuinely understands LogQL is Loki itself, so this boots the pinned Loki +# image against a throwaway config with the rules mounted, and fails if the +# ruler reports a parse error. +# +# `-verify-config` alone is NOT sufficient — it validates the config file and +# never looks at the rule files. Verified: a rule file containing +# `count_over_time({{{BROKEN` passes -verify-config and is only caught here. +# +# Usage: scripts/check_loki_rules.sh + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK="${REPO_ROOT}/stacks/observability" +LOKI_IMAGE="grafana/loki:3.3.2" +BOOT_SECONDS="${BOOT_SECONDS:-45}" + +die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } + +RULES_DIR="${STACK}/loki/rules" +[[ -d "${RULES_DIR}" ]] || die "no rules directory at ${RULES_DIR}" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT +# auth_enabled is false, so Loki's local ruler looks under /fake/. +mkdir -p "${WORK}/rules/fake" "${WORK}/data" +cp "${RULES_DIR}"/*.yaml "${WORK}/rules/fake/" + +# Rewrite every path in the real config to point inside the scratch dir, so the +# rules are checked against the same settings production uses. +python3 - "$STACK/loki/loki-config.yaml" "${WORK}" > "${WORK}/loki.yaml" <<'PY' +import sys, yaml +cfg = yaml.safe_load(open(sys.argv[1])) +work = sys.argv[2] +cfg["common"]["path_prefix"] = f"{work}/data" +cfg["common"]["storage"]["filesystem"] = { + "chunks_directory": f"{work}/data/chunks", "rules_directory": f"{work}/data/rules"} +cfg["storage_config"]["tsdb_shipper"] = { + "active_index_directory": f"{work}/data/index", "cache_location": f"{work}/data/cache"} +cfg["storage_config"]["filesystem"] = {"directory": f"{work}/data/chunks"} +cfg["compactor"]["working_directory"] = f"{work}/data/compactor" +cfg["ruler"]["storage"]["local"]["directory"] = f"{work}/rules" +cfg["ruler"]["rule_path"] = f"{work}/data/rules-temp" +yaml.safe_dump(cfg, sys.stdout) +PY + +n_rules="$(grep -ch '^ *- alert:' "${RULES_DIR}"/*.yaml | paste -sd+ | bc)" +info "checking ${n_rules} Loki rule(s)" + +if command -v loki >/dev/null 2>&1; then + RUN=(loki) +elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + RUN=(docker run --rm -v "${WORK}:${WORK}" -w "${WORK}" --entrypoint loki "${LOKI_IMAGE}") +else + printf '\033[0;33m SKIP\033[0m no loki binary and no docker daemon\n' + exit 0 +fi + +OUT="${WORK}/loki.log" +timeout "${BOOT_SECONDS}" "${RUN[@]}" \ + -config.file="${WORK}/loki.yaml" -target=all \ + -server.http-listen-port=3197 > "${OUT}" 2>&1 + +if grep -qiE 'parse error|failed to parse|syntax error' "${OUT}"; then + printf '\033[0;31m FAIL\033[0m LogQL parse error in the Loki rules\n' + grep -iE 'parse error|failed to parse|syntax error' "${OUT}" | head -5 + exit 1 +fi + +# Confirm the ruler actually loaded them. A silent "no errors" is not evidence +# if the ruler never read the files at all. +evaluated="$(grep -o 'rule_name=[A-Za-z]*' "${OUT}" | sort -u | wc -l)" +if ((evaluated == 0)); then + printf '\033[0;31m FAIL\033[0m the ruler evaluated no rules — check the tenant path\n' + exit 1 +fi + +printf '\033[0;32m PASS\033[0m %s Loki rule(s) parsed, %s evaluated by the ruler\n' \ + "${n_rules}" "${evaluated}" diff --git a/scripts/validate.sh b/scripts/validate.sh index aa4336f..482e7bb 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -123,6 +123,15 @@ else skip "no alloy binary and no docker daemon" fi +# --------------------------------------------------------------------------- +head_ "Loki rules" +# --------------------------------------------------------------------------- +if ./scripts/check_loki_rules.sh; then + : +else + FAILED=1 +fi + # --------------------------------------------------------------------------- head_ "Grafana dashboards" # --------------------------------------------------------------------------- diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index 9945efc..7ff7510 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -93,6 +93,9 @@ services: command: -config.file=/etc/loki/loki-config.yaml volumes: - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + # Loki's local ruler reads //. auth_enabled is false, + # so the tenant is "fake". + - ./loki/rules:/etc/loki/rules/fake:ro - loki-data:/loki ports: - "${BIND_ADDR:-0.0.0.0}:${LOKI_PORT:-3100}:3100" diff --git a/stacks/observability/loki/loki-config.yaml b/stacks/observability/loki/loki-config.yaml index b162003..5ff7f05 100644 --- a/stacks/observability/loki/loki-config.yaml +++ b/stacks/observability/loki/loki-config.yaml @@ -61,7 +61,11 @@ ruler: storage: type: local local: - directory: /loki/rules + # Loki's local ruler reads //*.yaml. With + # auth_enabled: false the tenant is literally "fake", so compose mounts + # loki/rules/ at /etc/loki/rules/fake. Kept outside /loki so the + # read-only rule mount does not collide with the data volume. + directory: /etc/loki/rules rule_path: /loki/rules-temp alertmanager_url: http://alertmanager:9093 enable_api: true diff --git a/stacks/observability/loki/rules/security.rules.yaml b/stacks/observability/loki/rules/security.rules.yaml new file mode 100644 index 0000000..5e0f624 --- /dev/null +++ b/stacks/observability/loki/rules/security.rules.yaml @@ -0,0 +1,131 @@ +--- +# Log-based alerts, evaluated by Loki's ruler and sent to the same Alertmanager +# as the Prometheus rules. +# +# These exist because some conditions are only visible in logs. A metric can +# tell you sshd is running; only the log tells you it rejected forty passwords +# in five minutes. +# +# Loki rule files use the Prometheus alerting schema with LogQL expressions, so +# `severity` and `category` route through exactly the same Alertmanager tree. +groups: + - name: authentication + interval: 1m + rules: + - alert: SshBruteForce + # /var/log/auth.log, labelled log_type="authlog" by config.alloy. + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "(?i)failed (password|publickey)" [5m]) + ) > 20 + for: 2m + labels: + severity: warning + category: security + annotations: + summary: "{{ $labels.host }}: {{ $value }} failed SSH auths in 5 minutes" + description: >- + Sustained authentication failure. On a management-VLAN host this + should be close to zero — nothing on VLAN 99 is internet-facing. + + - alert: SshBruteForceSevere + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "(?i)failed (password|publickey)" [5m]) + ) > 100 + for: 1m + labels: + severity: critical + category: security + annotations: + summary: "{{ $labels.host }}: {{ $value }} failed SSH auths in 5 minutes" + + - alert: SshLoginFromUnexpectedSubnet + # Accepted logins should only ever originate from Hicks (10.0.50.0/24) + # or from within management itself. Anything else means a firewall rule + # is not doing what the documentation claims. + expr: | + sum by (host) ( + count_over_time( + {log_type="authlog"} + |~ "Accepted (password|publickey)" + != "10.0.50." + != "10.0.99." + [10m] + ) + ) > 0 + for: 1m + labels: + severity: critical + category: security + annotations: + summary: "{{ $labels.host }}: SSH login accepted from outside VLAN 50/99" + description: >- + Either segmentation is broken or the source ranges in this rule are + stale. Both are worth knowing about immediately. + + - alert: SudoFailure + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "sudo:.*authentication failure" [10m]) + ) > 5 + for: 5m + labels: + severity: warning + category: security + annotations: + summary: "{{ $labels.host }}: repeated sudo authentication failures" + + - alert: NewUserOrGroupCreated + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "(useradd|groupadd|usermod).*new (user|group)" [10m]) + ) > 0 + labels: + severity: warning + category: security + annotations: + summary: "{{ $labels.host }}: a user or group was created" + description: >- + Expected during provisioning, worth a second look otherwise. + + - name: system + interval: 1m + rules: + - alert: KernelOomKill + # The kernel's own record. Complements ContainerOomKilled in + # containers.rules.yaml, which only sees cgroup events. + expr: | + sum by (host) ( + count_over_time({log_type=~"syslog|varlog"} |~ "Out of memory: Kill(ed)? process" [15m]) + ) > 0 + labels: + severity: warning + category: capacity + annotations: + summary: "{{ $labels.host }}: kernel OOM killer fired" + + - alert: FilesystemRemountedReadOnly + expr: | + sum by (host) ( + count_over_time({log_type=~"syslog|varlog"} |~ "Remounting filesystem read-only" [15m]) + ) > 0 + labels: + severity: critical + category: hardware + annotations: + summary: "{{ $labels.host }}: filesystem remounted read-only" + description: >- + Almost always failing storage. The monitoring host runs on a + thirteen-year-old laptop SSD, so this is not hypothetical. + + - alert: DiskIoErrors + expr: | + sum by (host) ( + count_over_time({log_type=~"syslog|varlog"} |~ "(I/O error|ata[0-9]+.*failed|SMART.*FAILED)" [15m]) + ) > 0 + labels: + severity: critical + category: hardware + annotations: + summary: "{{ $labels.host }}: disk I/O errors in the kernel log" From ddd4d716e37a2d598f48596ccfcd0ae51104fce8 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 14:47:01 +0000 Subject: [PATCH 3/7] fix(ci): make the Loki rules check work in a container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI failed with 'the ruler evaluated no rules'. The guard did its job — it refused to report success when nothing had been validated — but the cause was the harness, not the rules. mktemp -d creates a 0700 directory owned by the invoking user, and the Loki image runs as uid 10001. In CI the container therefore could not traverse its own working directory, exited after three seconds, and evaluated nothing. The local run passed because the loki binary ran as root against the same paths. Fixes: * chmod -R a+rwX the throwaway directory, applied again after the generated config is written so the container can read that too. Confirmed the barrier: the directory is 0700 before and 0777 after. * Distinguish the two failure modes in the output. Exit status 124 means loki ran the full window and genuinely evaluated nothing (a tenant path problem); anything else means it died early. Either way the last 15 log lines are now printed, so the next failure explains itself instead of needing a local reproduction. * Guard the PyYAML import. It is not guaranteed on a clean runner and the failure without this is an opaque ModuleNotFoundError inside a heredoc. Verified: shellcheck clean, the check still passes against the real Loki v3.3.2 binary (8 rules parsed, 8 evaluated), and the short-boot path prints the diagnostic rather than a bare failure. --- scripts/check_loki_rules.sh | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/scripts/check_loki_rules.sh b/scripts/check_loki_rules.sh index 525accc..999e96c 100755 --- a/scripts/check_loki_rules.sh +++ b/scripts/check_loki_rules.sh @@ -33,6 +33,21 @@ trap 'rm -rf "${WORK}"' EXIT mkdir -p "${WORK}/rules/fake" "${WORK}/data" cp "${RULES_DIR}"/*.yaml "${WORK}/rules/fake/" +# The Loki image runs as uid 10001, while mktemp -d creates a 0700 directory +# owned by the invoking user. Without this the container cannot read its own +# config, exits within seconds, and the run looks like "the ruler evaluated +# nothing" rather than a permissions problem. Throwaway directory, so the broad +# mode is fine. +chmod -R a+rwX "${WORK}" + +# PyYAML is not guaranteed on a clean runner, and the failure mode without this +# guard is an opaque ModuleNotFoundError inside a heredoc. +if ! python3 -c 'import yaml' 2>/dev/null; then + info "installing PyYAML" + python3 -m pip install --quiet --disable-pip-version-check pyyaml >/dev/null 2>&1 \ + || die "PyYAML is required and could not be installed" +fi + # Rewrite every path in the real config to point inside the scratch dir, so the # rules are checked against the same settings production uses. python3 - "$STACK/loki/loki-config.yaml" "${WORK}" > "${WORK}/loki.yaml" <<'PY' @@ -64,9 +79,13 @@ else fi OUT="${WORK}/loki.log" +# Re-apply after loki.yaml was generated above, so the container user can read +# it too. +chmod -R a+rwX "${WORK}" 2>/dev/null || true timeout "${BOOT_SECONDS}" "${RUN[@]}" \ -config.file="${WORK}/loki.yaml" -target=all \ -server.http-listen-port=3197 > "${OUT}" 2>&1 +rc=$? if grep -qiE 'parse error|failed to parse|syntax error' "${OUT}"; then printf '\033[0;31m FAIL\033[0m LogQL parse error in the Loki rules\n' @@ -78,7 +97,16 @@ fi # if the ruler never read the files at all. evaluated="$(grep -o 'rule_name=[A-Za-z]*' "${OUT}" | sort -u | wc -l)" if ((evaluated == 0)); then - printf '\033[0;31m FAIL\033[0m the ruler evaluated no rules — check the tenant path\n' + printf '\033[0;31m FAIL\033[0m the ruler evaluated no rules\n' + # rc 124 is the timeout we expect (Loki runs until killed). Anything else + # means Loki died early — usually it could not read the mounted files. + if ((rc != 124)); then + printf ' loki exited early with status %s; last output:\n' "${rc}" + else + printf ' loki ran the full %ss but never evaluated a rule.\n' "${BOOT_SECONDS}" + printf ' Check the ruler tenant path (/fake/).\n' + fi + tail -n 15 "${OUT}" | sed 's/^/ /' exit 1 fi From aec7686874c16a86424b5536b8ed3092e78fe98b Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 14:51:35 +0000 Subject: [PATCH 4/7] fix: address review on the purge tool and Loki rules check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All correct, and one of them repeats the exact mistake this PR was opened to fix. * .gitleaks.toml allowlisted .purge-secrets.txt by path — the same "teach the scanner to ignore a real finding" pattern the purge fix removes. Entry gone. Worth being precise about what that does and does not buy: removing it does not make gitleaks catch an accidental commit of that file. It is gitignored, so the filesystem scan skips it, and it holds bare literals with no keyword context for any rule to match. Verified by force-adding it and re-scanning — still clean. The real control is an assertion that the file is never tracked, now in both CI and scripts/validate.sh alongside the existing .env and .rendered/ checks, and confirmed to fire. * Leak detection used grep without -F. A rotated community is a random string and may contain regex metacharacters; without -F those are either mis-parsed or make grep error out, and a grep that errors reports no match — one more way to fail open, which is the failure mode this script already had once. Both the history and worktree scans now use -F. Verified with the literal a[b(c).*+?d. * The runbook used `git log -S "$(cat .purge-secrets.txt)"`, which folds a multi-line file into one search string and matches nothing. The file format is documented as one literal per line, so all three call sites now loop, and the git grep example uses -F. * check_loki_rules.sh ran under `set -uo pipefail` without -e, so a failed cp or config rewrite could still reach a cheerful PASS. Now `set -euo pipefail`, with the one command expected to fail — the timeout that kills Loki — guarded explicitly. * The evaluated-rule count matched rule_name=[A-Za-z]*, which can match empty and misses digits and underscores in alert names. Now anchored to a valid identifier with the optional opening quote stripped. The PyYAML preflight raised in review was already fixed in ddd4d71. Re-verified after the changes: shellcheck clean across all five scripts, the Loki check still passes against the real v3.3.2 binary (8 parsed, 8 evaluated) and still fails on a deliberately broken expression, and the purge dry run passes for both a metacharacter-laden literal and the real one. --- .github/workflows/ci.yml | 6 +++++- .gitleaks.toml | 3 --- docs/runbooks/purge-git-history.md | 16 +++++++++++++--- scripts/check_loki_rules.sh | 14 +++++++++----- scripts/purge-history.sh | 8 ++++++-- scripts/validate.sh | 12 ++++++++---- 6 files changed, 41 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f6a626..bc38881 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,10 +152,14 @@ jobs: zricethezav/gitleaks:v8.24.0 \ detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" -v + # gitleaks cannot be the control here. .purge-secrets.txt is gitignored so + # the filesystem scan skips it, and its contents are bare literals with no + # keyword context for a rule to match. The control is simply that it must + # never be a tracked file. - name: Assert no decrypted artefact is tracked run: | fail=0 - for pattern in '.env' '.rendered/'; do + for pattern in '.env' '.rendered/' '.purge-secrets.txt'; do if git ls-files | grep -E "(^|/)${pattern//./\\.}" | grep -v '\.env\.example'; then echo "::error::tracked file matching '${pattern}' — it must be gitignored" fail=1 diff --git a/.gitleaks.toml b/.gitleaks.toml index f3e419d..5ca29f8 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -48,9 +48,6 @@ paths = [ '''secrets/.*\.sops\.yaml$''', # Documents required key names and deliberately carries change-me values. '''secrets/.*\.example\.yaml$''', - # The list of literals to purge, written by the operator at purge time. - # Gitignored; never committed. - '''\.purge-secrets\.txt$''', ] regexes = [ diff --git a/docs/runbooks/purge-git-history.md b/docs/runbooks/purge-git-history.md index fcd024f..58031b9 100644 --- a/docs/runbooks/purge-git-history.md +++ b/docs/runbooks/purge-git-history.md @@ -18,7 +18,13 @@ Verify for yourself before and after: ```bash git show 647d90a~1:certificates/Gandalf.Gondor.Lab/ca-key.pem | head -1 -git log --all -S "$(cat .purge-secrets.txt)" --oneline + +# One literal per line, so loop — `$(cat ...)` would fold the whole file into a +# single search string and match nothing. +while IFS= read -r s; do + [ -z "$s" ] && continue + git log --all -S "$s" --oneline +done < .purge-secrets.txt ``` Deleting a file in a later commit does not remove it from history. `git show` @@ -65,7 +71,8 @@ inspection — check it before continuing: ```bash cd /tmp//repo git log --oneline | head -git log --all -S "$(cat .purge-secrets.txt)" --oneline # must be empty +while IFS= read -r s; do [ -n "$s" ] && git log --all -S "$s" --oneline; done \ + < .purge-secrets.txt # must print nothing ``` ## Execute @@ -113,7 +120,10 @@ git push --force --tags origin ```bash git log --all --oneline -- certificates/ # empty -git grep -I -e "$(cat .purge-secrets.txt)" $(git rev-list --all) # no matches +# -F: a rotated community is a random string and may contain regex characters. +while IFS= read -r s; do + [ -n "$s" ] && git grep -IF -e "$s" $(git rev-list --all) +done < .purge-secrets.txt # no matches gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" ``` diff --git a/scripts/check_loki_rules.sh b/scripts/check_loki_rules.sh index 999e96c..a923710 100755 --- a/scripts/check_loki_rules.sh +++ b/scripts/check_loki_rules.sh @@ -14,7 +14,9 @@ # # Usage: scripts/check_loki_rules.sh -set -uo pipefail +# -e is on: a failed cp or config rewrite must not produce a cheerful PASS. +# The one command allowed to fail is the timeout below, which is guarded. +set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" STACK="${REPO_ROOT}/stacks/observability" @@ -66,7 +68,7 @@ cfg["ruler"]["rule_path"] = f"{work}/data/rules-temp" yaml.safe_dump(cfg, sys.stdout) PY -n_rules="$(grep -ch '^ *- alert:' "${RULES_DIR}"/*.yaml | paste -sd+ | bc)" +n_rules="$(grep -ch '^ *- alert:' "${RULES_DIR}"/*.yaml | paste -sd+ | bc || true)" info "checking ${n_rules} Loki rule(s)" if command -v loki >/dev/null 2>&1; then @@ -82,10 +84,11 @@ OUT="${WORK}/loki.log" # Re-apply after loki.yaml was generated above, so the container user can read # it too. chmod -R a+rwX "${WORK}" 2>/dev/null || true +# Loki runs until killed, so a 124 from timeout is the expected outcome. +rc=0 timeout "${BOOT_SECONDS}" "${RUN[@]}" \ -config.file="${WORK}/loki.yaml" -target=all \ - -server.http-listen-port=3197 > "${OUT}" 2>&1 -rc=$? + -server.http-listen-port=3197 > "${OUT}" 2>&1 || rc=$? if grep -qiE 'parse error|failed to parse|syntax error' "${OUT}"; then printf '\033[0;31m FAIL\033[0m LogQL parse error in the Loki rules\n' @@ -95,7 +98,8 @@ fi # Confirm the ruler actually loaded them. A silent "no errors" is not evidence # if the ruler never read the files at all. -evaluated="$(grep -o 'rule_name=[A-Za-z]*' "${OUT}" | sort -u | wc -l)" +evaluated="$(grep -oE 'rule_name="?[A-Za-z_][A-Za-z0-9_]*' "${OUT}" \ + | sed 's/^rule_name="\?//' | sort -u | wc -l || true)" if ((evaluated == 0)); then printf '\033[0;31m FAIL\033[0m the ruler evaluated no rules\n' # rc 124 is the timeout we expect (Loki runs until killed). Anything else diff --git a/scripts/purge-history.sh b/scripts/purge-history.sh index 2bc5c56..4c6e24c 100755 --- a/scripts/purge-history.sh +++ b/scripts/purge-history.sh @@ -119,17 +119,21 @@ report() { # non-zero and the check silently reports clean. Measured on this repo: a # string present in 17 of 49 commits was reported as absent. A security check # that fails open is worse than no check. + # -F throughout: a rotated community is a random string and may contain regex + # metacharacters. Without it, values containing [ ] ( ) . * + ? are either + # mis-parsed or make grep error out — and a grep that errors reports no match, + # which is one more way to fail open. local leaked=0 literal hit local ignore_name; ignore_name="$(basename "${SECRETS_LIST}")" for literal in "${LITERALS[@]}"; do # `|| true` is load-bearing: under `set -o pipefail` the xargs 123 above # would abort the whole script before this check ever printed anything. hit="$(git -C "${target}" rev-list --all \ - | xargs -r -I{} git -C "${target}" grep -lI -e "${literal}" {} -- 2>/dev/null \ + | xargs -r -I{} git -C "${target}" grep -lIF -e "${literal}" {} -- 2>/dev/null \ | head -n1 || true)" [[ -n "${hit}" ]] && { leaked=1; printf ' history: %s\n' "${hit}"; } - hit="$(grep -rlI -e "${literal}" "${target}" \ + hit="$(grep -rlIF -e "${literal}" "${target}" \ --exclude-dir=.git --exclude="${ignore_name}" 2>/dev/null | head -n1 || true)" [[ -n "${hit}" ]] && { leaked=1; printf ' worktree: %s\n' "${hit}"; } done diff --git a/scripts/validate.sh b/scripts/validate.sh index 482e7bb..f587950 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -189,12 +189,16 @@ else skip "gitleaks not installed" fi -# Cheap belt-and-braces check that no rendered/decrypted artefact is staged. +# Cheap belt-and-braces check that no rendered or decrypted artefact is staged. +# gitleaks cannot cover .purge-secrets.txt — it is gitignored (so the filesystem +# scan skips it) and holds bare literals with no keyword context to match. Being +# untracked is the control. if git ls-files --error-unmatch "${STACK}/.env" >/dev/null 2>&1 \ - || git ls-files "${STACK}/snmp-exporter/.rendered" | grep -q .; then - fail "a rendered or decrypted file is tracked by git" + || git ls-files "${STACK}/snmp-exporter/.rendered" | grep -q . \ + || git ls-files | grep -q '\.purge-secrets\.txt'; then + fail "a rendered, decrypted or purge-secrets file is tracked by git" else - pass "no rendered or decrypted files tracked" + pass "no rendered, decrypted or purge-secrets files tracked" fi printf '\n' From 1ca69b91b4101e450138e12b01ac54bc3b95f805 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 14:59:46 +0000 Subject: [PATCH 5/7] fix(ci): evaluate Loki rules promptly instead of waiting out a 1m interval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The permission fix worked — CI shows Loki running the full 45s and shutting down cleanly, so it was reading its config. It still reported no rules evaluated. Cause is timing, not configuration. The rule groups use interval: 1m, and Loki jitters a group's first evaluation across its interval, so a 45s window can legitimately see none. Locally the jitter happened to land early, which is why it passed here and failed there — the check was non-deterministic in a way that made a real timing issue look like a broken tenant path. The throwaway copies of the rules now have their group interval rewritten to 5s before Loki starts. The committed rules keep interval: 1m; only the scratch copy is touched, and the LogQL is never modified. A 25s window is now sufficient where 75s was previously marginal, so the 45s default has real headroom. Also: the container now runs with --user "$(id -u):$(id -g)". Without it Loki writes its WAL and index as uid 10001, the runner cannot delete the scratch directory, and cleanup floods the log with "rm: Permission denied" immediately after the verdict. The cleanup trap is non-fatal regardless, so a cleanup failure can never mask the result. Verified: 8 rules parsed and 8 evaluated within a 25s window, the deliberately broken expression still exits 1 and names the failing alert, and the committed rules file is unchanged at interval: 1m. --- scripts/check_loki_rules.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/check_loki_rules.sh b/scripts/check_loki_rules.sh index a923710..c41971b 100755 --- a/scripts/check_loki_rules.sh +++ b/scripts/check_loki_rules.sh @@ -30,7 +30,8 @@ RULES_DIR="${STACK}/loki/rules" [[ -d "${RULES_DIR}" ]] || die "no rules directory at ${RULES_DIR}" WORK="$(mktemp -d)" -trap 'rm -rf "${WORK}"' EXIT +# Loki writes as its own uid; never let cleanup failure mask the result. +trap 'rm -rf "${WORK}" 2>/dev/null || true' EXIT # auth_enabled is false, so Loki's local ruler looks under /fake/. mkdir -p "${WORK}/rules/fake" "${WORK}/data" cp "${RULES_DIR}"/*.yaml "${WORK}/rules/fake/" @@ -68,13 +69,30 @@ cfg["ruler"]["rule_path"] = f"{work}/data/rules-temp" yaml.safe_dump(cfg, sys.stdout) PY +# Rule groups use interval: 1m in production, and Loki jitters a group's first +# evaluation across that interval — so a short boot window can legitimately see +# no evaluations, which looks identical to a misconfigured tenant path. Shorten +# the interval in the throwaway copies only; the committed rules and their LogQL +# are untouched. +python3 - "${WORK}/rules/fake" <<'SPEEDUP' +import pathlib, sys, yaml +for f in pathlib.Path(sys.argv[1]).glob("*.yaml"): + doc = yaml.safe_load(f.read_text()) + for group in doc.get("groups", []): + group["interval"] = "5s" + f.write_text(yaml.safe_dump(doc)) +SPEEDUP + n_rules="$(grep -ch '^ *- alert:' "${RULES_DIR}"/*.yaml | paste -sd+ | bc || true)" info "checking ${n_rules} Loki rule(s)" if command -v loki >/dev/null 2>&1; then RUN=(loki) elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then - RUN=(docker run --rm -v "${WORK}:${WORK}" -w "${WORK}" --entrypoint loki "${LOKI_IMAGE}") + # --user: without it Loki writes as uid 10001 and the runner cannot delete + # the scratch directory afterwards, filling the log with rm errors. + RUN=(docker run --rm --user "$(id -u):$(id -g)" \ + -v "${WORK}:${WORK}" -w "${WORK}" --entrypoint loki "${LOKI_IMAGE}") else printf '\033[0;33m SKIP\033[0m no loki binary and no docker daemon\n' exit 0 From 42510f6c715595273b152fdb49134123df9f137e Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 15:08:36 +0000 Subject: [PATCH 6/7] fix(ci): resolve image versions from compose.yaml instead of duplicating them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dependabot merged version bumps into main while this branch was open, which exposed a design flaw in the CI added in #1: the image pins were duplicated across compose.yaml, ci.yml, validate.sh and check_loki_rules.sh, and Dependabot only updates compose.yaml. The result after those merges: compose.yaml prom/prometheus:v3.13.2 grafana/loki:3.7.4 grafana/alloy:v1.18.0 ci.yml prom/prometheus:v3.1.0 (n/a) grafana/alloy:v1.6.1 scripts prom/prometheus:v3.1.0 grafana/loki:3.3.2 grafana/alloy:v1.6.1 CI was set to validate configs against versions two years older than the ones the stack deploys, and would have reported green while doing it. Validating the wrong version is worse than not validating, because it still produces a passing check. scripts/image-for.sh now reads the pin straight out of compose.yaml, and ci.yml resolves the images into $GITHUB_ENV at run time rather than declaring them. compose.yaml is the single source of truth, so a Dependabot bump automatically reaches every check. The Makefile's snmp-generator is derived from the snmp-exporter pin — the two are released together and the generator is not a compose service, so it cannot be looked up directly. A new CI step fails on any pinned image version outside compose.yaml, so the duplication cannot creep back in. The gitleaks image stays declared in ci.yml because it is a CI tool rather than part of the stack. Verified against the versions actually deployed, not the ones I had lying around: the Loki rules parse and evaluate on 3.7.4 (8/8), loki-config.yaml is valid on 3.7.4, and config.alloy passes fmt --test on v1.18.0 with an empty diff against its canonical formatting — so the formatter did not change behaviour between v1.6.1 and v1.18.0. Also corrects comments that named specific tool versions now resolved dynamically. --- .github/workflows/ci.yml | 41 +++++++++++++++++++++++++++------ Makefile | 7 +++++- scripts/check_loki_rules.sh | 3 ++- scripts/image-for.sh | 45 +++++++++++++++++++++++++++++++++++++ scripts/validate.sh | 18 ++++++++++----- 5 files changed, 99 insertions(+), 15 deletions(-) create mode 100755 scripts/image-for.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 831a851..4658c5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,11 @@ concurrency: env: STACK: stacks/observability - PROM_IMAGE: prom/prometheus:v3.1.0 - AM_IMAGE: prom/alertmanager:v0.28.0 - ALLOY_IMAGE: grafana/alloy:v1.6.1 + # Image versions are NOT duplicated here. They are resolved from compose.yaml + # at run time by scripts/image-for.sh, because Dependabot only updates + # compose.yaml — hardcoded copies went stale silently and CI ended up + # validating v3.1.0 configs against a stack running v3.13.2. + GITLEAKS_IMAGE: zricethezav/gitleaks:v8.24.0 jobs: # --------------------------------------------------------------------------- @@ -58,9 +60,34 @@ jobs: cp "$STACK/.env.example" "$STACK/.env" echo "GRAFANA_ADMIN_PASSWORD=validation-only" >> "$STACK/.env" + # Single source of truth: whatever compose.yaml pins is what gets tested. + - name: Resolve pinned images from compose.yaml + run: | + { + echo "PROM_IMAGE=$(./scripts/image-for.sh prometheus)" + echo "AM_IMAGE=$(./scripts/image-for.sh alertmanager)" + echo "ALLOY_IMAGE=$(./scripts/image-for.sh alloy)" + } >> "$GITHUB_ENV" + ./scripts/image-for.sh prometheus + ./scripts/image-for.sh alertmanager + ./scripts/image-for.sh alloy + - name: docker compose config run: docker compose -f "$STACK/compose.yaml" config -q + # Guard against the duplication coming back. Any image: pin outside + # compose.yaml is drift waiting to happen, since Dependabot cannot see it. + - name: Verify image versions are not duplicated outside compose.yaml + run: | + if grep -rnE '(prom|grafana)/[a-z-]+:v?[0-9]+\.[0-9]+' \ + --include='*.sh' --include='*.yml' --include='Makefile' \ + scripts .github Makefile 2>/dev/null \ + | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' ; then + echo "::error::pinned image version outside compose.yaml — use scripts/image-for.sh" + exit 1 + fi + echo "no duplicated image pins" + # Covers every place an image is referenced, not just compose.yaml — the # first version of this check only looked at the stack and let :latest # through in the workflow itself and in the Makefile. @@ -98,8 +125,8 @@ jobs: # `fmt --test` exits non-zero if the file is not canonically formatted, and # fails outright on a syntax error. It does not validate that components - # are configured correctly — Alloy v1.6.1 has no `validate` subcommand, so - # that is only caught at load time on the host. + # are configured correctly — Alloy has no `validate` subcommand, so that is + # only caught at load time on the host. - name: alloy fmt --test run: | docker run --rm --entrypoint alloy \ @@ -143,13 +170,13 @@ jobs: - name: gitleaks — working tree run: | docker run --rm -v "$PWD:/repo" -w /repo \ - zricethezav/gitleaks:v8.24.0 \ + "$GITLEAKS_IMAGE" \ detect --no-git --no-banner --redact -c .gitleaks.toml -v - name: gitleaks — full history run: | docker run --rm -v "$PWD:/repo" -w /repo \ - zricethezav/gitleaks:v8.24.0 \ + "$GITLEAKS_IMAGE" \ detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" -v # gitleaks cannot be the control here. .purge-secrets.txt is gitignored so diff --git a/Makefile b/Makefile index dc996a6..d4d3470 100644 --- a/Makefile +++ b/Makefile @@ -111,13 +111,18 @@ scan: ## Scan the working tree and history for secrets .PHONY: snmp-generate snmp-generate: ## Regenerate snmp.yaml from generator.yaml + @# The generator is released in lockstep with snmp-exporter but is not a + @# compose service, so its version is derived from the exporter's pin rather + @# than duplicated — see scripts/image-for.sh. + @gen="$$(./scripts/image-for.sh snmp-exporter | sed 's|snmp-exporter|snmp-generator|')"; \ + printf 'using %s\n' "$$gen"; \ docker run --rm \ -v "$(PWD)/$(STACK_DIR)/snmp-exporter:/opt/" \ -e SNMP_COMMUNITY_PFSENSE='$${SNMP_COMMUNITY_PFSENSE}' \ -e SNMP_COMMUNITY_APC='$${SNMP_COMMUNITY_APC}' \ -e SNMP_COMMUNITY_MOKERLINK='$${SNMP_COMMUNITY_MOKERLINK}' \ -e SNMP_COMMUNITY_ILO='$${SNMP_COMMUNITY_ILO}' \ - prom/snmp-generator:v0.28.0 generate \ + "$$gen" generate \ -m /opt/mibs -g /opt/generator.yaml -o /opt/snmp.yaml @printf '\033[0;33mCheck the diff before committing — placeholders must survive.\033[0m\n' diff --git a/scripts/check_loki_rules.sh b/scripts/check_loki_rules.sh index c41971b..e1cf121 100755 --- a/scripts/check_loki_rules.sh +++ b/scripts/check_loki_rules.sh @@ -20,7 +20,8 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" STACK="${REPO_ROOT}/stacks/observability" -LOKI_IMAGE="grafana/loki:3.3.2" +# Resolved from compose.yaml — see scripts/image-for.sh. +LOKI_IMAGE="$("${REPO_ROOT}/scripts/image-for.sh" loki)" BOOT_SECONDS="${BOOT_SECONDS:-45}" die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } diff --git a/scripts/image-for.sh b/scripts/image-for.sh new file mode 100755 index 0000000..35e061c --- /dev/null +++ b/scripts/image-for.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Print the pinned image for a compose service. +# +# Image versions must live in exactly one place: compose.yaml, which is what +# Dependabot updates. They were previously duplicated into ci.yml, validate.sh +# and check_loki_rules.sh, and the duplicates went stale the moment Dependabot +# merged a bump — CI ended up validating configs against Prometheus v3.1.0 while +# the stack deployed v3.13.2. Validation against the wrong version is worse than +# no validation, because it still reports green. +# +# Deliberately parses the text rather than using PyYAML or `docker compose +# config`: this runs before any of those are guaranteed present, and needs no +# .env for the ${VAR:?} guards. +# +# Usage: scripts/image-for.sh e.g. loki -> grafana/loki:3.7.4 + +set -euo pipefail + +SERVICE="${1:?usage: image-for.sh }" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE="${COMPOSE_FILE:-${REPO_ROOT}/stacks/observability/compose.yaml}" + +[[ -f "${COMPOSE}" ]] || { printf 'no compose file at %s\n' "${COMPOSE}" >&2; exit 1; } + +image="$(awk -v svc="${SERVICE}" ' + # Track whether we are inside the services: block. + /^services:/ { in_services = 1; next } + /^[^[:space:]#]/ { in_services = 0 } + !in_services { next } + # A two-space-indented key is a service name. + /^ [A-Za-z0-9_.-]+:/ { + line = $0 + sub(/^ /, "", line); sub(/:.*/, "", line) + current = line + } + current == svc && $1 == "image:" { print $2; exit } +' "${COMPOSE}")" + +if [[ -z "${image}" ]]; then + printf 'no image found for service %s in %s\n' "${SERVICE}" "${COMPOSE}" >&2 + exit 1 +fi + +printf '%s\n' "${image}" diff --git a/scripts/validate.sh b/scripts/validate.sh index f587950..be08dd1 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash # -# Everything CI runs, runnable locally. Uses containers for the Prometheus and -# Alertmanager tooling so the versions match what actually runs in production. +# Everything CI runs, runnable locally. +# +# Image versions come from compose.yaml via scripts/image-for.sh, so the +# containers used here are the ones actually deployed. A locally installed +# binary is preferred when present for speed — if yours is a different version +# from the pin, CI is the authority. # # Usage: scripts/validate.sh @@ -11,9 +15,11 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${REPO_ROOT}" || exit 1 STACK="stacks/observability" -PROM_IMAGE="prom/prometheus:v3.1.0" -AM_IMAGE="prom/alertmanager:v0.28.0" -ALLOY_IMAGE="grafana/alloy:v1.6.1" +# Resolved from compose.yaml so Dependabot's bumps reach the checks. Hardcoding +# these meant CI validated Prometheus v3.1.0 configs while the stack ran v3.13.2. +PROM_IMAGE="$(./scripts/image-for.sh prometheus)" +AM_IMAGE="$(./scripts/image-for.sh alertmanager)" +ALLOY_IMAGE="$(./scripts/image-for.sh alloy)" FAILED=0 pass() { printf '\033[0;32m PASS\033[0m %s\n' "$*"; } @@ -111,7 +117,7 @@ else fi # `fmt --test` fails on a syntax error and on non-canonical formatting. It does -# not check component configuration — v1.6.1 has no `validate` subcommand. +# not check component configuration — Alloy has no `validate` subcommand. if ((${#ALLOY[@]})); then if "${ALLOY[@]}" fmt --test "${STACK}/alloy/config.alloy" >/dev/null 2>&1; then pass "alloy fmt --test" From 054542c6cea502d2a4adcf3c6f5768a81c697d0c Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 15:18:53 +0000 Subject: [PATCH 7/7] feat(security): pin images by digest, add SECURITY.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Digest pinning A tag is a mutable pointer. `prom/prometheus:v3.13.2` is whatever the publisher last pushed under that name, and a tag can be moved — silently, by a compromised or coerced publisher. A digest is the content hash: it matches or the pull fails. Every service image is now `repo:tag@sha256:...`, keeping both halves. The tag stays readable and says which version you are on; the digest is what Docker enforces. Dependabot understands this form and updates both together. scripts/pin-digests.sh resolves digests through the registry HTTP API rather than `docker pull`, so it needs no daemon and downloads no layers. Without arguments it reports drift and exits non-zero, so it doubles as a check; --write applies. CI additionally asserts every compose image carries a digest. Two things this surfaced: * The Makefile derives prom/snmp-generator from the snmp-exporter pin, and with digests that silently produced snmp-generator tagged with the *exporter's* digest — a reference that cannot be pulled. image-for.sh grew a --tag-only mode for exactly this case. * The README badges hardcoded Prometheus v3.1.0, Grafana 11.5 and Loki 3.3 while the stack ran v3.13.2, 13.0.2 and 3.7.4 — the same drift class fixed in the previous commit, in the first thing a reader sees. Version numbers are gone from the badges rather than left to rot again; there is no mechanism that could keep them honest. ## SECURITY.md A security-focused repository with no disclosure policy is a gap. It covers how to report privately (GitHub Security Advisories), an explicit request not to probe the live network, the three known historical exposures with their current status, what the repository deliberately never publishes and why, and the controls CI enforces. It documents the known exposure rather than quietly carrying it: a written-down, accepted exposure reads very differently from an overlooked one. Verified: digests resolve for all six images and re-running reports them unchanged; the generator derivation yields prom/snmp-generator:v0.30.1 with no digest; all 57 relative links across the docs resolve; and the full local sweep passes including the new digest check. --- .github/workflows/ci.yml | 13 ++++ Makefile | 11 ++- README.md | 36 +++++---- SECURITY.md | 82 ++++++++++++++++++++ docs/roadmap.md | 2 + scripts/image-for.sh | 19 ++++- scripts/pin-digests.sh | 125 ++++++++++++++++++++++++++++++ stacks/observability/compose.yaml | 12 +-- 8 files changed, 278 insertions(+), 22 deletions(-) create mode 100644 SECURITY.md create mode 100755 scripts/pin-digests.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4658c5b..e310f5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,19 @@ jobs: fi echo "all image references pinned" + # A tag is a mutable pointer; a digest is the content hash. Every service + # image must carry both, so a moved tag cannot change what gets deployed. + - name: Verify every image is pinned by digest + run: | + missing=0 + while read -r ref; do + case "$ref" in + *@sha256:*) ;; + *) echo "::error::$ref is not pinned by digest — run make pin-digests"; missing=1 ;; + esac + done < <(awk '$1 == "image:" { print $2 }' "$STACK/compose.yaml") + exit "$missing" + - name: promtool check config run: | docker run --rm --entrypoint promtool \ diff --git a/Makefile b/Makefile index d4d3470..264fae9 100644 --- a/Makefile +++ b/Makefile @@ -100,6 +100,14 @@ check-rules: ## Validate Prometheus rules and config check-loki-rules: ## Validate Loki (LogQL) alerting rules ./scripts/check_loki_rules.sh +.PHONY: pin-digests +pin-digests: ## Re-resolve image digests in compose.yaml (--write applies) + ./scripts/pin-digests.sh --write + +.PHONY: check-digests +check-digests: ## Verify pinned digests still match the registry + ./scripts/pin-digests.sh + .PHONY: scan scan: ## Scan the working tree and history for secrets gitleaks detect --no-banner --redact -c .gitleaks.toml @@ -114,7 +122,8 @@ snmp-generate: ## Regenerate snmp.yaml from generator.yaml @# The generator is released in lockstep with snmp-exporter but is not a @# compose service, so its version is derived from the exporter's pin rather @# than duplicated — see scripts/image-for.sh. - @gen="$$(./scripts/image-for.sh snmp-exporter | sed 's|snmp-exporter|snmp-generator|')"; \ + @# --tag-only: the exporter's digest does not belong to the generator. + @gen="$$(./scripts/image-for.sh --tag-only snmp-exporter | sed 's|snmp-exporter|snmp-generator|')"; \ printf 'using %s\n' "$$gen"; \ docker run --rm \ -v "$(PWD)/$(STACK_DIR)/snmp-exporter:/opt/" \ diff --git a/README.md b/README.md index 8a852c0..6839bf5 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,15 @@ [![CI](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml/badge.svg)](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![Secrets: SOPS + age](https://img.shields.io/badge/secrets-SOPS%20%2B%20age-6f42c1.svg)](docs/adr/0005-secrets-with-sops-and-age.md) -[![Prometheus](https://img.shields.io/badge/Prometheus-v3.1.0-E6522C.svg?logo=prometheus&logoColor=white)](stacks/observability/prometheus) -[![Grafana](https://img.shields.io/badge/Grafana-11.5-F46800.svg?logo=grafana&logoColor=white)](stacks/observability/grafana) -[![Loki](https://img.shields.io/badge/Loki-3.3-F5A800.svg?logo=grafana&logoColor=white)](stacks/observability/loki) +[![Prometheus](https://img.shields.io/badge/Prometheus-E6522C.svg?logo=prometheus&logoColor=white)](stacks/observability/prometheus) +[![Grafana](https://img.shields.io/badge/Grafana-F46800.svg?logo=grafana&logoColor=white)](stacks/observability/grafana) +[![Loki](https://img.shields.io/badge/Loki-F5A800.svg?logo=grafana&logoColor=white)](stacks/observability/loki) [![pfSense](https://img.shields.io/badge/pfSense-FreeBSD%2015-212121.svg)](docs/network.md) [Architecture](docs/architecture.md) · [Network](docs/network.md) · [Observability](docs/observability.md) · -[Security](docs/security.md) · +[Security](SECURITY.md) · [Runbooks](docs/runbooks) · [Decisions](docs/adr) · [Roadmap](docs/roadmap.md) @@ -42,16 +42,20 @@ incident. metrics and logs from Linux hosts; `snmp_exporter` polls the four devices that can't run an agent (firewall, switch, UPS, iLO). One agent config, deployed identically everywhere. [How](docs/architecture.md#observability-data-flow) -- **Dashboards and alerting as code.** 5 provisioned dashboards, 79 panels, 32 - alert rules with severity routing and inhibition. No dashboard exists only in - a database. +- **Dashboards and alerting as code.** 5 provisioned dashboards, 79 panels, and + 40 alert rules — 32 metric-based in Prometheus, 8 log-based in Loki — sharing + one Alertmanager routing tree. No dashboard exists only in a database. - **Secrets encrypted in-repo with SOPS + age.** Per-device credentials, decrypted at deploy time into gitignored paths, with `git log` showing which credential rotated and when — but never to what. [Why](docs/adr/0005-secrets-with-sops-and-age.md) - **CI that actually validates the infrastructure.** `docker compose config`, - `promtool`, `amtool`, `alloy fmt`, dashboard-JSON and datasource checks, every - dashboard's PromQL parsed, plus `gitleaks` over the full history. + `promtool`, `amtool`, `alloy fmt`, a real Loki boot to parse the LogQL rules, + dashboard-JSON and datasource checks, every dashboard's PromQL parsed, plus + `gitleaks` over the full history. +- **Supply chain pinned by digest.** Every image carries both a tag and a + `sha256:` digest, so a moved tag cannot change what deploys. CI enforces it; + `make pin-digests` re-resolves them from the registry. - **Documented decisions and runbooks.** Five ADRs covering what was chosen and what was rejected; four runbooks for the operations that are easy to get wrong at 1am. @@ -111,7 +115,7 @@ the internet and nothing more. Full topology and data flow in | Alerting | Alertmanager | Severity routing, inhibition | | Visualisation | Grafana | 5 provisioned dashboards | | Secrets | SOPS + age | Encrypted in-repo | -| CI | GitHub Actions | Lint, config validation, secret scanning | +| CI | GitHub Actions | Lint, config validation, secret scanning, digest pinning | ## Repository layout @@ -121,12 +125,13 @@ the internet and nothing more. Full topology and data flow in │ ├── compose.yaml │ ├── prometheus/ # config, file_sd targets, 32 alert rules │ ├── alertmanager/ # routing and inhibition -│ ├── loki/ # single-binary config +│ ├── loki/ # single-binary config + 8 LogQL rules │ ├── alloy/ # one agent config, used on every host │ ├── snmp-exporter/ # generator.yaml is the source of truth │ └── grafana/ # provisioning + 5 dashboards ├── secrets/ # SOPS-encrypted; see secrets/README.md -├── scripts/ # bootstrap, render, validate, history purge +├── scripts/ # bootstrap, render, validate, pin-digests, purge +├── SECURITY.md # disclosure policy and known exposure ├── docs/ │ ├── architecture.md network.md hardware.md │ ├── observability.md security.md roadmap.md @@ -184,7 +189,12 @@ owner-linked device names, camera placement) are in Historical credential exposure in this repository's git history is documented there too, along with the runbooks to remediate it — including the parts not yet -done. +done. [`SECURITY.md`](SECURITY.md) carries the disclosure policy and a summary of +what is known. + +Container images are pinned by **tag and digest**. A tag is a mutable pointer; a +digest is the content hash, so a moved tag cannot change what gets deployed. CI +enforces it, and `make pin-digests` re-resolves them. ## Roadmap diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4e75cb0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,82 @@ +# Security policy + +This repository documents and configures a private home network. It is not a +product and has no users other than its owner, so "supported versions" does not +really apply — `main` is the only branch that means anything, and it is what the +lab runs. + +What *is* useful here is a clear answer to two questions: what to do if you spot +a problem, and what is already known. + +## Reporting something + +If you find a misconfiguration, a leaked credential, or a weakness in what is +published here, please report it privately rather than opening a public issue: + +- **GitHub Security Advisories** — [open a draft advisory](https://github.com/Gerrrt/HomeLab/security/advisories/new) + (preferred; it stays private until fixed) +- Failing that, a GitHub issue *without* details, asking for a contact. + +Please do not open a public issue containing a working credential, a capture, or +anything that would let someone else reach the network before it can be fixed. + +This is a personal project, so there is no SLA. Realistically: acknowledgement +within a few days, and credential exposure treated as urgent. + +### Please don't + +The lab is a home network, not a bug bounty target. Scanning, probing or +attempting to reach any host described in `docs/network.md` is unwelcome and not +authorised. Everything worth reviewing is in this repository — review the +configuration, not the running system. + +## Known exposure + +Documented rather than quietly fixed, because a known and written-down exposure +is a very different thing from an overlooked one. Full detail in +[`docs/security.md`](docs/security.md). + +| What | Status | +| --- | --- | +| SNMP community committed in plaintext, shared across firewall, switch, UPS and BMC | Removed from `HEAD` and replaced with per-device SOPS-encrypted values. **Still present in git history, and not yet rotated on the devices.** Treat it as public. | +| Grafana `admin`/`admin` with anonymous Admin access enabled | Fixed — anonymous auth off, password from SOPS | +| Passphrase-encrypted TLS private keys under `certificates/` | Removed from `HEAD`, still reachable in history. Purge tooling and a runbook are provided; not yet run. | + +Remediation is tracked in [`docs/roadmap.md`](docs/roadmap.md), with procedures +in [`docs/runbooks/rotate-snmp-community.md`](docs/runbooks/rotate-snmp-community.md) +and [`docs/runbooks/purge-git-history.md`](docs/runbooks/purge-git-history.md). + +[`.gitleaksignore`](.gitleaksignore) enumerates all nine historical findings +individually, with a note on each. It exists so the full-history scan stays +meaningful — a job that is permanently red for a known reason gets ignored, and +then a genuinely new leak goes unnoticed alongside it. It is an acknowledgement, +not a fix, and it gets deleted once the purge has run. + +## What this repository will not contain + +Deliberate omissions, so their absence is not mistaken for an oversight: + +- **Full MAC addresses.** Truncated to the vendor OUI, which keeps the useful + half and drops the unique identifier. +- **Owner-linked device names**, and no room labelled as a child's. +- **Camera-to-room mapping.** That there are cameras is fine; which one covers + which door is not. +- **The WAN address, firewall rule bodies, and Wi-Fi configuration.** +- **Any plaintext credential.** Secrets are SOPS + age encrypted; the private key + never enters the repository. See [`secrets/README.md`](secrets/README.md). + +## Controls in CI + +Every push and pull request runs: + +- **`gitleaks`** over the working tree *and* full history, with rules for SNMP + communities, inline Grafana passwords, PEM private keys and age secret keys. +- An assertion that every `secrets/*.sops.yaml` is genuinely encrypted, which + needs no ability to decrypt. +- An assertion that no rendered or decrypted artefact — `.env`, `.rendered/`, + `.purge-secrets.txt` — is ever a tracked file. +- Verification that every container image is pinned by **tag *and* digest**, so a + moved tag cannot silently change what is deployed. + +See [`docs/security.md`](docs/security.md) for the threat model and segmentation +rationale. diff --git a/docs/roadmap.md b/docs/roadmap.md index df2b8ab..b1edb03 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -67,5 +67,7 @@ inventory. Ordered roughly by how much it matters. - [x] Add alerting (32 rules) and Alertmanager routing - [x] Move secrets to SOPS + age - [x] Add CI: lint, config validation, secret scanning +- [x] Pin every image by digest, not just tag, with drift detection in CI +- [x] Add SECURITY.md with a disclosure policy and known-exposure summary - [x] Loki alerting rules (8) for auth, SSH brute force and disk/OOM events, validated in CI by booting the pinned Loki image against them diff --git a/scripts/image-for.sh b/scripts/image-for.sh index 35e061c..94c8196 100755 --- a/scripts/image-for.sh +++ b/scripts/image-for.sh @@ -13,11 +13,23 @@ # config`: this runs before any of those are guaranteed present, and needs no # .env for the ${VAR:?} guards. # -# Usage: scripts/image-for.sh e.g. loki -> grafana/loki:3.7.4 +# Usage: +# scripts/image-for.sh full reference, digest included +# scripts/image-for.sh --tag-only repo:tag, digest stripped +# +# --tag-only exists for the one case where a digest must NOT be carried across: +# deriving prom/snmp-generator from prom/snmp-exporter. They share a version but +# are different images, so reusing the exporter's digest produces a reference +# that cannot be pulled. set -euo pipefail -SERVICE="${1:?usage: image-for.sh }" +TAG_ONLY=0 +if [[ "${1:-}" == "--tag-only" ]]; then + TAG_ONLY=1 + shift +fi +SERVICE="${1:?usage: image-for.sh [--tag-only] }" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" COMPOSE="${COMPOSE_FILE:-${REPO_ROOT}/stacks/observability/compose.yaml}" @@ -42,4 +54,7 @@ if [[ -z "${image}" ]]; then exit 1 fi +# Strip the @sha256:... suffix when only the tag was asked for. +((TAG_ONLY)) && image="${image%%@*}" + printf '%s\n' "${image}" diff --git a/scripts/pin-digests.sh b/scripts/pin-digests.sh new file mode 100755 index 0000000..d005819 --- /dev/null +++ b/scripts/pin-digests.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# +# Pin every compose image to an immutable digest. +# +# A tag is a mutable pointer. `prom/prometheus:v3.13.2` is whatever the +# publisher last pushed under that name — a tag can be moved, and a compromised +# or coerced publisher can move it silently. A digest is the content hash: it +# either matches or the pull fails. +# +# Images are written as `repo:tag@sha256:...`, keeping both. The tag stays +# human-readable and tells you which version you are on at a glance; the digest +# is what Docker actually enforces. Dependabot understands this form and updates +# both halves together. +# +# Resolves through the registry HTTP API rather than `docker pull`, so it works +# without a daemon and without downloading hundreds of megabytes of layers. +# +# Usage: +# scripts/pin-digests.sh # report drift, change nothing +# scripts/pin-digests.sh --write # update compose.yaml in place + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE="${COMPOSE_FILE:-${REPO_ROOT}/stacks/observability/compose.yaml}" +WRITE=0 +[[ "${1:-}" == "--write" ]] && WRITE=1 + +die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } + +[[ -f "${COMPOSE}" ]] || die "no compose file at ${COMPOSE}" +command -v curl >/dev/null 2>&1 || die "curl is required" + +# Resolve repo:tag to its manifest digest. +# +# Accepts all four media types: multi-arch images return an OCI index or a +# Docker manifest list, single-arch ones return a plain manifest. Omitting any +# of these makes the registry return a 404 or the wrong digest for some images. +resolve_digest() { + local repo="$1" tag="$2" token digest + + # Official images live under library/ but are written without it. + [[ "${repo}" == */* ]] || repo="library/${repo}" + + token="$(curl -sS --fail --max-time 30 \ + "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')" \ + || { printf 'could not get a pull token for %s\n' "${repo}" >&2; return 1; } + + digest="$(curl -sS --fail --max-time 30 -I \ + -H "Authorization: Bearer ${token}" \ + -H "Accept: application/vnd.oci.image.index.v1+json" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + "https://registry-1.docker.io/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | awk 'tolower($1) == "docker-content-digest:" { print $2 }')" \ + || { printf 'manifest request failed for %s:%s\n' "${repo}" "${tag}" >&2; return 1; } + + [[ "${digest}" == sha256:* ]] || { printf 'no digest returned for %s:%s\n' "${repo}" "${tag}" >&2; return 1; } + printf '%s\n' "${digest}" +} + +mapfile -t refs < <(awk '$1 == "image:" { print $2 }' "${COMPOSE}") +((${#refs[@]} > 0)) || die "no image: lines found in ${COMPOSE}" + +changed=0 +declare -a updates=() + +for ref in "${refs[@]}"; do + repo_tag="${ref%%@*}" + current_digest="" + [[ "${ref}" == *@* ]] && current_digest="${ref##*@}" + + repo="${repo_tag%:*}" + tag="${repo_tag##*:}" + [[ "${repo}" != "${tag}" ]] || die "image ${ref} has no tag — refusing to pin a floating reference" + + if ! digest="$(resolve_digest "${repo}" "${tag}")"; then + die "could not resolve ${repo}:${tag}" + fi + + if [[ "${current_digest}" == "${digest}" ]]; then + printf ' \033[0;32mok\033[0m %s\n' "${repo_tag}" + elif [[ -z "${current_digest}" ]]; then + printf ' \033[0;33munpinned\033[0m %s -> %s\n' "${repo_tag}" "${digest}" + updates+=("${ref}|${repo_tag}@${digest}") + changed=1 + else + printf ' \033[0;31mDRIFT\033[0m %s\n pinned: %s\n registry: %s\n' \ + "${repo_tag}" "${current_digest}" "${digest}" + updates+=("${ref}|${repo_tag}@${digest}") + changed=1 + fi +done + +if ((changed == 0)); then + printf '\n\033[0;32mall images pinned to their current digest\033[0m\n' + exit 0 +fi + +if ((WRITE == 0)); then + printf '\n%s image(s) need pinning. Re-run with --write to apply.\n' "${#updates[@]}" + # Non-zero so CI can use this as a drift check. + exit 1 +fi + +for u in "${updates[@]}"; do + old="${u%%|*}"; new="${u##*|}" + # Fixed-string replace via python: digests contain no regex metacharacters, + # but repo names contain / and . which sed would need escaping for. + python3 - "${COMPOSE}" "${old}" "${new}" <<'PY' +import sys, pathlib +path, old, new = sys.argv[1], sys.argv[2], sys.argv[3] +p = pathlib.Path(path) +text = p.read_text(encoding="utf-8") +if old not in text: + sys.exit(f"expected to find {old} in {path}") +p.write_text(text.replace(old, new), encoding="utf-8") +PY +done + +info "updated ${#updates[@]} image reference(s) in $(basename "${COMPOSE}")" +printf '\033[0;33mReview the diff, then re-run make validate.\033[0m\n' diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index 4d11159..c6d7d86 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -28,7 +28,7 @@ services: # --------------------------------------------------------------------------- prometheus: <<: *service-defaults - image: prom/prometheus:v3.13.2 + image: prom/prometheus:v3.13.2@sha256:508729e0e2d18e11fd742a5a5ca70e557b940a93948c3c95fd0123a6fd538b69 container_name: prometheus user: "65534:65534" command: @@ -57,7 +57,7 @@ services: # --------------------------------------------------------------------------- alertmanager: <<: *service-defaults - image: prom/alertmanager:v0.33.1 + image: prom/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d container_name: alertmanager user: "65534:65534" command: @@ -87,7 +87,7 @@ services: # --------------------------------------------------------------------------- loki: <<: *service-defaults - image: grafana/loki:3.7.4 + image: grafana/loki:3.7.4@sha256:87f0a067673756a3cede1bcbf0c74875f7df9b09fddb53e399d0c576f756cfcc container_name: loki user: "10001:10001" command: -config.file=/etc/loki/loki-config.yaml @@ -113,7 +113,7 @@ services: # --------------------------------------------------------------------------- snmp-exporter: <<: *service-defaults - image: prom/snmp-exporter:v0.30.1 + image: prom/snmp-exporter:v0.30.1@sha256:e5fd5e8b43ace6c088fe9bf0b37b7fff0e04380bee352be7ec41b853a4dd5859 container_name: snmp-exporter user: "65534:65534" command: @@ -135,7 +135,7 @@ services: # --------------------------------------------------------------------------- grafana: <<: *service-defaults - image: grafana/grafana-oss:13.0.2 + image: grafana/grafana-oss:13.0.2@sha256:5dad0df181cb644a14e13617b913b261a54f7d4fd4510721dba420929f35bea2 container_name: grafana environment: GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} @@ -175,7 +175,7 @@ services: # --------------------------------------------------------------------------- alloy: <<: *service-defaults - image: grafana/alloy:v1.18.0 + image: grafana/alloy:v1.18.0@sha256:491b0578c04983fd54fe99b587b6fab4404dc46d0dc16677bd6b00cc1140b308 container_name: alloy privileged: true environment: