diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e1f032..e310f5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,11 @@ concurrency: env: STACK: stacks/observability - PROM_IMAGE: prom/prometheus:v3.1.0 - AM_IMAGE: prom/alertmanager:v0.28.0 - ALLOY_IMAGE: grafana/alloy:v1.6.1 + # Image versions are NOT duplicated here. They are resolved from compose.yaml + # at run time by scripts/image-for.sh, because Dependabot only updates + # compose.yaml — hardcoded copies went stale silently and CI ended up + # validating v3.1.0 configs against a stack running v3.13.2. + GITLEAKS_IMAGE: zricethezav/gitleaks:v8.24.0 jobs: # --------------------------------------------------------------------------- @@ -58,9 +60,34 @@ jobs: cp "$STACK/.env.example" "$STACK/.env" echo "GRAFANA_ADMIN_PASSWORD=validation-only" >> "$STACK/.env" + # Single source of truth: whatever compose.yaml pins is what gets tested. + - name: Resolve pinned images from compose.yaml + run: | + { + echo "PROM_IMAGE=$(./scripts/image-for.sh prometheus)" + echo "AM_IMAGE=$(./scripts/image-for.sh alertmanager)" + echo "ALLOY_IMAGE=$(./scripts/image-for.sh alloy)" + } >> "$GITHUB_ENV" + ./scripts/image-for.sh prometheus + ./scripts/image-for.sh alertmanager + ./scripts/image-for.sh alloy + - name: docker compose config run: docker compose -f "$STACK/compose.yaml" config -q + # Guard against the duplication coming back. Any image: pin outside + # compose.yaml is drift waiting to happen, since Dependabot cannot see it. + - name: Verify image versions are not duplicated outside compose.yaml + run: | + if grep -rnE '(prom|grafana)/[a-z-]+:v?[0-9]+\.[0-9]+' \ + --include='*.sh' --include='*.yml' --include='Makefile' \ + scripts .github Makefile 2>/dev/null \ + | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' ; then + echo "::error::pinned image version outside compose.yaml — use scripts/image-for.sh" + exit 1 + fi + echo "no duplicated image pins" + # Covers every place an image is referenced, not just compose.yaml — the # first version of this check only looked at the stack and let :latest # through in the workflow itself and in the Makefile. @@ -76,6 +103,19 @@ jobs: fi echo "all image references pinned" + # A tag is a mutable pointer; a digest is the content hash. Every service + # image must carry both, so a moved tag cannot change what gets deployed. + - name: Verify every image is pinned by digest + run: | + missing=0 + while read -r ref; do + case "$ref" in + *@sha256:*) ;; + *) echo "::error::$ref is not pinned by digest — run make pin-digests"; missing=1 ;; + esac + done < <(awk '$1 == "image:" { print $2 }' "$STACK/compose.yaml") + exit "$missing" + - name: promtool check config run: | docker run --rm --entrypoint promtool \ @@ -98,14 +138,20 @@ jobs: # `fmt --test` exits non-zero if the file is not canonically formatted, and # fails outright on a syntax error. It does not validate that components - # are configured correctly — Alloy v1.6.1 has no `validate` subcommand, so - # that is only caught at load time on the host. + # are configured correctly — Alloy has no `validate` subcommand, so that is + # only caught at load time on the host. - name: alloy fmt --test run: | docker run --rm --entrypoint alloy \ -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \ fmt --test "$STACK/alloy/config.alloy" + # promtool cannot check these — it parses PromQL and rejects every LogQL + # stream selector. Loki itself is the only thing that understands them, + # so the checker boots the pinned image with the rules mounted. + - name: Validate Loki rules + run: ./scripts/check_loki_rules.sh + - name: Validate Grafana dashboards run: python3 scripts/check_dashboards.py @@ -137,19 +183,23 @@ jobs: - name: gitleaks — working tree run: | docker run --rm -v "$PWD:/repo" -w /repo \ - zricethezav/gitleaks:v8.24.0 \ + "$GITLEAKS_IMAGE" \ detect --no-git --no-banner --redact -c .gitleaks.toml -v - name: gitleaks — full history run: | docker run --rm -v "$PWD:/repo" -w /repo \ - zricethezav/gitleaks:v8.24.0 \ + "$GITLEAKS_IMAGE" \ detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" -v + # gitleaks cannot be the control here. .purge-secrets.txt is gitignored so + # the filesystem scan skips it, and its contents are bare literals with no + # keyword context for a rule to match. The control is simply that it must + # never be a tracked file. - name: Assert no decrypted artefact is tracked run: | fail=0 - for pattern in '.env' '.rendered/'; do + for pattern in '.env' '.rendered/' '.purge-secrets.txt'; do if git ls-files | grep -E "(^|/)${pattern//./\\.}" | grep -v '\.env\.example'; then echo "::error::tracked file matching '${pattern}' — it must be gitignored" fail=1 diff --git a/.gitignore b/.gitignore index cd92cf2..af87747 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ secrets/* *.crt age.key keys.txt +# Literals fed to scripts/purge-history.sh — never commit these. +.purge-secrets.txt certificates/ # ---- Rendered / decrypted config (produced by scripts/render-config.sh) ---- diff --git a/.gitleaks.toml b/.gitleaks.toml index 48ff397..5ca29f8 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -48,9 +48,6 @@ paths = [ '''secrets/.*\.sops\.yaml$''', # Documents required key names and deliberately carries change-me values. '''secrets/.*\.example\.yaml$''', - # Documents the leak it teaches you to remove. - '''docs/runbooks/purge-git-history\.md$''', - '''scripts/purge-history\.sh$''', ] regexes = [ diff --git a/Makefile b/Makefile index 8a26606..264fae9 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,18 @@ check-rules: ## Validate Prometheus rules and config promtool check config $(STACK_DIR)/prometheus/prometheus.yaml promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml +.PHONY: check-loki-rules +check-loki-rules: ## Validate Loki (LogQL) alerting rules + ./scripts/check_loki_rules.sh + +.PHONY: pin-digests +pin-digests: ## Re-resolve image digests in compose.yaml (--write applies) + ./scripts/pin-digests.sh --write + +.PHONY: check-digests +check-digests: ## Verify pinned digests still match the registry + ./scripts/pin-digests.sh + .PHONY: scan scan: ## Scan the working tree and history for secrets gitleaks detect --no-banner --redact -c .gitleaks.toml @@ -107,13 +119,19 @@ scan: ## Scan the working tree and history for secrets .PHONY: snmp-generate snmp-generate: ## Regenerate snmp.yaml from generator.yaml + @# The generator is released in lockstep with snmp-exporter but is not a + @# compose service, so its version is derived from the exporter's pin rather + @# than duplicated — see scripts/image-for.sh. + @# --tag-only: the exporter's digest does not belong to the generator. + @gen="$$(./scripts/image-for.sh --tag-only snmp-exporter | sed 's|snmp-exporter|snmp-generator|')"; \ + printf 'using %s\n' "$$gen"; \ docker run --rm \ -v "$(PWD)/$(STACK_DIR)/snmp-exporter:/opt/" \ -e SNMP_COMMUNITY_PFSENSE='$${SNMP_COMMUNITY_PFSENSE}' \ -e SNMP_COMMUNITY_APC='$${SNMP_COMMUNITY_APC}' \ -e SNMP_COMMUNITY_MOKERLINK='$${SNMP_COMMUNITY_MOKERLINK}' \ -e SNMP_COMMUNITY_ILO='$${SNMP_COMMUNITY_ILO}' \ - prom/snmp-generator:v0.28.0 generate \ + "$$gen" generate \ -m /opt/mibs -g /opt/generator.yaml -o /opt/snmp.yaml @printf '\033[0;33mCheck the diff before committing — placeholders must survive.\033[0m\n' diff --git a/README.md b/README.md index 8a852c0..6839bf5 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,15 @@ [![CI](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml/badge.svg)](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![Secrets: SOPS + age](https://img.shields.io/badge/secrets-SOPS%20%2B%20age-6f42c1.svg)](docs/adr/0005-secrets-with-sops-and-age.md) -[![Prometheus](https://img.shields.io/badge/Prometheus-v3.1.0-E6522C.svg?logo=prometheus&logoColor=white)](stacks/observability/prometheus) -[![Grafana](https://img.shields.io/badge/Grafana-11.5-F46800.svg?logo=grafana&logoColor=white)](stacks/observability/grafana) -[![Loki](https://img.shields.io/badge/Loki-3.3-F5A800.svg?logo=grafana&logoColor=white)](stacks/observability/loki) +[![Prometheus](https://img.shields.io/badge/Prometheus-E6522C.svg?logo=prometheus&logoColor=white)](stacks/observability/prometheus) +[![Grafana](https://img.shields.io/badge/Grafana-F46800.svg?logo=grafana&logoColor=white)](stacks/observability/grafana) +[![Loki](https://img.shields.io/badge/Loki-F5A800.svg?logo=grafana&logoColor=white)](stacks/observability/loki) [![pfSense](https://img.shields.io/badge/pfSense-FreeBSD%2015-212121.svg)](docs/network.md) [Architecture](docs/architecture.md) · [Network](docs/network.md) · [Observability](docs/observability.md) · -[Security](docs/security.md) · +[Security](SECURITY.md) · [Runbooks](docs/runbooks) · [Decisions](docs/adr) · [Roadmap](docs/roadmap.md) @@ -42,16 +42,20 @@ incident. metrics and logs from Linux hosts; `snmp_exporter` polls the four devices that can't run an agent (firewall, switch, UPS, iLO). One agent config, deployed identically everywhere. [How](docs/architecture.md#observability-data-flow) -- **Dashboards and alerting as code.** 5 provisioned dashboards, 79 panels, 32 - alert rules with severity routing and inhibition. No dashboard exists only in - a database. +- **Dashboards and alerting as code.** 5 provisioned dashboards, 79 panels, and + 40 alert rules — 32 metric-based in Prometheus, 8 log-based in Loki — sharing + one Alertmanager routing tree. No dashboard exists only in a database. - **Secrets encrypted in-repo with SOPS + age.** Per-device credentials, decrypted at deploy time into gitignored paths, with `git log` showing which credential rotated and when — but never to what. [Why](docs/adr/0005-secrets-with-sops-and-age.md) - **CI that actually validates the infrastructure.** `docker compose config`, - `promtool`, `amtool`, `alloy fmt`, dashboard-JSON and datasource checks, every - dashboard's PromQL parsed, plus `gitleaks` over the full history. + `promtool`, `amtool`, `alloy fmt`, a real Loki boot to parse the LogQL rules, + dashboard-JSON and datasource checks, every dashboard's PromQL parsed, plus + `gitleaks` over the full history. +- **Supply chain pinned by digest.** Every image carries both a tag and a + `sha256:` digest, so a moved tag cannot change what deploys. CI enforces it; + `make pin-digests` re-resolves them from the registry. - **Documented decisions and runbooks.** Five ADRs covering what was chosen and what was rejected; four runbooks for the operations that are easy to get wrong at 1am. @@ -111,7 +115,7 @@ the internet and nothing more. Full topology and data flow in | Alerting | Alertmanager | Severity routing, inhibition | | Visualisation | Grafana | 5 provisioned dashboards | | Secrets | SOPS + age | Encrypted in-repo | -| CI | GitHub Actions | Lint, config validation, secret scanning | +| CI | GitHub Actions | Lint, config validation, secret scanning, digest pinning | ## Repository layout @@ -121,12 +125,13 @@ the internet and nothing more. Full topology and data flow in │ ├── compose.yaml │ ├── prometheus/ # config, file_sd targets, 32 alert rules │ ├── alertmanager/ # routing and inhibition -│ ├── loki/ # single-binary config +│ ├── loki/ # single-binary config + 8 LogQL rules │ ├── alloy/ # one agent config, used on every host │ ├── snmp-exporter/ # generator.yaml is the source of truth │ └── grafana/ # provisioning + 5 dashboards ├── secrets/ # SOPS-encrypted; see secrets/README.md -├── scripts/ # bootstrap, render, validate, history purge +├── scripts/ # bootstrap, render, validate, pin-digests, purge +├── SECURITY.md # disclosure policy and known exposure ├── docs/ │ ├── architecture.md network.md hardware.md │ ├── observability.md security.md roadmap.md @@ -184,7 +189,12 @@ owner-linked device names, camera placement) are in Historical credential exposure in this repository's git history is documented there too, along with the runbooks to remediate it — including the parts not yet -done. +done. [`SECURITY.md`](SECURITY.md) carries the disclosure policy and a summary of +what is known. + +Container images are pinned by **tag and digest**. A tag is a mutable pointer; a +digest is the content hash, so a moved tag cannot change what gets deployed. CI +enforces it, and `make pin-digests` re-resolves them. ## Roadmap diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4e75cb0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,82 @@ +# Security policy + +This repository documents and configures a private home network. It is not a +product and has no users other than its owner, so "supported versions" does not +really apply — `main` is the only branch that means anything, and it is what the +lab runs. + +What *is* useful here is a clear answer to two questions: what to do if you spot +a problem, and what is already known. + +## Reporting something + +If you find a misconfiguration, a leaked credential, or a weakness in what is +published here, please report it privately rather than opening a public issue: + +- **GitHub Security Advisories** — [open a draft advisory](https://github.com/Gerrrt/HomeLab/security/advisories/new) + (preferred; it stays private until fixed) +- Failing that, a GitHub issue *without* details, asking for a contact. + +Please do not open a public issue containing a working credential, a capture, or +anything that would let someone else reach the network before it can be fixed. + +This is a personal project, so there is no SLA. Realistically: acknowledgement +within a few days, and credential exposure treated as urgent. + +### Please don't + +The lab is a home network, not a bug bounty target. Scanning, probing or +attempting to reach any host described in `docs/network.md` is unwelcome and not +authorised. Everything worth reviewing is in this repository — review the +configuration, not the running system. + +## Known exposure + +Documented rather than quietly fixed, because a known and written-down exposure +is a very different thing from an overlooked one. Full detail in +[`docs/security.md`](docs/security.md). + +| What | Status | +| --- | --- | +| SNMP community committed in plaintext, shared across firewall, switch, UPS and BMC | Removed from `HEAD` and replaced with per-device SOPS-encrypted values. **Still present in git history, and not yet rotated on the devices.** Treat it as public. | +| Grafana `admin`/`admin` with anonymous Admin access enabled | Fixed — anonymous auth off, password from SOPS | +| Passphrase-encrypted TLS private keys under `certificates/` | Removed from `HEAD`, still reachable in history. Purge tooling and a runbook are provided; not yet run. | + +Remediation is tracked in [`docs/roadmap.md`](docs/roadmap.md), with procedures +in [`docs/runbooks/rotate-snmp-community.md`](docs/runbooks/rotate-snmp-community.md) +and [`docs/runbooks/purge-git-history.md`](docs/runbooks/purge-git-history.md). + +[`.gitleaksignore`](.gitleaksignore) enumerates all nine historical findings +individually, with a note on each. It exists so the full-history scan stays +meaningful — a job that is permanently red for a known reason gets ignored, and +then a genuinely new leak goes unnoticed alongside it. It is an acknowledgement, +not a fix, and it gets deleted once the purge has run. + +## What this repository will not contain + +Deliberate omissions, so their absence is not mistaken for an oversight: + +- **Full MAC addresses.** Truncated to the vendor OUI, which keeps the useful + half and drops the unique identifier. +- **Owner-linked device names**, and no room labelled as a child's. +- **Camera-to-room mapping.** That there are cameras is fine; which one covers + which door is not. +- **The WAN address, firewall rule bodies, and Wi-Fi configuration.** +- **Any plaintext credential.** Secrets are SOPS + age encrypted; the private key + never enters the repository. See [`secrets/README.md`](secrets/README.md). + +## Controls in CI + +Every push and pull request runs: + +- **`gitleaks`** over the working tree *and* full history, with rules for SNMP + communities, inline Grafana passwords, PEM private keys and age secret keys. +- An assertion that every `secrets/*.sops.yaml` is genuinely encrypted, which + needs no ability to decrypt. +- An assertion that no rendered or decrypted artefact — `.env`, `.rendered/`, + `.purge-secrets.txt` — is ever a tracked file. +- Verification that every container image is pinned by **tag *and* digest**, so a + moved tag cannot silently change what is deployed. + +See [`docs/security.md`](docs/security.md) for the threat model and segmentation +rationale. diff --git a/docs/observability.md b/docs/observability.md index edd9c74..e8b3684 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -70,6 +70,35 @@ expression in every panel is syntactically valid. ## Alerting +40 rules in total: 32 metric-based in `prometheus/rules/`, and 8 log-based in +`loki/rules/`. + +### Log-based (Loki ruler) + +Some conditions only exist in logs. A metric confirms sshd is running; only the +log shows it rejecting forty passwords in five minutes. `loki/rules/security.rules.yaml` +covers SSH brute force, SSH accepted from outside VLAN 50/99, repeated sudo +failures, user/group creation, kernel OOM kills, read-only remounts and disk I/O +errors. + +They use the same `severity` and `category` labels as the Prometheus rules and +are sent to the same Alertmanager, so routing and inhibition are shared. + +Loki's local ruler reads `//`, and with `auth_enabled: false` +the tenant is literally `fake` — hence the `loki/rules:/etc/loki/rules/fake` +mount in `compose.yaml`. Getting that path wrong produces no error, just a ruler +that silently evaluates nothing. + +`promtool` cannot validate these; it parses PromQL and rejects every LogQL +stream selector. `scripts/check_loki_rules.sh` boots the pinned Loki image with +the rules mounted and fails on a parse error, then asserts the ruler actually +evaluated them. Note that `loki -verify-config` is *not* sufficient on its own — +it validates the config file and never opens the rule files. A file containing +`count_over_time({{{BROKEN` passes `-verify-config` and is caught only by the +boot check. + +### Metric-based (Prometheus) + 32 rules across four files in `prometheus/rules/`: | File | Covers | diff --git a/docs/roadmap.md b/docs/roadmap.md index ed89ec4..b1edb03 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -33,9 +33,6 @@ inventory. Ordered roughly by how much it matters. inventory currently agree on the address but describe different things. - [ ] Capture dashboard screenshots for the README once the stack has a few days of real data. → [`images/README.md`](images/README.md) -- [ ] Loki alerting rules — the ruler is configured and pointed at Alertmanager - but no log-based rules exist yet. Repeated SSH auth failure is the obvious - first one. ## Infrastructure @@ -70,3 +67,7 @@ inventory. Ordered roughly by how much it matters. - [x] Add alerting (32 rules) and Alertmanager routing - [x] Move secrets to SOPS + age - [x] Add CI: lint, config validation, secret scanning +- [x] Pin every image by digest, not just tag, with drift detection in CI +- [x] Add SECURITY.md with a disclosure policy and known-exposure summary +- [x] Loki alerting rules (8) for auth, SSH brute force and disk/OOM events, + validated in CI by booting the pinned Loki image against them diff --git a/docs/runbooks/purge-git-history.md b/docs/runbooks/purge-git-history.md index 753248c..58031b9 100644 --- a/docs/runbooks/purge-git-history.md +++ b/docs/runbooks/purge-git-history.md @@ -18,7 +18,13 @@ Verify for yourself before and after: ```bash git show 647d90a~1:certificates/Gandalf.Gondor.Lab/ca-key.pem | head -1 -git log --all -S '7H3r315N05p00N' --oneline + +# One literal per line, so loop — `$(cat ...)` would fold the whole file into a +# single search string and match nothing. +while IFS= read -r s; do + [ -z "$s" ] && continue + git log --all -S "$s" --oneline +done < .purge-secrets.txt ``` Deleting a file in a later commit does not remove it from history. `git show` @@ -37,9 +43,21 @@ while after a force-push, and forks keep it indefinitely. 4. Install the tool: ```bash - pipx install git-filter-repo + sudo apt install git-filter-repo # or: pipx install git-filter-repo + ``` + +5. **Write the literals to redact into `.purge-secrets.txt`**, one per line: + + ```bash + printf '%s\n' 'the-old-snmp-community' > .purge-secrets.txt ``` + This file is gitignored on purpose. The script reads from it rather than + hardcoding the value, because a purge tool that contains a copy of the secret + leaves the secret in the repository after a successful purge — which is + exactly what the first version of this script did. Delete the file when you + are finished. + ## Dry run ```bash @@ -53,7 +71,8 @@ inspection — check it before continuing: ```bash cd /tmp//repo git log --oneline | head -git log --all -S '7H3r315N05p00N' --oneline # must be empty +while IFS= read -r s; do [ -n "$s" ] && git log --all -S "$s" --oneline; done \ + < .purge-secrets.txt # must print nothing ``` ## Execute @@ -101,7 +120,10 @@ git push --force --tags origin ```bash git log --all --oneline -- certificates/ # empty -git grep -I '7H3r315N05p00N' $(git rev-list --all) # no matches +# -F: a rotated community is a random string and may contain regex characters. +while IFS= read -r s; do + [ -n "$s" ] && git grep -IF -e "$s" $(git rev-list --all) +done < .purge-secrets.txt # no matches gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" ``` diff --git a/scripts/check_loki_rules.sh b/scripts/check_loki_rules.sh new file mode 100755 index 0000000..e1cf121 --- /dev/null +++ b/scripts/check_loki_rules.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# +# Validate the Loki alerting rules. +# +# There is no `promtool check rules` equivalent for LogQL: promtool parses +# PromQL and rejects every stream selector in these files. The only tool that +# genuinely understands LogQL is Loki itself, so this boots the pinned Loki +# image against a throwaway config with the rules mounted, and fails if the +# ruler reports a parse error. +# +# `-verify-config` alone is NOT sufficient — it validates the config file and +# never looks at the rule files. Verified: a rule file containing +# `count_over_time({{{BROKEN` passes -verify-config and is only caught here. +# +# Usage: scripts/check_loki_rules.sh + +# -e is on: a failed cp or config rewrite must not produce a cheerful PASS. +# The one command allowed to fail is the timeout below, which is guarded. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK="${REPO_ROOT}/stacks/observability" +# Resolved from compose.yaml — see scripts/image-for.sh. +LOKI_IMAGE="$("${REPO_ROOT}/scripts/image-for.sh" loki)" +BOOT_SECONDS="${BOOT_SECONDS:-45}" + +die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } + +RULES_DIR="${STACK}/loki/rules" +[[ -d "${RULES_DIR}" ]] || die "no rules directory at ${RULES_DIR}" + +WORK="$(mktemp -d)" +# Loki writes as its own uid; never let cleanup failure mask the result. +trap 'rm -rf "${WORK}" 2>/dev/null || true' EXIT +# auth_enabled is false, so Loki's local ruler looks under /fake/. +mkdir -p "${WORK}/rules/fake" "${WORK}/data" +cp "${RULES_DIR}"/*.yaml "${WORK}/rules/fake/" + +# The Loki image runs as uid 10001, while mktemp -d creates a 0700 directory +# owned by the invoking user. Without this the container cannot read its own +# config, exits within seconds, and the run looks like "the ruler evaluated +# nothing" rather than a permissions problem. Throwaway directory, so the broad +# mode is fine. +chmod -R a+rwX "${WORK}" + +# PyYAML is not guaranteed on a clean runner, and the failure mode without this +# guard is an opaque ModuleNotFoundError inside a heredoc. +if ! python3 -c 'import yaml' 2>/dev/null; then + info "installing PyYAML" + python3 -m pip install --quiet --disable-pip-version-check pyyaml >/dev/null 2>&1 \ + || die "PyYAML is required and could not be installed" +fi + +# Rewrite every path in the real config to point inside the scratch dir, so the +# rules are checked against the same settings production uses. +python3 - "$STACK/loki/loki-config.yaml" "${WORK}" > "${WORK}/loki.yaml" <<'PY' +import sys, yaml +cfg = yaml.safe_load(open(sys.argv[1])) +work = sys.argv[2] +cfg["common"]["path_prefix"] = f"{work}/data" +cfg["common"]["storage"]["filesystem"] = { + "chunks_directory": f"{work}/data/chunks", "rules_directory": f"{work}/data/rules"} +cfg["storage_config"]["tsdb_shipper"] = { + "active_index_directory": f"{work}/data/index", "cache_location": f"{work}/data/cache"} +cfg["storage_config"]["filesystem"] = {"directory": f"{work}/data/chunks"} +cfg["compactor"]["working_directory"] = f"{work}/data/compactor" +cfg["ruler"]["storage"]["local"]["directory"] = f"{work}/rules" +cfg["ruler"]["rule_path"] = f"{work}/data/rules-temp" +yaml.safe_dump(cfg, sys.stdout) +PY + +# Rule groups use interval: 1m in production, and Loki jitters a group's first +# evaluation across that interval — so a short boot window can legitimately see +# no evaluations, which looks identical to a misconfigured tenant path. Shorten +# the interval in the throwaway copies only; the committed rules and their LogQL +# are untouched. +python3 - "${WORK}/rules/fake" <<'SPEEDUP' +import pathlib, sys, yaml +for f in pathlib.Path(sys.argv[1]).glob("*.yaml"): + doc = yaml.safe_load(f.read_text()) + for group in doc.get("groups", []): + group["interval"] = "5s" + f.write_text(yaml.safe_dump(doc)) +SPEEDUP + +n_rules="$(grep -ch '^ *- alert:' "${RULES_DIR}"/*.yaml | paste -sd+ | bc || true)" +info "checking ${n_rules} Loki rule(s)" + +if command -v loki >/dev/null 2>&1; then + RUN=(loki) +elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + # --user: without it Loki writes as uid 10001 and the runner cannot delete + # the scratch directory afterwards, filling the log with rm errors. + RUN=(docker run --rm --user "$(id -u):$(id -g)" \ + -v "${WORK}:${WORK}" -w "${WORK}" --entrypoint loki "${LOKI_IMAGE}") +else + printf '\033[0;33m SKIP\033[0m no loki binary and no docker daemon\n' + exit 0 +fi + +OUT="${WORK}/loki.log" +# Re-apply after loki.yaml was generated above, so the container user can read +# it too. +chmod -R a+rwX "${WORK}" 2>/dev/null || true +# Loki runs until killed, so a 124 from timeout is the expected outcome. +rc=0 +timeout "${BOOT_SECONDS}" "${RUN[@]}" \ + -config.file="${WORK}/loki.yaml" -target=all \ + -server.http-listen-port=3197 > "${OUT}" 2>&1 || rc=$? + +if grep -qiE 'parse error|failed to parse|syntax error' "${OUT}"; then + printf '\033[0;31m FAIL\033[0m LogQL parse error in the Loki rules\n' + grep -iE 'parse error|failed to parse|syntax error' "${OUT}" | head -5 + exit 1 +fi + +# Confirm the ruler actually loaded them. A silent "no errors" is not evidence +# if the ruler never read the files at all. +evaluated="$(grep -oE 'rule_name="?[A-Za-z_][A-Za-z0-9_]*' "${OUT}" \ + | sed 's/^rule_name="\?//' | sort -u | wc -l || true)" +if ((evaluated == 0)); then + printf '\033[0;31m FAIL\033[0m the ruler evaluated no rules\n' + # rc 124 is the timeout we expect (Loki runs until killed). Anything else + # means Loki died early — usually it could not read the mounted files. + if ((rc != 124)); then + printf ' loki exited early with status %s; last output:\n' "${rc}" + else + printf ' loki ran the full %ss but never evaluated a rule.\n' "${BOOT_SECONDS}" + printf ' Check the ruler tenant path (/fake/).\n' + fi + tail -n 15 "${OUT}" | sed 's/^/ /' + exit 1 +fi + +printf '\033[0;32m PASS\033[0m %s Loki rule(s) parsed, %s evaluated by the ruler\n' \ + "${n_rules}" "${evaluated}" diff --git a/scripts/image-for.sh b/scripts/image-for.sh new file mode 100755 index 0000000..94c8196 --- /dev/null +++ b/scripts/image-for.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Print the pinned image for a compose service. +# +# Image versions must live in exactly one place: compose.yaml, which is what +# Dependabot updates. They were previously duplicated into ci.yml, validate.sh +# and check_loki_rules.sh, and the duplicates went stale the moment Dependabot +# merged a bump — CI ended up validating configs against Prometheus v3.1.0 while +# the stack deployed v3.13.2. Validation against the wrong version is worse than +# no validation, because it still reports green. +# +# Deliberately parses the text rather than using PyYAML or `docker compose +# config`: this runs before any of those are guaranteed present, and needs no +# .env for the ${VAR:?} guards. +# +# Usage: +# scripts/image-for.sh full reference, digest included +# scripts/image-for.sh --tag-only repo:tag, digest stripped +# +# --tag-only exists for the one case where a digest must NOT be carried across: +# deriving prom/snmp-generator from prom/snmp-exporter. They share a version but +# are different images, so reusing the exporter's digest produces a reference +# that cannot be pulled. + +set -euo pipefail + +TAG_ONLY=0 +if [[ "${1:-}" == "--tag-only" ]]; then + TAG_ONLY=1 + shift +fi +SERVICE="${1:?usage: image-for.sh [--tag-only] }" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE="${COMPOSE_FILE:-${REPO_ROOT}/stacks/observability/compose.yaml}" + +[[ -f "${COMPOSE}" ]] || { printf 'no compose file at %s\n' "${COMPOSE}" >&2; exit 1; } + +image="$(awk -v svc="${SERVICE}" ' + # Track whether we are inside the services: block. + /^services:/ { in_services = 1; next } + /^[^[:space:]#]/ { in_services = 0 } + !in_services { next } + # A two-space-indented key is a service name. + /^ [A-Za-z0-9_.-]+:/ { + line = $0 + sub(/^ /, "", line); sub(/:.*/, "", line) + current = line + } + current == svc && $1 == "image:" { print $2; exit } +' "${COMPOSE}")" + +if [[ -z "${image}" ]]; then + printf 'no image found for service %s in %s\n' "${SERVICE}" "${COMPOSE}" >&2 + exit 1 +fi + +# Strip the @sha256:... suffix when only the tag was asked for. +((TAG_ONLY)) && image="${image%%@*}" + +printf '%s\n' "${image}" diff --git a/scripts/pin-digests.sh b/scripts/pin-digests.sh new file mode 100755 index 0000000..d005819 --- /dev/null +++ b/scripts/pin-digests.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# +# Pin every compose image to an immutable digest. +# +# A tag is a mutable pointer. `prom/prometheus:v3.13.2` is whatever the +# publisher last pushed under that name — a tag can be moved, and a compromised +# or coerced publisher can move it silently. A digest is the content hash: it +# either matches or the pull fails. +# +# Images are written as `repo:tag@sha256:...`, keeping both. The tag stays +# human-readable and tells you which version you are on at a glance; the digest +# is what Docker actually enforces. Dependabot understands this form and updates +# both halves together. +# +# Resolves through the registry HTTP API rather than `docker pull`, so it works +# without a daemon and without downloading hundreds of megabytes of layers. +# +# Usage: +# scripts/pin-digests.sh # report drift, change nothing +# scripts/pin-digests.sh --write # update compose.yaml in place + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE="${COMPOSE_FILE:-${REPO_ROOT}/stacks/observability/compose.yaml}" +WRITE=0 +[[ "${1:-}" == "--write" ]] && WRITE=1 + +die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } + +[[ -f "${COMPOSE}" ]] || die "no compose file at ${COMPOSE}" +command -v curl >/dev/null 2>&1 || die "curl is required" + +# Resolve repo:tag to its manifest digest. +# +# Accepts all four media types: multi-arch images return an OCI index or a +# Docker manifest list, single-arch ones return a plain manifest. Omitting any +# of these makes the registry return a 404 or the wrong digest for some images. +resolve_digest() { + local repo="$1" tag="$2" token digest + + # Official images live under library/ but are written without it. + [[ "${repo}" == */* ]] || repo="library/${repo}" + + token="$(curl -sS --fail --max-time 30 \ + "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')" \ + || { printf 'could not get a pull token for %s\n' "${repo}" >&2; return 1; } + + digest="$(curl -sS --fail --max-time 30 -I \ + -H "Authorization: Bearer ${token}" \ + -H "Accept: application/vnd.oci.image.index.v1+json" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + "https://registry-1.docker.io/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | awk 'tolower($1) == "docker-content-digest:" { print $2 }')" \ + || { printf 'manifest request failed for %s:%s\n' "${repo}" "${tag}" >&2; return 1; } + + [[ "${digest}" == sha256:* ]] || { printf 'no digest returned for %s:%s\n' "${repo}" "${tag}" >&2; return 1; } + printf '%s\n' "${digest}" +} + +mapfile -t refs < <(awk '$1 == "image:" { print $2 }' "${COMPOSE}") +((${#refs[@]} > 0)) || die "no image: lines found in ${COMPOSE}" + +changed=0 +declare -a updates=() + +for ref in "${refs[@]}"; do + repo_tag="${ref%%@*}" + current_digest="" + [[ "${ref}" == *@* ]] && current_digest="${ref##*@}" + + repo="${repo_tag%:*}" + tag="${repo_tag##*:}" + [[ "${repo}" != "${tag}" ]] || die "image ${ref} has no tag — refusing to pin a floating reference" + + if ! digest="$(resolve_digest "${repo}" "${tag}")"; then + die "could not resolve ${repo}:${tag}" + fi + + if [[ "${current_digest}" == "${digest}" ]]; then + printf ' \033[0;32mok\033[0m %s\n' "${repo_tag}" + elif [[ -z "${current_digest}" ]]; then + printf ' \033[0;33munpinned\033[0m %s -> %s\n' "${repo_tag}" "${digest}" + updates+=("${ref}|${repo_tag}@${digest}") + changed=1 + else + printf ' \033[0;31mDRIFT\033[0m %s\n pinned: %s\n registry: %s\n' \ + "${repo_tag}" "${current_digest}" "${digest}" + updates+=("${ref}|${repo_tag}@${digest}") + changed=1 + fi +done + +if ((changed == 0)); then + printf '\n\033[0;32mall images pinned to their current digest\033[0m\n' + exit 0 +fi + +if ((WRITE == 0)); then + printf '\n%s image(s) need pinning. Re-run with --write to apply.\n' "${#updates[@]}" + # Non-zero so CI can use this as a drift check. + exit 1 +fi + +for u in "${updates[@]}"; do + old="${u%%|*}"; new="${u##*|}" + # Fixed-string replace via python: digests contain no regex metacharacters, + # but repo names contain / and . which sed would need escaping for. + python3 - "${COMPOSE}" "${old}" "${new}" <<'PY' +import sys, pathlib +path, old, new = sys.argv[1], sys.argv[2], sys.argv[3] +p = pathlib.Path(path) +text = p.read_text(encoding="utf-8") +if old not in text: + sys.exit(f"expected to find {old} in {path}") +p.write_text(text.replace(old, new), encoding="utf-8") +PY +done + +info "updated ${#updates[@]} image reference(s) in $(basename "${COMPOSE}")" +printf '\033[0;33mReview the diff, then re-run make validate.\033[0m\n' diff --git a/scripts/purge-history.sh b/scripts/purge-history.sh index 2a87cc1..4c6e24c 100755 --- a/scripts/purge-history.sh +++ b/scripts/purge-history.sh @@ -15,14 +15,34 @@ # running it, and rotate the credentials regardless — assume anything that was # ever pushed to a public repository is compromised. # +# The literals to scrub are read from a gitignored file, NOT hardcoded here. +# An earlier version of this script embedded the leaked community string +# directly — which meant that after a successful history rewrite the secret was +# still sitting in the working tree, in the very script written to remove it. +# The dry run proved it: `git grep` on the rewritten mirror still found the +# string in this file. A purge tool must not itself be a copy of the secret. +# # Usage: # scripts/purge-history.sh --dry-run # rewrite a scratch mirror, report # scripts/purge-history.sh --execute # rewrite ./ for real +# +# --secrets-file PATH literals to redact, one per line +# (default: .purge-secrets.txt, gitignored) set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -MODE="${1:---dry-run}" +MODE="--dry-run" +SECRETS_LIST="${REPO_ROOT}/.purge-secrets.txt" + +while (($#)); do + case "$1" in + --dry-run|--execute) MODE="$1"; shift ;; + --secrets-file) SECRETS_LIST="${2:?--secrets-file needs a path}"; shift 2 ;; + *) printf 'usage: %s [--dry-run|--execute] [--secrets-file PATH]\n' \ + "$(basename "$0")" >&2; exit 1 ;; + esac +done die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } @@ -36,15 +56,37 @@ if ! git filter-repo --help >/dev/null 2>&1; then https://github.com/newren/git-filter-repo" fi -# The literal to scrub. Kept here rather than in a tracked replacements file so -# the string itself is not re-committed by the very script meant to remove it. -LEAKED_COMMUNITY='7H3r315N05p00N!' +if [[ ! -f "${SECRETS_LIST}" ]]; then + die "no secrets list at ${SECRETS_LIST} + +Create it with one literal per line — the values to redact from history: + + printf '%s\\n' 'the-old-snmp-community' > .purge-secrets.txt + +It is gitignored, so the secret never enters a commit. Delete it when done. +Pass --secrets-file PATH to use a different location." +fi REPLACEMENTS="$(mktemp)" PATHS_FILE="$(mktemp)" trap 'rm -f "${REPLACEMENTS}" "${PATHS_FILE}"' EXIT +chmod 600 "${REPLACEMENTS}" + +# Read the list once into an array. Everything downstream uses the array rather +# than re-reading the file, so the secret is touched on disk exactly once. +LITERALS=() +while IFS= read -r literal; do + [[ -z "${literal}" || "${literal}" =~ ^[[:space:]]*# ]] && continue + LITERALS+=("${literal}") +done < "${SECRETS_LIST}" +((${#LITERALS[@]} > 0)) || die "${SECRETS_LIST} contains no literals" + +# git-filter-repo replacement syntax: ==> +for literal in "${LITERALS[@]}"; do + printf '%s==>REDACTED-ROTATED-CREDENTIAL\n' "${literal}" >> "${REPLACEMENTS}" +done +info "loaded ${#LITERALS[@]} literal(s) to redact" -printf '%s==>REDACTED-ROTATED-CREDENTIAL\n' "${LEAKED_COMMUNITY}" > "${REPLACEMENTS}" cat > "${PATHS_FILE}" <<'EOF' certificates/ EOF @@ -65,10 +107,41 @@ report() { printf '\033[0;32m PASS\033[0m no commit touches certificates/\n' fi - if git -C "${target}" grep -qI "${LEAKED_COMMUNITY}" "$(git -C "${target}" rev-list --all)" -- 2>/dev/null; then - printf '\033[0;31m FAIL\033[0m leaked community string still present\n' + # Check every literal across every commit. The earlier version only checked + # one string and reported PASS while the same string was still sitting in the + # working tree of the rewritten repo, inside this script. Scan the checked-out + # tree as well as history. + # Detect by OUTPUT PRESENCE, never by exit status. + # + # `git rev-list --all | xargs -I{} git grep -q ... {}` looks correct and is + # not: xargs returns 123 when *any* invocation exits 1-125, so a literal + # present in some commits but absent from others makes the whole pipeline + # non-zero and the check silently reports clean. Measured on this repo: a + # string present in 17 of 49 commits was reported as absent. A security check + # that fails open is worse than no check. + # -F throughout: a rotated community is a random string and may contain regex + # metacharacters. Without it, values containing [ ] ( ) . * + ? are either + # mis-parsed or make grep error out — and a grep that errors reports no match, + # which is one more way to fail open. + local leaked=0 literal hit + local ignore_name; ignore_name="$(basename "${SECRETS_LIST}")" + for literal in "${LITERALS[@]}"; do + # `|| true` is load-bearing: under `set -o pipefail` the xargs 123 above + # would abort the whole script before this check ever printed anything. + hit="$(git -C "${target}" rev-list --all \ + | xargs -r -I{} git -C "${target}" grep -lIF -e "${literal}" {} -- 2>/dev/null \ + | head -n1 || true)" + [[ -n "${hit}" ]] && { leaked=1; printf ' history: %s\n' "${hit}"; } + + hit="$(grep -rlIF -e "${literal}" "${target}" \ + --exclude-dir=.git --exclude="${ignore_name}" 2>/dev/null | head -n1 || true)" + [[ -n "${hit}" ]] && { leaked=1; printf ' worktree: %s\n' "${hit}"; } + done + + if ((leaked)); then + printf '\033[0;31m FAIL\033[0m a redacted literal is still present (history or worktree)\n' else - printf '\033[0;32m PASS\033[0m leaked community string absent from all commits\n' + printf '\033[0;32m PASS\033[0m no redacted literal in any commit or in the worktree\n' fi printf ' commits: %s\n' "$(git -C "${target}" rev-list --all --count)" diff --git a/scripts/validate.sh b/scripts/validate.sh index aa4336f..be08dd1 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash # -# Everything CI runs, runnable locally. Uses containers for the Prometheus and -# Alertmanager tooling so the versions match what actually runs in production. +# Everything CI runs, runnable locally. +# +# Image versions come from compose.yaml via scripts/image-for.sh, so the +# containers used here are the ones actually deployed. A locally installed +# binary is preferred when present for speed — if yours is a different version +# from the pin, CI is the authority. # # Usage: scripts/validate.sh @@ -11,9 +15,11 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${REPO_ROOT}" || exit 1 STACK="stacks/observability" -PROM_IMAGE="prom/prometheus:v3.1.0" -AM_IMAGE="prom/alertmanager:v0.28.0" -ALLOY_IMAGE="grafana/alloy:v1.6.1" +# Resolved from compose.yaml so Dependabot's bumps reach the checks. Hardcoding +# these meant CI validated Prometheus v3.1.0 configs while the stack ran v3.13.2. +PROM_IMAGE="$(./scripts/image-for.sh prometheus)" +AM_IMAGE="$(./scripts/image-for.sh alertmanager)" +ALLOY_IMAGE="$(./scripts/image-for.sh alloy)" FAILED=0 pass() { printf '\033[0;32m PASS\033[0m %s\n' "$*"; } @@ -111,7 +117,7 @@ else fi # `fmt --test` fails on a syntax error and on non-canonical formatting. It does -# not check component configuration — v1.6.1 has no `validate` subcommand. +# not check component configuration — Alloy has no `validate` subcommand. if ((${#ALLOY[@]})); then if "${ALLOY[@]}" fmt --test "${STACK}/alloy/config.alloy" >/dev/null 2>&1; then pass "alloy fmt --test" @@ -123,6 +129,15 @@ else skip "no alloy binary and no docker daemon" fi +# --------------------------------------------------------------------------- +head_ "Loki rules" +# --------------------------------------------------------------------------- +if ./scripts/check_loki_rules.sh; then + : +else + FAILED=1 +fi + # --------------------------------------------------------------------------- head_ "Grafana dashboards" # --------------------------------------------------------------------------- @@ -180,12 +195,16 @@ else skip "gitleaks not installed" fi -# Cheap belt-and-braces check that no rendered/decrypted artefact is staged. +# Cheap belt-and-braces check that no rendered or decrypted artefact is staged. +# gitleaks cannot cover .purge-secrets.txt — it is gitignored (so the filesystem +# scan skips it) and holds bare literals with no keyword context to match. Being +# untracked is the control. if git ls-files --error-unmatch "${STACK}/.env" >/dev/null 2>&1 \ - || git ls-files "${STACK}/snmp-exporter/.rendered" | grep -q .; then - fail "a rendered or decrypted file is tracked by git" + || git ls-files "${STACK}/snmp-exporter/.rendered" | grep -q . \ + || git ls-files | grep -q '\.purge-secrets\.txt'; then + fail "a rendered, decrypted or purge-secrets file is tracked by git" else - pass "no rendered or decrypted files tracked" + pass "no rendered, decrypted or purge-secrets files tracked" fi printf '\n' diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index 16d00e8..c6d7d86 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -28,7 +28,7 @@ services: # --------------------------------------------------------------------------- prometheus: <<: *service-defaults - image: prom/prometheus:v3.13.2 + image: prom/prometheus:v3.13.2@sha256:508729e0e2d18e11fd742a5a5ca70e557b940a93948c3c95fd0123a6fd538b69 container_name: prometheus user: "65534:65534" command: @@ -57,7 +57,7 @@ services: # --------------------------------------------------------------------------- alertmanager: <<: *service-defaults - image: prom/alertmanager:v0.33.1 + image: prom/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d container_name: alertmanager user: "65534:65534" command: @@ -87,12 +87,15 @@ services: # --------------------------------------------------------------------------- loki: <<: *service-defaults - image: grafana/loki:3.7.4 + image: grafana/loki:3.7.4@sha256:87f0a067673756a3cede1bcbf0c74875f7df9b09fddb53e399d0c576f756cfcc container_name: loki user: "10001:10001" command: -config.file=/etc/loki/loki-config.yaml volumes: - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + # Loki's local ruler reads //. auth_enabled is false, + # so the tenant is "fake". + - ./loki/rules:/etc/loki/rules/fake:ro - loki-data:/loki ports: - "${BIND_ADDR:-0.0.0.0}:${LOKI_PORT:-3100}:3100" @@ -110,7 +113,7 @@ services: # --------------------------------------------------------------------------- snmp-exporter: <<: *service-defaults - image: prom/snmp-exporter:v0.30.1 + image: prom/snmp-exporter:v0.30.1@sha256:e5fd5e8b43ace6c088fe9bf0b37b7fff0e04380bee352be7ec41b853a4dd5859 container_name: snmp-exporter user: "65534:65534" command: @@ -132,7 +135,7 @@ services: # --------------------------------------------------------------------------- grafana: <<: *service-defaults - image: grafana/grafana-oss:13.0.2 + image: grafana/grafana-oss:13.0.2@sha256:5dad0df181cb644a14e13617b913b261a54f7d4fd4510721dba420929f35bea2 container_name: grafana environment: GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} @@ -172,7 +175,7 @@ services: # --------------------------------------------------------------------------- alloy: <<: *service-defaults - image: grafana/alloy:v1.18.0 + image: grafana/alloy:v1.18.0@sha256:491b0578c04983fd54fe99b587b6fab4404dc46d0dc16677bd6b00cc1140b308 container_name: alloy privileged: true environment: diff --git a/stacks/observability/loki/loki-config.yaml b/stacks/observability/loki/loki-config.yaml index b162003..5ff7f05 100644 --- a/stacks/observability/loki/loki-config.yaml +++ b/stacks/observability/loki/loki-config.yaml @@ -61,7 +61,11 @@ ruler: storage: type: local local: - directory: /loki/rules + # Loki's local ruler reads //*.yaml. With + # auth_enabled: false the tenant is literally "fake", so compose mounts + # loki/rules/ at /etc/loki/rules/fake. Kept outside /loki so the + # read-only rule mount does not collide with the data volume. + directory: /etc/loki/rules rule_path: /loki/rules-temp alertmanager_url: http://alertmanager:9093 enable_api: true diff --git a/stacks/observability/loki/rules/security.rules.yaml b/stacks/observability/loki/rules/security.rules.yaml new file mode 100644 index 0000000..5e0f624 --- /dev/null +++ b/stacks/observability/loki/rules/security.rules.yaml @@ -0,0 +1,131 @@ +--- +# Log-based alerts, evaluated by Loki's ruler and sent to the same Alertmanager +# as the Prometheus rules. +# +# These exist because some conditions are only visible in logs. A metric can +# tell you sshd is running; only the log tells you it rejected forty passwords +# in five minutes. +# +# Loki rule files use the Prometheus alerting schema with LogQL expressions, so +# `severity` and `category` route through exactly the same Alertmanager tree. +groups: + - name: authentication + interval: 1m + rules: + - alert: SshBruteForce + # /var/log/auth.log, labelled log_type="authlog" by config.alloy. + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "(?i)failed (password|publickey)" [5m]) + ) > 20 + for: 2m + labels: + severity: warning + category: security + annotations: + summary: "{{ $labels.host }}: {{ $value }} failed SSH auths in 5 minutes" + description: >- + Sustained authentication failure. On a management-VLAN host this + should be close to zero — nothing on VLAN 99 is internet-facing. + + - alert: SshBruteForceSevere + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "(?i)failed (password|publickey)" [5m]) + ) > 100 + for: 1m + labels: + severity: critical + category: security + annotations: + summary: "{{ $labels.host }}: {{ $value }} failed SSH auths in 5 minutes" + + - alert: SshLoginFromUnexpectedSubnet + # Accepted logins should only ever originate from Hicks (10.0.50.0/24) + # or from within management itself. Anything else means a firewall rule + # is not doing what the documentation claims. + expr: | + sum by (host) ( + count_over_time( + {log_type="authlog"} + |~ "Accepted (password|publickey)" + != "10.0.50." + != "10.0.99." + [10m] + ) + ) > 0 + for: 1m + labels: + severity: critical + category: security + annotations: + summary: "{{ $labels.host }}: SSH login accepted from outside VLAN 50/99" + description: >- + Either segmentation is broken or the source ranges in this rule are + stale. Both are worth knowing about immediately. + + - alert: SudoFailure + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "sudo:.*authentication failure" [10m]) + ) > 5 + for: 5m + labels: + severity: warning + category: security + annotations: + summary: "{{ $labels.host }}: repeated sudo authentication failures" + + - alert: NewUserOrGroupCreated + expr: | + sum by (host) ( + count_over_time({log_type="authlog"} |~ "(useradd|groupadd|usermod).*new (user|group)" [10m]) + ) > 0 + labels: + severity: warning + category: security + annotations: + summary: "{{ $labels.host }}: a user or group was created" + description: >- + Expected during provisioning, worth a second look otherwise. + + - name: system + interval: 1m + rules: + - alert: KernelOomKill + # The kernel's own record. Complements ContainerOomKilled in + # containers.rules.yaml, which only sees cgroup events. + expr: | + sum by (host) ( + count_over_time({log_type=~"syslog|varlog"} |~ "Out of memory: Kill(ed)? process" [15m]) + ) > 0 + labels: + severity: warning + category: capacity + annotations: + summary: "{{ $labels.host }}: kernel OOM killer fired" + + - alert: FilesystemRemountedReadOnly + expr: | + sum by (host) ( + count_over_time({log_type=~"syslog|varlog"} |~ "Remounting filesystem read-only" [15m]) + ) > 0 + labels: + severity: critical + category: hardware + annotations: + summary: "{{ $labels.host }}: filesystem remounted read-only" + description: >- + Almost always failing storage. The monitoring host runs on a + thirteen-year-old laptop SSD, so this is not hypothetical. + + - alert: DiskIoErrors + expr: | + sum by (host) ( + count_over_time({log_type=~"syslog|varlog"} |~ "(I/O error|ata[0-9]+.*failed|SMART.*FAILED)" [15m]) + ) > 0 + labels: + severity: critical + category: hardware + annotations: + summary: "{{ $labels.host }}: disk I/O errors in the kernel log"