diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e310f5f..293f737 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,9 +146,16 @@ jobs: -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \ fmt --test "$STACK/alloy/config.alloy" + # `docker compose config` happily accepts a service_healthy dependency on + # a service that can never report healthy — the stack then hangs at deploy + # time with no error at all. Loki's distroless image has no wget to probe + # with, which is exactly how that happened here. + - name: Verify health dependencies are satisfiable + run: python3 scripts/check_compose_health.py + # promtool cannot check these — it parses PromQL and rejects every LogQL - # stream selector. Loki itself is the only thing that understands them, - # so the checker boots the pinned image with the rules mounted. + # stream selector. Loki itself is the only thing that understands them, so + # the checker boots the pinned image with the rules mounted. - name: Validate Loki rules run: ./scripts/check_loki_rules.sh diff --git a/.gitignore b/.gitignore index af87747..44f7a56 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ data/ # ---- Local tooling ---- .venv/ node_modules/ +__pycache__/ +*.py[cod] *.log *.tmp *.bak diff --git a/.yamllint.yaml b/.yamllint.yaml index 4a500f6..21e86b4 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -3,6 +3,9 @@ extends: default ignore: | stacks/observability/snmp-exporter/snmp.yaml + # SOPS output: machine-generated ciphertext, not hand-written YAML. It does + # not follow comment/indent conventions and never will. + secrets/*.sops.yaml stacks/observability/snmp-exporter/.rendered/ node_modules/ .venv/ diff --git a/Makefile b/Makefile index 264fae9..38442e8 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,10 @@ check-rules: ## Validate Prometheus rules and config promtool check config $(STACK_DIR)/prometheus/prometheus.yaml promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml +.PHONY: check-compose-health +check-compose-health: ## Verify compose health dependencies can be satisfied + python3 scripts/check_compose_health.py + .PHONY: check-loki-rules check-loki-rules: ## Validate Loki (LogQL) alerting rules ./scripts/check_loki_rules.sh diff --git a/scripts/check_compose_health.py b/scripts/check_compose_health.py new file mode 100755 index 0000000..9c4e37d --- /dev/null +++ b/scripts/check_compose_health.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Check that compose health dependencies can actually be satisfied. + +`depends_on: : condition: service_healthy` waits for to report +healthy. If declares no healthcheck it never will, and everything +downstream hangs forever with no error — the stack simply never finishes +starting. + +That happened here: Loki's image is gcr.io/distroless/static:nonroot, which +contains only /usr/bin/loki — no shell, no wget, no curl. The healthcheck +execed wget, which cannot exist, so Loki was permanently "starting" and both +Grafana and Alloy blocked on it. Nothing logged an error; `make up` just sat +there. + +Also flags healthchecks that exec a binary the image is unlikely to provide, +since that is the same failure wearing a different hat. + +Usage: scripts/check_compose_health.py [compose.yaml] +""" +from __future__ import annotations + +import pathlib +import subprocess +import sys + +# PyYAML is not guaranteed on a clean runner, and this script gates CI. Install +# it rather than failing a green compose file on a missing library — the same +# thing scripts/check_loki_rules.sh does, for the same reason. +try: + import yaml +except ModuleNotFoundError: + print("installing PyYAML", file=sys.stderr) + if subprocess.run( + [sys.executable, "-m", "pip", "install", "--quiet", + "--disable-pip-version-check", "pyyaml"], + check=False, + ).returncode: + sys.exit("PyYAML is required and could not be installed") + import yaml + +REPO = pathlib.Path(__file__).resolve().parent.parent +DEFAULT = REPO / "stacks/observability/compose.yaml" + +# An image reference says nothing about its base, so a substring heuristic alone +# would not have caught the bug this script exists to prevent: the offending +# image was `grafana/loki`, which contains none of the markers below. Known +# no-userland images therefore have to be listed by name. +# +# Verified against each project's published Dockerfile: +# grafana/loki gcr.io/distroless/static:nonroot — /usr/bin/loki only +# For contrast, and so nobody adds them here by pattern-matching on the vendor: +# prom/prometheus, prom/alertmanager, prom/snmp-exporter busybox +# grafana/grafana-oss alpine +# grafana/alloy ubuntu +# — all four can run a wget healthcheck, and do. +NO_USERLAND_IMAGES = frozenset({"grafana/loki"}) + +# Still worth keeping for images that name their own base. Catches anything +# pulled straight from a distroless or scratch reference. +DISTROLESS_MARKERS = ("distroless", "/static", "scratch") + + +def has_no_userland(image: str) -> bool: + """True if a healthcheck could not exec anything inside this image.""" + repository = image.split("@", 1)[0] + # Strip the tag, but only if the trailing colon is a tag separator and not + # the port in a registry host such as registry.local:5000/grafana/loki. + head, sep, tail = repository.rpartition(":") + if sep and "/" not in tail: + repository = head + repository = repository.lower() + # Match bare and registry-qualified forms alike, so docker.io/grafana/loki + # and registry.local:5000/grafana/loki are recognised as the same image. + if any( + repository == known or repository.endswith(f"/{known}") + for known in NO_USERLAND_IMAGES + ): + return True + return any(marker in image.lower() for marker in DISTROLESS_MARKERS) + + +def main() -> int: + path = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT + if not path.exists(): + print(f"no compose file at {path}", file=sys.stderr) + return 1 + + compose = yaml.safe_load(path.read_text(encoding="utf-8")) + services = compose.get("services") or {} + problems: list[str] = [] + + for name, svc in services.items(): + svc = svc or {} + depends = svc.get("depends_on") + if not isinstance(depends, dict): + continue + + for target, cfg in depends.items(): + condition = cfg.get("condition") if isinstance(cfg, dict) else cfg + if condition != "service_healthy": + continue + + target_svc = services.get(target) + if target_svc is None: + problems.append( + f"{name} depends on {target}, which is not defined in this file" + ) + elif not target_svc.get("healthcheck"): + problems.append( + f"{name} waits for {target} to become healthy, but {target} " + f"declares no healthcheck — it can never report healthy, so " + f"{name} will hang forever" + ) + + # A healthcheck on a distroless image is equally unsatisfiable. + for name, svc in services.items(): + svc = svc or {} + check = svc.get("healthcheck") + image = str(svc.get("image", "")) + if not check or check.get("disable"): + continue + test = check.get("test") + if isinstance(test, list) and test and test[0] in ("CMD", "CMD-SHELL"): + binary = test[1] if len(test) > 1 else "" + if has_no_userland(image): + problems.append( + f"{name} has a healthcheck exec'ing {binary!r}, but {image} " + f"has no shell and no userland — the probe can never run, so " + f"{name} stays 'starting' forever" + ) + + for problem in problems: + print(f" {problem}", file=sys.stderr) + + if problems: + print( + f"\n{len(problems)} unsatisfiable health dependency/dependencies " + f"in {path.name}", + file=sys.stderr, + ) + return 1 + + healthy_deps = sum( + 1 + for svc in services.values() + for cfg in ((svc or {}).get("depends_on") or {}).values() + if (cfg.get("condition") if isinstance(cfg, dict) else cfg) == "service_healthy" + ) + checks = sum(1 for svc in services.values() if (svc or {}).get("healthcheck")) + print( + f"{path.name} OK — {checks} healthcheck(s), " + f"{healthy_deps} service_healthy dependency/dependencies, all satisfiable" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate.sh b/scripts/validate.sh index be08dd1..0f17aa5 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -129,6 +129,19 @@ else skip "no alloy binary and no docker daemon" fi +# --------------------------------------------------------------------------- +head_ "Compose health dependencies" +# --------------------------------------------------------------------------- +if have python3; then + if python3 scripts/check_compose_health.py; then + pass "compose health dependencies" + else + fail "compose health dependencies" + fi +else + skip "python3 not installed" +fi + # --------------------------------------------------------------------------- head_ "Loki rules" # --------------------------------------------------------------------------- diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index c6d7d86..7f2b8cf 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -99,12 +99,22 @@ services: - loki-data:/loki ports: - "${BIND_ADDR:-0.0.0.0}:${LOKI_PORT:-3100}:3100" - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"] - interval: 30s - timeout: 5s - retries: 5 - start_period: 45s + + # No healthcheck on loki, deliberately. + # + # grafana/loki is built FROM gcr.io/distroless/static:nonroot. The image + # contains /usr/bin/loki and nothing else — no shell, no wget, no curl. A + # Docker healthcheck can only exec something inside the container, so there is + # nothing available to probe with, and any `test:` here fails forever. + # + # That is not cosmetic: an unhealthy service blocks anything waiting on it via + # `depends_on: condition: service_healthy`, so Grafana and Alloy never start + # and the stack appears to hang with Loki "starting" indefinitely. + # + # Readiness is instead observed from outside the container: + # curl -s localhost:3100/ready + # and continuously by Prometheus, which scrapes loki:3100 as the `loki` job — + # `InstanceDown{job="loki"}` and `LokiIngestionStalled` both cover it. # --------------------------------------------------------------------------- # SNMP polling proxy for pfSense, the MokerLink switch, the APC UPS and iLO. @@ -159,8 +169,12 @@ services: depends_on: prometheus: condition: service_healthy + # service_started, not service_healthy: Loki is distroless and cannot + # report health (see the loki service above). Grafana's Loki datasource is + # resolved lazily per query, so starting before Loki is fully ready costs + # nothing beyond an empty panel for a few seconds. loki: - condition: service_healthy + condition: service_started healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] interval: 30s @@ -196,8 +210,11 @@ services: ports: - "127.0.0.1:${ALLOY_PORT:-12345}:12345" depends_on: + # service_started for loki, for the same reason as grafana above: the + # distroless image cannot report health. Alloy retries a failed push with + # backoff and buffers in the meantime, so racing Loki's startup is safe. loki: - condition: service_healthy + condition: service_started prometheus: condition: service_healthy