fix(observability): remove impossible Loki healthcheck that hung the stack

.github/workflows/ci.yml

@@ -146,9 +146,16 @@ jobs:
             -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \
             fmt --test "$STACK/alloy/config.alloy"
 
+      # `docker compose config` happily accepts a service_healthy dependency on
+      # a service that can never report healthy — the stack then hangs at deploy
+      # time with no error at all. Loki's distroless image has no wget to probe
+      # with, which is exactly how that happened here.
+      - name: Verify health dependencies are satisfiable
+        run: python3 scripts/check_compose_health.py
+
       # promtool cannot check these — it parses PromQL and rejects every LogQL
-      # stream selector. Loki itself is the only thing that understands them,
-      # so the checker boots the pinned image with the rules mounted.
+      # stream selector. Loki itself is the only thing that understands them, so
+      # the checker boots the pinned image with the rules mounted.
       - name: Validate Loki rules
         run: ./scripts/check_loki_rules.sh
 

.gitignore

@@ -36,6 +36,8 @@ data/
 # ---- Local tooling ----
 .venv/
 node_modules/
+__pycache__/
+*.py[cod]
 *.log
 *.tmp
 *.bak

.yamllint.yaml

@@ -3,6 +3,9 @@ extends: default
 
 ignore: |
   stacks/observability/snmp-exporter/snmp.yaml
+  # SOPS output: machine-generated ciphertext, not hand-written YAML. It does
+  # not follow comment/indent conventions and never will.
+  secrets/*.sops.yaml
   stacks/observability/snmp-exporter/.rendered/
   node_modules/
   .venv/

Makefile

@@ -96,6 +96,10 @@ check-rules: ## Validate Prometheus rules and config
 	promtool check config $(STACK_DIR)/prometheus/prometheus.yaml
 	promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml
 
+.PHONY: check-compose-health
+check-compose-health: ## Verify compose health dependencies can be satisfied
+	python3 scripts/check_compose_health.py
+
 .PHONY: check-loki-rules
 check-loki-rules: ## Validate Loki (LogQL) alerting rules
 	./scripts/check_loki_rules.sh

scripts/check_compose_health.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Check that compose health dependencies can actually be satisfied.
+
+`depends_on: <svc>: condition: service_healthy` waits for <svc> to report
+healthy. If <svc> declares no healthcheck it never will, and everything
+downstream hangs forever with no error — the stack simply never finishes
+starting.
+
+That happened here: Loki's image is gcr.io/distroless/static:nonroot, which
+contains only /usr/bin/loki — no shell, no wget, no curl. The healthcheck
+execed wget, which cannot exist, so Loki was permanently "starting" and both
+Grafana and Alloy blocked on it. Nothing logged an error; `make up` just sat
+there.
+
+Also flags healthchecks that exec a binary the image is unlikely to provide,
+since that is the same failure wearing a different hat.
+
+Usage: scripts/check_compose_health.py [compose.yaml]
+"""
+from __future__ import annotations
+
+import pathlib
+import subprocess
+import sys
+
+# PyYAML is not guaranteed on a clean runner, and this script gates CI. Install
+# it rather than failing a green compose file on a missing library — the same
+# thing scripts/check_loki_rules.sh does, for the same reason.
+try:
+    import yaml
+except ModuleNotFoundError:
+    print("installing PyYAML", file=sys.stderr)
+    if subprocess.run(
+        [sys.executable, "-m", "pip", "install", "--quiet",
+         "--disable-pip-version-check", "pyyaml"],
+        check=False,
+    ).returncode:
+        sys.exit("PyYAML is required and could not be installed")
+    import yaml
+
+REPO = pathlib.Path(__file__).resolve().parent.parent
+DEFAULT = REPO / "stacks/observability/compose.yaml"
+
+# An image reference says nothing about its base, so a substring heuristic alone
+# would not have caught the bug this script exists to prevent: the offending
+# image was `grafana/loki`, which contains none of the markers below. Known
+# no-userland images therefore have to be listed by name.
+#
+# Verified against each project's published Dockerfile:
+#   grafana/loki       gcr.io/distroless/static:nonroot  — /usr/bin/loki only
+# For contrast, and so nobody adds them here by pattern-matching on the vendor:
+#   prom/prometheus, prom/alertmanager, prom/snmp-exporter  busybox
+#   grafana/grafana-oss                                     alpine
+#   grafana/alloy                                           ubuntu
+# — all four can run a wget healthcheck, and do.
+NO_USERLAND_IMAGES = frozenset({"grafana/loki"})
+
+# Still worth keeping for images that name their own base. Catches anything
+# pulled straight from a distroless or scratch reference.
+DISTROLESS_MARKERS = ("distroless", "/static", "scratch")
+
+
+def has_no_userland(image: str) -> bool:
+    """True if a healthcheck could not exec anything inside this image."""
+    repository = image.split("@", 1)[0]
+    # Strip the tag, but only if the trailing colon is a tag separator and not
+    # the port in a registry host such as registry.local:5000/grafana/loki.
+    head, sep, tail = repository.rpartition(":")
+    if sep and "/" not in tail:
+        repository = head
+    repository = repository.lower()
+    # Match bare and registry-qualified forms alike, so docker.io/grafana/loki
+    # and registry.local:5000/grafana/loki are recognised as the same image.
+    if any(
+        repository == known or repository.endswith(f"/{known}")
+        for known in NO_USERLAND_IMAGES
+    ):
+        return True
+    return any(marker in image.lower() for marker in DISTROLESS_MARKERS)
+
+
+def main() -> int:
+    path = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT
+    if not path.exists():
+        print(f"no compose file at {path}", file=sys.stderr)
+        return 1
+
+    compose = yaml.safe_load(path.read_text(encoding="utf-8"))
+    services = compose.get("services") or {}
+    problems: list[str] = []
+
+    for name, svc in services.items():
+        svc = svc or {}
+        depends = svc.get("depends_on")
+        if not isinstance(depends, dict):
+            continue
+
+        for target, cfg in depends.items():
+            condition = cfg.get("condition") if isinstance(cfg, dict) else cfg
+            if condition != "service_healthy":
+                continue
+
+            target_svc = services.get(target)
+            if target_svc is None:
+                problems.append(
+                    f"{name} depends on {target}, which is not defined in this file"
+                )
+            elif not target_svc.get("healthcheck"):
+                problems.append(
+                    f"{name} waits for {target} to become healthy, but {target} "
+                    f"declares no healthcheck — it can never report healthy, so "
+                    f"{name} will hang forever"
+                )
+
+    # A healthcheck on a distroless image is equally unsatisfiable.
+    for name, svc in services.items():
+        svc = svc or {}
+        check = svc.get("healthcheck")
+        image = str(svc.get("image", ""))
+        if not check or check.get("disable"):
+            continue
+        test = check.get("test")
+        if isinstance(test, list) and test and test[0] in ("CMD", "CMD-SHELL"):
+            binary = test[1] if len(test) > 1 else ""
+            if has_no_userland(image):
+                problems.append(
+                    f"{name} has a healthcheck exec'ing {binary!r}, but {image} "
+                    f"has no shell and no userland — the probe can never run, so "
+                    f"{name} stays 'starting' forever"
+                )
+
+    for problem in problems:
+        print(f"  {problem}", file=sys.stderr)
+
+    if problems:
+        print(
+            f"\n{len(problems)} unsatisfiable health dependency/dependencies "
+            f"in {path.name}",
+            file=sys.stderr,
+        )
+        return 1
+
+    healthy_deps = sum(
+        1
+        for svc in services.values()
+        for cfg in ((svc or {}).get("depends_on") or {}).values()
+        if (cfg.get("condition") if isinstance(cfg, dict) else cfg) == "service_healthy"
+    )
+    checks = sum(1 for svc in services.values() if (svc or {}).get("healthcheck"))
+    print(
+        f"{path.name} OK — {checks} healthcheck(s), "
+        f"{healthy_deps} service_healthy dependency/dependencies, all satisfiable"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/validate.sh

@@ -129,6 +129,19 @@ else
   skip "no alloy binary and no docker daemon"
 fi
 
+# ---------------------------------------------------------------------------
+head_ "Compose health dependencies"
+# ---------------------------------------------------------------------------
+if have python3; then
+  if python3 scripts/check_compose_health.py; then
+    pass "compose health dependencies"
+  else
+    fail "compose health dependencies"
+  fi
+else
+  skip "python3 not installed"
+fi
+
 # ---------------------------------------------------------------------------
 head_ "Loki rules"
 # ---------------------------------------------------------------------------

stacks/observability/compose.yaml

@@ -99,12 +99,22 @@ services:
       - loki-data:/loki
     ports:
       - "${BIND_ADDR:-0.0.0.0}:${LOKI_PORT:-3100}:3100"
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"]
-      interval: 30s
-      timeout: 5s
-      retries: 5
-      start_period: 45s
+
+  # No healthcheck on loki, deliberately.
+  #
+  # grafana/loki is built FROM gcr.io/distroless/static:nonroot. The image
+  # contains /usr/bin/loki and nothing else — no shell, no wget, no curl. A
+  # Docker healthcheck can only exec something inside the container, so there is
+  # nothing available to probe with, and any `test:` here fails forever.
+  #
+  # That is not cosmetic: an unhealthy service blocks anything waiting on it via
+  # `depends_on: condition: service_healthy`, so Grafana and Alloy never start
+  # and the stack appears to hang with Loki "starting" indefinitely.
+  #
+  # Readiness is instead observed from outside the container:
+  #   curl -s localhost:3100/ready
+  # and continuously by Prometheus, which scrapes loki:3100 as the `loki` job —
+  # `InstanceDown{job="loki"}` and `LokiIngestionStalled` both cover it.
 
   # ---------------------------------------------------------------------------
   # SNMP polling proxy for pfSense, the MokerLink switch, the APC UPS and iLO.
@@ -159,8 +169,12 @@ services:
     depends_on:
       prometheus:
         condition: service_healthy
+      # service_started, not service_healthy: Loki is distroless and cannot
+      # report health (see the loki service above). Grafana's Loki datasource is
+      # resolved lazily per query, so starting before Loki is fully ready costs
+      # nothing beyond an empty panel for a few seconds.
       loki:
-        condition: service_healthy
+        condition: service_started
     healthcheck:
       test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
       interval: 30s
@@ -196,8 +210,11 @@ services:
     ports:
       - "127.0.0.1:${ALLOY_PORT:-12345}:12345"
     depends_on:
+      # service_started for loki, for the same reason as grafana above: the
+      # distroless image cannot report health. Alloy retries a failed push with
+      # backoff and buffers in the meantime, so racing Loki's startup is safe.
       loki:
-        condition: service_healthy
+        condition: service_started
       prometheus:
         condition: service_healthy