From b4b74e1ee2def84726b452ea628bd8c1ccd27eba Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 22:46:10 +0000 Subject: [PATCH 1/3] fix(observability): remove impossible Loki healthcheck that hung the stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grafana/loki is built FROM gcr.io/distroless/static:nonroot — the image contains /usr/bin/loki and nothing else. No shell, no wget, no curl. A Docker healthcheck can only exec something inside the container, so the wget probe could never succeed and Loki sat in "starting" forever. That is not cosmetic. Grafana and Alloy both waited on Loki via `depends_on: condition: service_healthy`, so neither ever started and `make up` appeared to hang with no error logged anywhere. Loki itself was healthy the whole time; only the probe was impossible. - drop the healthcheck from loki, with a comment explaining why one cannot exist and where readiness is observed instead (curl :3100/ready, plus Prometheus scraping the loki job — InstanceDown and LokiIngestionStalled already cover it) - grafana and alloy now depend on loki with condition: service_started. Grafana resolves its Loki datasource lazily per query; Alloy retries pushes with backoff and buffers meanwhile. Racing Loki's startup is safe for both. Add scripts/check_compose_health.py so this cannot come back. It asserts that every service_healthy dependency targets a service that actually declares a healthcheck, and flags healthchecks that exec a binary on an image that looks distroless. `docker compose config` accepts both happily, which is why the existing CI was silent. Wired into CI, scripts/validate.sh and `make check-compose-health`. Also ignore secrets/*.sops.yaml in .yamllint.yaml. SOPS output is machine-generated ciphertext that does not follow the comment and indentation conventions and never will; committing the encrypted file turned the lint job red on main. --- .github/workflows/ci.yml | 11 ++- .yamllint.yaml | 3 + Makefile | 4 ++ scripts/check_compose_health.py | 112 ++++++++++++++++++++++++++++++ scripts/validate.sh | 13 ++++ stacks/observability/compose.yaml | 33 ++++++--- 6 files changed, 166 insertions(+), 10 deletions(-) create mode 100755 scripts/check_compose_health.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e310f5f..293f737 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,9 +146,16 @@ jobs: -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \ fmt --test "$STACK/alloy/config.alloy" + # `docker compose config` happily accepts a service_healthy dependency on + # a service that can never report healthy — the stack then hangs at deploy + # time with no error at all. Loki's distroless image has no wget to probe + # with, which is exactly how that happened here. + - name: Verify health dependencies are satisfiable + run: python3 scripts/check_compose_health.py + # promtool cannot check these — it parses PromQL and rejects every LogQL - # stream selector. Loki itself is the only thing that understands them, - # so the checker boots the pinned image with the rules mounted. + # stream selector. Loki itself is the only thing that understands them, so + # the checker boots the pinned image with the rules mounted. - name: Validate Loki rules run: ./scripts/check_loki_rules.sh diff --git a/.yamllint.yaml b/.yamllint.yaml index 4a500f6..21e86b4 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -3,6 +3,9 @@ extends: default ignore: | stacks/observability/snmp-exporter/snmp.yaml + # SOPS output: machine-generated ciphertext, not hand-written YAML. It does + # not follow comment/indent conventions and never will. + secrets/*.sops.yaml stacks/observability/snmp-exporter/.rendered/ node_modules/ .venv/ diff --git a/Makefile b/Makefile index 264fae9..38442e8 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,10 @@ check-rules: ## Validate Prometheus rules and config promtool check config $(STACK_DIR)/prometheus/prometheus.yaml promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml +.PHONY: check-compose-health +check-compose-health: ## Verify compose health dependencies can be satisfied + python3 scripts/check_compose_health.py + .PHONY: check-loki-rules check-loki-rules: ## Validate Loki (LogQL) alerting rules ./scripts/check_loki_rules.sh diff --git a/scripts/check_compose_health.py b/scripts/check_compose_health.py new file mode 100755 index 0000000..c189937 --- /dev/null +++ b/scripts/check_compose_health.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Check that compose health dependencies can actually be satisfied. + +`depends_on: : condition: service_healthy` waits for to report +healthy. If declares no healthcheck it never will, and everything +downstream hangs forever with no error — the stack simply never finishes +starting. + +That happened here: Loki's image is gcr.io/distroless/static:nonroot, which +contains only /usr/bin/loki — no shell, no wget, no curl. The healthcheck +execed wget, which cannot exist, so Loki was permanently "starting" and both +Grafana and Alloy blocked on it. Nothing logged an error; `make up` just sat +there. + +Also flags healthchecks that exec a binary the image is unlikely to provide, +since that is the same failure wearing a different hat. + +Usage: scripts/check_compose_health.py [compose.yaml] +""" +from __future__ import annotations + +import pathlib +import sys + +try: + import yaml +except ModuleNotFoundError: + sys.exit("PyYAML is required: python3 -m pip install pyyaml") + +REPO = pathlib.Path(__file__).resolve().parent.parent +DEFAULT = REPO / "stacks/observability/compose.yaml" + +# Images with no shell and no userland. A healthcheck cannot exec anything in +# these beyond the service binary itself. +DISTROLESS_MARKERS = ("distroless", "/static", "scratch") + + +def main() -> int: + path = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT + if not path.exists(): + return int(bool(print(f"no compose file at {path}", file=sys.stderr))) or 1 + + compose = yaml.safe_load(path.read_text(encoding="utf-8")) + services = compose.get("services") or {} + problems: list[str] = [] + + for name, svc in services.items(): + svc = svc or {} + depends = svc.get("depends_on") + if not isinstance(depends, dict): + continue + + for target, cfg in depends.items(): + condition = cfg.get("condition") if isinstance(cfg, dict) else cfg + if condition != "service_healthy": + continue + + target_svc = services.get(target) + if target_svc is None: + problems.append( + f"{name} depends on {target}, which is not defined in this file" + ) + elif not target_svc.get("healthcheck"): + problems.append( + f"{name} waits for {target} to become healthy, but {target} " + f"declares no healthcheck — it can never report healthy, so " + f"{name} will hang forever" + ) + + # A healthcheck on a distroless image is equally unsatisfiable. + for name, svc in services.items(): + svc = svc or {} + check = svc.get("healthcheck") + image = str(svc.get("image", "")) + if not check or check.get("disable"): + continue + test = check.get("test") + if isinstance(test, list) and test and test[0] in ("CMD", "CMD-SHELL"): + binary = test[1] if len(test) > 1 else "" + if any(m in image.lower() for m in DISTROLESS_MARKERS): + problems.append( + f"{name} has a healthcheck exec'ing {binary!r} but its image " + f"looks distroless — there is no userland to run it" + ) + + for problem in problems: + print(f" {problem}", file=sys.stderr) + + if problems: + print( + f"\n{len(problems)} unsatisfiable health dependency/dependencies " + f"in {path.name}", + file=sys.stderr, + ) + return 1 + + healthy_deps = sum( + 1 + for svc in services.values() + for cfg in ((svc or {}).get("depends_on") or {}).values() + if (cfg.get("condition") if isinstance(cfg, dict) else cfg) == "service_healthy" + ) + checks = sum(1 for svc in services.values() if (svc or {}).get("healthcheck")) + print( + f"{path.name} OK — {checks} healthcheck(s), " + f"{healthy_deps} service_healthy dependency/dependencies, all satisfiable" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate.sh b/scripts/validate.sh index be08dd1..06b9dad 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -129,6 +129,19 @@ else skip "no alloy binary and no docker daemon" fi +# --------------------------------------------------------------------------- +head_ "Compose health dependencies" +# --------------------------------------------------------------------------- +if have python3; then + if python3 scripts/check_compose_health.py; then + pass "health dependencies satisfiable" + else + fail "health dependencies satisfiable" + fi +else + skip "python3 not installed" +fi + # --------------------------------------------------------------------------- head_ "Loki rules" # --------------------------------------------------------------------------- diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index c6d7d86..7f2b8cf 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -99,12 +99,22 @@ services: - loki-data:/loki ports: - "${BIND_ADDR:-0.0.0.0}:${LOKI_PORT:-3100}:3100" - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"] - interval: 30s - timeout: 5s - retries: 5 - start_period: 45s + + # No healthcheck on loki, deliberately. + # + # grafana/loki is built FROM gcr.io/distroless/static:nonroot. The image + # contains /usr/bin/loki and nothing else — no shell, no wget, no curl. A + # Docker healthcheck can only exec something inside the container, so there is + # nothing available to probe with, and any `test:` here fails forever. + # + # That is not cosmetic: an unhealthy service blocks anything waiting on it via + # `depends_on: condition: service_healthy`, so Grafana and Alloy never start + # and the stack appears to hang with Loki "starting" indefinitely. + # + # Readiness is instead observed from outside the container: + # curl -s localhost:3100/ready + # and continuously by Prometheus, which scrapes loki:3100 as the `loki` job — + # `InstanceDown{job="loki"}` and `LokiIngestionStalled` both cover it. # --------------------------------------------------------------------------- # SNMP polling proxy for pfSense, the MokerLink switch, the APC UPS and iLO. @@ -159,8 +169,12 @@ services: depends_on: prometheus: condition: service_healthy + # service_started, not service_healthy: Loki is distroless and cannot + # report health (see the loki service above). Grafana's Loki datasource is + # resolved lazily per query, so starting before Loki is fully ready costs + # nothing beyond an empty panel for a few seconds. loki: - condition: service_healthy + condition: service_started healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] interval: 30s @@ -196,8 +210,11 @@ services: ports: - "127.0.0.1:${ALLOY_PORT:-12345}:12345" depends_on: + # service_started for loki, for the same reason as grafana above: the + # distroless image cannot report health. Alloy retries a failed push with + # backoff and buffers in the meantime, so racing Loki's startup is safe. loki: - condition: service_healthy + condition: service_started prometheus: condition: service_healthy From 65c416f03f2ca16d14551cde8a11348558dfd4b7 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 22:51:46 +0000 Subject: [PATCH 2/3] fix(ci): close the distroless gap the health guard was meant to catch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review caught that the guard did not actually cover its own motivating case. The distroless detection matched on substrings in the image reference — "distroless", "/static", "scratch" — but the image that caused the outage is `grafana/loki`, which contains none of them. Re-adding a wget healthcheck to Loki would have satisfied the service_healthy check and hung the stack again, with the guard silent. An image reference says nothing about its base, so known no-userland images have to be listed by name. NO_USERLAND_IMAGES holds grafana/loki, verified against its published Dockerfile, and the substring markers stay for images that name their own base. Matching is on the repository component and accepts registry-qualified forms, so docker.io/grafana/loki and registry.local:5000/grafana/loki resolve the same way. Verified with a 13-case table over the reference forms, and against a fixture that re-adds the wget healthcheck to Loki — previously silent, now: loki has a healthcheck exec'ing 'wget', but grafana/loki:3.7.4@sha256:... has no shell and no userland — the probe can never run, so loki stays 'starting' forever Also from review: - Install PyYAML if absent instead of exiting. The script now gates CI, and a clean runner without the library would have failed a perfectly good compose file. Matches what check_loki_rules.sh already does. Exercised both ways in a throwaway venv: installs and passes where pip exists, and reports "PyYAML is required and could not be installed" rather than a traceback where it does not. - Replace `int(bool(print(...))) or 1` with a plain print and return. - Rename the validate.sh label to "compose health dependencies", so the failure line no longer reads "FAIL health dependencies satisfiable". --- .../check_compose_health.cpython-311.pyc | Bin 0 -> 8237 bytes scripts/check_compose_health.py | 60 ++++++++++++++++-- scripts/validate.sh | 4 +- 3 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 scripts/__pycache__/check_compose_health.cpython-311.pyc diff --git a/scripts/__pycache__/check_compose_health.cpython-311.pyc b/scripts/__pycache__/check_compose_health.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e601f34a465d9d3a1b7f6a2cae5cc4e2281fdd6c GIT binary patch literal 8237 zcmb_BTWlLwc6T_#Hz|@5skiKrtyoqgiH@DDWBH*dk}Nrv?L<=ISlyVWI3sB!YRJ7a zlp-$GWdQ?A8!D{2PQ`U%n7hcP4&q{sqKo>cKwF?d`bWCch8R?U&?3NY`$HpH!0=bk z9X?04(*)>nc;?Q1oO@pP+4bLBlKV7rF>~E2vOfWpb*+Z9GXEK z#TmlXvSG$Rkun{oXK3xo%rM&1IAgRHTTC-1ae6=E&0PpjP{@FI3&-%rMQa)TedA{a zguZ}(@zVM*i=|l+g1+{08)xFooQ1P;HqJh8<{bB(`mu|Sa!>4rnT(6h@;g`e8BqTP z{EJtKZlLZG5}8%>A$;fMQ%L^>WM|qLs#gM>DY$zzQ!H^sahQn*@EYC_o zS&-%gp6j>St`vJpfrvQ3o{&~Trv?C*$O*DQY9t=72q8XDM7wr{T@4Dd#Lh)LB(DgdZI2dBD4I5Ku0etMKn4=@Ddy4@p4e)MH!~uS7vUXCcxQdVxIHaAbZSYQZivVIF6%EC-i(HWt0YF2*F8 zScnaTi+PaTP#8#<3kN|HHH1q=vm@lQK^E8#;x$bl6=}xAu&~4frC=ga99a=KzSkxR zVu&ve1%Ye&3oi5QTu=zdFwd^?K}_fivYaqC2lIemQ@qx$jYB5(@I7aWqehl-iQ|pFxI$^*j&X}NAC~Qbj z8ivY#Q5g;ey-&Q($mVKmikYK0(^ZRXsn(6`8lj&b8#wb6(&0dW!vZ)eyAwj+jbjS+ zSmSR@;~1!L4y?6iVbmzNxH!b3Un9DPB>*-L7NO~;|0(sx?}^AT;rb~x@RaiFlw*YI zBSbmVQ22;CizwKr6X=6|>(n(6SaB1HHeEAhp#9m4Lka3*bjP4)|2?!tQD|d7M*$NC z?1gdX!D=nAT}Z2fya0w}=s%|cm1^WgS)>>r2H zL~z#sf2_-=))VvKct4EdQ}KNZK`9_c0%Uv`e1HE5@KM3AbgIA7=Pj5?dKbWx-=W{3 zBq|`>Vu7<^I)?)zUYB z7)|RWO_qTs=@BT@NV;{MN`S00Q2XqqH_H`Nl587?K(Z7Osxcf{<#7*D%@vgJK(UwJwoc>v3P#j`*8)*npBVy|kwTKd}8T`+=y zJSJB4Edfxnkdg(Aj4^r(7UcG(mijltaKz zMVc7&eO5%j@-{}^`Pc)K(R$nu}CE>>(+#o zvn~;rjxe?HI>_m{l1=R9)|Rj(+Y|KjgQf)YG2(0qGeG;DUE4nFIZFxK^Jn24dJ3h^ z@0`J@%}}R7mo}&S&MqEdwqI@h&-d2;e|&H4@1k_wF7GdE8=)84jEAe&+k&3obw|Pu z`m?`4It}{Xps!0W?CaI{U3cp2$0>JJsnt`2FD0D#V#3M!s$PU^TtWC;t@lbe_4q5E z$^s*6{EtX+_3L$VcXhVJF3I8V)^Ss!?qdVjoN$0GwP-qSeU6T|6Ev({cHO1(RBZ{@ z5AjqRKnZ?LhoN1^Gp*+tDC$U{MRIb|E|`n?L0<9Y@<{M`d#)m|(#C2{wY1wu>EHD< zt-Iw~tb6~TuzG5xqUV`o6CRL6XTl9|-<@3^82rBvPC@(6<9c8y-az8tQig)WSu6zO zm~3ZpV@#gwd%bu74MjKz$y7@z=^#ObqopxOazeFMvJ$GR)S;Y}h&xL84ib^G@%^PZ zorGQ_&f<7T2tf2Dh{eM$Q}Jme(LT6q42`9;T3LE(MoxnBpO3BPW9%gGvvd z8;M&1^i7SAPE5qxVXVaJ0}c{qbdy*%ZYmLaiV#XVmNOArMgU&QXYNY&1OjsiLu-hR ziA73VNMdD8V!O*}hVC-KaRbXfrEKw6*qB(#B$1%F#Arzqka=43>yl6)ech4U1T<<# z?th0uCA8tlCQmlt4Aee*aY%3WTF;upRds}Zv4E5K(YiqBYQg;di@FdXB- z$_2=ZfS4gOqB5KilFI=D4e2o;1o$l?f{(I7I zM{-@ql&)i0$D66)8)MhUHqYi9tm0s^4)&px9ee+g{r#-{{el@;Z7|AliGsJvU9grh zYKLW=5fB%k8NoOdDn`)p!;IiJEgV%6{2tB#Vy7{QkxIg*@KJbyNSDN5_&5NGtS$EY z2=gP-n+z`sGwYs&dsBKa{NT*P!+$sUH-rChGI!)1<;Xjv+EZ+vhicPv#c*#Ly#&Bhs)VpvgUf4R65Ly5QMEsDig%>E2=pflo!IntjfF% z1=TzZ0h#|$*5n3h zB$;29@G;;44**b2S`L-N2n-i7s*a*Z zUkd;_Ni}I%I7uQa4#Lr0S1~wc$Qr_zpvNTqrB}cjlW6Pvi7dOUrmnW?XDZguOGj5a^qwH8H}E7i!)pIN0|`- zaR0X9Z(FmL6FJKX#c~1)d6zr&ldtRQH^lVx-OINxKe(8_oa;ELbezo9ol@#fCCBov z#v4Dq{?qh8&UHY6INsHJZ(nje-`buW-|=;9UA}oa=X*)Hkx_vY%mmHO_KdE3>J?*8?wIoCnObr6DPt1s{H?zHW@d-C?lUkrXS zSU@f-U;=RMZ0hvp@b;mBhe!UI$sHP14vl7;&!_3T=G*3sJ=ff;H1}qkd%rxAI+J%b zXWRQ8x%#rMzP!76r=|05`|bA3T&|^0Y3WP7m3OsdU7gUf<+|z0H??KG({SHwEIt7~ zrVXC@ZxH|xTN#*8BTHD!oXhzRE55_vwVd@;B+%1$NCWr4p_R*D)|7Xe+6u;UgVyCS z&5(HjKwR&se^O{7+;*YHJ+-o|BeDdrTbAyIE%Qxt+L?17P}~Pn#%*`Q=J3|o&9U@Y zW-{kJsCWy5SRYw5224ZbS&q3NpXQSJR_*1g#Dur8?}c9zVubgB|#NCY1PjN%7MJ?B*)^{0N9bmnG!dT zro@!^>m#2IWG($UOTS|2hXUL?W>*23H2W_YDCeG!8aChm^Nw2`Fxa_A1Nr*KTRjE3 z{Q9-Wd#klzW~{cy2mq)Eb`A~LklO>Rqyb1TZyP%eZCiskVSnmU8oCNbsD4b^z9j%k zUQ+U!k`sS$pt_P1PA}Yz+>Ye-_?10=aL76Lqyo@6sW>N-r?(j^EZmJ_K$vxkD#G+! z?OWd71hiHfi%pMd+DMW_Mld-7EXd(Zy;4j7Spos#O9Fu&rs$%Gcf{nABPJ;n*nPBe z4=J~*=0JdpgaQH876{D6AT!Sg0+@v2cnE-MlVY>r)I#t93X@bB9wW64=OSDz%)=L) z(~+3SjgpUGc$74sCe1XCLAI701!FSV8c61rw?4wO_$zquhDM zl{{PgJ;|F}3&=vbt02!*GL*Fp)liW< zTkrr4Po6v}c#*}MT)mpu9M73M6m!Q{=9eCsU&@)g6>|40=GT%;-d30OyqdGUnlu;M zQRCjtOPRXtUVqMaNbw!YqIxi=#sTs)xv4W?l+VBe;}qpApl9HF2E;g0@kew^mTt*2 xj^yyw@ziu``r3ON@8uY;!g#ag8#3oiEA*1c7w^Ah+R(>Cbwj=MKfX)>{9nyu(@p>Y literal 0 HcmV?d00001 diff --git a/scripts/check_compose_health.py b/scripts/check_compose_health.py index c189937..9c4e37d 100755 --- a/scripts/check_compose_health.py +++ b/scripts/check_compose_health.py @@ -20,25 +20,70 @@ from __future__ import annotations import pathlib +import subprocess import sys +# PyYAML is not guaranteed on a clean runner, and this script gates CI. Install +# it rather than failing a green compose file on a missing library — the same +# thing scripts/check_loki_rules.sh does, for the same reason. try: import yaml except ModuleNotFoundError: - sys.exit("PyYAML is required: python3 -m pip install pyyaml") + print("installing PyYAML", file=sys.stderr) + if subprocess.run( + [sys.executable, "-m", "pip", "install", "--quiet", + "--disable-pip-version-check", "pyyaml"], + check=False, + ).returncode: + sys.exit("PyYAML is required and could not be installed") + import yaml REPO = pathlib.Path(__file__).resolve().parent.parent DEFAULT = REPO / "stacks/observability/compose.yaml" -# Images with no shell and no userland. A healthcheck cannot exec anything in -# these beyond the service binary itself. +# An image reference says nothing about its base, so a substring heuristic alone +# would not have caught the bug this script exists to prevent: the offending +# image was `grafana/loki`, which contains none of the markers below. Known +# no-userland images therefore have to be listed by name. +# +# Verified against each project's published Dockerfile: +# grafana/loki gcr.io/distroless/static:nonroot — /usr/bin/loki only +# For contrast, and so nobody adds them here by pattern-matching on the vendor: +# prom/prometheus, prom/alertmanager, prom/snmp-exporter busybox +# grafana/grafana-oss alpine +# grafana/alloy ubuntu +# — all four can run a wget healthcheck, and do. +NO_USERLAND_IMAGES = frozenset({"grafana/loki"}) + +# Still worth keeping for images that name their own base. Catches anything +# pulled straight from a distroless or scratch reference. DISTROLESS_MARKERS = ("distroless", "/static", "scratch") +def has_no_userland(image: str) -> bool: + """True if a healthcheck could not exec anything inside this image.""" + repository = image.split("@", 1)[0] + # Strip the tag, but only if the trailing colon is a tag separator and not + # the port in a registry host such as registry.local:5000/grafana/loki. + head, sep, tail = repository.rpartition(":") + if sep and "/" not in tail: + repository = head + repository = repository.lower() + # Match bare and registry-qualified forms alike, so docker.io/grafana/loki + # and registry.local:5000/grafana/loki are recognised as the same image. + if any( + repository == known or repository.endswith(f"/{known}") + for known in NO_USERLAND_IMAGES + ): + return True + return any(marker in image.lower() for marker in DISTROLESS_MARKERS) + + def main() -> int: path = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT if not path.exists(): - return int(bool(print(f"no compose file at {path}", file=sys.stderr))) or 1 + print(f"no compose file at {path}", file=sys.stderr) + return 1 compose = yaml.safe_load(path.read_text(encoding="utf-8")) services = compose.get("services") or {} @@ -77,10 +122,11 @@ def main() -> int: test = check.get("test") if isinstance(test, list) and test and test[0] in ("CMD", "CMD-SHELL"): binary = test[1] if len(test) > 1 else "" - if any(m in image.lower() for m in DISTROLESS_MARKERS): + if has_no_userland(image): problems.append( - f"{name} has a healthcheck exec'ing {binary!r} but its image " - f"looks distroless — there is no userland to run it" + f"{name} has a healthcheck exec'ing {binary!r}, but {image} " + f"has no shell and no userland — the probe can never run, so " + f"{name} stays 'starting' forever" ) for problem in problems: diff --git a/scripts/validate.sh b/scripts/validate.sh index 06b9dad..0f17aa5 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -134,9 +134,9 @@ head_ "Compose health dependencies" # --------------------------------------------------------------------------- if have python3; then if python3 scripts/check_compose_health.py; then - pass "health dependencies satisfiable" + pass "compose health dependencies" else - fail "health dependencies satisfiable" + fail "compose health dependencies" fi else skip "python3 not installed" From df46660f1043cc95efd270c75958e18c2282ec21 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 22:54:50 +0000 Subject: [PATCH 3/3] chore: gitignore __pycache__ and drop a committed .pyc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bytecode cache directory landed in the previous commit — importing check_compose_health.py to unit-test its image classification created scripts/__pycache__/, and it was swept in by `git add -A`. Remove it from the index and ignore __pycache__/ and *.py[cod], which the repository never had a pattern for despite carrying two Python scripts. --- .gitignore | 2 ++ .../check_compose_health.cpython-311.pyc | Bin 8237 -> 0 bytes 2 files changed, 2 insertions(+) delete mode 100644 scripts/__pycache__/check_compose_health.cpython-311.pyc diff --git a/.gitignore b/.gitignore index af87747..44f7a56 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ data/ # ---- Local tooling ---- .venv/ node_modules/ +__pycache__/ +*.py[cod] *.log *.tmp *.bak diff --git a/scripts/__pycache__/check_compose_health.cpython-311.pyc b/scripts/__pycache__/check_compose_health.cpython-311.pyc deleted file mode 100644 index e601f34a465d9d3a1b7f6a2cae5cc4e2281fdd6c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8237 zcmb_BTWlLwc6T_#Hz|@5skiKrtyoqgiH@DDWBH*dk}Nrv?L<=ISlyVWI3sB!YRJ7a zlp-$GWdQ?A8!D{2PQ`U%n7hcP4&q{sqKo>cKwF?d`bWCch8R?U&?3NY`$HpH!0=bk z9X?04(*)>nc;?Q1oO@pP+4bLBlKV7rF>~E2vOfWpb*+Z9GXEK z#TmlXvSG$Rkun{oXK3xo%rM&1IAgRHTTC-1ae6=E&0PpjP{@FI3&-%rMQa)TedA{a zguZ}(@zVM*i=|l+g1+{08)xFooQ1P;HqJh8<{bB(`mu|Sa!>4rnT(6h@;g`e8BqTP z{EJtKZlLZG5}8%>A$;fMQ%L^>WM|qLs#gM>DY$zzQ!H^sahQn*@EYC_o zS&-%gp6j>St`vJpfrvQ3o{&~Trv?C*$O*DQY9t=72q8XDM7wr{T@4Dd#Lh)LB(DgdZI2dBD4I5Ku0etMKn4=@Ddy4@p4e)MH!~uS7vUXCcxQdVxIHaAbZSYQZivVIF6%EC-i(HWt0YF2*F8 zScnaTi+PaTP#8#<3kN|HHH1q=vm@lQK^E8#;x$bl6=}xAu&~4frC=ga99a=KzSkxR zVu&ve1%Ye&3oi5QTu=zdFwd^?K}_fivYaqC2lIemQ@qx$jYB5(@I7aWqehl-iQ|pFxI$^*j&X}NAC~Qbj z8ivY#Q5g;ey-&Q($mVKmikYK0(^ZRXsn(6`8lj&b8#wb6(&0dW!vZ)eyAwj+jbjS+ zSmSR@;~1!L4y?6iVbmzNxH!b3Un9DPB>*-L7NO~;|0(sx?}^AT;rb~x@RaiFlw*YI zBSbmVQ22;CizwKr6X=6|>(n(6SaB1HHeEAhp#9m4Lka3*bjP4)|2?!tQD|d7M*$NC z?1gdX!D=nAT}Z2fya0w}=s%|cm1^WgS)>>r2H zL~z#sf2_-=))VvKct4EdQ}KNZK`9_c0%Uv`e1HE5@KM3AbgIA7=Pj5?dKbWx-=W{3 zBq|`>Vu7<^I)?)zUYB z7)|RWO_qTs=@BT@NV;{MN`S00Q2XqqH_H`Nl587?K(Z7Osxcf{<#7*D%@vgJK(UwJwoc>v3P#j`*8)*npBVy|kwTKd}8T`+=y zJSJB4Edfxnkdg(Aj4^r(7UcG(mijltaKz zMVc7&eO5%j@-{}^`Pc)K(R$nu}CE>>(+#o zvn~;rjxe?HI>_m{l1=R9)|Rj(+Y|KjgQf)YG2(0qGeG;DUE4nFIZFxK^Jn24dJ3h^ z@0`J@%}}R7mo}&S&MqEdwqI@h&-d2;e|&H4@1k_wF7GdE8=)84jEAe&+k&3obw|Pu z`m?`4It}{Xps!0W?CaI{U3cp2$0>JJsnt`2FD0D#V#3M!s$PU^TtWC;t@lbe_4q5E z$^s*6{EtX+_3L$VcXhVJF3I8V)^Ss!?qdVjoN$0GwP-qSeU6T|6Ev({cHO1(RBZ{@ z5AjqRKnZ?LhoN1^Gp*+tDC$U{MRIb|E|`n?L0<9Y@<{M`d#)m|(#C2{wY1wu>EHD< zt-Iw~tb6~TuzG5xqUV`o6CRL6XTl9|-<@3^82rBvPC@(6<9c8y-az8tQig)WSu6zO zm~3ZpV@#gwd%bu74MjKz$y7@z=^#ObqopxOazeFMvJ$GR)S;Y}h&xL84ib^G@%^PZ zorGQ_&f<7T2tf2Dh{eM$Q}Jme(LT6q42`9;T3LE(MoxnBpO3BPW9%gGvvd z8;M&1^i7SAPE5qxVXVaJ0}c{qbdy*%ZYmLaiV#XVmNOArMgU&QXYNY&1OjsiLu-hR ziA73VNMdD8V!O*}hVC-KaRbXfrEKw6*qB(#B$1%F#Arzqka=43>yl6)ech4U1T<<# z?th0uCA8tlCQmlt4Aee*aY%3WTF;upRds}Zv4E5K(YiqBYQg;di@FdXB- z$_2=ZfS4gOqB5KilFI=D4e2o;1o$l?f{(I7I zM{-@ql&)i0$D66)8)MhUHqYi9tm0s^4)&px9ee+g{r#-{{el@;Z7|AliGsJvU9grh zYKLW=5fB%k8NoOdDn`)p!;IiJEgV%6{2tB#Vy7{QkxIg*@KJbyNSDN5_&5NGtS$EY z2=gP-n+z`sGwYs&dsBKa{NT*P!+$sUH-rChGI!)1<;Xjv+EZ+vhicPv#c*#Ly#&Bhs)VpvgUf4R65Ly5QMEsDig%>E2=pflo!IntjfF% z1=TzZ0h#|$*5n3h zB$;29@G;;44**b2S`L-N2n-i7s*a*Z zUkd;_Ni}I%I7uQa4#Lr0S1~wc$Qr_zpvNTqrB}cjlW6Pvi7dOUrmnW?XDZguOGj5a^qwH8H}E7i!)pIN0|`- zaR0X9Z(FmL6FJKX#c~1)d6zr&ldtRQH^lVx-OINxKe(8_oa;ELbezo9ol@#fCCBov z#v4Dq{?qh8&UHY6INsHJZ(nje-`buW-|=;9UA}oa=X*)Hkx_vY%mmHO_KdE3>J?*8?wIoCnObr6DPt1s{H?zHW@d-C?lUkrXS zSU@f-U;=RMZ0hvp@b;mBhe!UI$sHP14vl7;&!_3T=G*3sJ=ff;H1}qkd%rxAI+J%b zXWRQ8x%#rMzP!76r=|05`|bA3T&|^0Y3WP7m3OsdU7gUf<+|z0H??KG({SHwEIt7~ zrVXC@ZxH|xTN#*8BTHD!oXhzRE55_vwVd@;B+%1$NCWr4p_R*D)|7Xe+6u;UgVyCS z&5(HjKwR&se^O{7+;*YHJ+-o|BeDdrTbAyIE%Qxt+L?17P}~Pn#%*`Q=J3|o&9U@Y zW-{kJsCWy5SRYw5224ZbS&q3NpXQSJR_*1g#Dur8?}c9zVubgB|#NCY1PjN%7MJ?B*)^{0N9bmnG!dT zro@!^>m#2IWG($UOTS|2hXUL?W>*23H2W_YDCeG!8aChm^Nw2`Fxa_A1Nr*KTRjE3 z{Q9-Wd#klzW~{cy2mq)Eb`A~LklO>Rqyb1TZyP%eZCiskVSnmU8oCNbsD4b^z9j%k zUQ+U!k`sS$pt_P1PA}Yz+>Ye-_?10=aL76Lqyo@6sW>N-r?(j^EZmJ_K$vxkD#G+! z?OWd71hiHfi%pMd+DMW_Mld-7EXd(Zy;4j7Spos#O9Fu&rs$%Gcf{nABPJ;n*nPBe z4=J~*=0JdpgaQH876{D6AT!Sg0+@v2cnE-MlVY>r)I#t93X@bB9wW64=OSDz%)=L) z(~+3SjgpUGc$74sCe1XCLAI701!FSV8c61rw?4wO_$zquhDM zl{{PgJ;|F}3&=vbt02!*GL*Fp)liW< zTkrr4Po6v}c#*}MT)mpu9M73M6m!Q{=9eCsU&@)g6>|40=GT%;-d30OyqdGUnlu;M zQRCjtOPRXtUVqMaNbw!YqIxi=#sTs)xv4W?l+VBe;}qpApl9HF2E;g0@kew^mTt*2 xj^yyw@ziu``r3ON@8uY;!g#ag8#3oiEA*1c7w^Ah+R(>Cbwj=MKfX)>{9nyu(@p>Y