From 824225f4b953f526eaca1f67ed6786d952844ee9 Mon Sep 17 00:00:00 2001 From: Derek Date: Sat, 21 Nov 2026 15:17:20 +1100 Subject: [PATCH] fix(hyperdx): rename the DocumentDB TLS files into place A restarting container could read a partial file when the one-shot overwrote the CA, key or leaf in place. The one-shot writes each file to a .new name before renaming it into place. It renames the key and leaf first and the CA last. The rewrite message printed "a running stack serves the old one" on a fresh volume. The message tells you to restart only if the stack is running. The tests run the one-shot script in a temp directory. A valid pair keeps its sha256 across two runs. If the CA, key or leaf is empty, the one-shot renews all three in the same directories. No .new file is left behind. openssl verify passes. The renew tests fail on main because the one-shot changes the directory inode. --- compose/one-shots.yml | 17 ++-- scripts/tests/test_documentdb_tls.py | 114 ++++++++++++++++++++++----- 2 files changed, 103 insertions(+), 28 deletions(-) diff --git a/compose/one-shots.yml b/compose/one-shots.yml index 5937e65..cc99923 100644 --- a/compose/one-shots.yml +++ b/compose/one-shots.yml @@ -175,23 +175,26 @@ services: set -eu umask 077 cd /tls - # Running containers mount ca and server by subpath, so the directories stay and only their files are replaced + # Running containers mount ca and server by subpath, so the directories stay and each file is renamed into place whole mkdir -p ca server if ! { [ -s ca/ca.crt ] && [ -s server/tls.key ] && [ -s server/tls.crt ] \ && openssl x509 -checkend 2592000 -noout -in server/tls.crt >/dev/null; }; then - rm -f ca/ca.crt server/tls.key server/tls.crt ca.key ca.srl server.csr server.ext + rm -f ca/ca.crt.new server/tls.key.new server/tls.crt.new ca.key ca.srl server.csr server.ext openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 -days 3650 \ -subj /CN=dfe-hyperdx-documentdb-ca \ -addext basicConstraints=critical,CA:TRUE -addext keyUsage=critical,keyCertSign,cRLSign \ - -keyout ca.key -out ca/ca.crt + -keyout ca.key -out ca/ca.crt.new openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 \ - -subj /CN=hyperdx-documentdb -keyout server/tls.key -out server.csr + -subj /CN=hyperdx-documentdb -keyout server/tls.key.new -out server.csr printf '%s\n' basicConstraints=critical,CA:FALSE keyUsage=critical,digitalSignature \ extendedKeyUsage=serverAuth subjectAltName=DNS:hyperdx-documentdb > server.ext - openssl x509 -req -in server.csr -CA ca/ca.crt -CAkey ca.key -CAcreateserial -CAserial ca.srl \ - -days 3650 -sha384 -extfile server.ext -out server/tls.crt + openssl x509 -req -in server.csr -CA ca/ca.crt.new -CAkey ca.key -CAcreateserial -CAserial ca.srl \ + -days 3650 -sha384 -extfile server.ext -out server/tls.crt.new + mv server/tls.key.new server/tls.key + mv server/tls.crt.new server/tls.crt + mv ca/ca.crt.new ca/ca.crt rm -f ca.key ca.srl server.csr server.ext - echo "hyperdx-documentdb-tls: wrote a new CA and gateway certificate. A running stack serves the old one until 'docker compose restart hyperdx-documentdb', then 'docker compose restart dfe-hyperdx'" + echo "hyperdx-documentdb-tls: wrote a new CA and gateway certificate. If the stack is running, run 'docker compose restart hyperdx-documentdb', then 'docker compose restart dfe-hyperdx'" fi chmod 0755 ca chmod 0644 ca/ca.crt diff --git a/scripts/tests/test_documentdb_tls.py b/scripts/tests/test_documentdb_tls.py index 5343a04..d99190a 100644 --- a/scripts/tests/test_documentdb_tls.py +++ b/scripts/tests/test_documentdb_tls.py @@ -9,34 +9,106 @@ """hyperdx-documentdb and dfe-hyperdx mount the one-shot's ca and server directories by subpath. Removing a directory leaves a running container holding the deleted one, so dfe-hyperdx then -fails to open its CA file. The one-shot keeps the directories and replaces their files. +fails to open its CA file. The one-shot keeps the directories, leaves a valid pair alone and +renames each new file into place whole. These run the one-shot's own script against a temp +directory, with chown stubbed because the test user is not root. """ +import hashlib +import shutil +import subprocess from pathlib import Path import pytest +import yaml ONE_SHOTS = Path(__file__).resolve().parents[2] / "compose" / "one-shots.yml" +FILES = ("ca/ca.crt", "server/tls.key", "server/tls.crt") +REWRITE = "wrote a new CA and gateway certificate" +pytestmark = pytest.mark.skipif(shutil.which("openssl") is None, reason="needs openssl") -def _script() -> str: - text = ONE_SHOTS.read_text(encoding="utf-8") - start = text.index(" hyperdx-documentdb-tls:") - end = text.index("restart:", start) - return text[start:end] - - -@pytest.mark.parametrize( - ("fragment", "present"), - [ - ("mkdir -p ca server", True), - ("rm -rf", False), - ("mkdir ca server", False), - ("docker compose restart hyperdx-documentdb", True), - ], - ids=["keeps-dirs", "no-dir-removal", "no-bare-mkdir", "says-to-restart"], -) -def test_the_one_shot_keeps_its_mounted_directories( - fragment: str, present: bool + +class _Tags(yaml.SafeLoader): + """Reads compose's !override and !reset tags as plain values.""" + + +_Tags.add_multi_constructor("!", lambda loader, suffix, node: None) + + +def _run(tls: Path, shims: Path) -> str: + services = yaml.load(ONE_SHOTS.read_text(encoding="utf-8"), Loader=_Tags)[ + "services" + ] + script = services["hyperdx-documentdb-tls"]["command"][0].replace( + "cd /tls", f"cd {tls}" + ) + result = subprocess.run( + ["sh", "-c", script], + capture_output=True, + check=True, + encoding="utf-8", + env={"PATH": f"{shims}:/usr/bin:/bin"}, + errors="replace", + text=True, + ) + return result.stdout + + +def _digests(tls: Path) -> dict[str, str]: + return { + name: hashlib.sha256((tls / name).read_bytes()).hexdigest() for name in FILES + } + + +@pytest.fixture +def tls(tmp_path: Path) -> tuple[Path, Path]: + shims = tmp_path / "shims" + shims.mkdir() + chown = shims / "chown" + chown.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + chown.chmod(0o755) + volume = tmp_path / "tls" + volume.mkdir() + return volume, shims + + +def test_a_valid_pair_is_left_alone(tls: tuple[Path, Path]) -> None: + volume, shims = tls + assert REWRITE in _run(volume, shims) + before = _digests(volume) + + assert REWRITE not in _run(volume, shims) + assert _digests(volume) == before + + +@pytest.mark.parametrize("broken", FILES, ids=["no-ca", "no-key", "no-leaf"]) +def test_a_missing_file_renews_the_pair_in_the_same_directories( + tls: tuple[Path, Path], broken: str ) -> None: - assert (fragment in _script()) is present + volume, shims = tls + _run(volume, shims) + inodes = {name: (volume / name).stat().st_ino for name in ("ca", "server")} + before = _digests(volume) + (volume / broken).write_bytes(b"") + + said = _run(volume, shims) + assert {name: (volume / name).stat().st_ino for name in ("ca", "server")} == inodes + assert all(_digests(volume)[name] != before[name] for name in FILES) + assert REWRITE in said + assert not list(volume.rglob("*.new")) + verify = subprocess.run( + [ + "openssl", + "verify", + "-CAfile", + str(volume / "ca/ca.crt"), + str(volume / "server/tls.crt"), + ], + capture_output=True, + check=False, + encoding="utf-8", + errors="replace", + text=True, + ) + assert verify.returncode == 0, verify.stderr