From 5639ef21c8bebc3f9e67633f5e75ff5a0cba2f91 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 25 Sep 2026 21:51:12 +0100 Subject: [PATCH] fix(release): await R2 readbacks across public-channel propagation lag Promote run 36187580126 aborted because the aws s3api put for terraphim-agent-1.21.16-aarch64-unknown-linux-musl.tar.gz was acknowledged but the immediate readback through downloads.terraphim.ai returned 404; the object surfaced on the public channel roughly five minutes later with cf-cache-status DYNAMIC, so the 404s were origin responses behind a propagation lag, not edge-cache staleness or a failed write. Post-put readbacks now retry every five seconds for R2_READBACK_WAIT seconds (default 600, 0 through 3600 validated) before declaring an object or stable pointer absent. The promotion contract gains a lag-simulating curl stub mode pinning both the retried success and the bounded-timeout failure that must stop before any stable write. Refs #337 --- docs/release-operator-checklist.md | 4 ++++ scripts/promote-release.sh | 36 ++++++++++++++++++++++++++---- tests/test_promotion_contract.py | 31 +++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/docs/release-operator-checklist.md b/docs/release-operator-checklist.md index 500f17c..366b6a4 100644 --- a/docs/release-operator-checklist.md +++ b/docs/release-operator-checklist.md @@ -105,6 +105,10 @@ order: strict `stable-v2.json` first and legacy `stable.json` last. conditional put in this flow, so a residual race remains between the final 404 and the put. Never claim an atomic no-clobber guarantee; investigate any readback mismatch immediately. +- [ ] A successful R2 write can take minutes to appear on the public channel + even though the S3 operation already acknowledged it. Every post-put + readback retries for `R2_READBACK_WAIT` seconds (default 600) before the + run declares the object absent and stops. - [ ] On any failure before the stable phase, verify that no stable pointer was written, correct the failure, and rerun from the same sealed stage. - [ ] If interruption occurs during the final stable-pointer loop, rerun from diff --git a/scripts/promote-release.sh b/scripts/promote-release.sh index 006a595..a779ddf 100755 --- a/scripts/promote-release.sh +++ b/scripts/promote-release.sh @@ -30,6 +30,10 @@ repo="terraphim/${target_repo}" base_url="${BASE_URL:-https://downloads.terraphim.ai}" bucket="${R2_BUCKET:-terraphim-releases}" r2_read_timeout="${R2_READ_TIMEOUT:-600}" +# Newly written objects can take minutes to appear on the public custom +# domain even though the S3 write already succeeded, so every post-put +# readback retries for this long before declaring the object absent. +r2_readback_wait="${R2_READBACK_WAIT:-600}" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" [[ "$base_url" == https://* ]] || { @@ -44,6 +48,10 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" echo "ERROR: R2_READ_TIMEOUT must be an integer from 30 through 3600 seconds" >&2 exit 2 } +[[ "$r2_readback_wait" =~ ^[0-9]+$ ]] && [ "$r2_readback_wait" -ge 0 ] && [ "$r2_readback_wait" -le 3600 ] || { + echo "ERROR: R2_READBACK_WAIT must be an integer from 0 through 3600 seconds" >&2 + exit 2 +} for command_name in gh wrangler curl cmp; do command -v "$command_name" >/dev/null || { @@ -181,6 +189,26 @@ r2_put() { fi } +# Await an object's appearance on the public channel. The write path (S3 API +# or wrangler) can acknowledge an object minutes before the custom domain +# serves it, so a single immediate re-read is not evidence of absence. +await_r2() { + local object_path="$1" + local destination="$2" + local waited=0 + local interval=5 + while :; do + if fetch_r2 "$object_path" "$destination"; then + return 0 + fi + if [ "$waited" -ge "$r2_readback_wait" ]; then + return 1 + fi + sleep "$interval" + waited=$((waited + interval)) + done +} + github_plan="$verification_dir/github-upload.tsv" r2_plan="$verification_dir/r2-upload.tsv" : > "$github_plan" @@ -250,8 +278,8 @@ while IFS=$'\t' read -r object_path local_path; do fi r2_put "$object_path" "$local_path" readback="$verification_dir/r2-readback/${object_path//\//_}" - fetch_r2 "$object_path" "$readback" || { - echo "ERROR: uploaded R2 object is absent: $object_path" >&2 + await_r2 "$object_path" "$readback" || { + echo "ERROR: uploaded R2 object is absent after ${r2_readback_wait}s: $object_path" >&2 exit 1 } cmp "$local_path" "$readback" || { @@ -298,8 +326,8 @@ advance_pointer() { rm -f "$existing" fi r2_put "$object_path" "$local_path" - fetch_r2 "$object_path" "$existing" || { - echo "ERROR: stable pointer readback is absent: $object_path" >&2 + await_r2 "$object_path" "$existing" || { + echo "ERROR: stable pointer readback is absent after ${r2_readback_wait}s: $object_path" >&2 exit 1 } cmp "$local_path" "$existing" || { diff --git a/tests/test_promotion_contract.py b/tests/test_promotion_contract.py index b2d39cf..936cbb9 100644 --- a/tests/test_promotion_contract.py +++ b/tests/test_promotion_contract.py @@ -202,6 +202,9 @@ def install_remote_tools(root: Path) -> tuple[Path, Path, Path, Path]: count_dir.mkdir(parents=True, exist_ok=True) count = int(count_file.read_text()) + 1 if count_file.exists() else 1 count_file.write_text(str(count)) +if key == os.environ.get("LAG_KEY") and count <= int(os.environ.get("LAG_READS", "0")): + sys.stdout.write("404") + sys.exit(0) if key == os.environ.get("APPEAR_ON_READ_KEY") and count == int(os.environ.get("APPEAR_ON_READ_NUMBER", "2")): source.parent.mkdir(parents=True, exist_ok=True) source.write_bytes(os.environ.get("APPEAR_BYTES", "different-race-winner").encode()) @@ -477,6 +480,34 @@ def test_partial_s3_credentials_are_rejected_before_any_remote_query(self) -> No self.assertIn("ERROR:", result.stderr) self.assertFalse(log.exists(), "S3 misconfiguration must fail before any remote query") + def test_public_channel_lag_is_awaited_before_declaring_absence(self) -> None: + path = "terraphim-agent/terraphim-agent-1.21.15-aarch64-apple-darwin.tar.gz" + for lag_reads, wait, expect_success in ((3, "600", True), (999, "0", False)): + with self.subTest(lag_reads=lag_reads, wait=wait), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + staged = prepare_complete_stage(root) + tools, gh_remote, r2_remote, log = install_remote_tools(root) + env = promotion_env(tools, gh_remote, r2_remote, log) + env.update( + { + "LAG_KEY": path, + "LAG_READS": str(lag_reads), + "R2_READBACK_WAIT": wait, + } + ) + result = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True) + if expect_success: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((r2_remote / path).read_bytes(), b"signed-final-terraphim-agent-aarch64-apple-darwin") + reads = (log.parent / "curl-counts" / path.replace("/", "_")).read_text() + self.assertEqual(int(reads), lag_reads + 1, "readback must retry past the lag window") + else: + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"uploaded R2 object is absent after {wait}s: {path}", result.stderr) + calls = log.read_text() if log.exists() else "" + self.assertNotIn("stable.json", calls) + self.assertNotIn("stable-v2.json", calls) + def test_rollback_requires_explicit_pointers_only_authorization_flag(self) -> None: result = subprocess.run( [str(ROLLBACK), VERSION, "/nonexistent", SOURCE_SHA, CORRELATION_ID],