Await R2 readbacks across public-channel propagation lag

docs/release-operator-checklist.md

@@ -105,6 +105,10 @@ order: strict `stable-v2.json` first and legacy `stable.json` last.
   conditional put in this flow, so a residual race remains between the final
   404 and the put. Never claim an atomic no-clobber guarantee; investigate any
   readback mismatch immediately.
+- [ ] A successful R2 write can take minutes to appear on the public channel
+  even though the S3 operation already acknowledged it. Every post-put
+  readback retries for `R2_READBACK_WAIT` seconds (default 600) before the
+  run declares the object absent and stops.
 - [ ] On any failure before the stable phase, verify that no stable pointer was
   written, correct the failure, and rerun from the same sealed stage.
 - [ ] If interruption occurs during the final stable-pointer loop, rerun from

scripts/promote-release.sh

@@ -30,6 +30,10 @@ repo="terraphim/${target_repo}"
 base_url="${BASE_URL:-https://downloads.terraphim.ai}"
 bucket="${R2_BUCKET:-terraphim-releases}"
 r2_read_timeout="${R2_READ_TIMEOUT:-600}"
+# Newly written objects can take minutes to appear on the public custom
+# domain even though the S3 write already succeeded, so every post-put
+# readback retries for this long before declaring the object absent.
+r2_readback_wait="${R2_READBACK_WAIT:-600}"
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 
 [[ "$base_url" == https://* ]] || {
@@ -44,6 +48,10 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
   echo "ERROR: R2_READ_TIMEOUT must be an integer from 30 through 3600 seconds" >&2
   exit 2
 }
+[[ "$r2_readback_wait" =~ ^[0-9]+$ ]] && [ "$r2_readback_wait" -ge 0 ] && [ "$r2_readback_wait" -le 3600 ] || {
+  echo "ERROR: R2_READBACK_WAIT must be an integer from 0 through 3600 seconds" >&2
+  exit 2
+}
 
 for command_name in gh wrangler curl cmp; do
   command -v "$command_name" >/dev/null || {
@@ -181,6 +189,26 @@ r2_put() {
   fi
 }
 
+# Await an object's appearance on the public channel. The write path (S3 API
+# or wrangler) can acknowledge an object minutes before the custom domain
+# serves it, so a single immediate re-read is not evidence of absence.
+await_r2() {
+  local object_path="$1"
+  local destination="$2"
+  local waited=0
+  local interval=5
+  while :; do
+    if fetch_r2 "$object_path" "$destination"; then
+      return 0
+    fi
+    if [ "$waited" -ge "$r2_readback_wait" ]; then
+      return 1
+    fi
+    sleep "$interval"
+    waited=$((waited + interval))
+  done
+}
+
 github_plan="$verification_dir/github-upload.tsv"
 r2_plan="$verification_dir/r2-upload.tsv"
 : > "$github_plan"
@@ -250,8 +278,8 @@ while IFS=$'\t' read -r object_path local_path; do
   fi
   r2_put "$object_path" "$local_path"
   readback="$verification_dir/r2-readback/${object_path//\//_}"
-  fetch_r2 "$object_path" "$readback" || {
-    echo "ERROR: uploaded R2 object is absent: $object_path" >&2
+  await_r2 "$object_path" "$readback" || {
+    echo "ERROR: uploaded R2 object is absent after ${r2_readback_wait}s: $object_path" >&2
     exit 1
   }
   cmp "$local_path" "$readback" || {
@@ -298,8 +326,8 @@ advance_pointer() {
     rm -f "$existing"
   fi
   r2_put "$object_path" "$local_path"
-  fetch_r2 "$object_path" "$existing" || {
-    echo "ERROR: stable pointer readback is absent: $object_path" >&2
+  await_r2 "$object_path" "$existing" || {
+    echo "ERROR: stable pointer readback is absent after ${r2_readback_wait}s: $object_path" >&2
     exit 1
   }
   cmp "$local_path" "$existing" || {

tests/test_promotion_contract.py

@@ -202,6 +202,9 @@ def install_remote_tools(root: Path) -> tuple[Path, Path, Path, Path]:
 count_dir.mkdir(parents=True, exist_ok=True)
 count = int(count_file.read_text()) + 1 if count_file.exists() else 1
 count_file.write_text(str(count))
+if key == os.environ.get("LAG_KEY") and count <= int(os.environ.get("LAG_READS", "0")):
+    sys.stdout.write("404")
+    sys.exit(0)
 if key == os.environ.get("APPEAR_ON_READ_KEY") and count == int(os.environ.get("APPEAR_ON_READ_NUMBER", "2")):
     source.parent.mkdir(parents=True, exist_ok=True)
     source.write_bytes(os.environ.get("APPEAR_BYTES", "different-race-winner").encode())
@@ -477,6 +480,34 @@ def test_partial_s3_credentials_are_rejected_before_any_remote_query(self) -> No
                 self.assertIn("ERROR:", result.stderr)
                 self.assertFalse(log.exists(), "S3 misconfiguration must fail before any remote query")
 
+    def test_public_channel_lag_is_awaited_before_declaring_absence(self) -> None:
+        path = "terraphim-agent/terraphim-agent-1.21.15-aarch64-apple-darwin.tar.gz"
+        for lag_reads, wait, expect_success in ((3, "600", True), (999, "0", False)):
+            with self.subTest(lag_reads=lag_reads, wait=wait), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                staged = prepare_complete_stage(root)
+                tools, gh_remote, r2_remote, log = install_remote_tools(root)
+                env = promotion_env(tools, gh_remote, r2_remote, log)
+                env.update(
+                    {
+                        "LAG_KEY": path,
+                        "LAG_READS": str(lag_reads),
+                        "R2_READBACK_WAIT": wait,
+                    }
+                )
+                result = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True)
+                if expect_success:
+                    self.assertEqual(result.returncode, 0, result.stderr)
+                    self.assertEqual((r2_remote / path).read_bytes(), b"signed-final-terraphim-agent-aarch64-apple-darwin")
+                    reads = (log.parent / "curl-counts" / path.replace("/", "_")).read_text()
+                    self.assertEqual(int(reads), lag_reads + 1, "readback must retry past the lag window")
+                else:
+                    self.assertNotEqual(result.returncode, 0)
+                    self.assertIn(f"uploaded R2 object is absent after {wait}s: {path}", result.stderr)
+                    calls = log.read_text() if log.exists() else ""
+                    self.assertNotIn("stable.json", calls)
+                    self.assertNotIn("stable-v2.json", calls)
+
     def test_rollback_requires_explicit_pointers_only_authorization_flag(self) -> None:
         result = subprocess.run(
             [str(ROLLBACK), VERSION, "/nonexistent", SOURCE_SHA, CORRELATION_ID],