Promote R2 uploads through the S3 API

.github/workflows/promote-release.yml

@@ -1,12 +1,15 @@
 name: Promote sealed release stage
 
 # Privileged operator promotion of an already-sealed client release stage,
-# executed inside Actions so the R2 credential (repo secret
-# CLOUDFLARE_API_TOKEN) never leaves GitHub. The wrapped
-# scripts/promote-release.sh re-validates provenance (version, source SHA,
-# correlation id) against the sealed stage before any remote write, and its
-# immutable-asset preflight makes re-runs idempotent: identical GitHub
-# assets and R2 objects are skipped byte-for-byte, differing bytes abort.
+# executed inside Actions so the R2 credentials (repo secrets
+# CLOUDFLARE_API_TOKEN, R2_ENDPOINT, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY)
+# never leave GitHub. R2 uploads use the S3 API through the R2_* secrets;
+# wrangler is installed only as the fallback transport when they are absent.
+# The wrapped scripts/promote-release.sh re-validates provenance (version,
+# source SHA, correlation id) against the sealed stage before any remote
+# write, and its immutable-asset preflight makes re-runs idempotent:
+# identical GitHub assets and R2 objects are skipped byte-for-byte,
+# differing bytes abort.
 on:
   workflow_dispatch:
     inputs:
@@ -62,6 +65,9 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           VERSION: ${{ inputs.version }}
           EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }}
           CORRELATION_ID: ${{ inputs.correlation_id }}

docs/release-operator-checklist.md

@@ -61,9 +61,13 @@ generation refuses to write `stable.json` or `stable-v2.json` by design.
 - [ ] Confirm the destination GitHub release exists and its tag is exact.
 - [ ] Confirm the release is neither `draft` nor `prerelease`; either state is
   forbidden from advancing stable manifests.
-- [ ] Configure `gh`, `wrangler`, R2 credentials, and the public `BASE_URL` in
-  the privileged operator environment. These credentials do not belong in the
-  producer workflow.
+- [ ] Configure `gh`, R2 credentials, and the public `BASE_URL` in the
+  privileged operator environment. Exporting `R2_ENDPOINT`,
+  `R2_ACCESS_KEY_ID`, and `R2_SECRET_ACCESS_KEY` uploads through the S3 API,
+  which is the transport used by `promote-release.yml`; without them the
+  script falls back to `wrangler r2 object put --remote`, which has reported
+  success for multi-megabyte objects that never became readable. These
+  credentials do not belong in the producer workflow.
 - [ ] Set `TMPDIR` to a protected filesystem with enough space for one largest
   remote object plus the small plans. Successful comparison/readback copies are
   removed immediately rather than retained until process exit. The stage
@@ -97,9 +101,10 @@ order: strict `stable-v2.json` first and legacy `stable.json` last.
   absence; redirects, authorization/rate-limit/server errors, malformed status,
   timeouts, and transport failures stop the run.
 - [ ] R2 immutables are re-read immediately before each put and every put is
-  read back. Wrangler does not expose an atomic conditional put in this flow,
-  so a residual race remains between the final 404 and the put. Never claim an
-  atomic no-clobber guarantee; investigate any readback mismatch immediately.
+  read back. Neither R2 transport (S3 API or wrangler) exposes an atomic
+  conditional put in this flow, so a residual race remains between the final
+  404 and the put. Never claim an atomic no-clobber guarantee; investigate any
+  readback mismatch immediately.
 - [ ] On any failure before the stable phase, verify that no stable pointer was
   written, correct the failure, and rerun from the same sealed stage.
 - [ ] If interruption occurs during the final stable-pointer loop, rerun from

scripts/promote-release.sh

@@ -52,6 +52,31 @@ for command_name in gh wrangler curl cmp; do
   }
 done
 
+# Wrangler's `r2 object put --remote` has reported "Upload complete." for
+# multi-megabyte objects that never became readable, so the S3 API is the
+# preferred upload transport whenever its credentials are present. Wrangler
+# stays the fallback for operator environments without S3 keys.
+r2_s3_endpoint="${R2_ENDPOINT:-}"
+r2_s3_access_key="${R2_ACCESS_KEY_ID:-}"
+r2_s3_secret_key="${R2_SECRET_ACCESS_KEY:-}"
+if [ -n "$r2_s3_endpoint$r2_s3_access_key$r2_s3_secret_key" ]; then
+  [[ "$r2_s3_endpoint" == https://* ]] || {
+    echo "ERROR: R2_ENDPOINT must use HTTPS" >&2
+    exit 2
+  }
+  [ -n "$r2_s3_access_key" ] && [ -n "$r2_s3_secret_key" ] || {
+    echo "ERROR: R2_ENDPOINT requires both R2_ACCESS_KEY_ID and R2_SECRET_ACCESS_KEY" >&2
+    exit 2
+  }
+  command -v aws >/dev/null || {
+    echo "ERROR: required command not found: aws" >&2
+    exit 2
+  }
+  echo "R2 upload transport: S3 API (aws s3api) via R2_ENDPOINT"
+else
+  echo "R2 upload transport: wrangler r2 object put --remote"
+fi
+
 verification_dir="$(mktemp -d)"
 snapshot_tmp=""
 cleanup() {
@@ -128,6 +153,34 @@ fetch_r2() {
   esac
 }
 
+r2_content_type() {
+  case "$1" in
+    *.json) echo "application/json" ;;
+    *.tar.gz) echo "application/gzip" ;;
+    *.zip) echo "application/zip" ;;
+    *.zst) echo "application/zstd" ;;
+    *) echo "application/octet-stream" ;;
+  esac
+}
+
+r2_put() {
+  local object_path="$1"
+  local local_path="$2"
+  local content_type
+  content_type="$(r2_content_type "$object_path")"
+  if [ -n "$r2_s3_endpoint" ]; then
+    AWS_ACCESS_KEY_ID="$r2_s3_access_key" \
+    AWS_SECRET_ACCESS_KEY="$r2_s3_secret_key" \
+      aws s3api put-object --bucket "$bucket" --key "$object_path" \
+        --body "$local_path" --content-type "$content_type" \
+        --endpoint-url "$r2_s3_endpoint" --region auto \
+        --output text --no-cli-pager >/dev/null
+  else
+    wrangler r2 object put "$bucket/$object_path" --file "$local_path" \
+      --content-type "$content_type" --remote
+  fi
+}
+
 github_plan="$verification_dir/github-upload.tsv"
 r2_plan="$verification_dir/r2-upload.tsv"
 : > "$github_plan"
@@ -180,7 +233,7 @@ while IFS= read -r local_asset; do
   rmdir "$verification_dir/github-readback/$name.dir"
 done < "$github_plan"
 
-# Wrangler does not expose an atomic if-none-match put for this command. Re-read
+# Neither R2 transport exposes an atomic if-none-match put. Re-read
 # immediately before each put, skip an identical race winner, and fail on a
 # differing winner. A sub-request race between the final 404 and put remains a
 # documented provider limitation; every put is nevertheless read back exactly.
@@ -195,7 +248,7 @@ while IFS=$'\t' read -r object_path local_path; do
     rm -f "$immediate"
     continue
   fi
-  wrangler r2 object put "$bucket/$object_path" --file "$local_path" --remote
+  r2_put "$object_path" "$local_path"
   readback="$verification_dir/r2-readback/${object_path//\//_}"
   fetch_r2 "$object_path" "$readback" || {
     echo "ERROR: uploaded R2 object is absent: $object_path" >&2
@@ -244,7 +297,7 @@ advance_pointer() {
     fi
     rm -f "$existing"
   fi
-  wrangler r2 object put "$bucket/$object_path" --file "$local_path" --content-type application/json --remote
+  r2_put "$object_path" "$local_path"
   fetch_r2 "$object_path" "$existing" || {
     echo "ERROR: stable pointer readback is absent: $object_path" >&2
     exit 1

tests/test_promotion_contract.py

@@ -227,6 +227,35 @@ def install_remote_tools(root: Path) -> tuple[Path, Path, Path, Path]:
     return tools, gh_remote, r2_remote, log
 
 
+def install_aws_s3_stub(tools: Path) -> None:
+    executable(
+        tools / "aws",
+        r'''#!/usr/bin/env python3
+import os, pathlib, shutil, sys
+args = sys.argv[1:]
+if args[:2] != ["s3api", "put-object"]: sys.exit(2)
+if "--endpoint-url" not in args or "--content-type" not in args: sys.exit(2)
+key = args[args.index("--key") + 1]
+expected_types = {".json": "application/json", ".tar.gz": "application/gzip", ".zip": "application/zip", ".zst": "application/zstd"}
+expected = next((value for suffix, value in expected_types.items() if key.endswith(suffix)), "application/octet-stream")
+if args[args.index("--content-type") + 1] != expected: sys.exit(5)
+failure = os.environ.get("FAIL_OBJECT_ONCE", os.environ.get("FAIL_POINTER_ONCE", ""))
+marker = pathlib.Path(os.environ.get("FAILURE_MARKER", "/nonexistent"))
+if key == failure and not marker.exists():
+    marker.write_text("failed\n")
+    sys.exit(19)
+remote = pathlib.Path(os.environ["R2_REMOTE"]) / key
+remote.parent.mkdir(parents=True, exist_ok=True)
+shutil.copyfile(args[args.index("--body") + 1], remote)
+with pathlib.Path(os.environ["CALL_LOG"]).open("a") as handle: handle.write(f"r2-put {key}\n")
+''',
+    )
+
+
+def poison_wrangler(tools: Path) -> None:
+    executable(tools / "wrangler", "#!/bin/sh\nexit 3\n")
+
+
 def promotion_env(tools: Path, gh_remote: Path, r2_remote: Path, log: Path) -> dict[str, str]:
     env = os.environ.copy()
     env.update(
@@ -375,6 +404,79 @@ def test_successful_promotion_releases_each_verification_copy_immediately(self)
             self.assertNotIn("scratch-leak", log.read_text())
             self.assertEqual(list(scratch.iterdir()), [])
 
+    def test_s3_api_transport_promotes_without_wrangler_and_is_idempotent(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            staged = prepare_complete_stage(root)
+            tools, gh_remote, r2_remote, log = install_remote_tools(root)
+            install_aws_s3_stub(tools)
+            poison_wrangler(tools)
+            env = promotion_env(tools, gh_remote, r2_remote, log)
+            env.update(
+                {
+                    "R2_ENDPOINT": "https://s3.invalid",
+                    "R2_ACCESS_KEY_ID": "test-access-key",
+                    "R2_SECRET_ACCESS_KEY": "test-secret-key",
+                }
+            )
+            first = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True)
+            self.assertEqual(first.returncode, 0, first.stderr)
+            self.assertIn("R2 upload transport: S3 API", first.stdout)
+            writes = [line for line in log.read_text().splitlines() if line.startswith("r2-put")]
+            self.assertIn(f"r2-put terraphim-agent/manifests/v2/{VERSION}.json", writes)
+            self.assertIn("r2-put terraphim-agent/stable.json", writes)
+            for binary in ("terraphim-agent", "terraphim-cli", "terraphim-grep"):
+                expected = (staged / "manifests" / f"{binary}.v1.candidate.json").read_bytes()
+                self.assertEqual((r2_remote / binary / "stable.json").read_bytes(), expected)
+            second = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True)
+            self.assertEqual(second.returncode, 0, second.stderr)
+            self.assertEqual(
+                [line for line in log.read_text().splitlines() if line.startswith("r2-put")],
+                writes,
+            )
+
+    def test_s3_api_transport_failure_never_advances_stable_manifest(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            staged = prepare_complete_stage(root)
+            tools, gh_remote, r2_remote, log = install_remote_tools(root)
+            install_aws_s3_stub(tools)
+            env = promotion_env(tools, gh_remote, r2_remote, log)
+            env.update(
+                {
+                    "R2_ENDPOINT": "https://s3.invalid",
+                    "R2_ACCESS_KEY_ID": "test-access-key",
+                    "R2_SECRET_ACCESS_KEY": "test-secret-key",
+                    "FAIL_OBJECT_ONCE": f"terraphim-agent/manifests/v1/{VERSION}.json",
+                    "FAILURE_MARKER": str(root / "failed-once"),
+                }
+            )
+            result = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True)
+            self.assertNotEqual(result.returncode, 0)
+            calls = log.read_text() if log.exists() else ""
+            self.assertNotIn("stable.json", calls)
+            self.assertNotIn("stable-v2.json", calls)
+
+    def test_partial_s3_credentials_are_rejected_before_any_remote_query(self) -> None:
+        # A missing aws binary cannot be simulated portably: hosted runners
+        # ship a real aws on PATH, which is exactly the production setup.
+        for missing in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY"):
+            with self.subTest(missing=missing), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                staged = prepare_complete_stage(root)
+                tools, gh_remote, r2_remote, log = install_remote_tools(root)
+                install_aws_s3_stub(tools)
+                env = promotion_env(tools, gh_remote, r2_remote, log)
+                env["R2_ENDPOINT"] = "https://s3.invalid"
+                if missing != "R2_ACCESS_KEY_ID":
+                    env["R2_ACCESS_KEY_ID"] = "test-access-key"
+                if missing != "R2_SECRET_ACCESS_KEY":
+                    env["R2_SECRET_ACCESS_KEY"] = "test-secret-key"
+                result = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("ERROR:", result.stderr)
+                self.assertFalse(log.exists(), "S3 misconfiguration must fail before any remote query")
+
     def test_rollback_requires_explicit_pointers_only_authorization_flag(self) -> None:
         result = subprocess.run(
             [str(ROLLBACK), VERSION, "/nonexistent", SOURCE_SHA, CORRELATION_ID],