From 701c637cfac2b289d972359698c1fd194b185bcd Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 25 Sep 2026 21:33:28 +0100 Subject: [PATCH 1/2] fix(release): promote R2 uploads through the S3 API Wrangler 4.140.0 r2 object put --remote reported Upload complete. for the 8.4 MB v1.21.16 terraphim-agent archive, yet the object never became readable through downloads.terraphim.ai while 8 KB manifests uploaded and read back fine (workers-sdk issues #12982, #13034, #6642 family). Route every R2 put through aws s3api put-object when R2_ENDPOINT, R2_ACCESS_KEY_ID, and R2_SECRET_ACCESS_KEY are exported, keeping wrangler as the fallback transport. promote-release.yml now passes the existing R2_* repo secrets so Actions uses the S3 path; each put keeps its immediate public-URL readback and byte comparison. Contract tests gain an aws stub covering the transport, its failure semantics, idempotent rerun, and partial-credential rejection. Refs #337 --- .github/workflows/promote-release.yml | 18 +++-- docs/release-operator-checklist.md | 17 +++-- scripts/promote-release.sh | 59 ++++++++++++++- tests/test_promotion_contract.py | 101 ++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 15 deletions(-) diff --git a/.github/workflows/promote-release.yml b/.github/workflows/promote-release.yml index b6c77e2..b92bab3 100644 --- a/.github/workflows/promote-release.yml +++ b/.github/workflows/promote-release.yml @@ -1,12 +1,15 @@ name: Promote sealed release stage # Privileged operator promotion of an already-sealed client release stage, -# executed inside Actions so the R2 credential (repo secret -# CLOUDFLARE_API_TOKEN) never leaves GitHub. The wrapped -# scripts/promote-release.sh re-validates provenance (version, source SHA, -# correlation id) against the sealed stage before any remote write, and its -# immutable-asset preflight makes re-runs idempotent: identical GitHub -# assets and R2 objects are skipped byte-for-byte, differing bytes abort. +# executed inside Actions so the R2 credentials (repo secrets +# CLOUDFLARE_API_TOKEN, R2_ENDPOINT, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY) +# never leave GitHub. R2 uploads use the S3 API through the R2_* secrets; +# wrangler is installed only as the fallback transport when they are absent. +# The wrapped scripts/promote-release.sh re-validates provenance (version, +# source SHA, correlation id) against the sealed stage before any remote +# write, and its immutable-asset preflight makes re-runs idempotent: +# identical GitHub assets and R2 objects are skipped byte-for-byte, +# differing bytes abort. on: workflow_dispatch: inputs: @@ -62,6 +65,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} VERSION: ${{ inputs.version }} EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} CORRELATION_ID: ${{ inputs.correlation_id }} diff --git a/docs/release-operator-checklist.md b/docs/release-operator-checklist.md index 960976b..500f17c 100644 --- a/docs/release-operator-checklist.md +++ b/docs/release-operator-checklist.md @@ -61,9 +61,13 @@ generation refuses to write `stable.json` or `stable-v2.json` by design. - [ ] Confirm the destination GitHub release exists and its tag is exact. - [ ] Confirm the release is neither `draft` nor `prerelease`; either state is forbidden from advancing stable manifests. -- [ ] Configure `gh`, `wrangler`, R2 credentials, and the public `BASE_URL` in - the privileged operator environment. These credentials do not belong in the - producer workflow. +- [ ] Configure `gh`, R2 credentials, and the public `BASE_URL` in the + privileged operator environment. Exporting `R2_ENDPOINT`, + `R2_ACCESS_KEY_ID`, and `R2_SECRET_ACCESS_KEY` uploads through the S3 API, + which is the transport used by `promote-release.yml`; without them the + script falls back to `wrangler r2 object put --remote`, which has reported + success for multi-megabyte objects that never became readable. These + credentials do not belong in the producer workflow. - [ ] Set `TMPDIR` to a protected filesystem with enough space for one largest remote object plus the small plans. Successful comparison/readback copies are removed immediately rather than retained until process exit. The stage @@ -97,9 +101,10 @@ order: strict `stable-v2.json` first and legacy `stable.json` last. absence; redirects, authorization/rate-limit/server errors, malformed status, timeouts, and transport failures stop the run. - [ ] R2 immutables are re-read immediately before each put and every put is - read back. Wrangler does not expose an atomic conditional put in this flow, - so a residual race remains between the final 404 and the put. Never claim an - atomic no-clobber guarantee; investigate any readback mismatch immediately. + read back. Neither R2 transport (S3 API or wrangler) exposes an atomic + conditional put in this flow, so a residual race remains between the final + 404 and the put. Never claim an atomic no-clobber guarantee; investigate any + readback mismatch immediately. - [ ] On any failure before the stable phase, verify that no stable pointer was written, correct the failure, and rerun from the same sealed stage. - [ ] If interruption occurs during the final stable-pointer loop, rerun from diff --git a/scripts/promote-release.sh b/scripts/promote-release.sh index c2824e1..006a595 100755 --- a/scripts/promote-release.sh +++ b/scripts/promote-release.sh @@ -52,6 +52,31 @@ for command_name in gh wrangler curl cmp; do } done +# Wrangler's `r2 object put --remote` has reported "Upload complete." for +# multi-megabyte objects that never became readable, so the S3 API is the +# preferred upload transport whenever its credentials are present. Wrangler +# stays the fallback for operator environments without S3 keys. +r2_s3_endpoint="${R2_ENDPOINT:-}" +r2_s3_access_key="${R2_ACCESS_KEY_ID:-}" +r2_s3_secret_key="${R2_SECRET_ACCESS_KEY:-}" +if [ -n "$r2_s3_endpoint$r2_s3_access_key$r2_s3_secret_key" ]; then + [[ "$r2_s3_endpoint" == https://* ]] || { + echo "ERROR: R2_ENDPOINT must use HTTPS" >&2 + exit 2 + } + [ -n "$r2_s3_access_key" ] && [ -n "$r2_s3_secret_key" ] || { + echo "ERROR: R2_ENDPOINT requires both R2_ACCESS_KEY_ID and R2_SECRET_ACCESS_KEY" >&2 + exit 2 + } + command -v aws >/dev/null || { + echo "ERROR: required command not found: aws" >&2 + exit 2 + } + echo "R2 upload transport: S3 API (aws s3api) via R2_ENDPOINT" +else + echo "R2 upload transport: wrangler r2 object put --remote" +fi + verification_dir="$(mktemp -d)" snapshot_tmp="" cleanup() { @@ -128,6 +153,34 @@ fetch_r2() { esac } +r2_content_type() { + case "$1" in + *.json) echo "application/json" ;; + *.tar.gz) echo "application/gzip" ;; + *.zip) echo "application/zip" ;; + *.zst) echo "application/zstd" ;; + *) echo "application/octet-stream" ;; + esac +} + +r2_put() { + local object_path="$1" + local local_path="$2" + local content_type + content_type="$(r2_content_type "$object_path")" + if [ -n "$r2_s3_endpoint" ]; then + AWS_ACCESS_KEY_ID="$r2_s3_access_key" \ + AWS_SECRET_ACCESS_KEY="$r2_s3_secret_key" \ + aws s3api put-object --bucket "$bucket" --key "$object_path" \ + --body "$local_path" --content-type "$content_type" \ + --endpoint-url "$r2_s3_endpoint" --region auto \ + --output text --no-cli-pager >/dev/null + else + wrangler r2 object put "$bucket/$object_path" --file "$local_path" \ + --content-type "$content_type" --remote + fi +} + github_plan="$verification_dir/github-upload.tsv" r2_plan="$verification_dir/r2-upload.tsv" : > "$github_plan" @@ -180,7 +233,7 @@ while IFS= read -r local_asset; do rmdir "$verification_dir/github-readback/$name.dir" done < "$github_plan" -# Wrangler does not expose an atomic if-none-match put for this command. Re-read +# Neither R2 transport exposes an atomic if-none-match put. Re-read # immediately before each put, skip an identical race winner, and fail on a # differing winner. A sub-request race between the final 404 and put remains a # documented provider limitation; every put is nevertheless read back exactly. @@ -195,7 +248,7 @@ while IFS=$'\t' read -r object_path local_path; do rm -f "$immediate" continue fi - wrangler r2 object put "$bucket/$object_path" --file "$local_path" --remote + r2_put "$object_path" "$local_path" readback="$verification_dir/r2-readback/${object_path//\//_}" fetch_r2 "$object_path" "$readback" || { echo "ERROR: uploaded R2 object is absent: $object_path" >&2 @@ -244,7 +297,7 @@ advance_pointer() { fi rm -f "$existing" fi - wrangler r2 object put "$bucket/$object_path" --file "$local_path" --content-type application/json --remote + r2_put "$object_path" "$local_path" fetch_r2 "$object_path" "$existing" || { echo "ERROR: stable pointer readback is absent: $object_path" >&2 exit 1 diff --git a/tests/test_promotion_contract.py b/tests/test_promotion_contract.py index 2760b40..e07b81b 100644 --- a/tests/test_promotion_contract.py +++ b/tests/test_promotion_contract.py @@ -227,6 +227,35 @@ def install_remote_tools(root: Path) -> tuple[Path, Path, Path, Path]: return tools, gh_remote, r2_remote, log +def install_aws_s3_stub(tools: Path) -> None: + executable( + tools / "aws", + r'''#!/usr/bin/env python3 +import os, pathlib, shutil, sys +args = sys.argv[1:] +if args[:2] != ["s3api", "put-object"]: sys.exit(2) +if "--endpoint-url" not in args or "--content-type" not in args: sys.exit(2) +key = args[args.index("--key") + 1] +expected_types = {".json": "application/json", ".tar.gz": "application/gzip", ".zip": "application/zip", ".zst": "application/zstd"} +expected = next((value for suffix, value in expected_types.items() if key.endswith(suffix)), "application/octet-stream") +if args[args.index("--content-type") + 1] != expected: sys.exit(5) +failure = os.environ.get("FAIL_OBJECT_ONCE", os.environ.get("FAIL_POINTER_ONCE", "")) +marker = pathlib.Path(os.environ.get("FAILURE_MARKER", "/nonexistent")) +if key == failure and not marker.exists(): + marker.write_text("failed\n") + sys.exit(19) +remote = pathlib.Path(os.environ["R2_REMOTE"]) / key +remote.parent.mkdir(parents=True, exist_ok=True) +shutil.copyfile(args[args.index("--body") + 1], remote) +with pathlib.Path(os.environ["CALL_LOG"]).open("a") as handle: handle.write(f"r2-put {key}\n") +''', + ) + + +def poison_wrangler(tools: Path) -> None: + executable(tools / "wrangler", "#!/bin/sh\nexit 3\n") + + def promotion_env(tools: Path, gh_remote: Path, r2_remote: Path, log: Path) -> dict[str, str]: env = os.environ.copy() env.update( @@ -375,6 +404,78 @@ def test_successful_promotion_releases_each_verification_copy_immediately(self) self.assertNotIn("scratch-leak", log.read_text()) self.assertEqual(list(scratch.iterdir()), []) + def test_s3_api_transport_promotes_without_wrangler_and_is_idempotent(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + staged = prepare_complete_stage(root) + tools, gh_remote, r2_remote, log = install_remote_tools(root) + install_aws_s3_stub(tools) + poison_wrangler(tools) + env = promotion_env(tools, gh_remote, r2_remote, log) + env.update( + { + "R2_ENDPOINT": "https://s3.invalid", + "R2_ACCESS_KEY_ID": "test-access-key", + "R2_SECRET_ACCESS_KEY": "test-secret-key", + } + ) + first = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True) + self.assertEqual(first.returncode, 0, first.stderr) + self.assertIn("R2 upload transport: S3 API", first.stdout) + writes = [line for line in log.read_text().splitlines() if line.startswith("r2-put")] + self.assertIn(f"r2-put terraphim-agent/manifests/v2/{VERSION}.json", writes) + self.assertIn("r2-put terraphim-agent/stable.json", writes) + for binary in ("terraphim-agent", "terraphim-cli", "terraphim-grep"): + expected = (staged / "manifests" / f"{binary}.v1.candidate.json").read_bytes() + self.assertEqual((r2_remote / binary / "stable.json").read_bytes(), expected) + second = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True) + self.assertEqual(second.returncode, 0, second.stderr) + self.assertEqual( + [line for line in log.read_text().splitlines() if line.startswith("r2-put")], + writes, + ) + + def test_s3_api_transport_failure_never_advances_stable_manifest(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + staged = prepare_complete_stage(root) + tools, gh_remote, r2_remote, log = install_remote_tools(root) + install_aws_s3_stub(tools) + env = promotion_env(tools, gh_remote, r2_remote, log) + env.update( + { + "R2_ENDPOINT": "https://s3.invalid", + "R2_ACCESS_KEY_ID": "test-access-key", + "R2_SECRET_ACCESS_KEY": "test-secret-key", + "FAIL_OBJECT_ONCE": f"terraphim-agent/manifests/v1/{VERSION}.json", + "FAILURE_MARKER": str(root / "failed-once"), + } + ) + result = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True) + self.assertNotEqual(result.returncode, 0) + calls = log.read_text() if log.exists() else "" + self.assertNotIn("stable.json", calls) + self.assertNotIn("stable-v2.json", calls) + + def test_partial_s3_credentials_are_rejected_before_any_remote_query(self) -> None: + for missing in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "aws"): + with self.subTest(missing=missing), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + staged = prepare_complete_stage(root) + tools, gh_remote, r2_remote, log = install_remote_tools(root) + if missing != "aws": + install_aws_s3_stub(tools) + env = promotion_env(tools, gh_remote, r2_remote, log) + env["R2_ENDPOINT"] = "https://s3.invalid" + if missing != "R2_ACCESS_KEY_ID": + env["R2_ACCESS_KEY_ID"] = "test-access-key" + if missing != "R2_SECRET_ACCESS_KEY": + env["R2_SECRET_ACCESS_KEY"] = "test-secret-key" + result = subprocess.run(promotion_command(staged), env=env, text=True, capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("ERROR:", result.stderr) + self.assertFalse(log.exists(), "S3 misconfiguration must fail before any remote query") + def test_rollback_requires_explicit_pointers_only_authorization_flag(self) -> None: result = subprocess.run( [str(ROLLBACK), VERSION, "/nonexistent", SOURCE_SHA, CORRELATION_ID], From fe5dbb1f72dbe0b8d31ce1cd0a57a744e49a001c Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 25 Sep 2026 21:40:15 +0100 Subject: [PATCH 2/2] test(release): drop unportable aws-absence subtest Hosted runners ship a real aws binary on PATH, so omitting the stub cannot simulate its absence; the run reached the real client and failed on connection instead of the command guard. The two partial-credential subtests remain and exercise the same fail-closed guard block. Refs #337 --- tests/test_promotion_contract.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_promotion_contract.py b/tests/test_promotion_contract.py index e07b81b..b2d39cf 100644 --- a/tests/test_promotion_contract.py +++ b/tests/test_promotion_contract.py @@ -458,13 +458,14 @@ def test_s3_api_transport_failure_never_advances_stable_manifest(self) -> None: self.assertNotIn("stable-v2.json", calls) def test_partial_s3_credentials_are_rejected_before_any_remote_query(self) -> None: - for missing in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "aws"): + # A missing aws binary cannot be simulated portably: hosted runners + # ship a real aws on PATH, which is exactly the production setup. + for missing in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY"): with self.subTest(missing=missing), tempfile.TemporaryDirectory() as directory: root = Path(directory) staged = prepare_complete_stage(root) tools, gh_remote, r2_remote, log = install_remote_tools(root) - if missing != "aws": - install_aws_s3_stub(tools) + install_aws_s3_stub(tools) env = promotion_env(tools, gh_remote, r2_remote, log) env["R2_ENDPOINT"] = "https://s3.invalid" if missing != "R2_ACCESS_KEY_ID":