From 460239ba1fb8a60e272500a2f7bcdb408281ac2a Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 9 Sep 2026 14:14:00 +0800
Subject: [PATCH 1/3] fix(ci): isolate Azure live-test resources

---
 .github/azure-live-tests/azure-pipelines.sh   | 124 +++++++++
 .github/azure-live-tests/imds-vm.sh           |  97 +++++++
 .github/azure-live-tests/test_scripts.py      | 246 ++++++++++++++++++
 .github/azure-live-tests/with-probe.sh        |  68 +++++
 .github/workflows/azure_storage.yml           | 240 +++++------------
 azure-pipelines.yml                           |   3 +-
 services/azure-storage/tests/README.md        |  39 ++-
 .../tests/credential_providers/mod.rs         |   4 +-
 .../azure-storage/tests/signing/sas_token.rs  |   6 +-
 .../azure-storage/tests/signing/shared_key.rs |   2 +-
 10 files changed, 631 insertions(+), 198 deletions(-)
 create mode 100644 .github/azure-live-tests/azure-pipelines.sh
 create mode 100644 .github/azure-live-tests/imds-vm.sh
 create mode 100644 .github/azure-live-tests/test_scripts.py
 create mode 100644 .github/azure-live-tests/with-probe.sh

diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh
new file mode 100644
index 00000000..e7d45001
--- /dev/null
+++ b/.github/azure-live-tests/azure-pipelines.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Keep the fixture alive while Azure Pipelines uses its read-only identity.
+set -euo pipefail
+run_id=""
+completed=false
+
+# Called by the EXIT handler.
+# shellcheck disable=SC2329
+cancel_unfinished_run() {
+  access_token=$(az account get-access-token \
+    --resource https://app.vssps.visualstudio.com/ \
+    --query accessToken \
+    --output tsv) || return
+  state=$(curl --fail-with-body --silent --show-error \
+    --header "Authorization: Bearer $access_token" \
+    "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1" |
+    jq --raw-output '.state') || return
+  if [[ "$state" != "completed" ]]; then
+    curl --fail-with-body --silent --show-error \
+      --request PATCH \
+      --header "Authorization: Bearer $access_token" \
+      --header "Content-Type: application/json" \
+      --data '{"status":"cancelling"}' \
+      "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/build/builds/${run_id}?api-version=7.1" \
+      >/dev/null
+  fi
+}
+# shellcheck disable=SC2329
+cleanup() {
+  local result=$?
+  trap - EXIT
+  if [[ -n "$run_id" && "$completed" != true ]]; then
+    if ! cancel_unfinished_run; then
+      echo "Failed to cancel the unfinished Azure Pipelines run" >&2
+      if (( result == 0 )); then result=1; fi
+    fi
+  fi
+  exit "$result"
+}
+trap 'cleanup' EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+access_token=$(az account get-access-token \
+  --resource https://app.vssps.visualstudio.com/ \
+  --query accessToken \
+  --output tsv)
+test -n "$access_token"
+if [[ "${GITHUB_ACTIONS:-}" == true ]]; then
+  echo "::add-mask::$access_token"
+fi
+
+# Pass the probe as a run variable so the bootstrap YAML needs no new parameter.
+request_body=$(jq --null-input \
+  --arg github_sha "$GITHUB_SHA" \
+  --arg github_ref "$GITHUB_REF" \
+  --arg probe_url "$REQSIGN_AZURE_STORAGE_PROBE_URL" \
+  '{templateParameters: {githubSha: $github_sha, githubRef: $github_ref},
+    variables: {REQSIGN_AZURE_STORAGE_PROBE_URL: {value: $probe_url}}}')
+response_file="$RUNNER_TEMP/azure-pipelines-queue-response.json"
+if ! http_status=$(curl --silent --show-error \
+  --output "$response_file" \
+  --write-out '%{http_code}' \
+  --request POST \
+  --header "Authorization: Bearer $access_token" \
+  --header "Content-Type: application/json" \
+  --data "$request_body" \
+  "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs?api-version=7.1"); then
+  echo "Failed to send the Azure Pipelines queue request" >&2
+  exit 1
+fi
+if (( http_status < 200 || http_status >= 300 )); then
+  message=$(jq --raw-output '.message // .error.message // "No error message returned"' \
+    "$response_file" 2>/dev/null || echo "Non-JSON error response")
+  message=$(tr '\r\n' '  ' <<< "$message" | cut -c1-1000)
+  echo "Azure Pipelines queue request failed with HTTP $http_status: $message" >&2
+  exit 1
+fi
+response=$(<"$response_file")
+
+run_id=$(jq --raw-output '.id // empty' <<< "$response")
+run_url=$(jq --raw-output '._links.web.href // empty' <<< "$response")
+test -n "$run_id"
+test -n "$run_url"
+echo "Queued Azure Pipelines run: $run_url"
+
+deadline=$((SECONDS + 3300))
+while (( SECONDS < deadline )); do
+  access_token=$(az account get-access-token \
+    --resource https://app.vssps.visualstudio.com/ \
+    --query accessToken \
+    --output tsv)
+  response=$(curl --fail-with-body --silent --show-error \
+    --header "Authorization: Bearer $access_token" \
+    "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1")
+  state=$(jq --raw-output '.state' <<< "$response")
+  result=$(jq --raw-output '.result // empty' <<< "$response")
+  if [[ "$state" == "completed" ]]; then
+    completed=true
+    echo "Azure Pipelines run completed with result '$result': $run_url"
+    test "$result" = "succeeded"
+    exit
+  fi
+  sleep 15
+done
+echo "Azure Pipelines run did not complete before the timeout: $run_url" >&2
+exit 1
diff --git a/.github/azure-live-tests/imds-vm.sh b/.github/azure-live-tests/imds-vm.sh
new file mode 100644
index 00000000..d6211fdc
--- /dev/null
+++ b/.github/azure-live-tests/imds-vm.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+: "${AZURE_RUNTIME_RESOURCE_GROUP:?}"
+: "${RUNNER_TEMP:?}"
+attempts_file="$RUNNER_TEMP/azure-imds-vm-attempts"
+
+case "${1:-}" in
+  create)
+    : "${GITHUB_RUN_ID:?}"
+    : "${GITHUB_RUN_ATTEMPT:?}"
+    : "${GITHUB_OUTPUT:?}"
+    : "${AZURE_IMDS_IDENTITY_ID:?}"
+    ssh-keygen -q -t ed25519 -N '' -f "$RUNNER_TEMP/azure-imds-vm-key"
+    : > "$attempts_file"
+    attempt=0
+    while read -r location vnet size; do
+      attempt=$((attempt + 1))
+      vm_name="reqsign-imds-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${location}-${attempt}"
+      # Record before provisioning so cleanup also covers partial deployments.
+      echo "$vm_name" >> "$attempts_file"
+      if az vm create \
+        --resource-group "$AZURE_RUNTIME_RESOURCE_GROUP" \
+        --name "$vm_name" --location "$location" \
+        --image Ubuntu2404 --size "$size" \
+        --admin-username azureuser \
+        --ssh-key-values "$RUNNER_TEMP/azure-imds-vm-key.pub" \
+        --assign-identity "$AZURE_IMDS_IDENTITY_ID" \
+        --vnet-name "$vnet" --subnet reqsign-ci-subnet \
+        --nsg "" --public-ip-address "" --security-type Standard \
+        --os-disk-name "${vm_name}-os" \
+        --nic-delete-option Delete --os-disk-delete-option Delete \
+        --tags Project=reqsign Purpose=live-test GitHubRunId="$GITHUB_RUN_ID" \
+        --only-show-errors --output none; then
+        echo "name=$vm_name" >> "$GITHUB_OUTPUT"
+        exit 0
+      fi
+    done <<'CANDIDATES'
+eastus reqsign-ci-vnet Standard_B1s
+eastus reqsign-ci-vnet Standard_B2s
+eastus reqsign-ci-vnet Standard_D2s_v6
+eastus2 reqsign-ci-vnet-eastus2 Standard_B1s
+eastus2 reqsign-ci-vnet-eastus2 Standard_B2s
+eastus2 reqsign-ci-vnet-eastus2 Standard_D2s_v7
+eastus2 reqsign-ci-vnet-eastus2 Standard_D2s_v6
+CANDIDATES
+    echo "All configured Azure VM deployment attempts failed; see the errors above" >&2
+    exit 1
+    ;;
+  cleanup)
+    [[ -f "$attempts_file" ]] || exit 0
+    result=0
+    while read -r vm_name; do
+      if ! vm_id=$(az vm list --resource-group "$AZURE_RUNTIME_RESOURCE_GROUP" \
+        --query "[?name=='${vm_name}'].id | [0]" --output tsv --only-show-errors); then
+        result=1
+        continue
+      fi
+      if [[ -n "$vm_id" ]]; then
+        if ! az vm delete --ids "$vm_id" --yes --force-deletion true --only-show-errors; then
+          result=1
+          continue
+        fi
+      fi
+      # A failed deployment may create a NIC or disk without creating a VM.
+      if ! resource_ids=$(az resource list --resource-group "$AZURE_RUNTIME_RESOURCE_GROUP" \
+        --query "[?name=='${vm_name}VMNic' || name=='${vm_name}-os'].id" \
+        --output tsv --only-show-errors); then
+        result=1
+        continue
+      fi
+      while IFS= read -r resource_id; do
+        [[ -n "$resource_id" ]] || continue
+        az resource delete --ids "$resource_id" --only-show-errors || result=1
+      done <<< "$resource_ids"
+    done < "$attempts_file"
+    exit "$result"
+    ;;
+  *) echo "Usage: imds-vm.sh create|cleanup" >&2; exit 2 ;;
+esac
diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py
new file mode 100644
index 00000000..c6c4e9ba
--- /dev/null
+++ b/.github/azure-live-tests/test_scripts.py
@@ -0,0 +1,246 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Exercise fixture ownership and partial VM deployment cleanup without Azure."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+SCRIPTS = Path(__file__).resolve().parent
+FAKE_AZ = r"""#!/usr/bin/env python3
+import json, os, pathlib, sys
+args = sys.argv[1:]
+root = pathlib.Path(os.environ['FAKE_AZ_STATE'])
+def option(name, default=''):
+    return args[args.index(name)+1] if name in args else default
+with (root / 'calls.jsonl').open('a') as f:
+    f.write(json.dumps(args) + '\n')
+if args[:2] == ['account', 'get-access-token']:
+    print('fake-devops-token')
+elif args[:3] == ['storage', 'blob', 'upload']:
+    if os.getenv('FAIL_UPLOAD'): sys.exit(1)
+    body = pathlib.Path(option('--file')).read_bytes()
+    assert body == b'reqsign-live-azure-ok\n'
+    (root / option('--name')).write_bytes(body)
+elif args[:3] == ['storage', 'blob', 'generate-sas']:
+    if os.getenv('FAIL_SAS'): sys.exit(1)
+    assert option('--permissions') == 'r'
+    print('fake-read-only-token')
+elif args[:3] == ['storage', 'blob', 'delete']:
+    if os.getenv('FAIL_DELETE'): sys.exit(1)
+    (root / option('--name')).unlink()
+elif args[:2] == ['vm', 'create']:
+    name = option('--name')
+    location = option('--location')
+    nic = root / (name + 'VMNic')
+    # Model NIC creation preceding a capacity failure, and Azure's region rule.
+    if nic.exists() and nic.read_text() != location:
+        print('InvalidResourceLocation', file=sys.stderr)
+        sys.exit(1)
+    nic.write_text(location)
+    (root / option('--os-disk-name')).write_text(location)
+    if location == 'eastus' or os.getenv('FAIL_ALL_VMS'):
+        print('SkuNotAvailable', file=sys.stderr)
+        sys.exit(1)
+    (root / name).write_text(location)
+elif args[:2] in (['vm', 'list'], ['resource', 'list']):
+    import re
+    for name in re.findall("name=='([^']+)'", option('--query')):
+        if (root / name).exists(): print('/fake/' + name)
+elif args[:2] in (['vm', 'delete'], ['resource', 'delete']):
+    (root / option('--ids').split('/')[-1]).unlink()
+else:
+    raise AssertionError(args)
+"""
+
+
+FAKE_CURL = r"""#!/usr/bin/env python3
+import json, os, pathlib, sys
+args = sys.argv[1:]
+root = pathlib.Path(os.environ['FAKE_AZ_STATE'])
+def option(name, default=''):
+    return args[args.index(name)+1] if name in args else default
+method = option('--request', 'GET')
+# Both waiting and cancellation must finish before fixture cleanup.
+assert list(root.glob('ci-probe-*'))
+if method == 'POST':
+    request = json.loads(option('--data'))
+    params = request['templateParameters']
+    assert params['githubSha'] == 'a' * 40
+    assert params['githubRef'] == 'refs/heads/test'
+    assert (root / request['variables']['REQSIGN_AZURE_STORAGE_PROBE_URL']['value'].rsplit('/', 1)[1]).exists()
+    assert set(params) == {'githubSha', 'githubRef'}
+    (root / 'queued.json').write_text(json.dumps(params))
+    pathlib.Path(option('--output')).write_text(json.dumps({
+        'id': 23, '_links': {'web': {'href': 'https://example.test/run/23'}}}))
+    print('200')
+elif method == 'PATCH':
+    assert json.loads(option('--data')) == {'status': 'cancelling'}
+    (root / 'cancelled').touch()
+elif os.getenv('FAIL_PIPELINE_POLL'):
+    if not (root / 'poll-failed').exists():
+        (root / 'poll-failed').touch()
+        sys.exit(22)
+    print(json.dumps({'state': 'inProgress'}))
+else:
+    print(json.dumps({'state': 'completed',
+                      'result': os.getenv('PIPELINE_RESULT', 'succeeded')}))
+"""
+
+
+class LiveScriptTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.state = self.root / 'state'
+        self.state.mkdir()
+        binary = self.root / 'az'
+        binary.write_text(FAKE_AZ)
+        binary.chmod(0o755)
+        curl = self.root / 'curl'
+        curl.write_text(FAKE_CURL)
+        curl.chmod(0o755)
+        self.env = {
+            **os.environ,
+            'PATH': str(self.root) + os.pathsep + os.environ['PATH'],
+            'FAKE_AZ_STATE': str(self.state),
+            'AZURE_STORAGE_ACCOUNT': 'fixture',
+            'AZURE_STORAGE_CONTAINER': 'tests',
+            'AZURE_STORAGE_AUTH_MODE': 'login',
+            'GITHUB_ACTIONS': 'false',
+            'GITHUB_SHA': 'a' * 40,
+            'GITHUB_REF': 'refs/heads/test',
+            'AZURE_DEVOPS_ORGANIZATION': 'fixture',
+            'AZURE_DEVOPS_PROJECT': 'tests',
+            'AZURE_DEVOPS_PIPELINE_ID': '1',
+            'AZURE_RUNTIME_RESOURCE_GROUP': 'fixture-rg',
+            'AZURE_IMDS_IDENTITY_ID': '/fake/identity',
+            'RUNNER_TEMP': str(self.root),
+            'GITHUB_RUN_ID': '123',
+            'GITHUB_RUN_ATTEMPT': '1',
+            'GITHUB_OUTPUT': str(self.root / 'outputs'),
+        }
+
+    def run_script(self, script, *args):
+        return subprocess.run(['bash', str(SCRIPTS / script), *args],
+                              env=self.env, capture_output=True, text=True)
+
+    def calls(self):
+        return [json.loads(line) for line in
+                (self.state / 'calls.jsonl').read_text().splitlines()]
+
+    def resources(self):
+        return [p.name for p in self.state.iterdir() if p.name != 'calls.jsonl']
+
+    def test_probe_body_sas_and_cleanup(self):
+        self.env.update(AZURE_STORAGE_AUTH_MODE='key',
+                        REQSIGN_AZURE_STORAGE_ACCOUNT_KEY='fake-key')
+        child = """import os, pathlib
+name = os.environ['REQSIGN_AZURE_STORAGE_PROBE_URL'].rsplit('/', 1)[1]
+assert (pathlib.Path(os.environ['FAKE_AZ_STATE']) / name).read_bytes() == b'reqsign-live-azure-ok\\n'
+assert os.environ['REQSIGN_AZURE_STORAGE_SAS_TOKEN'] == 'fake-read-only-token'
+"""
+        result = self.run_script('with-probe.sh', 'python3', '-c', child)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.resources(), [])
+        self.assertNotIn('fake-read-only-token', result.stdout + result.stderr)
+
+    def test_failed_test_is_cleaned_and_preserves_status(self):
+        result = self.run_script('with-probe.sh', 'bash', '-c', 'exit 42')
+        self.assertEqual(result.returncode, 42, result.stderr)
+        self.assertEqual(self.resources(), [])
+
+    def test_failed_setup_does_not_run_test(self):
+        self.env['FAIL_UPLOAD'] = '1'
+        result = self.run_script('with-probe.sh', 'touch', str(self.root / 'ran'))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertFalse((self.root / 'ran').exists())
+
+    def test_failed_sas_setup_cleans_blob(self):
+        self.env.update(AZURE_STORAGE_AUTH_MODE='key', FAIL_SAS='1',
+                        REQSIGN_AZURE_STORAGE_ACCOUNT_KEY='fake-key')
+        result = self.run_script('with-probe.sh', 'touch', str(self.root / 'ran'))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertFalse((self.root / 'ran').exists())
+        self.assertEqual(self.resources(), [])
+
+    def test_cleanup_failure_fails_successful_test(self):
+        self.env['FAIL_DELETE'] = '1'
+        result = self.run_script('with-probe.sh', 'true')
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn('Failed to delete', result.stderr)
+
+    def test_repeated_jobs_get_different_objects(self):
+        for _ in range(2):
+            result = self.run_script('with-probe.sh', 'true')
+            self.assertEqual(result.returncode, 0, result.stderr)
+        uploads = [c[c.index('--name') + 1] for c in self.calls()
+                   if c[:3] == ['storage', 'blob', 'upload']]
+        self.assertEqual(len(set(uploads)), 2)
+        self.assertEqual(self.resources(), [])
+
+    def test_region_fallback_and_partial_resources_cleanup(self):
+        result = self.run_script('imds-vm.sh', 'create')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn('InvalidResourceLocation', result.stderr)
+        self.assertIn('eastus2', (self.root / 'outputs').read_text())
+        creates = [c for c in self.calls() if c[:2] == ['vm', 'create']]
+        self.assertEqual(len(creates), 4)
+        names = [c[c.index('--name') + 1] for c in creates]
+        self.assertEqual(len(set(names)), 4)
+        result = self.run_script('imds-vm.sh', 'cleanup')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.resources(), [])
+
+    def test_all_vm_attempts_fail_and_are_cleaned(self):
+        self.env['FAIL_ALL_VMS'] = '1'
+        result = self.run_script('imds-vm.sh', 'create')
+        self.assertNotEqual(result.returncode, 0)
+        self.assertFalse((self.root / 'outputs').exists())
+        result = self.run_script('imds-vm.sh', 'cleanup')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.resources(), [])
+
+    def test_pipeline_reads_temporary_probe_until_completed(self):
+        result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue((self.state / 'queued.json').exists())
+        self.assertFalse((self.state / 'cancelled').exists())
+        self.assertEqual(list(self.state.glob('ci-probe-*')), [])
+
+    def test_failed_pipeline_fails_job_and_cleans_probe(self):
+        self.env['PIPELINE_RESULT'] = 'failed'
+        result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertFalse((self.state / 'cancelled').exists())
+        self.assertEqual(list(self.state.glob('ci-probe-*')), [])
+
+    def test_poll_failure_cancels_pipeline_before_probe_cleanup(self):
+        self.env['FAIL_PIPELINE_POLL'] = '1'
+        result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
+        self.assertEqual(result.returncode, 22, result.stderr)
+        self.assertTrue((self.state / 'cancelled').exists())
+        self.assertEqual(list(self.state.glob('ci-probe-*')), [])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/.github/azure-live-tests/with-probe.sh b/.github/azure-live-tests/with-probe.sh
new file mode 100644
index 00000000..a3e572bb
--- /dev/null
+++ b/.github/azure-live-tests/with-probe.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Run a live test with its own private blob. Setup credentials stay separate
+# from the credential provider exercised by the test command.
+set -euo pipefail
+
+: "${AZURE_STORAGE_ACCOUNT:?Set the fixture storage account}"
+: "${AZURE_STORAGE_CONTAINER:?Set the fixture container}"
+(( $# > 0 )) || { echo "Usage: with-probe.sh command [args...]" >&2; exit 2; }
+
+auth_mode=${AZURE_STORAGE_AUTH_MODE:-login}
+if [[ "$auth_mode" == key ]]; then
+  export AZURE_STORAGE_KEY="${REQSIGN_AZURE_STORAGE_ACCOUNT_KEY:?Set the fixture account key}"
+fi
+storage_args=(--auth-mode "$auth_mode" --account-name "$AZURE_STORAGE_ACCOUNT"
+  --container-name "$AZURE_STORAGE_CONTAINER" --only-show-errors)
+blob_name="ci-probe-$(python3 -c 'import uuid; print(uuid.uuid4())').txt"
+probe_dir=$(mktemp -d)
+created=false
+cleanup() {
+  local result=$?
+  trap - EXIT
+  if [[ "$created" == true ]]; then
+    if ! az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none; then
+      echo "Failed to delete the live-test probe" >&2
+      if (( result == 0 )); then result=1; fi
+    fi
+  fi
+  rm -rf "$probe_dir"
+  exit "$result"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+printf 'reqsign-live-azure-ok\n' > "$probe_dir/probe.txt"
+az storage blob upload "${storage_args[@]}" --name "$blob_name" \
+  --file "$probe_dir/probe.txt" --overwrite false --output none
+created=true
+export REQSIGN_AZURE_STORAGE_PROBE_URL="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_STORAGE_CONTAINER}/${blob_name}"
+
+# SAS cases use a short-lived, read-only token for this exact object.
+if [[ "$auth_mode" == key ]]; then
+  expiry=$(python3 -c 'from datetime import datetime, timedelta, timezone; print((datetime.now(timezone.utc) + timedelta(hours=2)).strftime("%Y-%m-%dT%H:%MZ"))')
+  sas_token=$(az storage blob generate-sas "${storage_args[@]}" --name "$blob_name" \
+    --permissions r --expiry "$expiry" --https-only --output tsv)
+  test -n "$sas_token"
+  if [[ "${GITHUB_ACTIONS:-}" == true ]]; then
+    echo "::add-mask::$sas_token"
+  fi
+  export REQSIGN_AZURE_STORAGE_SAS_TOKEN="$sas_token"
+fi
+"$@"
diff --git a/.github/workflows/azure_storage.yml b/.github/workflows/azure_storage.yml
index 52e5165a..c186480d 100644
--- a/.github/workflows/azure_storage.yml
+++ b/.github/workflows/azure_storage.yml
@@ -31,6 +31,9 @@ concurrency:
 
 env:
   RUST_BACKTRACE: "1"
+  AZURE_STORAGE_ACCOUNT: opendalci
+  AZURE_STORAGE_CONTAINER: reqsign-live-test
+  AZURE_SUBSCRIPTION_ID: 9e6b5234-646f-4f48-b5cf-9761a56f0917
 
 permissions: {}
 
@@ -41,6 +44,8 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+      - name: Test Azure live-test lifecycle scripts
+        run: python3 .github/azure-live-tests/test_scripts.py
       - name: Run unit tests
         run: |
           cargo test -p reqsign-azure-storage --lib --no-fail-fast
@@ -68,8 +73,6 @@ jobs:
     needs: check_secrets
     if: needs.check_secrets.outputs.trusted == 'true'
     runs-on: ubuntu-latest
-    env:
-      AZURE_SUBSCRIPTION_ID: 9e6b5234-646f-4f48-b5cf-9761a56f0917
     permissions:
       contents: read
       id-token: write
@@ -87,10 +90,8 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           REQSIGN_AZURE_STORAGE_ACCOUNT_NAME: op://reqsign/azure-storage/account_name
           REQSIGN_AZURE_STORAGE_ACCOUNT_KEY: op://reqsign/azure-storage/account_key
-          REQSIGN_AZURE_STORAGE_SAS_TOKEN: op://reqsign/azure-storage/sas_token
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
       - name: Log in to Azure with GitHub OIDC
@@ -110,16 +111,15 @@ jobs:
           echo "REQSIGN_AZURE_STORAGE_BEARER_TOKEN=$bearer_token" >> "$GITHUB_ENV"
       - name: Test StaticCredentialProvider against Azure Blob Storage
         env:
+          AZURE_STORAGE_AUTH_MODE: key
           REQSIGN_AZURE_STORAGE_TEST: "on"
         run: |
-          cargo test -p reqsign-azure-storage --test main credential_providers::static_provider:: -- --no-capture
+          bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main credential_providers::static_provider:: -- --no-capture
 
   test_env_provider:
     needs: check_secrets
     if: needs.check_secrets.outputs.trusted == 'true'
     runs-on: ubuntu-latest
-    env:
-      AZURE_SUBSCRIPTION_ID: 9e6b5234-646f-4f48-b5cf-9761a56f0917
     permissions:
       contents: read
       id-token: write
@@ -137,10 +137,8 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           REQSIGN_AZURE_STORAGE_ACCOUNT_NAME: op://reqsign/azure-storage/account_name
           REQSIGN_AZURE_STORAGE_ACCOUNT_KEY: op://reqsign/azure-storage/account_key
-          REQSIGN_AZURE_STORAGE_SAS_TOKEN: op://reqsign/azure-storage/sas_token
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
       - name: Log in to Azure with GitHub OIDC
@@ -160,14 +158,18 @@ jobs:
           echo "REQSIGN_AZURE_STORAGE_BEARER_TOKEN=$bearer_token" >> "$GITHUB_ENV"
       - name: Test EnvCredentialProvider against Azure Blob Storage
         env:
+          AZURE_STORAGE_AUTH_MODE: key
           REQSIGN_AZURE_STORAGE_TEST_ENV: "on"
         run: |
-          cargo test -p reqsign-azure-storage --test main credential_providers::env:: -- --no-capture
+          bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main credential_providers::env:: -- --no-capture
 
   test_client_secret_provider:
     needs: check_secrets
     if: needs.check_secrets.outputs.trusted == 'true'
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -182,20 +184,28 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
           AZURE_CLIENT_SECRET: op://reqsign/azure-storage/client_secret
+      - name: Log in to Azure with GitHub OIDC
+        uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0
+        with:
+          client-id: ${{ env.AZURE_CLIENT_ID }}
+          tenant-id: ${{ env.AZURE_TENANT_ID }}
+          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}
       - name: Test ClientSecretCredentialProvider against Azure Blob Storage
         env:
           REQSIGN_AZURE_STORAGE_TEST_CLIENT_SECRET: "on"
         run: |
-          cargo test -p reqsign-azure-storage --test main credential_providers::client_secret:: -- --no-capture
+          bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main credential_providers::client_secret:: -- --no-capture
 
   test_client_certificate_provider:
     needs: check_secrets
     if: needs.check_secrets.outputs.trusted == 'true'
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -210,7 +220,6 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
           AZURE_CLIENT_CERTIFICATE_BASE64: op://reqsign/azure-storage/certificate_pem_base64
@@ -221,18 +230,22 @@ jobs:
           test -s "$certificate_path"
           chmod 600 "$certificate_path"
           echo "AZURE_CLIENT_CERTIFICATE_PATH=$certificate_path" >> "$GITHUB_ENV"
+      - name: Log in to Azure with GitHub OIDC
+        uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0
+        with:
+          client-id: ${{ env.AZURE_CLIENT_ID }}
+          tenant-id: ${{ env.AZURE_TENANT_ID }}
+          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}
       - name: Test ClientCertificateCredentialProvider against Azure Blob Storage
         env:
           REQSIGN_AZURE_STORAGE_TEST_CLIENT_CERTIFICATE: "on"
         run: |
-          cargo test -p reqsign-azure-storage --test main credential_providers::client_certificate:: -- --no-capture
+          bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main credential_providers::client_certificate:: -- --no-capture
 
   test_azure_cli_provider:
     needs: check_secrets
     if: needs.check_secrets.outputs.trusted == 'true'
     runs-on: ubuntu-latest
-    env:
-      AZURE_SUBSCRIPTION_ID: 9e6b5234-646f-4f48-b5cf-9761a56f0917
     permissions:
       contents: read
       id-token: write
@@ -250,7 +263,6 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
       - name: Log in to Azure with GitHub OIDC
@@ -263,7 +275,7 @@ jobs:
         env:
           REQSIGN_AZURE_STORAGE_TEST_CLI: "on"
         run: |
-          cargo test -p reqsign-azure-storage --test main credential_providers::azure_cli:: -- --no-capture
+          bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main credential_providers::azure_cli:: -- --no-capture
 
   test_workload_identity_provider:
     needs: check_secrets
@@ -286,9 +298,14 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
+      - name: Log in to Azure with GitHub OIDC
+        uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0
+        with:
+          client-id: ${{ env.AZURE_CLIENT_ID }}
+          tenant-id: ${{ env.AZURE_TENANT_ID }}
+          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}
       - name: Get GitHub workload identity token
         run: |
           token_file="$RUNNER_TEMP/azure-federated-token"
@@ -305,14 +322,12 @@ jobs:
         env:
           REQSIGN_AZURE_STORAGE_TEST_WORKLOAD_IDENTITY: "on"
         run: |
-          cargo test -p reqsign-azure-storage --test main credential_providers::workload_identity:: -- --no-capture
+          bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main credential_providers::workload_identity:: -- --no-capture
 
   test_default_provider:
     needs: check_secrets
     if: needs.check_secrets.outputs.trusted == 'true'
     runs-on: ubuntu-latest
-    env:
-      AZURE_SUBSCRIPTION_ID: 9e6b5234-646f-4f48-b5cf-9761a56f0917
     permissions:
       contents: read
       id-token: write
@@ -330,7 +345,6 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
       - name: Log in to Azure with GitHub OIDC
@@ -342,7 +356,7 @@ jobs:
       - name: Test DefaultCredentialProvider against Azure Blob Storage
         env:
           REQSIGN_AZURE_STORAGE_TEST_DEFAULT: "on"
-        run: cargo test -p reqsign-azure-storage --test main credential_providers::default::test_default_provider -- --no-capture
+        run: bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main credential_providers::default::test_default_provider -- --no-capture
 
   test_imds_provider:
     name: ImdsCredentialProvider live test
@@ -350,9 +364,6 @@ jobs:
     if: needs.check_secrets.outputs.trusted == 'true'
     runs-on: ubuntu-latest
     env:
-      AZURE_SUBSCRIPTION_ID: 9e6b5234-646f-4f48-b5cf-9761a56f0917
-      AZURE_STORAGE_ACCOUNT: opendalci
-      AZURE_STORAGE_CONTAINER: reqsign-live-test
       AZURE_IMDS_CLIENT_ID: c4544d03-a76a-4908-b5c0-047b8c7e432c
       AZURE_IMDS_IDENTITY_ID: /subscriptions/9e6b5234-646f-4f48-b5cf-9761a56f0917/resourceGroups/opendal/providers/Microsoft.ManagedIdentity/userAssignedIdentities/reqsign-test
       AZURE_RUNTIME_RESOURCE_GROUP: reqsign-ci-runtime
@@ -373,7 +384,6 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id
           AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id
       - name: Log in to Azure with GitHub OIDC
@@ -408,52 +418,13 @@ jobs:
           echo "name=$artifact_name" >> "$GITHUB_OUTPUT"
       - name: Create private Azure VM
         id: vm
-        run: |
-          vm_name="reqsign-imds-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          ssh-keygen -q -t ed25519 -N '' -f "$RUNNER_TEMP/azure-imds-vm-key"
-
-          create_vm() {
-            local location=$1
-            local vnet=$2
-            local size=$3
-            az vm create \
-              --resource-group "$AZURE_RUNTIME_RESOURCE_GROUP" \
-              --name "$vm_name" \
-              --location "$location" \
-              --image Ubuntu2404 \
-              --size "$size" \
-              --admin-username azureuser \
-              --ssh-key-values "$RUNNER_TEMP/azure-imds-vm-key.pub" \
-              --assign-identity "$AZURE_IMDS_IDENTITY_ID" \
-              --vnet-name "$vnet" \
-              --subnet reqsign-ci-subnet \
-              --nsg "" \
-              --public-ip-address "" \
-              --security-type Standard \
-              --nic-delete-option Delete \
-              --os-disk-delete-option Delete \
-              --tags Project=reqsign Purpose=live-test GitHubRunId="$GITHUB_RUN_ID" \
-              --only-show-errors \
-              --output none
-          }
-
-          if create_vm eastus reqsign-ci-vnet Standard_B1s || \
-             create_vm eastus reqsign-ci-vnet Standard_B2s || \
-             create_vm eastus reqsign-ci-vnet Standard_D2s_v6 || \
-             create_vm eastus2 reqsign-ci-vnet-eastus2 Standard_B1s || \
-             create_vm eastus2 reqsign-ci-vnet-eastus2 Standard_B2s || \
-             create_vm eastus2 reqsign-ci-vnet-eastus2 Standard_D2s_v7 || \
-             create_vm eastus2 reqsign-ci-vnet-eastus2 Standard_D2s_v6; then
-            echo "name=$vm_name" >> "$GITHUB_OUTPUT"
-          else
-            echo "No configured Azure VM size had capacity" >&2
-            exit 1
-          fi
+        run: bash .github/azure-live-tests/imds-vm.sh create
       - name: Test ImdsCredentialProvider on the Azure VM
         env:
           ARTIFACT_NAME: ${{ steps.artifact.outputs.name }}
           VM_NAME: ${{ steps.vm.outputs.name }}
         run: |
+          bash .github/azure-live-tests/with-probe.sh bash -euo pipefail <<'SCRIPT'
           artifact_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_STORAGE_CONTAINER}/${ARTIFACT_NAME}"
           # shellcheck disable=SC2016
           remote_script='set -eu
@@ -461,7 +432,7 @@ jobs:
           storage_token=$(python3 -c '\''import json,sys; print(json.loads(sys.argv[1])["access_token"])'\'' "$imds_response")
           curl --connect-timeout 15 --fail --silent --show-error --header "Authorization: Bearer $storage_token" --header "x-ms-version: 2023-11-03" "'"$artifact_url"'" --output /tmp/reqsign-azure-imds-live-test
           chmod 700 /tmp/reqsign-azure-imds-live-test
-          REQSIGN_AZURE_STORAGE_TEST_IMDS=on REQSIGN_AZURE_STORAGE_URL='"$REQSIGN_AZURE_STORAGE_URL"' AZURE_CLIENT_ID='"$AZURE_IMDS_CLIENT_ID"' /tmp/reqsign-azure-imds-live-test credential_providers::imds::test_imds_provider --exact
+          REQSIGN_AZURE_STORAGE_TEST_IMDS=on REQSIGN_AZURE_STORAGE_PROBE_URL='"$REQSIGN_AZURE_STORAGE_PROBE_URL"' AZURE_CLIENT_ID='"$AZURE_IMDS_CLIENT_ID"' /tmp/reqsign-azure-imds-live-test credential_providers::imds::test_imds_provider --exact
           echo REQSIGN_IMDS_TEST_SUCCEEDED'
 
           result=$(az vm run-command invoke \
@@ -474,28 +445,21 @@ jobs:
             jq --raw-output '.value[0].message' <<< "$result"
             exit 1
           fi
-      - name: Delete ephemeral Azure resources
+          SCRIPT
+      - name: Delete ephemeral Azure VMs
         if: always()
+        run: bash .github/azure-live-tests/imds-vm.sh cleanup
+      - name: Delete IMDS live test binary
+        if: always() && steps.artifact.outputs.name != ''
         env:
           ARTIFACT_NAME: ${{ steps.artifact.outputs.name }}
-          VM_NAME: ${{ steps.vm.outputs.name }}
         run: |
-          if [[ -n "$VM_NAME" ]] && az vm show --resource-group "$AZURE_RUNTIME_RESOURCE_GROUP" --name "$VM_NAME" >/dev/null 2>&1; then
-            az vm delete \
-              --resource-group "$AZURE_RUNTIME_RESOURCE_GROUP" \
-              --name "$VM_NAME" \
-              --yes \
-              --force-deletion true \
-              --only-show-errors
-          fi
-          if [[ -n "$ARTIFACT_NAME" ]]; then
-            az storage blob delete \
-              --auth-mode login \
-              --account-name "$AZURE_STORAGE_ACCOUNT" \
-              --container-name "$AZURE_STORAGE_CONTAINER" \
-              --name "$ARTIFACT_NAME" \
-              --only-show-errors
-          fi
+          az storage blob delete \
+            --auth-mode login \
+            --account-name "$AZURE_STORAGE_ACCOUNT" \
+            --container-name "$AZURE_STORAGE_CONTAINER" \
+            --name "$ARTIFACT_NAME" \
+            --only-show-errors
 
   test_azure_pipelines_provider:
     name: AzurePipelinesCredentialProvider live test
@@ -504,7 +468,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
-      AZURE_SUBSCRIPTION_ID: 9e6b5234-646f-4f48-b5cf-9761a56f0917
       AZURE_DEVOPS_ORGANIZATION: opendal
       AZURE_DEVOPS_PROJECT: reqsign
       AZURE_DEVOPS_PIPELINE_ID: "1"
@@ -512,6 +475,9 @@ jobs:
       contents: read
       id-token: write
     steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
       - name: Configure 1Password Connect
         uses: 1Password/load-secrets-action/configure@70062d7a876d3eb6334754fa26efd2fbd90c32f2 # v5.0.1
         with:
@@ -530,95 +496,8 @@ jobs:
           client-id: ${{ env.AZURE_CLIENT_ID }}
           tenant-id: ${{ env.AZURE_TENANT_ID }}
           subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}
-      - name: Queue Azure Pipelines provider test
-        id: queue
-        run: |
-          access_token=$(az account get-access-token \
-            --resource https://app.vssps.visualstudio.com/ \
-            --query accessToken \
-            --output tsv)
-          test -n "$access_token"
-          echo "::add-mask::$access_token"
-
-          request_body=$(jq --null-input \
-            --arg github_sha "$GITHUB_SHA" \
-            --arg github_ref "$GITHUB_REF" \
-            '{templateParameters: {githubSha: $github_sha, githubRef: $github_ref}}')
-          response_file="$RUNNER_TEMP/azure-pipelines-queue-response.json"
-          if ! http_status=$(curl --silent --show-error \
-            --output "$response_file" \
-            --write-out '%{http_code}' \
-            --request POST \
-            --header "Authorization: Bearer $access_token" \
-            --header "Content-Type: application/json" \
-            --data "$request_body" \
-            "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs?api-version=7.1"); then
-            echo "Failed to send the Azure Pipelines queue request" >&2
-            exit 1
-          fi
-          if (( http_status < 200 || http_status >= 300 )); then
-            message=$(jq --raw-output '.message // .error.message // "No error message returned"' \
-              "$response_file" 2>/dev/null || echo "Non-JSON error response")
-            message=$(tr '\r\n' '  ' <<< "$message" | cut -c1-1000)
-            echo "Azure Pipelines queue request failed with HTTP $http_status: $message" >&2
-            exit 1
-          fi
-          response=$(<"$response_file")
-
-          run_id=$(jq --raw-output '.id // empty' <<< "$response")
-          run_url=$(jq --raw-output '._links.web.href // empty' <<< "$response")
-          test -n "$run_id"
-          test -n "$run_url"
-          echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
-          echo "run_url=$run_url" >> "$GITHUB_OUTPUT"
-          echo "Queued Azure Pipelines run: $run_url"
-      - name: Wait for Azure Pipelines provider test
-        env:
-          RUN_ID: ${{ steps.queue.outputs.run_id }}
-          RUN_URL: ${{ steps.queue.outputs.run_url }}
-        run: |
-          deadline=$((SECONDS + 3300))
-          while (( SECONDS < deadline )); do
-            access_token=$(az account get-access-token \
-              --resource https://app.vssps.visualstudio.com/ \
-              --query accessToken \
-              --output tsv)
-            response=$(curl --fail-with-body --silent --show-error \
-              --header "Authorization: Bearer $access_token" \
-              "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${RUN_ID}?api-version=7.1")
-            state=$(jq --raw-output '.state' <<< "$response")
-            result=$(jq --raw-output '.result // empty' <<< "$response")
-            if [[ "$state" == "completed" ]]; then
-              echo "Azure Pipelines run completed with result '$result': $RUN_URL"
-              test "$result" = "succeeded"
-              exit
-            fi
-            sleep 15
-          done
-          echo "Azure Pipelines run did not complete before the timeout: $RUN_URL" >&2
-          exit 1
-      - name: Cancel unfinished Azure Pipelines run
-        if: always() && steps.queue.outputs.run_id != ''
-        env:
-          RUN_ID: ${{ steps.queue.outputs.run_id }}
-        run: |
-          access_token=$(az account get-access-token \
-            --resource https://app.vssps.visualstudio.com/ \
-            --query accessToken \
-            --output tsv)
-          state=$(curl --fail-with-body --silent --show-error \
-            --header "Authorization: Bearer $access_token" \
-            "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${RUN_ID}?api-version=7.1" |
-            jq --raw-output '.state')
-          if [[ "$state" != "completed" ]]; then
-            curl --fail-with-body --silent --show-error \
-              --request PATCH \
-              --header "Authorization: Bearer $access_token" \
-              --header "Content-Type: application/json" \
-              --data '{"status":"cancelling"}' \
-              "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/build/builds/${RUN_ID}?api-version=7.1" \
-              >/dev/null
-          fi
+      - name: Test AzurePipelinesCredentialProvider with an ephemeral probe
+        run: bash .github/azure-live-tests/with-probe.sh bash .github/azure-live-tests/azure-pipelines.sh
 
   signing_test:
     needs: check_secrets
@@ -638,15 +517,14 @@ jobs:
         with:
           export-env: true
         env:
-          REQSIGN_AZURE_STORAGE_URL: op://reqsign/azure-storage/url
           REQSIGN_AZURE_STORAGE_ACCOUNT_NAME: op://reqsign/azure-storage/account_name
           REQSIGN_AZURE_STORAGE_ACCOUNT_KEY: op://reqsign/azure-storage/account_key
-          REQSIGN_AZURE_STORAGE_SAS_TOKEN: op://reqsign/azure-storage/sas_token
       - name: Test Azure Storage signing
         env:
+          AZURE_STORAGE_AUTH_MODE: key
           REQSIGN_AZURE_STORAGE_TEST: "on"
         run: |
-          cargo test -p reqsign-azure-storage --test main signing:: -- --no-capture
+          bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main signing:: -- --no-capture
 
   summary:
     name: Test Summary
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 09be55d0..97b7e70a 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -81,9 +81,10 @@ steps:
         test -n "$SYSTEM_ACCESSTOKEN"
         export REQSIGN_AZURE_STORAGE_TEST_PIPELINES=on
         test "$REQSIGN_AZURE_STORAGE_TEST_PIPELINES" = "on"
+        [[ "$REQSIGN_AZURE_STORAGE_PROBE_URL" =~ ^https://opendalci[.]blob[.]core[.]windows[.]net/reqsign-live-test/ci-probe-[0-9a-f-]+[.]txt$ ]] || exit 1
         cargo test -p reqsign-azure-storage --test main \
           credential_providers::azure_pipelines::test_azure_pipelines_provider \
           -- --exact --no-capture
     env:
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
-      REQSIGN_AZURE_STORAGE_URL: https://opendalci.blob.core.windows.net/reqsign-live-test/probe.txt
+      REQSIGN_AZURE_STORAGE_PROBE_URL: $(REQSIGN_AZURE_STORAGE_PROBE_URL)
diff --git a/services/azure-storage/tests/README.md b/services/azure-storage/tests/README.md
index 22afed69..da076a4b 100644
--- a/services/azure-storage/tests/README.md
+++ b/services/azure-storage/tests/README.md
@@ -3,7 +3,7 @@
 The Azure Storage test suite separates deterministic protocol parsing from live
 credential-provider acceptance tests. Unit tests parse sanitized responses that
 were captured from Azure. Integration tests obtain credentials from the real
-provider and use `RequestSigner` to read a fixed private blob.
+provider and use `RequestSigner` to read a private blob prepared for that test run.
 
 ## Test Structure
 
@@ -42,15 +42,32 @@ set to `on`. For example:
 
 ```bash
 REQSIGN_AZURE_STORAGE_TEST_CLI=on \
-REQSIGN_AZURE_STORAGE_URL=https://example.blob.core.windows.net/container/blob \
-cargo test -p reqsign-azure-storage --test main \
+AZURE_STORAGE_ACCOUNT=example \
+AZURE_STORAGE_CONTAINER=container \
+bash .github/azure-live-tests/with-probe.sh cargo test -p reqsign-azure-storage --test main \
   credential_providers::azure_cli::test_azure_cli_provider -- --exact
 ```
 
 ## Live Test Configuration
 
-Every live provider test requires `REQSIGN_AZURE_STORAGE_URL`. The URL must
-identify a blob that contains `reqsign-live-azure-ok\n`.
+Every live provider test requires `REQSIGN_AZURE_STORAGE_PROBE_URL`. The URL must
+identify a blob that contains `reqsign-live-azure-ok\n`. Run live tests through
+`.github/azure-live-tests/with-probe.sh` to create a uniquely named private blob,
+set this URL for the child command, and delete the blob when the command exits,
+including on test failure. Concurrent jobs and failed-job reruns use independent
+objects. The storage account and private container must already exist.
+
+The wrapper uses an existing Azure CLI login with permission to create and delete
+fixture blobs. Set `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_CONTAINER` to the
+fixture location. Fixture setup credentials are separate from the provider being
+tested; providers only need read access. For Shared Key and SAS tests, set
+`AZURE_STORAGE_AUTH_MODE=key` and `REQSIGN_AZURE_STORAGE_ACCOUNT_KEY`. The wrapper
+also generates a two-hour, read-only SAS for the new blob and exports it as
+`REQSIGN_AZURE_STORAGE_SAS_TOKEN`. Direct Cargo invocations can still use a
+caller-prepared blob through `REQSIGN_AZURE_STORAGE_PROBE_URL`.
+
+Lifecycle policies may remove abandoned fixtures after interrupted jobs; no test
+depends on a blob surviving between runs.
 
 | Variable | Provider |
 | --- | --- |
@@ -102,12 +119,16 @@ for:
 
 The IMDS job uploads the test binary to a private blob, creates a VM without a
 public IP, runs the exact provider test through Azure Run Command, and removes
-the VM and uploaded binary in an unconditional cleanup step.
+the VM and uploaded binary in unconditional cleanup steps. Each VM deployment
+attempt has a distinct name. Cleanup covers every attempted deployment, including
+NICs and disks left behind when VM creation fails.
 
 GitHub Actions queues `AzurePipelinesCredentialProvider` through the Azure
 DevOps REST API and waits for the result as part of the GitHub check. The queued
-run receives the exact GitHub ref and commit, then uses an Azure Resource Manager
-workload-identity service connection for the provider test. Azure DevOps has no
+run receives the exact GitHub ref and commit plus a temporary probe URL created
+by the GitHub job. Its Azure Resource Manager workload-identity service connection
+keeps Blob Reader access; the GitHub job owns fixture creation and cleanup and
+cancels an unfinished pipeline before releasing the fixture. Azure DevOps has no
 repository or scheduled trigger for this pipeline. It uses a dedicated Azure VM
 Scale Set agent pool with one-node maximum capacity, zero standby agents, and
 automatic recycling after every job.
@@ -121,10 +142,8 @@ GitHub Actions reads the existing `reqsign/azure-storage` item through
 1Password Connect. The workflow uses these existing fields without renaming or
 creating fields:
 
-- `url`
 - `account_name`
 - `account_key`
-- `sas_token`
 - `tenant_id`
 - `client_id`
 - `client_secret`
diff --git a/services/azure-storage/tests/credential_providers/mod.rs b/services/azure-storage/tests/credential_providers/mod.rs
index 30ac3145..7eb61167 100644
--- a/services/azure-storage/tests/credential_providers/mod.rs
+++ b/services/azure-storage/tests/credential_providers/mod.rs
@@ -51,8 +51,8 @@ pub async fn assert_provider_reads_probe(
     provider: impl ProvideCredential<Credential = Credential> + 'static,
     ctx: Context,
 ) -> anyhow::Result<()> {
-    let url = std::env::var("REQSIGN_AZURE_STORAGE_URL")
-        .map_err(|_| anyhow::anyhow!("REQSIGN_AZURE_STORAGE_URL must be set"))?;
+    let url = std::env::var("REQSIGN_AZURE_STORAGE_PROBE_URL")
+        .map_err(|_| anyhow::anyhow!("REQSIGN_AZURE_STORAGE_PROBE_URL must be set"))?;
     let signer = Signer::new(ctx.clone(), provider, RequestSigner::new());
     let request = http::Request::get(url)
         .header("x-ms-version", "2023-11-03")
diff --git a/services/azure-storage/tests/signing/sas_token.rs b/services/azure-storage/tests/signing/sas_token.rs
index 098aaa95..e020838a 100644
--- a/services/azure-storage/tests/signing/sas_token.rs
+++ b/services/azure-storage/tests/signing/sas_token.rs
@@ -42,7 +42,7 @@ async fn test_sas_token_signing() -> anyhow::Result<()> {
         return Ok(());
     }
 
-    let url = required_env("REQSIGN_AZURE_STORAGE_URL");
+    let url = required_env("REQSIGN_AZURE_STORAGE_PROBE_URL");
     let sas_token = load_sas_token();
 
     let ctx = Context::new()
@@ -95,7 +95,7 @@ async fn test_sas_token_with_existing_query() {
         return;
     }
 
-    let base_url = required_env("REQSIGN_AZURE_STORAGE_URL");
+    let base_url = required_env("REQSIGN_AZURE_STORAGE_PROBE_URL");
     let sas_token = load_sas_token();
 
     let ctx = Context::new()
@@ -135,7 +135,7 @@ async fn test_sas_token_preserves_headers() {
         return;
     }
 
-    let url = required_env("REQSIGN_AZURE_STORAGE_URL");
+    let url = required_env("REQSIGN_AZURE_STORAGE_PROBE_URL");
     let sas_token = load_sas_token();
 
     let ctx = Context::new()
diff --git a/services/azure-storage/tests/signing/shared_key.rs b/services/azure-storage/tests/signing/shared_key.rs
index 7d998404..fdc17e16 100644
--- a/services/azure-storage/tests/signing/shared_key.rs
+++ b/services/azure-storage/tests/signing/shared_key.rs
@@ -34,7 +34,7 @@ fn get_test_config() -> Option<(String, String, String, String, String)> {
         return None;
     }
 
-    let url = required_env("REQSIGN_AZURE_STORAGE_URL");
+    let url = required_env("REQSIGN_AZURE_STORAGE_PROBE_URL");
     let account_name = required_env("REQSIGN_AZURE_STORAGE_ACCOUNT_NAME");
     let account_key = required_env("REQSIGN_AZURE_STORAGE_ACCOUNT_KEY");
     let service =

From 37c97138dae4a76586ed8253a80e1563b027a39d Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 10 Sep 2026 15:40:31 +0800
Subject: [PATCH 2/3] docs(ci): document Azure probe queue-time variable setup

---
 services/azure-storage/tests/README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/services/azure-storage/tests/README.md b/services/azure-storage/tests/README.md
index da076a4b..9dd75210 100644
--- a/services/azure-storage/tests/README.md
+++ b/services/azure-storage/tests/README.md
@@ -133,6 +133,14 @@ repository or scheduled trigger for this pipeline. It uses a dedicated Azure VM
 Scale Set agent pool with one-node maximum capacity, zero standby agents, and
 automatic recycling after every job.
 
+The Azure DevOps pipeline definition must declare the non-secret variable
+`REQSIGN_AZURE_STORAGE_PROBE_URL` with an empty default and **Let users override
+this value when running this pipeline** enabled (`allowOverride: true` in the
+Build Definitions API). GitHub supplies this value when queuing each run. Keep
+the organization-level restriction on other queue-time variables enabled.
+Azure DevOps YAML previews do not validate this permission; verify it with an
+actual queued run.
+
 The final summary reports and enforces every provider and signing job
 independently for trusted changes.
 

From 2b7cab47c810d7f698057f0b3a7d32883af8c4d9 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 10 Sep 2026 15:46:37 +0800
Subject: [PATCH 3/3] fix(ci): provide swap for AWS IMDS test bootstrapping

---
 .github/workflows/aws_v4.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/aws_v4.yml b/.github/workflows/aws_v4.yml
index 61a361e1..a1e3e071 100644
--- a/.github/workflows/aws_v4.yml
+++ b/.github/workflows/aws_v4.yml
@@ -742,6 +742,11 @@ jobs:
           log_file=/var/log/reqsign-live-test.log
           (
             set -euo pipefail
+            # Package installation can exceed the memory available on t3.nano.
+            fallocate -l 1G /var/reqsign-ci.swap
+            chmod 600 /var/reqsign-ci.swap
+            mkswap /var/reqsign-ci.swap
+            swapon /var/reqsign-ci.swap
             dnf install --assumeyes docker
             systemctl enable --now docker
             aws ecr get-login-password --region '${AWS_REGION}' | docker login --username AWS --password-stdin '${IMAGE_URI%/*}'