diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d13ac32..e0b52ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,3 +244,18 @@ jobs: image-tag: ${{ github.sha }} image-digests: ${{ needs.publish.outputs.digests }} secrets: inherit + + deploy-prod: + name: Promote to production (approval) + needs: [publish, deploy-staging] + if: needs.deploy-staging.outputs.result == 'deployed' + permissions: + contents: read + id-token: write + actions: read + uses: ./.github/workflows/deploy-prod.yml + with: + image-tag: ${{ github.sha }} + image-digests: ${{ needs.publish.outputs.digests }} + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml new file mode 100644 index 0000000..2291e40 --- /dev/null +++ b/.github/workflows/deploy-prod.yml @@ -0,0 +1,230 @@ +name: deploy-prod + +# Promotes a release that passed staging to the production GKE cluster behind the `production` GitHub environment +# (s4.3, replaces the Concourse prod deploy). No rebuild: the exact image digests staging deployed are copied from the +# staging to the production Artifact Registry (ci/deploy/promote-images.sh, skopeo copy --preserve-digests), then +# ci/deploy/helm-deploy.sh runs helm upgrade --install with values-prod.yaml and those digests, waits for the rollouts, +# runs the Postman smoke (newman) + ci/tasks/e2e-test-both-sides.sh against prod and helm-rolls back on any failure. +# Proved offline by ci/kind/verify-prod-promotion.sh. Release and manual rollback procedure: docs/deploy-gke.md. +# +# Called by ci.yml once deploy-staging passes on master; workflow_dispatch promotes the staging-release artifact of an +# earlier master run (e.g. to re-promote a previous release). The promote job waits for the required reviewers of the +# `production` environment (create it with required reviewers and a `master` deployment branch policy). +# Skipped with a notice until these repository variables are set (terraform outputs of infra/terraform/envs/prod): +# PROD_GCP_WORKLOAD_IDENTITY_PROVIDER ci_workload_identity_provider (pool restricted to master + `production`) +# PROD_GCP_CI_SERVICE_ACCOUNT ci_service_account (roles/container.developer, Artifact Registry writer) +# PROD_GCP_ARTIFACT_REGISTRY artifact_registry_url +# PROD_GKE_CLUSTER projects//locations//clusters/ +# and then needs: +# PROD_CLOUDSQL_INSTANCE_CONNECTION_NAME cloudsql_instance_connection_name +# PROD_CLOUDSQL_PSA_CIDR private services access range of the Cloud SQL instance +# PROD_URL https:// served by the platform Gateway (gateway_address) +# plus the staging GCP_* variables (deploy-staging.yml) to read the staging images. Google auth is Workload Identity +# Federation only. Optional secret SLACK_WEBHOOK_URL. + +on: + workflow_call: + inputs: + image-tag: + description: Git SHA the images were tagged with + type: string + required: true + image-digests: + description: 'JSON {"": "sha256:..."} that deploy-staging just deployed' + type: string + required: true + secrets: + SLACK_WEBHOOK_URL: + required: false + workflow_dispatch: + inputs: + staging-run-id: + description: Id of a master ci/deploy-staging run whose staging deploy passed (its staging-release artifact is promoted) + type: string + required: true + +permissions: + contents: read + +concurrency: + group: deploy-prod + cancel-in-progress: false + +jobs: + release: + name: Staging-verified release + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + actions: read + outputs: + enabled: ${{ steps.config.outputs.enabled }} + image-tag: ${{ steps.release.outputs.image-tag }} + image-digests: ${{ steps.release.outputs.image-digests }} + env: + WIF_PROVIDER: ${{ vars.PROD_GCP_WORKLOAD_IDENTITY_PROVIDER }} + CI_SERVICE_ACCOUNT: ${{ vars.PROD_GCP_CI_SERVICE_ACCOUNT }} + REGISTRY: ${{ vars.PROD_GCP_ARTIFACT_REGISTRY }} + GKE_CLUSTER: ${{ vars.PROD_GKE_CLUSTER }} + CLOUDSQL_INSTANCE: ${{ vars.PROD_CLOUDSQL_INSTANCE_CONNECTION_NAME }} + CLOUDSQL_PSA_CIDR: ${{ vars.PROD_CLOUDSQL_PSA_CIDR }} + PROD_URL: ${{ vars.PROD_URL }} + STAGING_WIF_PROVIDER: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + STAGING_CI_SERVICE_ACCOUNT: ${{ vars.GCP_CI_SERVICE_ACCOUNT }} + STAGING_REGISTRY: ${{ vars.GCP_ARTIFACT_REGISTRY }} + steps: + - name: Check the GCP variables + id: config + run: | + if [ -n "$WIF_PROVIDER" ] && [ -n "$CI_SERVICE_ACCOUNT" ] && [ -n "$REGISTRY" ] && [ -n "$GKE_CLUSTER" ]; then + missing="" + for v in CLOUDSQL_INSTANCE CLOUDSQL_PSA_CIDR PROD_URL STAGING_WIF_PROVIDER STAGING_CI_SERVICE_ACCOUNT STAGING_REGISTRY; do + [ -n "${!v}" ] || missing="$missing $v" + done + if [ -n "$missing" ]; then + echo "::error::prod promotion needs PROD_CLOUDSQL_INSTANCE_CONNECTION_NAME, PROD_CLOUDSQL_PSA_CIDR, PROD_URL and the staging GCP_* variables (unset:$missing)" + exit 1 + fi + echo "enabled=true" >> "$GITHUB_OUTPUT" + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + msg="Production promotion skipped: set the repository variables PROD_GCP_WORKLOAD_IDENTITY_PROVIDER, PROD_GCP_CI_SERVICE_ACCOUNT, PROD_GCP_ARTIFACT_REGISTRY and PROD_GKE_CLUSTER (terraform outputs of infra/terraform/envs/prod)." + echo "::notice::$msg" + echo "$msg" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Check the staging run + if: steps.config.outputs.enabled == 'true' && github.event_name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ inputs.staging-run-id }} + run: | + [[ "$RUN_ID" =~ ^[0-9]+$ ]] || { echo "::error::staging-run-id must be a numeric run id"; exit 1; } + branch=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID" --jq .head_branch) + [ "$branch" = master ] || { echo "::error::run $RUN_ID is on '$branch'; only master releases are promoted"; exit 1; } + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + if: steps.config.outputs.enabled == 'true' && github.event_name == 'workflow_dispatch' + with: + name: staging-release + path: ${{ runner.temp }}/staging-release + run-id: ${{ inputs.staging-run-id }} + github-token: ${{ github.token }} + - name: Release to promote + id: release + if: steps.config.outputs.enabled == 'true' + env: + INPUT_TAG: ${{ inputs.image-tag }} + INPUT_DIGESTS: ${{ inputs.image-digests }} + RELEASE_FILE: ${{ runner.temp }}/staging-release/release.json + run: | + set -euo pipefail + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + tag=$(jq -r .image_tag "$RELEASE_FILE"); digests=$(jq -c .image_digests "$RELEASE_FILE") + else + tag=$INPUT_TAG; digests=$(jq -c . <<<"$INPUT_DIGESTS") + fi + [[ "$tag" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::image tag '$tag' is not a git SHA"; exit 1; } + { + echo "### Release waiting for \`production\` approval: \`$tag\`" + echo + echo "Exact digests that passed staging (promoted to \`$REGISTRY\` without a rebuild):" + echo + echo "| Image | Digest |" + echo "|---|---|" + } >> "$GITHUB_STEP_SUMMARY" + for s in trading-engine trader-app discovery-server trader-ui; do + d=$(jq -r --arg s "$s" '.[$s] // empty' <<<"$digests") + [[ "$d" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "::error::no sha256 digest for $s in the release"; exit 1; } + echo "| \`$s\` | \`$d\` |" >> "$GITHUB_STEP_SUMMARY" + done + echo "image-tag=$tag" >> "$GITHUB_OUTPUT" + echo "image-digests=$digests" >> "$GITHUB_OUTPUT" + + promote: + name: Promote digests + helm upgrade + smoke + e2e (rollback on failure) + needs: release + if: needs.release.outputs.enabled == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: + name: production + url: ${{ vars.PROD_URL }} + permissions: + contents: read + id-token: write + env: + REGISTRY: ${{ vars.PROD_GCP_ARTIFACT_REGISTRY }} + STAGING_REGISTRY: ${{ vars.GCP_ARTIFACT_REGISTRY }} + CLOUDSQL_INSTANCE: ${{ vars.PROD_CLOUDSQL_INSTANCE_CONNECTION_NAME }} + CLOUDSQL_PSA_CIDR: ${{ vars.PROD_CLOUDSQL_PSA_CIDR }} + PROD_URL: ${{ vars.PROD_URL }} + IMAGE_TAG: ${{ needs.release.outputs.image-tag }} + IMAGE_DIGESTS: ${{ needs.release.outputs.image-digests }} + steps: + # The chart and values of the release itself, as staging deployed them. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ needs.release.outputs.image-tag }} + persist-credentials: false + - name: Staging registry token (Workload Identity Federation) + id: staging-auth + uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13 + with: + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_CI_SERVICE_ACCOUNT }} + token_format: access_token + create_credentials_file: false + - name: Authenticate to the production project (Workload Identity Federation) + id: auth + uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13 + with: + workload_identity_provider: ${{ vars.PROD_GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.PROD_GCP_CI_SERVICE_ACCOUNT }} + token_format: access_token + - name: Promote the staging digests (no rebuild) + env: + SOURCE_TOKEN: ${{ steps.staging-auth.outputs.access_token }} + TARGET_TOKEN: ${{ steps.auth.outputs.access_token }} + run: SOURCE_REGISTRY="$STAGING_REGISTRY" TARGET_REGISTRY="$REGISTRY" bash ci/deploy/promote-images.sh + - name: GKE credentials + uses: google-github-actions/get-gke-credentials@64bc7249bbcf78056bb92f14d3cedc2da193946c # v2.3.5 + with: + cluster_name: ${{ vars.PROD_GKE_CLUSTER }} + - name: Deploy, smoke, e2e (helm rollback on failure) + id: deploy + env: + SMOKE_LOGS: ${{ runner.temp }}/smoke + run: | + IMAGE_REGISTRY="$REGISTRY" PUBLIC_URL="$PROD_URL" bash ci/deploy/helm-deploy.sh \ + -f deploy/helm/axon-trader/values-prod.yaml \ + --set "trading-engine.cloudSqlProxy.instanceConnectionName=$CLOUDSQL_INSTANCE" \ + --set "trader-app.cloudSqlProxy.instanceConnectionName=$CLOUDSQL_INSTANCE" \ + --set "networkPolicy.mysql.peers[0].ipBlock.cidr=$CLOUDSQL_PSA_CIDR" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: prod-smoke + path: ${{ runner.temp }}/smoke + if-no-files-found: ignore + - name: Slack + if: always() + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + OUTCOME: ${{ steps.deploy.outcome }} + RESULT: ${{ steps.deploy.outputs.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "::notice::Slack notification skipped: set the repository secret SLACK_WEBHOOK_URL." + exit 0 + fi + case "$OUTCOME/$RESULT" in + success/*) text=":white_check_mark: Production promotion of \`${IMAGE_TAG:0:7}\` _passed_ (staging digests, helm upgrade, smoke, e2e): <$RUN_URL|run>" ;; + */rolled-back-to-*) state="helm rolled back to revision ${RESULT#rolled-back-to-}" ;; + */rollback-failed) state="helm rollback FAILED, production needs a look NOW" ;; + */no-rollback) state="no earlier revision to roll back to" ;; + */not-deployed | skipped/*) state="failed before the helm upgrade, production unchanged" ;; + *) state="deploy step died ($OUTCOME), production state unknown" ;; + esac + [ "$OUTCOME" = success ] || text=" *FAILED* :x: Production promotion of \`${IMAGE_TAG:0:7}\` ($state): <$RUN_URL|run>" + jq -n --arg text "$text" '{text: $text}' | curl -fsS -X POST -H 'Content-Type: application/json' --data @- "$SLACK_WEBHOOK_URL" >/dev/null diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 15c2f2a..b37c51f 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -7,6 +7,8 @@ name: deploy-staging # ci/kind/verify-deploy-rollback.sh). Results go to the run summary and to Slack. # # Called by ci.yml after a master push publishes images; workflow_dispatch redeploys a published tag + digests. +# A passing deploy uploads the staging-release artifact (tag + digests) that deploy-prod.yml promotes, and returns +# result=deployed so ci.yml chains the approval-gated production promotion. # Skipped with a notice until these repository variables are set (terraform outputs of infra/terraform/envs/staging): # GCP_WORKLOAD_IDENTITY_PROVIDER ci_workload_identity_provider # GCP_CI_SERVICE_ACCOUNT ci_service_account (roles/container.developer) @@ -29,6 +31,10 @@ on: description: 'JSON {"": "sha256:..."} for trading-engine, trader-app, discovery-server and trader-ui' type: string required: true + outputs: + result: + description: helm-deploy.sh result (deployed, rolled-back-to-, ...); empty when skipped + value: ${{ jobs.deploy.outputs.result }} workflow_dispatch: inputs: image-tag: @@ -55,6 +61,8 @@ jobs: permissions: contents: read id-token: write + outputs: + result: ${{ steps.deploy.outputs.result }} env: WIF_PROVIDER: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} CI_SERVICE_ACCOUNT: ${{ vars.GCP_CI_SERVICE_ACCOUNT }} @@ -107,6 +115,20 @@ jobs: --set "trading-engine.cloudSqlProxy.instanceConnectionName=$CLOUDSQL_INSTANCE" \ --set "trader-app.cloudSqlProxy.instanceConnectionName=$CLOUDSQL_INSTANCE" \ --set "networkPolicy.mysql.peers[0].ipBlock.cidr=$CLOUDSQL_PSA_CIDR" + - name: Record the staging-verified release + if: steps.deploy.outputs.result == 'deployed' + env: + RELEASE_DIR: ${{ runner.temp }}/staging-release + run: | + mkdir -p "$RELEASE_DIR" + jq -n --arg tag "$IMAGE_TAG" --argjson digests "$IMAGE_DIGESTS" --arg sha "$GITHUB_SHA" --arg run "$GITHUB_RUN_ID" \ + '{image_tag: $tag, image_digests: $digests, git_sha: $sha, staging_run_id: $run}' > "$RELEASE_DIR/release.json" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: steps.deploy.outputs.result == 'deployed' + with: + name: staging-release + path: ${{ runner.temp }}/staging-release/release.json + retention-days: 90 - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() && steps.config.outputs.enabled == 'true' with: diff --git a/README.md b/README.md index 755a7d6..aa00e17 100755 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ On `master` the scanned images are pushed to Artifact Registry through Workload | `GCP_CI_SERVICE_ACCOUNT` | `ci_service_account` | `ci-publisher@my-project.iam.gserviceaccount.com` | | `GCP_ARTIFACT_REGISTRY` | `artifact_registry_url` | `europe-west1-docker.pkg.dev/my-project/esarch` | -The pushed digests are listed in the publish job summary. The Concourse pipeline (`ci/pipeline.yml`) still exists alongside it. +The pushed digests are listed in the publish job summary. They are then deployed to staging and, behind the `production` environment approval, promoted to production without a rebuild: see [docs/deploy-gke.md](docs/deploy-gke.md) for the release, rollback (`helm rollback`, previous digest) and local `kind` workflow. The Concourse pipeline (`ci/pipeline.yml`) still exists alongside it. ## CQRS & Event Sourcing Architectural Overview diff --git a/ci/deploy/promote-images.sh b/ci/deploy/promote-images.sh new file mode 100755 index 0000000..e97850d --- /dev/null +++ b/ci/deploy/promote-images.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Promotes the exact images that passed staging into the production Artifact Registry, without rebuilding (s4.3): +# for each service, copies /@ (the whole index, attestations included) to +# /: with `skopeo copy --all --preserve-digests`, which fails rather than rewrite +# a manifest, then checks the target tag serves that same digest. A tag already serving the digest is left alone (prod tags +# are immutable, so re-promoting a release, e.g. to roll back, must not push it again); any other digest fails. +# +# Env: SOURCE_REGISTRY, TARGET_REGISTRY e.g. us-central1-docker.pkg.dev//axon-trader +# IMAGE_TAG git SHA the images were published with +# IMAGE_DIGESTS JSON {"": "sha256:..."} (ci.yml publish output, deploy-staging artifact) +# SOURCE_TOKEN, TARGET_TOKEN optional OAuth access tokens (google-github-actions/auth token_format: +# access_token); each registry gets its own throwaway auth file +# REGISTRY_TLS_VERIFY false for plain-HTTP local registries (ci/kind/verify-prod-promotion.sh) +# SKOPEO_IMAGE pinned skopeo image (distro skopeo 1.13 has no --preserve-digests); runs on docker +# Prints " " for every promoted image; ci/kind/verify-prod-promotion.sh proves it offline. +set -euo pipefail + +SERVICES=${SERVICES:-"trading-engine trader-app discovery-server trader-ui"} +SKOPEO_IMAGE=${SKOPEO_IMAGE:-quay.io/skopeo/stable:v1.16.1@sha256:94f5c5e26997e2e78c234ec9abf19a391c234b39eb22e6d1210d0b527c97dcc8} +: "${SOURCE_REGISTRY:?SOURCE_REGISTRY is required}" "${TARGET_REGISTRY:?TARGET_REGISTRY is required}" +: "${IMAGE_TAG:?IMAGE_TAG is required}" "${IMAGE_DIGESTS:?IMAGE_DIGESTS is required}" +tls=${REGISTRY_TLS_VERIFY:-true} + +auth=$(mktemp -d) +trap 'rm -rf "$auth"' EXIT +skopeo() { docker run --rm -i --network host -v "$auth:/auth" "$SKOPEO_IMAGE" "$@"; } +docker pull -q "$SKOPEO_IMAGE" >/dev/null +src_auth=() dst_auth=() dst_inspect_auth=() +if [ -n "${SOURCE_TOKEN:-}" ]; then + skopeo login --tls-verify="$tls" --authfile /auth/source.json -u oauth2accesstoken --password-stdin "${SOURCE_REGISTRY%%/*}" <<<"$SOURCE_TOKEN" >/dev/null + src_auth=(--src-authfile /auth/source.json) +fi +if [ -n "${TARGET_TOKEN:-}" ]; then + skopeo login --tls-verify="$tls" --authfile /auth/target.json -u oauth2accesstoken --password-stdin "${TARGET_REGISTRY%%/*}" <<<"$TARGET_TOKEN" >/dev/null + dst_auth=(--dest-authfile /auth/target.json) + dst_inspect_auth=(--authfile /auth/target.json) +fi + +for s in $SERVICES; do + d=$(jq -r --arg s "$s" '.[$s] // empty' <<<"$IMAGE_DIGESTS") + [[ "$d" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "FAIL: IMAGE_DIGESTS has no sha256 digest for $s" >&2; exit 1; } + if skopeo inspect --tls-verify="$tls" "${dst_inspect_auth[@]}" --raw "docker://$TARGET_REGISTRY/$s:$IMAGE_TAG" "$auth/existing" 2>/dev/null; then + # Prod tags are immutable: an existing tag is either this exact release (re-promotion / rollback) or a conflict. + have=$(sha256sum <"$auth/existing" | cut -d' ' -f1) + [ "sha256:$have" = "$d" ] || { echo "FAIL: $TARGET_REGISTRY/$s:$IMAGE_TAG already holds sha256:$have, staging passed $d" >&2; exit 1; } + echo "$s $d (already promoted)" + continue + fi + skopeo copy --src-tls-verify="$tls" --dest-tls-verify="$tls" "${src_auth[@]}" "${dst_auth[@]}" --all --preserve-digests --quiet \ + "docker://$SOURCE_REGISTRY/$s@$d" "docker://$TARGET_REGISTRY/$s:$IMAGE_TAG" &2; exit 1; } + echo "$s $d" +done diff --git a/ci/kind/setup.sh b/ci/kind/setup.sh index f56bd17..021e15c 100644 --- a/ci/kind/setup.sh +++ b/ci/kind/setup.sh @@ -2,7 +2,7 @@ # kind_setup: kind cluster, local images, throwaway Secrets, the MySQL/RabbitMQ stand-ins (ci/kind/kustomization.yaml), # then `helm upgrade --install` of deploy/helm/axon-trader with ci/kind/values-kind.yaml. With ESARCH_KIND_GATEWAY=1 # it also installs Envoy Gateway and renders the chart's Gateway + HTTPRoute (ci/kind/values-kind-gateway.yaml). -# ESARCH_HELM_ARGS adds extra helm flags (word-split), e.g. --set trading-engine.replicaCount=2. +# KIND_CONFIG is an optional kind cluster config file. ESARCH_HELM_ARGS adds extra helm flags (word-split), e.g. --set trading-engine.replicaCount=2. # The passwords are generated per run and only ever live in the cluster. Never touches GCP or TAS. ENVOY_GATEWAY_VERSION=${ENVOY_GATEWAY_VERSION:-v1.5.4} KIND_IMAGES="esarch/discovery-server:latest esarch/trader-app:latest esarch/trading-engine:latest esarch/trader-ui:latest @@ -10,7 +10,7 @@ KIND_IMAGES="esarch/discovery-server:latest esarch/trader-app:latest esarch/trad kind_cluster() { echo "== kind cluster '$CLUSTER', images, namespace and throwaway secrets" - kind get clusters 2>/dev/null | grep -qx "$CLUSTER" || kind create cluster --name "$CLUSTER" --wait 120s + kind get clusters 2>/dev/null | grep -qx "$CLUSTER" || kind create cluster --name "$CLUSTER" ${KIND_CONFIG:+--config "$KIND_CONFIG"} --wait 120s local img for img in $KIND_IMAGES; do docker image inspect "$img" >/dev/null 2>&1 || docker pull -q "$img" diff --git a/ci/kind/verify-prod-promotion.sh b/ci/kind/verify-prod-promotion.sh new file mode 100755 index 0000000..0e18016 --- /dev/null +++ b/ci/kind/verify-prod-promotion.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# s4.3 offline proof of the production promotion (.github/workflows/deploy-prod.yml) on kind, with two local registries +# standing in for the staging and production Artifact Registries. The cluster can only pull from the "prod" one, and +# values-prod.yaml (plus the kind stand-ins of values-kind.yaml) is what gets deployed: +# 1. publish the CI images to the staging registry and promote that release with ci/deploy/promote-images.sh: same +# digests in prod, no rebuild; deploy them with ci/deploy/helm-deploy.sh (rollout + smoke + e2e) and check every +# pod runs exactly the staging digest, trading-engine at one replica (d-engine-replicas); +# 2. promote a release whose trader-app crash-loops: the rollout fails and helm rolls back to release 1 automatically; +# 3. promote a second good release, then follow the manual rollback of docs/deploy-gke.md: find the previous digests +# in the helm history, `helm rollback` to that revision and re-run the smoke + e2e. +# Build the images first (see verify-axon-routing.sh). Needs docker, kind, kubectl, helm, jq, node (npx newman) +# and uuidgen. ESARCH_KIND_KEEP=1 keeps the cluster and registries. Never touches GCP or TAS. +set -euo pipefail +cd "$(dirname "$0")/../.." + +CLUSTER=${ESARCH_KIND_CLUSTER:-esarch-prod} +NS=axon-trader +LOGS=${ESARCH_LOGS:-/tmp/esarch-kind-prod} +STAGING_REGISTRY=localhost:5001 +PROD_REGISTRY=localhost:5002 +SERVICES="trading-engine trader-app discovery-server trader-ui" +mkdir -p "$LOGS" +rm -rf "${LOGS:?}"/* +k() { kubectl --context "kind-$CLUSTER" -n "$NS" "$@"; } +hm() { helm --kube-context "kind-$CLUSTER" -n "$NS" "$@"; } +cleanup() { + if [ "${ESARCH_KIND_KEEP:-0}" != 1 ]; then + kind delete cluster --name "$CLUSTER" >/dev/null 2>&1 || true + docker rm -f esarch-staging-registry esarch-prod-registry >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } +export KUBE_CONTEXT=kind-$CLUSTER NS +revision() { hm history axon-trader --max 1 -o json | jq -r '.[0] | "\(.revision) \(.status) \(.description)"'; } +image_of() { k get deploy "$1" -o jsonpath='{.spec.template.spec.containers[0].image}'; } + +echo "== 0. staging + prod registries, release images" +for r in staging:5001 prod:5002; do + docker rm -f "esarch-${r%:*}-registry" >/dev/null 2>&1 || true + docker run -d --name "esarch-${r%:*}-registry" -p "127.0.0.1:${r#*:}:5000" registry:2 >/dev/null +done +# The two local registries speak plain HTTP (Artifact Registry is HTTPS + an OAuth token in CI). +export REGISTRY_TLS_VERIFY=false +docker build -q -t esarch/trader-app:broken - >/dev/null <<'DOCKERFILE' +FROM esarch/trader-app:latest +ENTRYPOINT ["java", "-XX:+NoSuchJvmOption", "-version"] +DOCKERFILE +# Release 2: a real new trader-ui image (new digest), otherwise identical. +docker build -q -t esarch/trader-ui:release2 - >/dev/null <<'DOCKERFILE' +FROM esarch/trader-ui:latest +LABEL org.opencontainers.image.version=release2 +DOCKERFILE + +# What ci.yml's publish job does: push : to the staging registry, emit {"": "sha256:..."}. +publish() { + local tag=$1 digests='{}' s src target digest + shift + for s in $SERVICES; do + src="esarch/$s:latest" + for o in "$@"; do [ "${o%%=*}" = "$s" ] && src=${o#*=}; done + target="$STAGING_REGISTRY/$s:$tag" + docker tag "$src" "$target" + docker push -q "$target" >/dev/null + digest=$(docker inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$target" | grep -F "$STAGING_REGISTRY/$s@" | head -1) + digests=$(jq -c --arg s "$s" --arg d "${digest#*@}" '. + {($s): $d}' <<<"$digests") + done + echo "$digests" +} +TAG1=$(git rev-parse HEAD) +TAG2=$(printf '%040d' 2) +TAG_BROKEN=$(printf '%040d' 9) +REL1=$(publish "$TAG1") +REL_BROKEN=$(publish "$TAG_BROKEN" trader-app=esarch/trader-app:broken) +REL2=$(publish "$TAG2" trader-ui=esarch/trader-ui:release2) +echo "release 1 ($TAG1): $REL1" +echo "release broken: $REL_BROKEN" +echo "release 2: $REL2" + +echo +echo "== kind cluster pulling from the prod registry only" +cat >"$LOGS/kind.yaml" <<'KIND' +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +KIND +# shellcheck source=ci/kind/setup.sh +. ci/kind/setup.sh +KIND_CONFIG=$LOGS/kind.yaml +KIND_IMAGES="mysql:5.7 rabbitmq:3.7-management curlimages/curl:8.16.0" +kind_cluster +docker network connect kind esarch-prod-registry 2>/dev/null || true +docker exec "$CLUSTER-control-plane" mkdir -p "/etc/containerd/certs.d/$PROD_REGISTRY" +printf '[host."http://esarch-prod-registry:5000"]\n' | docker exec -i "$CLUSTER-control-plane" tee "/etc/containerd/certs.d/$PROD_REGISTRY/hosts.toml" >/dev/null +for s in $SERVICES; do docker exec "$CLUSTER-control-plane" crictl rmi "esarch/$s:latest" >/dev/null 2>&1 || true; done +# values-prod.yaml expects the platform (Terraform foundation) to own the Workload Identity KSAs. +for s in trading-engine trader-app; do k create serviceaccount "$s" --dry-run=client -o yaml | k apply -f - >/dev/null; done +kind_backing + +# deploy-prod.yml's promote job: promote-images.sh, then helm-deploy.sh with values-prod.yaml. +promote_and_deploy() { + local name=$1 tag=$2 digests=$3 + shift 3 + echo "-- promote $tag" + SOURCE_REGISTRY=$STAGING_REGISTRY TARGET_REGISTRY=$PROD_REGISTRY IMAGE_TAG=$tag IMAGE_DIGESTS=$digests \ + bash ci/deploy/promote-images.sh >"$LOGS/promote-$name.txt" || { cat "$LOGS/promote-$name.txt"; return 1; } + cat "$LOGS/promote-$name.txt" + echo "-- deploy $tag" + SMOKE_LOGS="$LOGS/$name" IMAGE_REGISTRY=$PROD_REGISTRY IMAGE_TAG=$tag IMAGE_DIGESTS=$digests \ + bash ci/deploy/helm-deploy.sh -f deploy/helm/axon-trader/values-prod.yaml -f ci/kind/values-kind.yaml "$@" +} +# Every Deployment spec and every running pod is on /@. +expect_release() { + local digests=$1 s d ids + for s in $SERVICES; do + d=$(jq -r --arg s "$s" '.[$s]' <<<"$digests") + [ "$(image_of "$s")" = "$PROD_REGISTRY/$s@$d" ] || fail "$s spec image is $(image_of "$s"), expected $PROD_REGISTRY/$s@$d" + ids=$(k get pods -l "app.kubernetes.io/name=$s" -o jsonpath='{range .items[*]}{.status.containerStatuses[0].imageID}{"\n"}{end}' | sort -u) + [ "$ids" = "$PROD_REGISTRY/$s@$d" ] || fail "$s pods run '$ids', expected $PROD_REGISTRY/$s@$d" + echo "$s: spec + pods on $PROD_REGISTRY/$s@$d" + done +} + +echo +echo "== 1. promote release 1 (staging digests, no rebuild) and deploy with values-prod.yaml" +promote_and_deploy rel1 "$TAG1" "$REL1" || fail "the release 1 promotion failed" +for s in $SERVICES; do + got=$(docker run --rm --network host quay.io/skopeo/stable:v1.16.1 inspect --tls-verify=false --raw "docker://$PROD_REGISTRY/$s:$TAG1" | sha256sum | cut -d' ' -f1) + [ "sha256:$got" = "$(jq -r --arg s "$s" '.[$s]' <<<"$REL1")" ] || fail "prod registry $s:$TAG1 is sha256:$got" +done +echo "prod registry serves the staging digests under $TAG1" +expect_release "$REL1" +[ "$(k get deploy trading-engine -o jsonpath='{.spec.replicas}')" = 1 ] || fail "trading-engine is not at 1 replica" +k get hpa +echo "release: $(revision)" + +echo +echo "== 2. promote a release whose trader-app crash-loops: expect rollout failure + automatic rollback to release 1" +status=0 +ROLLOUT_TIMEOUT=240s promote_and_deploy broken "$TAG_BROKEN" "$REL_BROKEN" || status=$? +[ "$status" != 0 ] || fail "the broken promotion exited 0" +echo "exit status: $status (expected non-zero)" +rev=$(revision); echo "release: $rev" +grep -q 'Rollback to 1' <<<"$rev" || fail "expected the latest revision to be 'Rollback to 1', got: $rev" +k rollout status deploy/trader-app --timeout=300s +expect_release "$REL1" + +echo +echo "== 3. promote release 2, then the manual rollback of docs/deploy-gke.md back to release 1" +promote_and_deploy rel2 "$TAG2" "$REL2" || fail "the release 2 promotion failed" +expect_release "$REL2" +hm history axon-trader +# docs/deploy-gke.md: the last deployed revision before the current one, and the digests it ran. +previous=$(hm history axon-trader -o json | jq -r '[.[] | select(.status == "superseded")] | last | .revision') +prev_digests=$(hm get values axon-trader --revision "$previous" -o json | + jq -c '{"trading-engine", "trader-app", "discovery-server", "trader-ui"} | map_values(.image.digest)') +echo "previous revision $previous ran: $prev_digests" +[ "$prev_digests" = "$(jq -c '{"trading-engine", "trader-app", "discovery-server", "trader-ui"}' <<<"$REL1")" ] || + fail "helm history says revision $previous ran $prev_digests, not release 1" +hm rollback axon-trader "$previous" --wait --timeout 600s +for d in discovery-server trading-engine trader-app trader-ui; do k rollout status "deploy/$d" --timeout=600s; done +expect_release "$REL1" +SMOKE_LOGS="$LOGS/after-manual-rollback" bash ci/deploy/smoke.sh || fail "smoke/e2e failed after the manual rollback" + +echo +hm history axon-trader +k get deploy -o custom-columns=NAME:.metadata.name,READY:.status.readyReplicas,REPLICAS:.spec.replicas,IMAGE:.spec.template.spec.containers[0].image +[ "$(k get deploy trading-engine -o jsonpath='{.spec.replicas}')" = 1 ] || fail "trading-engine is not at 1 replica" +echo "PASS: staging digests promoted without rebuild and deployed with values-prod.yaml (smoke + e2e green); crash-looping release rolled back automatically; manual helm rollback to the previous digests green" diff --git a/docs/deploy-gke.md b/docs/deploy-gke.md new file mode 100644 index 0000000..49ea807 --- /dev/null +++ b/docs/deploy-gke.md @@ -0,0 +1,154 @@ +# Deploying Axon Trader to GKE + +How a change gets from `master` to production on GKE, how to roll it back, and how to run the same pipeline on a +local `kind` cluster. Replaces the Concourse `push-to-pcf` jobs (see `docs/migration/tas-gke-mapping.md`, section 8). + +## Pipeline + +``` +merge to master + └─ ci.yml: build, test, Trivy scan, publish → staging Artifact Registry, : + digests + └─ deploy-staging.yml → helm upgrade (values-staging.yaml) + smoke + e2e, rollback on failure + └─ deploy-prod.yml → waits for `production` approval + └─ promote job → copy the staging digests to the prod registry (no rebuild), + helm upgrade (values-prod.yaml) + smoke + e2e, rollback on failure +``` + +| Piece | What it does | +|---|---| +| `.github/workflows/ci.yml` | Builds and scans the four images, pushes them on `master` and emits `{"": "sha256:..."}` | +| `.github/workflows/deploy-staging.yml` | Deploys those digests to staging; on success uploads the `staging-release` artifact (`release.json`: tag + digests, kept 90 days) and returns `result=deployed` | +| `.github/workflows/deploy-prod.yml` | `release` job shows the release (tag + digests) in its summary; `promote` job runs in the `production` environment | +| `ci/deploy/promote-images.sh` | `skopeo copy --all --preserve-digests` (pinned skopeo 1.16 image) of each `/@` to `/:`, then checks the prod registry serves the same digest | +| `ci/deploy/helm-deploy.sh` | `helm upgrade --install` pinned by digest, `kubectl rollout status`, `ci/deploy/smoke.sh` (newman `ESArchSmokeTest.postman_collection.json` + `ci/tasks/e2e-test-both-sides.sh` + the public `/api/command/api` route); any failure runs `helm rollback` to the revision deployed before | + +Production never builds anything: the image bytes that run in prod are the ones staging tested, addressed by digest. +`trading-engine` stays at one replica in every environment (decision d-engine-replicas); only `trader-app` and +`trader-ui` autoscale in prod. + +## One-time setup + +### `production` environment (repository admin) + +Settings → Environments → New environment `production`: + +- **Required reviewers**: the release approvers (at least one person other than the usual committer). +- **Deployment branches and tags**: selected branches, `master` only. +- Optionally **Prevent self-review**. + +Or with the API (needs admin on the repository; reviewer ids from `gh api users/ --jq .id` or `gh api orgs/COG-GTM/teams/ --jq .id`): + +```bash +gh api -X PUT repos/COG-GTM/ESarch/environments/production --input - <<'JSON' +{ + "prevent_self_review": true, + "reviewers": [{"type": "Team", "id": 1234567}], + "deployment_branch_policy": {"protected_branches": false, "custom_branch_policies": true} +} +JSON +gh api -X POST repos/COG-GTM/ESarch/environments/production/deployment-branch-policies -f name=master -f type=branch +``` + +### Workload Identity Federation (Terraform) + +`infra/terraform/envs/prod` creates the prod WIF pool for this repository with `ci_allowed_refs = ["refs/heads/master"]` +and `ci_allowed_environments = ["production"]`: Google only exchanges GitHub tokens minted for a `master` job of the +`production` environment, so a prod token cannot exist without the reviewers' approval. No service account key is +ever created or stored. + +### Repository variables + +Values are Terraform outputs of `infra/terraform/envs/prod` (none of them is a secret). Until the four `PROD_GCP_*` / +`PROD_GKE_CLUSTER` variables are set, `deploy-prod.yml` is skipped with a notice and nothing waits for approval. + +| Variable | Terraform output / value | +|---|---| +| `PROD_GCP_WORKLOAD_IDENTITY_PROVIDER` | `ci_workload_identity_provider` | +| `PROD_GCP_CI_SERVICE_ACCOUNT` | `ci_service_account` (`roles/container.developer`, Artifact Registry writer on the prod repository) | +| `PROD_GCP_ARTIFACT_REGISTRY` | `artifact_registry_url` | +| `PROD_GKE_CLUSTER` | `projects//locations//clusters/` | +| `PROD_CLOUDSQL_INSTANCE_CONNECTION_NAME` | `cloudsql_instance_connection_name` | +| `PROD_CLOUDSQL_PSA_CIDR` | private services access range of the Cloud SQL instance | +| `PROD_URL` | `https://` served by the platform Gateway | + +The promote job also reads the staging images with the staging `GCP_WORKLOAD_IDENTITY_PROVIDER`, +`GCP_CI_SERVICE_ACCOUNT` and `GCP_ARTIFACT_REGISTRY` (README, GitHub Actions section). `SLACK_WEBHOOK_URL` (optional +secret) gets the result of both deploys. + +## Releasing + +1. Merge to `master`. `ci.yml` publishes, `deploy-staging` deploys and tests staging. +2. When staging passes, the `Promote to production (approval)` job of the same run waits. Open the run: the + `Staging-verified release` summary lists the git SHA and the four digests that will go to production. +3. A required reviewer clicks **Review deployments → production → Approve**. Only the latest waiting release stays + pending (the `deploy-prod` concurrency group replaces older waiting ones), and the approval expires after 30 days. +4. The promote job copies the digests, deploys with `values-prod.yaml`, runs the smoke + e2e against prod and posts to + Slack. Its summary shows the helm result; logs are in the `prod-smoke` artifact. + +To promote an earlier release (the latest pending one was replaced, or to roll forward/back to a known release): +Actions → `deploy-prod` → **Run workflow** on `master` with `staging-run-id` = the id of the `ci` (or `deploy-staging`) +run on `master` whose staging deploy passed. It promotes that run's `staging-release` artifact, behind the same approval. + +Note: the smoke collection writes test data (`/actuator/data-initializer`, orders) into the prod databases, as the +Concourse smoke did. Run the first promotion before cutover traffic (s6.3). + +## Rolling back + +### Automatic + +Any failure after `helm upgrade` starts (a rollout that does not complete within `ROLLOUT_TIMEOUT`, the newman smoke, +the e2e, the public route) runs `helm rollback axon-trader --wait` and fails the job. The job +summary and Slack say which: `rolled-back-to-`, `rollback-failed` (look now), `no-rollback` (first install) or +`not-deployed` (failed before touching the cluster, prod unchanged). + +### Manual: `helm rollback` + +From a shell with access to the prod cluster (`gcloud container clusters get-credentials`, `roles/container.developer`): + +```bash +helm -n axon-trader history axon-trader +# the revision to go back to (the last superseded one is the release before the current) +previous=$(helm -n axon-trader history axon-trader -o json | jq -r '[.[] | select(.status == "superseded")] | last | .revision') +# the digests it ran +helm -n axon-trader get values axon-trader --revision "$previous" -o json | + jq '{"trading-engine", "trader-app", "discovery-server", "trader-ui"} | map_values(.image.digest)' +helm -n axon-trader rollback axon-trader "$previous" --wait --timeout 600s +for d in discovery-server trading-engine trader-app trader-ui; do kubectl -n axon-trader rollout status deploy/$d; done +NS=axon-trader PUBLIC_URL=https:// bash ci/deploy/smoke.sh +``` + +### Manual: redeploy the previous digests through the pipeline + +Preferred when there is time, because it is approved, audited and smoke-tested like any release: run `deploy-prod` +by hand with the `staging-run-id` of the release to go back to (see Releasing). The digests that release ran are in +its `staging-release` artifact, in the `Staging-verified release` summary of its run and in `helm get values` above. + +Database changes are not rolled back by either path: Flyway migrations must stay backward compatible with the +previous release. + +## Local `kind` workflow + +The same scripts run against a local `kind` cluster with MySQL/RabbitMQ stand-ins (`ci/kind/`), with no GCP or TAS +access. Needs docker, kind, kubectl, helm, jq, node (`npx newman`), and uuidgen (the promotion proof runs skopeo from a pinned container image). + +```bash +# build the four images as CI does +./mvnw -B -DskipTests package jib:dockerBuild +docker build -t esarch/trader-ui:latest trader-app-ui + +deploy/helm/scripts/validate.sh # helm lint + kubeconform for every values file +bash ci/kind/verify-helm-smoke.sh # install the chart, smoke + e2e +bash ci/kind/verify-deploy-rollback.sh # staging deploy logic: good deploy, two broken images rolled back +bash ci/kind/verify-prod-promotion.sh # prod promotion: staging digests -> prod registry -> values-prod.yaml, + # automatic rollback of a broken release, manual helm rollback +``` + +`verify-prod-promotion.sh` runs two local registries standing in for the staging and prod Artifact Registries; the +cluster can only pull from the prod one, and the script checks every Deployment and pod runs +`/@`. `ESARCH_KIND_KEEP=1` keeps the cluster (`kind-esarch-prod`) and +registries for poking around; `kind delete cluster --name esarch-prod && docker rm -f esarch-staging-registry esarch-prod-registry` cleans up. + +## Not proved yet + +No GCP project exists yet: the staging deploy and the dry-run promotion to the real prod cluster (before cutover +traffic) are still owed once the Terraform is applied and the variables above are set. diff --git a/docs/migration/tas-gke-mapping.md b/docs/migration/tas-gke-mapping.md index 01d2805..80fd20f 100644 --- a/docs/migration/tas-gke-mapping.md +++ b/docs/migration/tas-gke-mapping.md @@ -135,6 +135,8 @@ Allowances that NET-1/NET-2 need under default-deny (owner s3.5, verified on sta **Done (s4.2), staging run still owed:** `.github/workflows/deploy-staging.yml`, called by `ci.yml` after `publish` pushes images, deploys their digests with `ci/deploy/helm-deploy.sh` (helm upgrade --install, rollout wait, `ci/deploy/smoke.sh`: newman smoke and `e2e-test-both-sides.sh` over `kubectl port-forward` plus the public `/api/command/api` route; `helm rollback` on any failure) and posts to Slack (`SLACK_WEBHOOK_URL`). Skipped with a notice until `GCP_WORKLOAD_IDENTITY_PROVIDER`, `GCP_CI_SERVICE_ACCOUNT`, `GCP_ARTIFACT_REGISTRY` and `GKE_CLUSTER` are set. Proved offline by `ci/kind/verify-deploy-rollback.sh` (CI-5, CI-6, CI-7, CI-10). +**Done (s4.3), prod dry-run still owed:** `.github/workflows/deploy-prod.yml`, chained by `ci.yml` after a passing staging deploy, waits for the `production` environment reviewers, copies the exact staging digests to the prod Artifact Registry (`ci/deploy/promote-images.sh`, no rebuild) and runs `ci/deploy/helm-deploy.sh` with `values-prod.yaml` (smoke, e2e, automatic `helm rollback`). The prod WIF pool only accepts `master` jobs of the `production` environment (`ci_allowed_environments`). Skipped with a notice until the `PROD_GCP_*` / `PROD_GKE_CLUSTER` variables are set. Proved offline by `ci/kind/verify-prod-promotion.sh`; release and rollback procedure in `docs/deploy-gke.md` (CI-1, CI-10). + | ID | TAS item | GKE target | Owner | Files that change | |---|---|---|---|---| | CI-1 | `just-build-it` manual gate | Dropped. Builds run on every PR and merge; production promotion is the manual gate (s4.3) | s4.1 | `ci/pipeline.yml` deleted in s6.4 | @@ -146,7 +148,7 @@ Allowances that NET-1/NET-2 need under default-deny (owner s3.5, verified on sta | CI-7 | `e2e-test` | Post-deploy job against staging, then the fuller parity suite | s4.2 (s5.1 extends) | `.github/workflows/deploy-staging.yml`, `ci/tasks/e2e-test-both-sides.sh` | | CI-8 | `manualtrigger` `time` resource | Dropped (`workflow_dispatch` covers manual runs) | s4.1 | none | | CI-9 | `source-code` git resource with deploy key | Native checkout in GitHub Actions; the deploy key is not needed | s4.1 | none | -| CI-10 | `push-to-pcf` `cf` resource, `slack-alert` resource | `helm upgrade --install` with the `-ci-deployer` account (`roles/container.developer`) via WIF; GitHub environments `staging` and `production` (required reviewers on production). Slack notification through a GitHub Action with the webhook as a repository secret | s4.2, s4.3 | `.github/workflows/deploy-staging.yml`, `.github/workflows/deploy-prod.yml` (new) | +| CI-10 | `push-to-pcf` `cf` resource, `slack-alert` resource | `helm upgrade --install` with the `-ci-deployer` account (`roles/container.developer`) via WIF; GitHub environments `staging` and `production` (required reviewers on production). Slack notification through a GitHub Action with the webhook as a repository secret | s4.2, s4.3 | `.github/workflows/deploy-staging.yml`, `.github/workflows/deploy-prod.yml` | | CI-11 | `build-trading-engine.sh` | Replaced by the Dockerfile build stage (CHG-13). Fix of the `${OLDPATH}` echo (L6) not needed once deleted | s4.1 (deleted s6.4) | `ci/tasks/build-trading-engine.{sh,yml}` | | CI-12 | `build-trader-app.sh` | As CI-11 | s4.1 (deleted s6.4) | `ci/tasks/build-trader-app.{sh,yml}` | | CI-13 | `build-trader-ui.sh` (copies `manifest.yml`, `Staticfile`) | Replaced by the UI Dockerfile (CHG-14) | s4.1 (deleted s6.4) | `ci/tasks/build-trader-ui.{sh,yml}` | diff --git a/infra/terraform/envs/prod/main.tf b/infra/terraform/envs/prod/main.tf index 03c5ea6..2cdaadd 100644 --- a/infra/terraform/envs/prod/main.tf +++ b/infra/terraform/envs/prod/main.tf @@ -24,6 +24,7 @@ module "foundation" { github_repository_id = var.github_repository_id github_repository_owner_id = var.github_repository_owner_id ci_allowed_refs = var.ci_allowed_refs + ci_allowed_environments = var.ci_allowed_environments } module "gke" { diff --git a/infra/terraform/envs/prod/terraform.tfvars.example b/infra/terraform/envs/prod/terraform.tfvars.example index 8f0dbf6..8edb743 100644 --- a/infra/terraform/envs/prod/terraform.tfvars.example +++ b/infra/terraform/envs/prod/terraform.tfvars.example @@ -18,6 +18,8 @@ github_repository = "COG-GTM/ESarch" github_repository_id = "REPLACE_WITH_NUMERIC_REPOSITORY_ID" github_repository_owner_id = "REPLACE_WITH_NUMERIC_OWNER_ID" ci_allowed_refs = ["refs/heads/master"] +# Only jobs of the `production` GitHub environment (required reviewers) get prod tokens (.github/workflows/deploy-prod.yml). +ci_allowed_environments = ["production"] # GKE Autopilot (regional, private nodes). Add the operator/CI egress CIDRs that run kubectl. gke_master_ipv4_cidr_block = "172.16.0.16/28" diff --git a/infra/terraform/envs/prod/variables.tf b/infra/terraform/envs/prod/variables.tf index b2b2001..d34adca 100644 --- a/infra/terraform/envs/prod/variables.tf +++ b/infra/terraform/envs/prod/variables.tf @@ -79,6 +79,12 @@ variable "ci_allowed_refs" { default = [] } +variable "ci_allowed_environments" { + description = "GitHub environments whose jobs may authenticate; production keeps the deploy-prod.yml approval gate in front of every prod token." + type = list(string) + default = ["production"] +} + variable "gke_master_ipv4_cidr_block" { description = "/28 control plane range of the GKE cluster (must not overlap the VPC ranges)." type = string diff --git a/infra/terraform/modules/foundation/main.tf b/infra/terraform/modules/foundation/main.tf index 3a8242e..12177f0 100644 --- a/infra/terraform/modules/foundation/main.tf +++ b/infra/terraform/modules/foundation/main.tf @@ -105,6 +105,7 @@ module "ci_workload_identity" { github_repository_id = var.github_repository_id github_repository_owner_id = var.github_repository_owner_id allowed_refs = var.ci_allowed_refs + allowed_environments = var.ci_allowed_environments ci_service_account_name = module.service_accounts.names["ci-deployer"] depends_on = [module.project_services] diff --git a/infra/terraform/modules/foundation/tests/foundation.tftest.hcl b/infra/terraform/modules/foundation/tests/foundation.tftest.hcl index 3981c04..2351043 100644 --- a/infra/terraform/modules/foundation/tests/foundation.tftest.hcl +++ b/infra/terraform/modules/foundation/tests/foundation.tftest.hcl @@ -87,12 +87,13 @@ run "prod_ref_restriction_and_ksa_bindings" { variables { environment = "prod" ci_allowed_refs = ["refs/heads/master"] + ci_allowed_environments = ["production"] enable_gke_workload_identity_bindings = true } assert { - condition = module.ci_workload_identity.attribute_condition == "assertion.repository == 'COG-GTM/ESarch' && assertion.repository_id == '1340826081' && assertion.repository_owner_id == '211054393' && assertion.ref in ['refs/heads/master']" - error_message = "Ref restriction missing from the WIF attribute condition." + condition = module.ci_workload_identity.attribute_condition == "assertion.repository == 'COG-GTM/ESarch' && assertion.repository_id == '1340826081' && assertion.repository_owner_id == '211054393' && assertion.ref in ['refs/heads/master'] && assertion.environment in ['production']" + error_message = "Ref or production environment restriction missing from the WIF attribute condition." } assert { diff --git a/infra/terraform/modules/foundation/variables.tf b/infra/terraform/modules/foundation/variables.tf index 2e38c2d..0285436 100644 --- a/infra/terraform/modules/foundation/variables.tf +++ b/infra/terraform/modules/foundation/variables.tf @@ -150,3 +150,9 @@ variable "ci_allowed_refs" { type = list(string) default = [] } + +variable "ci_allowed_environments" { + description = "GitHub environments whose jobs may authenticate from CI. Empty allows any job." + type = list(string) + default = [] +} diff --git a/infra/terraform/modules/workload-identity-federation/main.tf b/infra/terraform/modules/workload-identity-federation/main.tf index cf079ff..361843b 100644 --- a/infra/terraform/modules/workload-identity-federation/main.tf +++ b/infra/terraform/modules/workload-identity-federation/main.tf @@ -10,6 +10,10 @@ locals { "assertion.ref in [%s]", join(", ", [for r in var.allowed_refs : format("'%s'", r)]) )], + length(var.allowed_environments) == 0 ? [] : [format( + "assertion.environment in [%s]", + join(", ", [for e in var.allowed_environments : format("'%s'", e)]) + )], ) } @@ -35,7 +39,7 @@ resource "google_iam_workload_identity_pool_provider" "github" { "attribute.ref" = "assertion.ref" } - # Only tokens minted for this repository (and, if set, these refs) can be exchanged at all. + # Only tokens minted for this repository (and, if set, these refs and GitHub environments) can be exchanged at all. attribute_condition = join(" && ", local.conditions) oidc { diff --git a/infra/terraform/modules/workload-identity-federation/variables.tf b/infra/terraform/modules/workload-identity-federation/variables.tf index a90ed2f..38a1504 100644 --- a/infra/terraform/modules/workload-identity-federation/variables.tf +++ b/infra/terraform/modules/workload-identity-federation/variables.tf @@ -56,6 +56,12 @@ variable "allowed_refs" { default = [] } +variable "allowed_environments" { + description = "GitHub environments (e.g. production) whose jobs may authenticate; their protection rules (required reviewers) then gate every token. Empty allows any job." + type = list(string) + default = [] +} + variable "ci_service_account_name" { description = "Resource name (projects/

/serviceAccounts/) of the service account CI impersonates." type = string