From e287591fb383ab14d5eab27d86e98be19b2f44bb Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 23 Jul 2026 16:46:21 +0300 Subject: [PATCH] Reconcile Cloud Run scaling settings into deploy config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The production Cloud Run warm floor (service-level minScale=2) was set by hand and lived only as live service state — invisible to the repo and reliant on CI never asserting a service-level min. Codify it: the deploy now passes `--min ${CLOUD_RUN_SERVICE_MIN_INSTANCES}` (production 2, staging 0) alongside the unchanged revision-level `--min-instances 0`, so the floor is asserted on every deploy without the per-tag cost bomb. Also raise production max-instances 4 -> 8 for the 100% Cloud Run cutover: at the 50/50 split the service already sat pinned at the ceiling of 4 during the daytime peak, so carrying all traffic needs the headroom. Updates the deploy-script unit tests, the runbook, and adds a changelog fragment. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/cloud_run_env.sh | 10 ++++++ .github/scripts/deploy_cloud_run_candidate.sh | 1 + .github/workflows/push.yml | 17 +++++++--- ...ud-run-scaling-config-reconcile.changed.md | 1 + docs/migration/cloud-run-operations.md | 28 ++++++++++----- tests/unit/test_cloud_run_deploy_scripts.py | 34 +++++++++++++++++-- 6 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 changelog.d/cloud-run-scaling-config-reconcile.changed.md diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index 17c70e127..8aad5620f 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -13,7 +13,16 @@ cloud_run_set_defaults() { CLOUD_RUN_CPU="${CLOUD_RUN_CPU:-4}" CLOUD_RUN_MEMORY="${CLOUD_RUN_MEMORY:-16Gi}" CLOUD_RUN_TIMEOUT="${CLOUD_RUN_TIMEOUT:-300}" + # Revision-level minimum (gcloud --min-instances). MUST stay 0 everywhere: a + # positive value is set immutably on every new revision, so each tagged + # no-traffic candidate would pin its own warm instances (a per-tag cost bomb). CLOUD_RUN_MIN_INSTANCES="${CLOUD_RUN_MIN_INSTANCES:-0}" + # Service-level warm floor across all revisions (gcloud --min). This is the + # correct place to keep instances warm: one floor for whichever revision is + # serving, with no per-revision pinning. Production sets 2 so the service never + # cold-starts from zero under the overnight bot trough; staging leaves it 0 + # (scale-to-zero). Rationale in docs/migration/cloud-run-operations.md. + CLOUD_RUN_SERVICE_MIN_INSTANCES="${CLOUD_RUN_SERVICE_MIN_INSTANCES:-0}" CLOUD_RUN_MAX_INSTANCES="${CLOUD_RUN_MAX_INSTANCES:-1}" # Stage 2-qualified runtime shape, pinned explicitly on every deploy — # rationale in docs/migration/cloud-run-operations.md ("Runtime shape"). @@ -50,6 +59,7 @@ cloud_run_set_defaults() { export CLOUD_RUN_MEMORY export CLOUD_RUN_TIMEOUT export CLOUD_RUN_MIN_INSTANCES + export CLOUD_RUN_SERVICE_MIN_INSTANCES export CLOUD_RUN_MAX_INSTANCES export CLOUD_RUN_CONCURRENCY export CLOUD_RUN_WEB_CONCURRENCY diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index c06373987..5b0383e4e 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -52,6 +52,7 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \ --memory "${CLOUD_RUN_MEMORY}" \ --timeout "${CLOUD_RUN_TIMEOUT}" \ --min-instances "${CLOUD_RUN_MIN_INSTANCES}" \ + --min "${CLOUD_RUN_SERVICE_MIN_INSTANCES}" \ --max-instances "${CLOUD_RUN_MAX_INSTANCES}" \ --concurrency "${CLOUD_RUN_CONCURRENCY}" \ --startup-probe "${CLOUD_RUN_STARTUP_PROBE}" \ diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 47980029c..513417d1f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -214,8 +214,10 @@ jobs: env: CLOUD_RUN_SERVICE: policyengine-api-staging # Staging stays scale-to-zero, single instance: it exists for per-push - # validation, not capacity. + # validation, not capacity. Both the revision-level (--min-instances) and + # service-level (--min) floors are 0. CLOUD_RUN_MIN_INSTANCES: "0" + CLOUD_RUN_SERVICE_MIN_INSTANCES: "0" CLOUD_RUN_MAX_INSTANCES: "1" permissions: contents: read @@ -542,10 +544,15 @@ jobs: environment: production env: CLOUD_RUN_SERVICE: policyengine-api - # Sized by the Stage 2 qualification — rationale and numbers in - # docs/migration/cloud-run-operations.md ("Runtime shape and scaling"). - # Never pin a min-instances value here: warm capacity is service-level. - CLOUD_RUN_MAX_INSTANCES: "4" + # Sized by the Stage 2 qualification and the PR 4 host cutover — rationale + # and numbers in docs/migration/cloud-run-operations.md ("Runtime shape and + # scaling"). Warm capacity is expressed service-level (--min); the + # revision-level floor stays 0 (default in cloud_run_env.sh) to avoid + # per-tag pinning. Max raised to 8 for the 100% Cloud Run cutover: at 50/50 + # the service was already pinned at the previous ceiling of 4 during the + # daytime peak, so full traffic needs the extra headroom. + CLOUD_RUN_SERVICE_MIN_INSTANCES: "2" + CLOUD_RUN_MAX_INSTANCES: "8" permissions: contents: read id-token: write diff --git a/changelog.d/cloud-run-scaling-config-reconcile.changed.md b/changelog.d/cloud-run-scaling-config-reconcile.changed.md new file mode 100644 index 000000000..5b307a6e3 --- /dev/null +++ b/changelog.d/cloud-run-scaling-config-reconcile.changed.md @@ -0,0 +1 @@ +Raise the production Cloud Run service to max-instances 8 and apply the service-level warm floor (`--min`) from the deploy config so it is asserted on every deploy instead of only as live service state. diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index 4dd8b0e97..666491a62 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -124,17 +124,27 @@ Values measured and justified in covering every boot. Note that lazily deferring the build does **not** help: it relocates the cost onto the first request, where readiness would lie and a user would absorb it. -- **Scaling pins live in `push.yml` per job**: production `max-instances 4` (peak real - traffic ~11 RPS, mostly cached/light; ~1–2 concurrent uncached calculates per - instance), staging `min 0 / max 1`. -- **Warm capacity is a service-level setting made manually, once** — CI keeps - revision-level `--min-instances 0`, because revision-level minimums keep a warm 16Gi - instance alive per accumulated `stage3-*` tag. +- **Scaling pins live in `push.yml` per job**: production `max-instances 8`, staging + `min 0 / max 1`. Production was `max-instances 4` through Stage 10; it was raised to + 8 for the 100% Cloud Run cutover because at the 50/50 split the service already sat + pinned at the ceiling of 4 during the daytime peak (`instance_count` flatlined at + 4.0), so carrying all traffic needs the extra headroom. +- **Warm capacity is applied by CI on every deploy as a service-level `--min`**, via + the `CLOUD_RUN_SERVICE_MIN_INSTANCES` env (production 2, staging 0). CI keeps the + revision-level `--min-instances 0`, because a revision-level minimum keeps a warm + 16Gi instance alive per accumulated `stage3-*` tag. **The flag names differ by one word and mean opposite things:** service-level warm capacity is `--min` (or the `run.googleapis.com/minScale` annotation on the *service* metadata); `--min-instances` is the *revision-level* setting and is the - per-tag cost bomb. `--min` requires a recent gcloud (it does not exist in 461). + per-tag cost bomb. `--min` requires a recent gcloud (it does not exist in 461); the + CI runners and the deploy scripts assume a current gcloud. The deploy passes both + flags on every deploy — `--min-instances 0 --min ${CLOUD_RUN_SERVICE_MIN_INSTANCES}` + — so the floor is asserted in code, not carried only as live service state. + + To change the floor, edit `CLOUD_RUN_SERVICE_MIN_INSTANCES` in `push.yml` (and the + default in `cloud_run_env.sh`); the next production deploy applies it. To apply it + out-of-band between deploys: ```bash gcloud run services update policyengine-api \ @@ -173,7 +183,9 @@ Values measured and justified in **Current value: 2.** Originally set to 1 on 2026-07-08 (via the YAML-replace fallback, since the gcloud in use lacked `--min`), raised to 2 on 2026-07-22 by - the same route. One warm instance meant every + the same route, and codified into the deploy config on 2026-07-23 (env + `CLOUD_RUN_SERVICE_MIN_INSTANCES`) so it is no longer live-state-only. One warm + instance meant every burst beyond a single instance's capacity landed on a cold boot; two warm instances carry the burst during the ~161s a scaled-out instance takes to become ready. Cost is ~$131/month per warm instance at 4 vCPU / 16Gi (idle CPU diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index d05c058d4..34d8b3e76 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -373,6 +373,33 @@ def test_deploy_cloud_run_candidate_pins_runtime_shape(): # docs/migration/cloud-run-operations.md ("Runtime shape and scaling"). assert "--concurrency 6 " in result.stdout assert "WEB_CONCURRENCY=2 " in result.stdout + # Revision-level floor (--min-instances) stays 0; the warm floor is applied + # service-level (--min), defaulting to 0 unless a job overrides it. + assert "--min-instances 0 " in result.stdout + assert "--min 0 " in result.stdout + + +def test_deploy_cloud_run_candidate_applies_service_level_min_floor(): + """The warm floor is service-level (--min), never revision-level. + + A positive --min-instances is immutably baked onto every new revision, so + each tagged no-traffic candidate would pin its own warm instances. --min + keeps one floor across whichever revision is serving, with no per-tag cost. + """ + result = _run_script( + ".github/scripts/deploy_cloud_run_candidate.sh", + _script_env( + **_required_runtime_env(), + CLOUD_RUN_IMAGE_URI="us-central1-docker.pkg.dev/project/repo/api:sha", + CLOUD_RUN_TAG="stage3-test", + CLOUD_RUN_SERVICE_MIN_INSTANCES="2", + ), + ) + + assert result.returncode == 0, result.stderr + # Service-level floor honoured, and the revision-level floor stays 0 so the + # candidate does not pin per-tag warm instances. + assert "--min 2 " in result.stdout assert "--min-instances 0 " in result.stdout @@ -435,11 +462,14 @@ def test_push_workflow_pins_cloud_run_scaling_per_job(): assert 'CLOUD_RUN_MIN_INSTANCES: "0"' in staging_deploy assert 'CLOUD_RUN_MAX_INSTANCES: "1"' in staging_deploy - assert 'CLOUD_RUN_MAX_INSTANCES: "4"' in production_deploy + assert 'CLOUD_RUN_MAX_INSTANCES: "8"' in production_deploy + # Production keeps a service-level warm floor of 2; staging stays at 0. + assert 'CLOUD_RUN_SERVICE_MIN_INSTANCES: "2"' in production_deploy + assert 'CLOUD_RUN_SERVICE_MIN_INSTANCES: "0"' in staging_deploy # Revision-level min-instances must stay 0 EVERYWHERE in the workflow — # including workflow-level env, which flows into every job. Any mention # of the variable must be an explicit "0" pin. - assert workflow.count("CLOUD_RUN_MIN_INSTANCES") == workflow.count( + assert workflow.count("CLOUD_RUN_MIN_INSTANCES:") == workflow.count( 'CLOUD_RUN_MIN_INSTANCES: "0"' ) # The runtime-shape values live only in cloud_run_env.sh (where the