From 513ccbfd4bb35d03c85470ccfe9ebf836e3ee181 Mon Sep 17 00:00:00 2001 From: Test Operator Date: Fri, 15 May 2026 15:06:15 +0800 Subject: [PATCH] Route production deploys to modular workflows --- docs/design-docs/current-feature-inventory.md | 2 +- ...ops-console-page-map-and-e2e-validation.md | 2 +- .../03_Synapse_GCE_Deployment_Governance.md | 11 +++- ...roduction_GCP_Isolation_and_E2E_Runbook.md | 17 +++++- server/services/releases/production_tasks.py | 23 +++++--- .../services/releases/production_workflows.py | 45 ++++++++++++++++ tests/test_release_production_tasks_api.py | 52 +++++++++++++++++-- 7 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 server/services/releases/production_workflows.py diff --git a/docs/design-docs/current-feature-inventory.md b/docs/design-docs/current-feature-inventory.md index 9b2bb234..682cb024 100644 --- a/docs/design-docs/current-feature-inventory.md +++ b/docs/design-docs/current-feature-inventory.md @@ -105,7 +105,7 @@ 5. PR / release 只信任 workflow evidence、verification verdict 和 human gate decision。 6. Drop Goal 不会自动 rollback;下线已发布功能必须创建 decommission requirement。 7. `release_pr_status_sweep` 会只读同步 GitHub PR 状态:merged PR 更新为 `merged`,closed / missing PR 收敛为 `pr_closed / abandoned`,但不会自动 deploy 或 rollback。 -8. `/releases` 的 `发布管理` 分面使用独立 Release Task 记录手动和需求流转发布意图;Release Task 不等于 Deployment Record。任务创建不会自动部署,分支选择从目标 GitHub 仓库 branch list 读取;`main` 分支可由 Operator 直接点击 `发布到 staging`,其它分支没有 open/merged PR 时只能提交 PR,PR checks 通过后才允许触发白名单发布 runner。Production 发布已接入 Human-Gated Release Task:只允许 `main`,必须同一 project / service / commit 已有 staging success,并要求 operator 输入 `DEPLOY_PRODUCTION` 后 dispatch `deploy-production.yml`。当前一等发布目标包含 `Synapse-Network` 的 gateway / gateway-admin / admin-front / user-front / docs-front,以及 `Synapse-Network-Provider` 的 provider-service。`Synapse-Network` staging dispatch 按 runtime family 选择 workflow:gateway -> `deploy-staging-gateway.yml`,gateway-admin/admin-front -> `deploy-staging-admin.yml`,user-front/docs-front -> `deploy-staging-frontends.yml`;目标仓库的 `deploy-staging.yml` 只作为 reusable core / compatibility runner。Production dispatch 使用目标仓库 `deploy-production.yml`,通过 GCP WIF 部署到 `synapse-network-prod`。 +8. `/releases` 的 `发布管理` 分面使用独立 Release Task 记录手动和需求流转发布意图;Release Task 不等于 Deployment Record。任务创建不会自动部署,分支选择从目标 GitHub 仓库 branch list 读取;`main` 分支可由 Operator 直接点击 `发布到 staging`,其它分支没有 open/merged PR 时只能提交 PR,PR checks 通过后才允许触发白名单发布 runner。Production 发布已接入 Human-Gated Release Task:只允许 `main`,必须同一 project / service / commit 已有 staging success,并要求 operator 输入 `DEPLOY_PRODUCTION` 后 dispatch 模块化 production workflow。当前一等发布目标包含 `Synapse-Network` 的 gateway / gateway-admin / admin-front / user-front / docs-front,以及 `Synapse-Network-Provider` 的 provider-service。`Synapse-Network` staging dispatch 按 runtime family 选择 workflow:gateway -> `deploy-staging-gateway.yml`,gateway-admin/admin-front -> `deploy-staging-admin.yml`,user-front/docs-front -> `deploy-staging-frontends.yml`;目标仓库的 `deploy-staging.yml` 只作为 reusable core / compatibility runner。Production dispatch 同样按模块选择 workflow:gateway -> `deploy-production-gateway.yml`,gateway-admin -> `deploy-production-admin.yml`,admin-front/user-front/docs-front -> `deploy-production-frontends.yml`,provider-service -> `deploy-production-provider.yml`;所有 production runner 通过 GCP WIF 部署到 `synapse-network-prod`。 9. Release Task detail modal 展示 PR gate、DeploymentRecord、GitHub Actions run link、health check、failure summary 和 deployment metadata;Synapse-Network 与 Synapse-Network-Provider staging runner 都使用 GCP Workload Identity Federation + Cloud Build + Cloud Run,不使用长生命周期 GCP service account JSON key 或 SSH deploy secrets;Provider DeploymentRecord 标记 `deploy_strategy=github_actions_wif_cloudrun`。 10. `/releases` 的 `上线记录` 分面只读展示成功的 `DeploymentRecord(status=succeeded)`,不混入 pending / failed / cancelled Release Task。手动标记 Release Task 已完成时,后端会创建或绑定一条 `manual_completion` 成功 DeploymentRecord 后再进入上线记录。上线记录用于确认 GitHub Actions、Cloud Run revision / image / manifest、health check 和 rollback target,并可按 `staging` / `production` 过滤。 11. 没有 webhook callback 的本地发起 staging 部署,由全局 `release_deployment_status_sweep` 定时拉取 GitHub Actions run 结果,也可通过 `/releases` 的 `同步状态` 人工触发同一幂等同步函数;同步只处理未终态 DeploymentRecord,只读吸收外部事实,不触发 deploy / merge / rollback。点击 `发布到 staging` 后 Release Task 进入 `deploying / 自动检查中`,健康检查事实由 GitHub Actions runner 内的 Cloud Run smoke check 负责,Ops Console 只同步 Actions success/failure 和 smoke metadata。人工 completed / failed / cancelled 终态优先,不会被后台同步回滚。 diff --git a/docs/ops-console-page-map-and-e2e-validation.md b/docs/ops-console-page-map-and-e2e-validation.md index 04bfc92e..5031e80e 100644 --- a/docs/ops-console-page-map-and-e2e-validation.md +++ b/docs/ops-console-page-map-and-e2e-validation.md @@ -82,7 +82,7 @@ Every page needs three levels of validation: | `/workflows` | Stats, filters, tabs and definitions render | Active/Needs Human/Failed/Completed/Archived/Definitions tabs | Completed run appears in completed tab and can be archived | | `/workflows/:id` | Overview and step rail render | Steps/Evidence/Gates/Debug tabs, human gate decision | Step artifacts, events and atom invocations exist | | `/schedules` | Health cards, registry and run history render | Dry Run, Run Now confirmation, select workflow | Run history row is created with dry-run/result payload | -| `/releases` | Release candidates, Release Tasks, scheduled Cloud Run Jobs, SDK packages and Online Records render | URL-backed tabs survive refresh, filters, open detail modal, scheduled-job create modal selects environment, branch, and job name; requested staging row exposes `发布到 staging`; production row exposes `发布到 production` only after main commit has matching staging success; failed deployment detail shows GitHub Actions failed-step excerpt or permission diagnostic | Release candidate links to requirement/workflow; production deploy creates `DeploymentRecord(environment=production, trigger_source=release_task)` and dispatches `deploy-production.yml`; scheduled-job create only writes `scheduled_job_release_tasks(status=requested)`; row publish creates linked `DeploymentRecord(trigger_source=scheduled_job_release)` and dispatches `deploy-staging-jobs.yml`; sync updates records and notes/metadata contain bounded Actions failure detail when GitHub permissions allow log reads | +| `/releases` | Release candidates, Release Tasks, scheduled Cloud Run Jobs, SDK packages and Online Records render | URL-backed tabs survive refresh, filters, open detail modal, scheduled-job create modal selects environment, branch, and job name; requested staging row exposes `发布到 staging`; production row exposes `发布到 production` only after main commit has matching staging success; failed deployment detail shows GitHub Actions failed-step excerpt or permission diagnostic | Release candidate links to requirement/workflow; production deploy creates `DeploymentRecord(environment=production, trigger_source=release_task)` and dispatches the service-specific production workflow (`deploy-production-gateway.yml`, `deploy-production-admin.yml`, `deploy-production-frontends.yml`, or Provider `deploy-production-provider.yml`); scheduled-job create only writes `scheduled_job_release_tasks(status=requested)`; row publish creates linked `DeploymentRecord(trigger_source=scheduled_job_release)` and dispatches `deploy-staging-jobs.yml`; sync updates records and notes/metadata contain bounded Actions failure detail when GitHub permissions allow log reads | | `/releases/:requirementId` | Release state and validation render | Prepare branch, open PR, deploy buttons, rollback prompt | Release/deployment rows update or return guarded errors | | `/publishing` | X account drafts, reply queue, repost candidates, published-effect daily metrics, distribution accounts, and distribution campaigns render | Generate from prompt, edit, reject, publish confirmation, generate reply, reply confirmation, date-filter metrics, page metric rows, open centered metric detail Modal, edit distribution account, create campaign, generate targets, copy title/body, mark live/removed | Real Codex E2E proves generated body is not prompt-copy; reply drafts are generated by `social_reply_agent_v1` with no template fallback; publish/reply/repost require operator confirmation and record external ids or bounded failures; published-effect UI reads local metrics only and does not trigger X read; distribution targets never call external posting APIs and require operator-pasted `published_url` for `live` | | `/publishing/trends` | Trend result filters and list render | Search, source kind filter, author filter, pagination, centered detail Modal | Query API returns only matching `social_trend_items`; local seeds show non-repost guidance | diff --git a/docs/ops/03_Synapse_GCE_Deployment_Governance.md b/docs/ops/03_Synapse_GCE_Deployment_Governance.md index 7f3deb9d..2a8361db 100644 --- a/docs/ops/03_Synapse_GCE_Deployment_Governance.md +++ b/docs/ops/03_Synapse_GCE_Deployment_Governance.md @@ -48,7 +48,14 @@ author: engineering-ops | `Synapse-Network` | `user-front`, `docs-front` | `deploy-staging-frontends.yml` -> reusable `deploy-staging.yml` | `GCP_WORKLOAD_IDENTITY_PROVIDER`, `GCP_DEPLOY_SERVICE_ACCOUNT` | | `Synapse-Network-Provider` | `provider-service` | GitHub Actions + WIF + Cloud Build + Cloud Run | `GCP_WORKLOAD_IDENTITY_PROVIDER`, `GCP_DEPLOY_SERVICE_ACCOUNT`, `STAGING_GCP_PROJECT_ID`, `STAGING_GCP_REGION`, `STAGING_ARTIFACT_REGISTRY_REPOSITORY`, `STAGING_EVIDENCE_BUCKET` | -Production runner 固定为目标仓库的 `deploy-production.yml`。它必须使用 GitHub Environment `production`,并通过独立 production secrets 访问 `synapse-network-prod`: +Production 使用和 staging 对齐的模块化 runner。`Synapse-Network` 的业务服务按 family dispatch 到清晰命名的 wrapper workflow,wrapper 再调用 reusable core `deploy-production.yml`;`Synapse-Network-Provider` 使用独立 provider workflow。所有 production workflow 都必须使用 GitHub Environment `production`,并通过独立 production secrets 访问 `synapse-network-prod`: + +| Project | Service key | Production runner | +|---|---|---| +| `Synapse-Network` | `gateway` | `deploy-production-gateway.yml` -> reusable `deploy-production.yml` | +| `Synapse-Network` | `gateway-admin` | `deploy-production-admin.yml` -> reusable `deploy-production.yml` | +| `Synapse-Network` | `admin-front`, `user-front`, `docs-front` | `deploy-production-frontends.yml` -> reusable `deploy-production.yml` | +| `Synapse-Network-Provider` | `provider-service` | `deploy-production-provider.yml` | | Secret | Purpose | |---|---| @@ -60,7 +67,7 @@ Production runner 固定为目标仓库的 `deploy-production.yml`。它必须 | `PROD_EVIDENCE_BUCKET` | Production deploy manifest / evidence bucket | Growing 的 `/releases` 仍然是一个统一发布入口,但后端会按 `service_key` -选择对应 staging family workflow。不要让 operator 手工维护这张映射,也不要把所有 +选择对应 staging / production family workflow。不要让 operator 手工维护这张映射,也不要把所有 服务继续塞进一个只靠条件分支区分的 mega workflow。 ## 3. Promotion Rule diff --git a/docs/ops/06_Production_GCP_Isolation_and_E2E_Runbook.md b/docs/ops/06_Production_GCP_Isolation_and_E2E_Runbook.md index ac621b6c..906ba924 100644 --- a/docs/ops/06_Production_GCP_Isolation_and_E2E_Runbook.md +++ b/docs/ops/06_Production_GCP_Isolation_and_E2E_Runbook.md @@ -54,7 +54,7 @@ Use `docs-front` as the low-risk first production pilot. 3. Confirm the Production Gate shows the same main commit and source staging DeploymentRecord. 4. Click `发布到 production`, type `DEPLOY_PRODUCTION`, and submit. 5. Approve the target repository GitHub Environment `production` review. -6. Wait for GitHub Actions `deploy-production.yml` to pass. +6. Wait for GitHub Actions `deploy-production-frontends.yml` to pass. The wrapper calls reusable `deploy-production.yml` but the visible Actions run should identify the module. 7. Click `/releases` `同步状态` or wait for the deployment sweep. 8. Verify `/releases -> 上线记录` includes an `environment=production` row with GitHub Actions URL and health passed. 9. Smoke `https://docs.synapse-network.ai/`. @@ -72,6 +72,21 @@ After the pilot succeeds, promote services in this order to reduce blast radius: If any deployment fails, keep the Release Task failed with the GitHub Actions URL and failure excerpt. Do not manually mark success. Fix the root cause and retry from the failed service. +## Modular Production Workflow Map + +Production deploy observability mirrors staging: + +| Service | Workflow | +|---|---| +| `gateway` | `deploy-production-gateway.yml` | +| `gateway-admin` | `deploy-production-admin.yml` | +| `admin-front` | `deploy-production-frontends.yml` | +| `user-front` | `deploy-production-frontends.yml` | +| `docs-front` | `deploy-production-frontends.yml` | +| `provider-service` | `deploy-production-provider.yml` in `Synapse-Network-Provider` | + +Growing records the concrete workflow file on each `DeploymentRecord.workflow_file` so failed production deploys can be traced to the exact module runner. + ## Rollback Production rollback is not automatic in v1. Runtime rollback must use a previous successful production Online Record as the target and pass a separate Human Gate. Code rollback must use a revert branch, PR checks, merge to `main`, staging verification, then production promotion. diff --git a/server/services/releases/production_tasks.py b/server/services/releases/production_tasks.py index da4a048f..ac53ffd9 100644 --- a/server/services/releases/production_tasks.py +++ b/server/services/releases/production_tasks.py @@ -27,10 +27,12 @@ _valid_service_keys_for_project, deploy_release_task_to_staging, ) +from server.services.releases.production_workflows import ( + production_workflow_file_for_service, +) PRODUCTION_GCP_PROJECT_ID = "synapse-network-prod" PRODUCTION_CONFIRM_PHRASE = "DEPLOY_PRODUCTION" -PRODUCTION_WORKFLOW_FILE = "deploy-production.yml" REQUIRED_GCP_PRODUCTION_DEPLOY_SECRETS = { "PROD_GCP_WORKLOAD_IDENTITY_PROVIDER", "PROD_GCP_DEPLOY_SERVICE_ACCOUNT", @@ -184,10 +186,13 @@ def release_task_production_gate( return gate -def _production_workflow_file(policy: Any) -> str: - return ( - _clean(getattr(policy, "production_workflow_file", None)) - or PRODUCTION_WORKFLOW_FILE +def _production_workflow_file( + policy: Any, *, project_key: str, service_key: str +) -> str: + return production_workflow_file_for_service( + configured_workflow_file=getattr(policy, "production_workflow_file", None), + project_key=project_key, + service_key=service_key, ) @@ -321,7 +326,9 @@ def _create_production_deployment_record( ) -> DeploymentRecordRow: service_key = _clean(getattr(row, "service_key", None)).lower() project_key = _clean(row.project_key) - workflow_file = _production_workflow_file(policy) + workflow_file = _production_workflow_file( + policy, project_key=project_key, service_key=service_key + ) record = DeploymentRecordRow( requirement_id=int(row.requirement_id) if row.requirement_id is not None @@ -405,7 +412,9 @@ def _deploy_release_task_to_production( actor=actor, note=note, ) - workflow_file = _production_workflow_file(policy) + workflow_file = _production_workflow_file( + policy, project_key=_clean(row.project_key), service_key=service_key + ) inputs = _production_dispatch_inputs( row=row, record=record, diff --git a/server/services/releases/production_workflows.py b/server/services/releases/production_workflows.py new file mode 100644 index 00000000..d5b16045 --- /dev/null +++ b/server/services/releases/production_workflows.py @@ -0,0 +1,45 @@ +"""Production GitHub Actions workflow routing for release tasks.""" + +from __future__ import annotations + +PRODUCTION_WORKFLOW_FILE = "deploy-production.yml" +PROVIDER_PRODUCTION_WORKFLOW_FILE = "deploy-production-provider.yml" + +_SYNAPSE_NETWORK_WORKFLOWS = { + "gateway": "deploy-production-gateway.yml", + "gateway-admin": "deploy-production-admin.yml", + "admin-front": "deploy-production-frontends.yml", + "user-front": "deploy-production-frontends.yml", + "docs-front": "deploy-production-frontends.yml", +} + +_PROJECT_WORKFLOWS = { + "synapse-network": _SYNAPSE_NETWORK_WORKFLOWS, + "synapse-network-provider": { + "provider-service": PROVIDER_PRODUCTION_WORKFLOW_FILE, + }, +} + + +def _clean(value: object) -> str: + return str(value or "").strip() + + +def production_workflow_file_for_service( + *, + configured_workflow_file: str | None, + project_key: str, + service_key: str, +) -> str: + """Resolve the concrete production workflow for a project service. + + Project policies historically stored ``deploy-production.yml`` as the generic + production runner. Treat that default as a signal to use the modular + service-specific workflow. Custom policy values remain honored. + """ + + configured = _clean(configured_workflow_file) + if configured and configured != PRODUCTION_WORKFLOW_FILE: + return configured + project_workflows = _PROJECT_WORKFLOWS.get(_clean(project_key).lower(), {}) + return project_workflows.get(_clean(service_key).lower(), PRODUCTION_WORKFLOW_FILE) diff --git a/tests/test_release_production_tasks_api.py b/tests/test_release_production_tasks_api.py index 0b94b06b..c5fa92e6 100644 --- a/tests/test_release_production_tasks_api.py +++ b/tests/test_release_production_tasks_api.py @@ -6,6 +6,9 @@ from server.services.releases import ( production_tasks as release_production_tasks_service, ) +from server.services.releases.production_workflows import ( + production_workflow_file_for_service, +) def test_release_task_production_deploy_requires_gate_and_dispatches_workflow( @@ -30,7 +33,7 @@ def test_release_task_production_deploy_requires_gate_and_dispatches_workflow( def fake_dispatch(**kwargs): calls.append(kwargs) - return "https://github.com/SynapseNetworkAI/Synapse-Network/actions/workflows/deploy-production.yml" + return "https://github.com/SynapseNetworkAI/Synapse-Network/actions/workflows/deploy-production-frontends.yml" monkeypatch.setattr( release_production_tasks_service, "_dispatch_github_workflow", fake_dispatch @@ -81,10 +84,10 @@ def fake_dispatch(**kwargs): payload = response.json() assert payload["status"] == "deploying" assert payload["deployment_record_id"] is not None - assert payload["deployment_workflow_file"] == "deploy-production.yml" + assert payload["deployment_workflow_file"] == "deploy-production-frontends.yml" assert payload["deployment_commit_sha"] == "abc123prod" assert calls[0]["repo_full_name"] == "SynapseNetworkAI/Synapse-Network" - assert calls[0]["workflow_file"] == "deploy-production.yml" + assert calls[0]["workflow_file"] == "deploy-production-frontends.yml" assert calls[0]["inputs"]["deploy_scope"] == "docs-front" assert calls[0]["inputs"]["source_ref"] == "main" assert calls[0]["inputs"]["commit_sha"] == "abc123prod" @@ -96,6 +99,49 @@ def fake_dispatch(**kwargs): assert metadata["environmentIsolationVersion"] == "v1" +def test_production_workflow_resolver_uses_modular_defaults(): + assert ( + production_workflow_file_for_service( + configured_workflow_file="deploy-production.yml", + project_key="synapse-network", + service_key="gateway", + ) + == "deploy-production-gateway.yml" + ) + assert ( + production_workflow_file_for_service( + configured_workflow_file="deploy-production.yml", + project_key="synapse-network", + service_key="gateway-admin", + ) + == "deploy-production-admin.yml" + ) + assert ( + production_workflow_file_for_service( + configured_workflow_file="deploy-production.yml", + project_key="synapse-network", + service_key="docs-front", + ) + == "deploy-production-frontends.yml" + ) + assert ( + production_workflow_file_for_service( + configured_workflow_file="deploy-production.yml", + project_key="synapse-network-provider", + service_key="provider-service", + ) + == "deploy-production-provider.yml" + ) + assert ( + production_workflow_file_for_service( + configured_workflow_file="custom-prod.yml", + project_key="synapse-network", + service_key="gateway", + ) + == "custom-prod.yml" + ) + + def test_release_task_production_deploy_blocks_without_matching_staging( monkeypatch, synced_client ):