From 5f932ee8d290ae4890fa76b32aa0cd04eeb3d7fc Mon Sep 17 00:00:00 2001 From: Test Operator Date: Fri, 15 May 2026 15:39:56 +0800 Subject: [PATCH] Pin production release tasks to staging evidence --- .../principles/release-governance.md | 3 +- docs/design-docs/system-design-principles.md | 1 + .../03_Synapse_GCE_Deployment_Governance.md | 4 +- server/database.py | 2 + server/schemas.py | 1 + .../releases/production_staging_evidence.py | 77 +++++++++++++++++++ server/services/releases/production_tasks.py | 74 ++++++------------ server/services/releases/tasks.py | 2 +- tests/test_release_production_tasks_api.py | 77 +++++++++++++++++++ 9 files changed, 188 insertions(+), 53 deletions(-) create mode 100644 server/services/releases/production_staging_evidence.py diff --git a/docs/design-docs/principles/release-governance.md b/docs/design-docs/principles/release-governance.md index 0718c8fc..4cb22867 100644 --- a/docs/design-docs/principles/release-governance.md +++ b/docs/design-docs/principles/release-governance.md @@ -17,6 +17,7 @@ Load this when touching release tasks, delivery, PR handoff, deploy, staging, pr 2. PR body is reviewer-first: business summary, safety review reason, and human merge checklist at top. 3. Evidence, logs, and retry history should be folded or linked to artifacts. 4. After execution approval, system prepares requirement branch from latest target repository `main` before starting execution; do not defer all branch/PR work to the end. +5. GitHub PR bodies and comments must be written with real newlines via `--body-file`, heredoc, or an equivalent multiline file. Do not pass escaped `\n` sequences in shell arguments; GitHub renders them as garbled literal text. ## Release Tasks And Deployment Records @@ -37,7 +38,7 @@ Load this when touching release tasks, delivery, PR handoff, deploy, staging, pr 5. Do not store GCP JSON keys or SSH deploy secrets in GitHub. 6. Production deploy and rollback require Human Gate. 7. Production cannot reuse staging API or ordinary PATCH to bypass release governance. -8. Staging and production conditions differ: production normally requires main/tag/verified staging manifest plus Human Gate. +8. Staging and production conditions differ: production normally requires a task-pinned main/tag/verified staging manifest plus Human Gate. Production tasks must promote the staged commit they selected; they must not silently drift to a newer moving branch head. 9. Successful staging does not auto-promote production. ## Online Records And Reconciliation diff --git a/docs/design-docs/system-design-principles.md b/docs/design-docs/system-design-principles.md index 3a3d2759..b1503d98 100644 --- a/docs/design-docs/system-design-principles.md +++ b/docs/design-docs/system-design-principles.md @@ -22,6 +22,7 @@ 14. 前端必须是 Enterprise Workbench,不得回到平铺卡片后台、长期内联表单、Drawer/Side Panel 或浏览器原生 confirm。 15. 上下文是预算,不是垃圾桶;长期规则、项目 policy、会话上下文和临时证据必须分层治理。 16. 治理优先于更强自主性;policy、validation、delivery、incident、observability 未收口前,不升级更复杂的 agent 自主决策。 +17. GitHub PR 描述和评论必须使用真实换行;用 `gh pr create/edit --body-file`、heredoc 或临时 Markdown 文件,不要在 shell 字符串里写 literal `\n`,避免 GitHub 页面出现乱码式单段文本。 ## 按需加载入口 diff --git a/docs/ops/03_Synapse_GCE_Deployment_Governance.md b/docs/ops/03_Synapse_GCE_Deployment_Governance.md index 2a8361db..e55ec507 100644 --- a/docs/ops/03_Synapse_GCE_Deployment_Governance.md +++ b/docs/ops/03_Synapse_GCE_Deployment_Governance.md @@ -72,7 +72,7 @@ Growing 的 `/releases` 仍然是一个统一发布入口,但后端会按 `ser ## 3. Promotion Rule -`production` promotion 前,必须已经存在同一 project / service / commit SHA 的 `staging success` 记录。 +`production` promotion 前,必须已经存在同一 project / service / commit SHA 的 `staging success` 记录。Production Release Task 必须绑定一个确定的 `target_commit_sha`;如果是旧任务没有该字段,Growing 只能用最近一次同服务 `main` staging success 作为兼容 evidence,并在 dispatch 前把该 commit 写回任务。Production gate 不允许每次刷新时无声追随最新 moving `main`。 这条规则由 Growing 在 dispatch GitHub Actions 前先阻断。 @@ -81,7 +81,7 @@ Production 只允许 `main`。Growing 的 Release Task production deploy endpoin 1. task environment 是 `production`。 2. branch 是 `main`。 3. operator 输入 `DEPLOY_PRODUCTION`。 -4. 同一 `project_key + service_key + main commit` 已有 staging `DeploymentRecord(status=succeeded, health_status=passed|healthy|manual_verified)`。 +4. 同一 `project_key + service_key + target_commit_sha` 已有 staging `DeploymentRecord(status=succeeded, health_status=passed|healthy|manual_verified)`。 5. 目标仓库具备 production WIF secrets。 Production DeploymentRecord metadata 必须写入 `gcpProject=synapse-network-prod`、`sourceStagingDeploymentRecordId`、`deployScope`、`environmentIsolationVersion=v1`、`deploy_repo_full_name` 和 `service_key`。 diff --git a/server/database.py b/server/database.py index 80a5155f..074ceb0a 100644 --- a/server/database.py +++ b/server/database.py @@ -563,6 +563,7 @@ class ReleaseTaskRow(Base): project_display_name = Column(Text) service_key = Column(Text) branch_name = Column(Text, nullable=False) + target_commit_sha = Column(Text) environment = Column(Text, nullable=False) status = Column(Text, nullable=False, default="requested") actor = Column(Text) @@ -1682,6 +1683,7 @@ def _migrate(engine_): existing = {c["name"] for c in insp.get_columns("release_tasks")} release_task_cols = { "service_key": "TEXT", + "target_commit_sha": "TEXT", } for col, col_type in release_task_cols.items(): if col not in existing: diff --git a/server/schemas.py b/server/schemas.py index 51f7f0b6..45e31513 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -1285,6 +1285,7 @@ class ReleaseTaskOut(BaseModel): project_display_name: Optional[str] = None service_key: Optional[str] = None branch_name: str + target_commit_sha: Optional[str] = None environment: str status: str = "requested" actor: Optional[str] = None diff --git a/server/services/releases/production_staging_evidence.py b/server/services/releases/production_staging_evidence.py new file mode 100644 index 00000000..10874789 --- /dev/null +++ b/server/services/releases/production_staging_evidence.py @@ -0,0 +1,77 @@ +"""Staging deployment evidence helpers for production promotion gates.""" + +from __future__ import annotations + +from sqlalchemy.orm import Session + +from server.database import DeploymentRecordRow +from server.services.releases.tasks import _clean, _json_load + + +def deployment_commit_sha(row: DeploymentRecordRow) -> str: + metadata = _json_load(getattr(row, "metadata_json", None), {}) + return ( + _clean(getattr(row, "commit_sha", None)) + or _clean(metadata.get("resolvedCommitSha")) + or _clean(metadata.get("resolved_commit_sha")) + or _clean(metadata.get("githubHeadSha")) + or _clean(metadata.get("headSha")) + or _clean(metadata.get("commitSha")) + or _clean(metadata.get("commit_sha")) + ) + + +def deployment_service_key(row: DeploymentRecordRow) -> str: + metadata = _json_load(getattr(row, "metadata_json", None), {}) + return _clean( + metadata.get("service_key") + or metadata.get("serviceKey") + or metadata.get("deployScope") + or metadata.get("deploy_scope") + ) + + +def successful_staging_deployment_for_commit( + db: Session, + *, + project_key: str, + service_key: str, + commit_sha: str, +) -> DeploymentRecordRow | None: + rows = _successful_staging_rows(db, project_key=project_key, branch_name=None) + for row in rows: + if deployment_service_key(row) != service_key: + continue + if deployment_commit_sha(row) == commit_sha: + return row + return None + + +def latest_successful_staging_deployment( + db: Session, + *, + project_key: str, + service_key: str, +) -> DeploymentRecordRow | None: + rows = _successful_staging_rows(db, project_key=project_key, branch_name="main") + for row in rows: + if deployment_service_key(row) == service_key and deployment_commit_sha(row): + return row + return None + + +def _successful_staging_rows( + db: Session, + *, + project_key: str, + branch_name: str | None, +) -> list[DeploymentRecordRow]: + query = db.query(DeploymentRecordRow).filter( + DeploymentRecordRow.project_key == project_key, + DeploymentRecordRow.environment == "staging", + DeploymentRecordRow.status == "succeeded", + DeploymentRecordRow.health_status.in_(["passed", "healthy", "manual_verified"]), + ) + if branch_name is not None: + query = query.filter(DeploymentRecordRow.branch_name == branch_name) + return query.order_by(DeploymentRecordRow.id.desc()).limit(50).all() diff --git a/server/services/releases/production_tasks.py b/server/services/releases/production_tasks.py index ac53ffd9..c1faa053 100644 --- a/server/services/releases/production_tasks.py +++ b/server/services/releases/production_tasks.py @@ -19,7 +19,6 @@ from server.services.releases.tasks import ( E2E_BRANCH_OPTIONS, _clean, - _json_load, _policy_for_project, _release_task_deploy_repo, _release_task_to_out, @@ -30,6 +29,11 @@ from server.services.releases.production_workflows import ( production_workflow_file_for_service, ) +from server.services.releases.production_staging_evidence import ( + deployment_commit_sha as _deployment_commit_sha, + latest_successful_staging_deployment as _latest_successful_staging_deployment, + successful_staging_deployment_for_commit as _successful_staging_deployment_for_task, +) PRODUCTION_GCP_PROJECT_ID = "synapse-network-prod" PRODUCTION_CONFIRM_PHRASE = "DEPLOY_PRODUCTION" @@ -89,47 +93,8 @@ def _github_branch_commit_sha( ), None if sha else "GitHub branch response did not include commit sha." -def _deployment_commit_sha(row: DeploymentRecordRow) -> str: - metadata = _json_load(getattr(row, "metadata_json", None), {}) - return ( - _clean(getattr(row, "commit_sha", None)) - or _clean(metadata.get("resolvedCommitSha")) - or _clean(metadata.get("githubHeadSha")) - ) - - -def _deployment_service_key(row: DeploymentRecordRow) -> str: - metadata = _json_load(getattr(row, "metadata_json", None), {}) - return _clean(metadata.get("service_key") or metadata.get("deployScope")) - - -def _successful_staging_deployment_for_task( - db: Session, - *, - project_key: str, - service_key: str, - commit_sha: str, -) -> DeploymentRecordRow | None: - rows = ( - db.query(DeploymentRecordRow) - .filter( - DeploymentRecordRow.project_key == project_key, - DeploymentRecordRow.environment == "staging", - DeploymentRecordRow.status == "succeeded", - DeploymentRecordRow.health_status.in_( - ["passed", "healthy", "manual_verified"] - ), - ) - .order_by(DeploymentRecordRow.id.desc()) - .limit(50) - .all() - ) - for row in rows: - if _deployment_service_key(row) != service_key: - continue - if _deployment_commit_sha(row) == commit_sha: - return row - return None +def _task_target_commit_sha(row: ReleaseTaskRow) -> str: + return _clean(getattr(row, "target_commit_sha", None)) def release_task_production_gate( @@ -152,10 +117,11 @@ def release_task_production_gate( if service_key not in _valid_service_keys_for_project(project_key): gate.blocked_reason = _service_key_error_for_project(project_key) return gate - commit_sha, commit_error = _github_branch_commit_sha(repo_full_name, branch_name) - gate.branch_exists = bool(commit_sha) - gate.resolved_commit_sha = commit_sha - if not commit_sha: + current_main_sha, commit_error = _github_branch_commit_sha(repo_full_name, branch_name) + gate.branch_exists = bool(current_main_sha) + target_commit_sha = _task_target_commit_sha(row) or current_main_sha + gate.resolved_commit_sha = target_commit_sha + if not current_main_sha: gate.blocked_reason = commit_error or "Unable to resolve main commit sha." gate.environment_gate_summary = gate.blocked_reason return gate @@ -163,8 +129,17 @@ def release_task_production_gate( db, project_key=project_key, service_key=service_key, - commit_sha=commit_sha, + commit_sha=target_commit_sha, ) + if staging_record is None and not _task_target_commit_sha(row): + staging_record = _latest_successful_staging_deployment( + db, + project_key=project_key, + service_key=service_key, + ) + if staging_record is not None: + target_commit_sha = _deployment_commit_sha(staging_record) + gate.resolved_commit_sha = target_commit_sha if staging_record is None: gate.blocked_reason = ( "No successful staging deployment exists for this service and main commit." @@ -176,10 +151,10 @@ def release_task_production_gate( gate.staging_deployment_record_id = int(staging_record.id) gate.checks_status = "success" gate.checks_summary = ( - f"Staging deployment #{int(staging_record.id)} passed for {commit_sha[:12]}." + f"Staging deployment #{int(staging_record.id)} passed for {target_commit_sha[:12]}." ) gate.environment_gate_summary = ( - "Production gate ready: main commit has passed staging." + "Production gate ready: selected main commit has passed staging." ) gate.can_deploy_production = True gate.blocked_reason = None @@ -360,6 +335,7 @@ def _create_production_deployment_record( db.commit() db.refresh(record) row.deployment_record_id = int(record.id) + row.target_commit_sha = gate.resolved_commit_sha row.status = "deploying" row.operator = actor row.updated_at = utcnow() diff --git a/server/services/releases/tasks.py b/server/services/releases/tasks.py index a5b0f371..ea1d8314 100644 --- a/server/services/releases/tasks.py +++ b/server/services/releases/tasks.py @@ -96,7 +96,7 @@ def _release_task_to_out(row: ReleaseTaskRow, *, db: Session | None = None, pr_g project_key=str(row.project_key or ""), project_display_name=_clean(row.project_display_name) or None, service_key=_clean(getattr(row, "service_key", None)) or None, - branch_name=str(row.branch_name or ""), + branch_name=str(row.branch_name or ""), target_commit_sha=_clean(getattr(row, "target_commit_sha", None)) or None, environment=str(row.environment or ""), status=str(row.status or "requested"), actor=_clean(row.actor) or None, diff --git a/tests/test_release_production_tasks_api.py b/tests/test_release_production_tasks_api.py index c5fa92e6..85519e8d 100644 --- a/tests/test_release_production_tasks_api.py +++ b/tests/test_release_production_tasks_api.py @@ -169,6 +169,83 @@ def test_release_task_production_deploy_blocks_without_matching_staging( assert "staging deployment" in response.json()["detail"] +def test_release_task_production_gate_uses_pinned_staging_commit( + monkeypatch, db_session +): + monkeypatch.setattr( + release_production_tasks_service, + "_github_branch_commit_sha", + lambda repo, branch: ("new-main-head", None), + ) + staging_record = DeploymentRecordRow( + project_key="synapse-network", + environment="staging", + action="deploy", + status="succeeded", + health_status="passed", + branch_name="main", + commit_sha="staged-main-sha", + metadata_json=json.dumps({"deployScope": "gateway"}), + ) + db_session.add(staging_record) + db_session.commit() + row = ReleaseTaskRow( + project_key="synapse-network", + service_key="gateway", + branch_name="main", + target_commit_sha="staged-main-sha", + environment="production", + status="requested", + ) + + gate = release_production_tasks_service.release_task_production_gate( + db_session, row, "SynapseNetworkAI/Synapse-Network" + ) + + assert gate.can_deploy_production + assert gate.resolved_commit_sha == "staged-main-sha" + assert gate.staging_deployment_record_id == int(staging_record.id) + + +def test_release_task_production_gate_recovers_legacy_unpinned_task_from_latest_staging( + monkeypatch, db_session +): + monkeypatch.setattr( + release_production_tasks_service, + "_github_branch_commit_sha", + lambda repo, branch: ("new-main-head", None), + ) + db_session.add( + DeploymentRecordRow( + project_key="synapse-network", + environment="staging", + action="deploy", + status="succeeded", + health_status="passed", + branch_name="main", + metadata_json=json.dumps( + {"serviceKey": "gateway", "commitSha": "legacy-staged-sha"} + ), + ) + ) + db_session.commit() + row = ReleaseTaskRow( + project_key="synapse-network", + service_key="gateway", + branch_name="main", + environment="production", + status="requested", + ) + + gate = release_production_tasks_service.release_task_production_gate( + db_session, row, "SynapseNetworkAI/Synapse-Network" + ) + + assert gate.can_deploy_production + assert gate.resolved_commit_sha == "legacy-staged-sha" + assert "passed staging" in gate.environment_gate_summary + + def test_release_task_production_gate_blocks_invalid_inputs(monkeypatch, db_session): row = ReleaseTaskRow( project_key="synapse-network",