diff --git a/dashboard/src/pages/ProductionDeployModal.tsx b/dashboard/src/pages/ProductionDeployModal.tsx index 9b07a77f..f0daed93 100644 --- a/dashboard/src/pages/ProductionDeployModal.tsx +++ b/dashboard/src/pages/ProductionDeployModal.tsx @@ -3,6 +3,8 @@ import { FieldLabel, OverlayModal } from '../components/workbench' export default function ProductionDeployModal({ task, + gate, + gateStatus, confirmPhrase, busyAction, onConfirmPhraseChange, @@ -10,13 +12,21 @@ export default function ProductionDeployModal({ onDeploy, }: Readonly<{ task: ReleaseTask | null + gate?: ReleaseTask['pr_gate'] + gateStatus?: 'idle' | 'loading' | 'ready' | 'error' confirmPhrase: string busyAction: string | null onConfirmPhraseChange: (value: string) => void onClose: () => void onDeploy: () => void }>) { + if (!task) { + return null + } const busyKey = task ? `release-task-${task.id}-deploy-production` : '' + const effectiveGate = gate || task?.pr_gate + const gateReady = Boolean(effectiveGate?.can_deploy_production) + const gateLoading = gateStatus === 'loading' return ( - {task ? ( + {(
Production Human Gate
@@ -37,9 +47,14 @@ export default function ProductionDeployModal({
- - + +
+ {!gateReady ? ( +
+ {gateLoading ? 'Production gate 检查中…' : effectiveGate?.blocked_reason || '请先检查 production gate。'} +
+ ) : null}
- ) : null} + )}
) } diff --git a/dashboard/src/pages/ReleasesPage.production.test.tsx b/dashboard/src/pages/ReleasesPage.production.test.tsx new file mode 100644 index 00000000..9bb74a81 --- /dev/null +++ b/dashboard/src/pages/ReleasesPage.production.test.tsx @@ -0,0 +1,150 @@ +// @vitest-environment jsdom + +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { MemoryRouter } from 'react-router-dom' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import ReleasesPage from './ReleasesPage' + +const apiMock = vi.hoisted(() => ({ + getReleaseCandidates: vi.fn(), + getReleaseTasksPage: vi.fn(), + getReleaseTaskGateStatus: vi.fn(), + deployReleaseTask: vi.fn(), + getOnlineReleaseRecords: vi.fn(), + getProjectPolicies: vi.fn(), + getSdkPackageReleases: vi.fn(), + getScheduledCloudRunJobsStatus: vi.fn(), +})) + +vi.mock('../api', () => ({ + api: { + getReleaseCandidates: apiMock.getReleaseCandidates, + getOnlineReleaseRecords: apiMock.getOnlineReleaseRecords, + getProjectPolicies: apiMock.getProjectPolicies, + getSdkPackageReleases: apiMock.getSdkPackageReleases, + getScheduledCloudRunJobsStatus: apiMock.getScheduledCloudRunJobsStatus, + }, +})) + +vi.mock('../apiReleaseTasks', () => ({ + getReleaseTasksPage: apiMock.getReleaseTasksPage, + getReleaseTaskGateStatus: apiMock.getReleaseTaskGateStatus, + deployReleaseTask: apiMock.deployReleaseTask, +})) + +function releaseTask(overrides: Record = {}) { + return { + id: 42, + source_type: 'manual', + requirement_id: null, + deployment_record_id: null, + project_key: 'synapse-network', + project_display_name: 'Synapse-Network', + service_key: 'user-front', + branch_name: 'main', + environment: 'production', + status: 'requested', + actor: 'operator', + operator: null, + note: 'Manual staging release', + deployment_status: null, + deployment_external_run_id: null, + deployment_external_url: null, + deployment_commit_sha: null, + deployment_health_status: null, + deployment_failure_summary: null, + deployment_workflow_file: null, + deployment_metadata: null, + pr_gate: { + branch_exists: true, + pr_exists: false, + pr_url: null, + pr_number: null, + pr_state: 'production', + checks_status: 'unknown', + checks_summary: null, + can_create_pr: false, + can_deploy_staging: false, + can_deploy_production: false, + resolved_commit_sha: null, + staging_deployment_record_id: null, + blocked_reason: 'Production gate pending.', + }, + created_at: '2026-05-15T10:00:00Z', + updated_at: '2026-05-15T10:00:00Z', + ...overrides, + } +} + +describe('ReleasesPage production deploy', () => { + beforeEach(() => { + vi.clearAllMocks() + apiMock.getReleaseCandidates.mockResolvedValue([]) + apiMock.getOnlineReleaseRecords.mockResolvedValue([]) + apiMock.getProjectPolicies.mockResolvedValue([]) + apiMock.getSdkPackageReleases.mockResolvedValue([]) + apiMock.getScheduledCloudRunJobsStatus.mockResolvedValue({ + items: [], + diagnostics: [], + }) + apiMock.getReleaseTasksPage.mockResolvedValue({ + items: [releaseTask()], + total: 1, + limit: 10, + offset: 0, + }) + }) + + it('uses live production gate and does not reload the task page after deploy', async () => { + apiMock.getReleaseTaskGateStatus.mockResolvedValue({ + branch_exists: true, + pr_exists: false, + pr_url: null, + pr_number: null, + pr_state: 'production', + checks_status: 'success', + checks_summary: 'Staging deployment #71 passed.', + can_create_pr: false, + can_deploy_staging: false, + can_deploy_production: true, + resolved_commit_sha: 'prod-main-sha', + staging_deployment_record_id: 71, + blocked_reason: null, + }) + apiMock.deployReleaseTask.mockResolvedValueOnce( + releaseTask({ + status: 'deploying', + deployment_record_id: 99, + deployment_status: 'requested', + deployment_health_status: 'checking', + deployment_metadata: { dispatchPending: true, dispatchStatus: 'pending' }, + }), + ) + const user = userEvent.setup() + render(, { wrapper: MemoryRouter }) + + await user.click(await screen.findByRole('button', { name: '发布管理' })) + await user.click(await screen.findByRole('button', { name: '发布到 production' })) + + expect(await screen.findByText('prod-main-sha')).toBeTruthy() + expect(screen.getByText('Deployment #71')).toBeTruthy() + const initialTaskPageCalls = apiMock.getReleaseTasksPage.mock.calls.length + await user.type(screen.getByLabelText('输入确认短语'), 'DEPLOY_PRODUCTION') + await user.click(screen.getByRole('button', { name: '确认发布到 production' })) + + await waitFor(() => { + expect(apiMock.deployReleaseTask).toHaveBeenCalledWith(42, { + actor: 'operator', + note: 'Manual staging release', + confirm_phrase: 'DEPLOY_PRODUCTION', + }) + }) + expect(apiMock.getReleaseTasksPage).toHaveBeenCalledTimes(initialTaskPageCalls) + await waitFor(() => { + expect(screen.queryByText('确认发布到 production')).toBeNull() + }) + expect(screen.getByText('Deploy: 待执行')).toBeTruthy() + expect(screen.getByText('Health: 检查中')).toBeTruthy() + }) +}) diff --git a/dashboard/src/pages/ReleasesPage.tsx b/dashboard/src/pages/ReleasesPage.tsx index 98993246..dc38a90c 100644 --- a/dashboard/src/pages/ReleasesPage.tsx +++ b/dashboard/src/pages/ReleasesPage.tsx @@ -432,12 +432,13 @@ export default function ReleasesPage() { if (busyAction) return setBusyAction(busyKey) try { - await api.deployReleaseTaskToStaging(task.id, { + const updated = await api.deployReleaseTaskToStaging(task.id, { actor: 'operator', note: task.note, }) toast.success('Staging 发布已触发') - await load() + setTasks((current) => current.map((item) => (item.id === updated.id ? updated : item))) + setTaskDetail((current) => (current?.id === updated.id ? updated : current)) } catch (error) { const message = error instanceof Error ? error.message : '触发 staging 发布失败' toast.error(message) @@ -467,15 +468,22 @@ export default function ReleasesPage() { const busyKey = `release-task-${productionDeployTask.id}-deploy-production` setBusyAction(busyKey) try { - await deployReleaseTask(productionDeployTask.id, { + const updated = await deployReleaseTask(productionDeployTask.id, { actor: 'operator', note: productionDeployTask.note, confirm_phrase: productionConfirmPhrase, }) toast.success('Production 发布已触发') + setTasks((current) => current.map((item) => (item.id === updated.id ? updated : item))) + setTaskDetail((current) => (current?.id === updated.id ? updated : current)) + setGateStatusByTaskId((current) => { + if (!(updated.id in current)) return current + const next = { ...current } + delete next[updated.id] + return next + }) setProductionDeployTask(null) setProductionConfirmPhrase('') - await load() } catch (error) { const message = error instanceof Error ? error.message : '触发 production 发布失败' toast.error(message) @@ -659,6 +667,7 @@ export default function ReleasesPage() { onDeployProduction={(task) => { setProductionConfirmPhrase('') setProductionDeployTask(task) + refreshTaskGateStatus(task).catch(() => {}) }} onRefreshGate={(task) => refreshTaskGateStatus(task)} onRefresh={() => load().catch(() => {})} @@ -682,6 +691,12 @@ export default function ReleasesPage() { bool: + return "database is locked" in str(exc).lower() + + +def commit_with_sqlite_lock_retry( + db: Session, + apply_changes: Callable[[], T], + *, + attempts: int = 5, +) -> T: + """Apply and commit writes, retrying transient SQLite writer contention.""" + + last_error: OperationalError | None = None + for attempt in range(max(1, attempts)): + try: + result = apply_changes() + db.commit() + return result + except OperationalError as exc: + if not _is_sqlite_locked_error(exc) or attempt >= attempts - 1: + raise + last_error = exc + db.rollback() + time.sleep(0.15 * (attempt + 1)) + if last_error is not None: # pragma: no cover - loop always raises above. + raise last_error + return apply_changes() # pragma: no cover - attempts is normalized to at least one. diff --git a/server/services/releases/production_dispatch.py b/server/services/releases/production_dispatch.py new file mode 100644 index 00000000..79c216c9 --- /dev/null +++ b/server/services/releases/production_dispatch.py @@ -0,0 +1,234 @@ +"""Durable production release task dispatch sweep.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy.orm import Session + +from server.database import DeploymentRecordRow, ReleaseTaskRow, utcnow +from server.services.delivery.deployment_governance import ( + _dispatch_github_workflow, + _json_dump, +) +from server.services.result_objects import ServiceObjectResult +from server.services.releases.db_retry import commit_with_sqlite_lock_retry +from server.services.releases.production_tasks import ( + _assert_production_deploy_secrets, + _production_deployment_metadata, + _production_dispatch_inputs, + _production_staging_evidence, + _production_workflow_file, + release_task_production_gate, +) +from server.services.releases.tasks import ( + _clean, + _json_load, + _merge_metadata, + _policy_for_project, + _release_task_deploy_repo, +) + + +def _fail_pending_dispatch( + db: Session, + *, + record: DeploymentRecordRow, + row: ReleaseTaskRow | None, + error: Exception, +) -> None: + def apply() -> None: + message = str(error)[:1000] + record.status = "failed" + record.health_status = "failed" + record.notes = message + record.finished_at = utcnow() + _merge_metadata( + record, + { + "dispatchPending": False, + "dispatchStatus": "failed", + "dispatchFailureSummary": message, + "githubDispatchFailedAt": utcnow().isoformat(), + }, + ) + if row is not None: + row.status = "failed" + row.note = message + row.updated_at = utcnow() + + commit_with_sqlite_lock_retry(db, apply) + + +def _dispatch_context( + db: Session, record: DeploymentRecordRow, row: ReleaseTaskRow +) -> tuple[Any, str, Any, DeploymentRecordRow, str]: + project_key = _clean(row.project_key) + policy = _policy_for_project(db, project_key) + repo_full_name = _release_task_deploy_repo(policy, project_key) + gate = release_task_production_gate(db, row, repo_full_name) + staging_record = _production_staging_evidence(db, gate) + _assert_production_deploy_secrets(repo_full_name) + actor = _clean(getattr(record, "actor", None)) or _clean(row.operator) or "operator" + return policy, repo_full_name, gate, staging_record, actor + + +def _pending_dispatch_records(db: Session, *, limit: int) -> list[DeploymentRecordRow]: + return ( + db.query(DeploymentRecordRow) + .filter( + DeploymentRecordRow.trigger_source == "release_task", + DeploymentRecordRow.environment == "production", + DeploymentRecordRow.status == "requested", + ) + .order_by(DeploymentRecordRow.created_at.asc(), DeploymentRecordRow.id.asc()) + .limit(limit) + .all() + ) + + +def _is_pending_production_dispatch(record: DeploymentRecordRow) -> bool: + metadata = _json_load(record.metadata_json, {}) + return ( + metadata.get("dispatchPending") is True + and _clean(metadata.get("dispatchStatus")) == "pending" + ) + + +def _release_task_for_record( + db: Session, record: DeploymentRecordRow +) -> ReleaseTaskRow | None: + return ( + db.query(ReleaseTaskRow) + .filter(ReleaseTaskRow.deployment_record_id == int(record.id)) + .one_or_none() + ) + + +def _mark_dispatched( + db: Session, + *, + record: DeploymentRecordRow, + row: ReleaseTaskRow, + repo_full_name: str, + gate: Any, + staging_record: DeploymentRecordRow, + actor: str, + service_key: str, +) -> None: + def apply() -> None: + next_metadata = _production_deployment_metadata( + row=row, + repo_full_name=repo_full_name, + service_key=service_key, + commit_sha=gate.resolved_commit_sha or "", + staging_record=staging_record, + ) + next_metadata.update( + { + "dispatchPending": False, + "dispatchStatus": "dispatched", + "githubWorkflowFile": record.workflow_file, + "githubDispatchAt": utcnow().isoformat(), + } + ) + record.metadata_json = _json_dump(next_metadata) + record.notes = "Production workflow dispatched; waiting for GitHub Actions." + row.status = "deploying" + row.operator = actor + row.target_commit_sha = gate.resolved_commit_sha + row.updated_at = utcnow() + + commit_with_sqlite_lock_retry(db, apply) + + +def _dispatch_one_pending_record( + db: Session, record: DeploymentRecordRow, row: ReleaseTaskRow +) -> None: + policy, repo_full_name, gate, staging_record, actor = _dispatch_context( + db, record, row + ) + service_key = _clean(getattr(row, "service_key", None)).lower() + workflow_file = _production_workflow_file( + policy, project_key=_clean(row.project_key), service_key=service_key + ) + record.workflow_file = workflow_file + record.commit_sha = gate.resolved_commit_sha + record.branch_name = "main" + inputs = _production_dispatch_inputs( + row=row, + record=record, + commit_sha=gate.resolved_commit_sha or "", + service_key=service_key, + actor=actor, + ) + record.external_url = _dispatch_github_workflow( + repo_full_name=repo_full_name, + workflow_file=workflow_file, + ref="main", + inputs=inputs, + ) + _mark_dispatched( + db, + record=record, + row=row, + repo_full_name=repo_full_name, + gate=gate, + staging_record=staging_record, + actor=actor, + service_key=service_key, + ) + + +def dispatch_pending_production_deployments( + db: Session, *, max_records: int = 5 +) -> ServiceObjectResult: + bounded_limit = max(1, min(int(max_records or 5), 10)) + result: dict[str, Any] = { + "processed_count": 0, + "dispatched_ids": [], + "failed_ids": [], + "skipped_ids": [], + } + for record in _pending_dispatch_records(db, limit=bounded_limit * 3): + if not _is_pending_production_dispatch(record): + continue + if result["processed_count"] >= bounded_limit: + break + result["processed_count"] += 1 + row = _release_task_for_record(db, record) + if row is None: + _fail_pending_dispatch( + db, + record=record, + row=None, + error=ValueError("Production dispatch failed: release task not found."), + ) + result["failed_ids"].append(int(record.id)) + continue + if _clean(row.status).lower() in {"completed", "cancelled"}: + result["skipped_ids"].append(int(record.id)) + continue + try: + _dispatch_one_pending_record(db, record, row) + result["dispatched_ids"].append(int(record.id)) + except Exception as exc: # noqa: BLE001 + _fail_pending_dispatch(db, record=record, row=row, error=exc) + result["failed_ids"].append(int(record.id)) + return ServiceObjectResult(result) + + +def append_pending_production_dispatches( + db: Session, result: dict[str, Any], *, max_records: int +) -> None: + try: + production_dispatch = dispatch_pending_production_deployments( + db, max_records=max_records + ) + except Exception as exc: # noqa: BLE001 + result["production_dispatch_error"] = str(exc)[:1000] + return + result["production_dispatch"] = production_dispatch + result["processed_count"] += int(production_dispatch.get("processed_count") or 0) + result["failed_ids"].extend(production_dispatch.get("failed_ids") or []) + result["pending_ids"].extend(production_dispatch.get("dispatched_ids") or []) diff --git a/server/services/releases/production_tasks.py b/server/services/releases/production_tasks.py index 4ff8f3a8..eb2eb4e9 100644 --- a/server/services/releases/production_tasks.py +++ b/server/services/releases/production_tasks.py @@ -12,7 +12,6 @@ from server.schemas import ReleaseTaskOut, ReleaseTaskPrGateOut from server.services.delivery.deployment_governance import ( _callback_url, - _dispatch_github_workflow, _json_dump, ) from server.services.releases.github_cli import run_gh_json as _run_gh_json @@ -217,26 +216,26 @@ def _production_deployment_metadata( } -def _mark_release_task_dispatch_failed( - db: Session, +def _pending_production_deployment_metadata( *, row: ReleaseTaskRow, - record: DeploymentRecordRow, - actor: str, - error: Exception, -) -> ReleaseTaskOut: - message = str(error)[:1000] - record.status = "failed" - record.health_status = "failed" - record.notes = message - record.finished_at = utcnow() - row.status = "failed" - row.operator = actor - row.note = message - row.updated_at = utcnow() - db.commit() - db.refresh(row) - return _release_task_to_out(row, db=db) + repo_full_name: str, + service_key: str, +) -> dict[str, Any]: + return { + "release_task_id": int(row.id), + "service_key": service_key, + "deploy_repo_full_name": repo_full_name, + "deploy_strategy": "github_actions_wif_cloudrun", + "sourceRefType": "branch", + "sourceRef": "main", + "deployScope": service_key, + "gcpProject": PRODUCTION_GCP_PROJECT_ID, + "region": "us-central1", + "environmentIsolationVersion": "v1", + "dispatchPending": True, + "dispatchStatus": "pending", + } def _production_task_row(db: Session, task_id: int, body: Any) -> ReleaseTaskRow: @@ -296,28 +295,12 @@ def _assert_production_deploy_secrets(repo_full_name: str) -> None: ) -def _production_gate_context( - db: Session, task_id: int, body: Any -) -> tuple[ReleaseTaskRow, Any, str, ReleaseTaskPrGateOut, DeploymentRecordRow, str]: - row = _production_task_row(db, task_id, body) - project_key = _clean(row.project_key) - policy = _policy_for_project(db, project_key) - repo_full_name = _release_task_deploy_repo(policy, project_key) - gate = release_task_production_gate(db, row, repo_full_name) - staging_record = _production_staging_evidence(db, gate) - _assert_production_deploy_secrets(repo_full_name) - actor = _clean(getattr(body, "actor", "operator")) or "operator" - return row, policy, repo_full_name, gate, staging_record, actor - - -def _create_production_deployment_record( +def _create_pending_production_deployment_record( db: Session, *, row: ReleaseTaskRow, policy: Any, repo_full_name: str, - gate: ReleaseTaskPrGateOut, - staging_record: DeploymentRecordRow, actor: str, note: str | None, ) -> DeploymentRecordRow: @@ -337,18 +320,15 @@ def _create_production_deployment_record( status="requested", health_status="checking", branch_name="main", - commit_sha=gate.resolved_commit_sha, workflow_file=workflow_file, actor=actor, trigger_source="release_task", - notes=note, + notes=note or "Production dispatch pending; sync will verify gate before dispatch.", metadata_json=_json_dump( - _production_deployment_metadata( + _pending_production_deployment_metadata( row=row, repo_full_name=repo_full_name, service_key=service_key, - commit_sha=gate.resolved_commit_sha, - staging_record=staging_record, ) ), started_at=utcnow(), @@ -357,7 +337,6 @@ def _create_production_deployment_record( db.commit() db.refresh(record) row.deployment_record_id = int(record.id) - row.target_commit_sha = gate.resolved_commit_sha row.status = "deploying" row.operator = actor row.note = note @@ -397,51 +376,29 @@ def _production_dispatch_inputs( def _deploy_release_task_to_production( db: Session, task_id: int, body: Any ) -> ReleaseTaskOut: - row, policy, repo_full_name, gate, staging_record, actor = _production_gate_context( - db, task_id, body - ) + row = _production_task_row(db, task_id, body) + project_key = _clean(row.project_key) + policy = _policy_for_project(db, project_key) + repo_full_name = _release_task_deploy_repo(policy, project_key) + actor = _clean(getattr(body, "actor", "operator")) or "operator" note = _clean(getattr(body, "note", None)) or _clean(row.note) or None - service_key = _clean(getattr(row, "service_key", None)).lower() - record = _create_production_deployment_record( + _create_pending_production_deployment_record( db, row=row, policy=policy, repo_full_name=repo_full_name, - gate=gate, - staging_record=staging_record, actor=actor, note=note, ) - workflow_file = _production_workflow_file( - policy, project_key=_clean(row.project_key), service_key=service_key - ) - inputs = _production_dispatch_inputs( - row=row, - record=record, - commit_sha=gate.resolved_commit_sha or "", - service_key=service_key, - actor=actor, + queued_gate = ReleaseTaskPrGateOut( + branch_exists=True, + pr_state="production", + checks_status="pending", + checks_summary="Production dispatch queued.", + environment_gate_summary="Production dispatch queued; sync will verify gate before dispatch.", + blocked_reason=None, ) - try: - record.external_url = _dispatch_github_workflow( - repo_full_name=repo_full_name, - workflow_file=workflow_file, - ref="main", - inputs=inputs, - ) - db.commit() - db.refresh(row) - return _release_task_to_out(row, db=db, pr_gate=gate) - except Exception as exc: # noqa: BLE001 - db.rollback() - record = ( - db.get(DeploymentRecordRow, int(row.deployment_record_id)) - if row.deployment_record_id - else record - ) - return _mark_release_task_dispatch_failed( - db, row=row, record=record, actor=actor, error=exc - ) + return _release_task_to_out(row, db=db, pr_gate=queued_gate) def deploy_release_task(db: Session, task_id: int, body: Any) -> ReleaseTaskOut: diff --git a/server/services/releases/tasks.py b/server/services/releases/tasks.py index 41c8baa8..16d291fa 100644 --- a/server/services/releases/tasks.py +++ b/server/services/releases/tasks.py @@ -924,6 +924,20 @@ def _apply_github_run_to_deployment( def sync_release_task_deployment_statuses(db: Session, *, max_records: int = 50) -> ServiceObjectResult: bounded_limit = max(1, min(int(max_records or 50), 50)) + result: dict[str, Any] = { + "processed_count": 0, + "succeeded_ids": [], + "failed_ids": [], + "running_ids": [], + "pending_ids": [], + "error_ids": [], + "skipped_terminal_ids": [], + "timeout_ids": [], + "removed_run_ids": [], + } + from server.services.releases.production_dispatch import append_pending_production_dispatches + + append_pending_production_dispatches(db, result, max_records=min(5, bounded_limit)) sync_cutoff = utcnow() - timedelta(days=7) active_status_rank = case( (DeploymentRecordRow.status.in_(["requested", "running", "pending"]), 0), @@ -951,17 +965,6 @@ def sync_release_task_deployment_statuses(db: Session, *, max_records: int = 50) .limit(bounded_limit) .all() ) - result: dict[str, Any] = { - "processed_count": 0, - "succeeded_ids": [], - "failed_ids": [], - "running_ids": [], - "pending_ids": [], - "error_ids": [], - "skipped_terminal_ids": [], - "timeout_ids": [], - "removed_run_ids": [], - } for record in rows: record = db.get(DeploymentRecordRow, int(record.id)) if record is None: diff --git a/tests/test_release_production_tasks_api.py b/tests/test_release_production_tasks_api.py index d5273b1f..a16dda0b 100644 --- a/tests/test_release_production_tasks_api.py +++ b/tests/test_release_production_tasks_api.py @@ -1,17 +1,73 @@ import json +from sqlalchemy.exc import OperationalError + from server import config from server.database import DeploymentRecordRow from server.database import ReleaseTaskRow from server.services.releases import ( + production_dispatch as release_production_dispatch_service, production_tasks as release_production_tasks_service, ) +from server.services.releases.db_retry import commit_with_sqlite_lock_retry from server.services.releases.production_workflows import ( production_workflow_file_for_service, ) -def test_release_task_production_deploy_requires_gate_and_dispatches_workflow( +class _RetryDb: + def __init__(self, failures: list[OperationalError]): + self.failures = failures + self.commits = 0 + self.rollbacks = 0 + + def commit(self): + self.commits += 1 + if self.failures: + raise self.failures.pop(0) + + def rollback(self): + self.rollbacks += 1 + + +def _operational_error(message: str) -> OperationalError: + return OperationalError("UPDATE deployment_records SET external_url=?", {}, Exception(message)) + + +def test_commit_with_sqlite_lock_retry_retries_transient_lock(monkeypatch): + db = _RetryDb([_operational_error("database is locked")]) + calls: list[str] = [] + sleeps: list[float] = [] + + monkeypatch.setattr("server.services.releases.db_retry.time.sleep", sleeps.append) + + result = commit_with_sqlite_lock_retry( + db, # type: ignore[arg-type] + lambda: calls.append("apply") or "ok", + ) + + assert result == "ok" + assert calls == ["apply", "apply"] + assert db.commits == 2 + assert db.rollbacks == 1 + assert sleeps == [0.15] + + +def test_commit_with_sqlite_lock_retry_does_not_retry_unrelated_db_error(): + db = _RetryDb([_operational_error("permission denied")]) + + try: + commit_with_sqlite_lock_retry(db, lambda: "ok") # type: ignore[arg-type] + except OperationalError as exc: + assert "permission denied" in str(exc) + else: # pragma: no cover - fail branch for assertion clarity. + raise AssertionError("expected OperationalError") + + assert db.commits == 1 + assert db.rollbacks == 0 + + +def test_release_task_production_deploy_queues_and_sync_dispatches_workflow( monkeypatch, synced_client, db_session ): calls = [] @@ -36,7 +92,7 @@ def fake_dispatch(**kwargs): return "https://github.com/SynapseNetworkAI/Synapse-Network/actions/workflows/deploy-production-frontends.yml" monkeypatch.setattr( - release_production_tasks_service, "_dispatch_github_workflow", fake_dispatch + release_production_dispatch_service, "_dispatch_github_workflow", fake_dispatch ) staging_record = DeploymentRecordRow( @@ -85,18 +141,44 @@ def fake_dispatch(**kwargs): assert payload["status"] == "deploying" assert payload["deployment_record_id"] is not None assert payload["deployment_workflow_file"] == "deploy-production-frontends.yml" - assert payload["deployment_commit_sha"] == "abc123prod" + assert payload["deployment_commit_sha"] is None + assert calls == [] + + deployment = db_session.get(DeploymentRecordRow, payload["deployment_record_id"]) + metadata = json.loads(deployment.metadata_json) + assert metadata["dispatchPending"] is True + assert metadata["dispatchStatus"] == "pending" + + real_commit = db_session.commit + lock_failures: list[str] = [] + + def flaky_commit_once(): + if not lock_failures: + lock_failures.append("locked") + raise OperationalError( + "UPDATE deployment_records SET external_url=?", + {}, + Exception("database is locked"), + ) + return real_commit() + + monkeypatch.setattr(db_session, "commit", flaky_commit_once) + sync_response = synced_client.post("/api/releases/tasks/sync-deployment-statuses") + assert sync_response.status_code == 200 + assert payload["deployment_record_id"] in sync_response.json()["production_dispatch"]["dispatched_ids"] + assert calls[0]["repo_full_name"] == "SynapseNetworkAI/Synapse-Network" assert calls[0]["workflow_file"] == "deploy-production-frontends.yml" assert calls[0]["inputs"]["deploy_scope"] == "docs-front" assert calls[0]["inputs"]["source_ref"] == "main" assert calls[0]["inputs"]["commit_sha"] == "abc123prod" - deployment = db_session.get(DeploymentRecordRow, payload["deployment_record_id"]) + db_session.refresh(deployment) metadata = json.loads(deployment.metadata_json) assert metadata["gcpProject"] == "synapse-network-prod" assert metadata["sourceStagingDeploymentRecordId"] == int(staging_record.id) assert metadata["environmentIsolationVersion"] == "v1" + assert metadata["dispatchStatus"] == "dispatched" def test_production_workflow_resolver_uses_modular_defaults(): @@ -200,8 +282,12 @@ def test_release_task_production_deploy_blocks_without_matching_staging( json={"confirm_phrase": "DEPLOY_PRODUCTION"}, ) - assert response.status_code == 400 - assert "staging deployment" in response.json()["detail"] + assert response.status_code == 200 + payload = response.json() + assert payload["status"] == "deploying" + sync_response = synced_client.post("/api/releases/tasks/sync-deployment-statuses") + assert sync_response.status_code == 200 + assert payload["deployment_record_id"] in sync_response.json()["production_dispatch"]["failed_ids"] def test_release_task_production_gate_uses_pinned_staging_commit( @@ -495,7 +581,7 @@ def test_release_task_production_dispatch_failure_marks_task_failed( lambda repo: ([], None), ) monkeypatch.setattr( - release_production_tasks_service, + release_production_dispatch_service, "_dispatch_github_workflow", lambda **kwargs: (_ for _ in ()).throw(RuntimeError("dispatch exploded")), ) @@ -529,8 +615,16 @@ def test_release_task_production_dispatch_failure_marks_task_failed( assert response.status_code == 200 payload = response.json() - assert payload["status"] == "failed" - assert "dispatch exploded" in payload["note"] + assert payload["status"] == "deploying" + + sync_response = synced_client.post("/api/releases/tasks/sync-deployment-statuses") + assert sync_response.status_code == 200 + assert payload["deployment_record_id"] in sync_response.json()["production_dispatch"]["failed_ids"] + updated = synced_client.get(f"/api/releases/tasks/{task['id']}/gate-status") + assert updated.status_code == 200 + failed_task = db_session.get(ReleaseTaskRow, task["id"]) + assert failed_task.status == "failed" + assert "dispatch exploded" in failed_task.note def test_failed_production_release_task_can_retry_with_new_deployment_record( @@ -555,7 +649,7 @@ def fake_dispatch(**kwargs): return "https://github.com/SynapseNetworkAI/Synapse-Network/actions/runs/9" monkeypatch.setattr( - release_production_tasks_service, + release_production_dispatch_service, "_dispatch_github_workflow", fake_dispatch, ) @@ -587,9 +681,13 @@ def fake_dispatch(**kwargs): json={"confirm_phrase": "DEPLOY_PRODUCTION"}, ) assert first_response.status_code == 200 - failed_payload = first_response.json() - assert failed_payload["status"] == "failed" - failed_record_id = failed_payload["deployment_record_id"] + first_payload = first_response.json() + assert first_payload["status"] == "deploying" + failed_record_id = first_payload["deployment_record_id"] + sync_response = synced_client.post("/api/releases/tasks/sync-deployment-statuses") + assert sync_response.status_code == 200 + failed_task = db_session.get(ReleaseTaskRow, task["id"]) + assert failed_task.status == "failed" retry_response = synced_client.post( f"/api/releases/tasks/{task['id']}/deploy", @@ -600,5 +698,7 @@ def fake_dispatch(**kwargs): retry_payload = retry_response.json() assert retry_payload["status"] == "deploying" assert retry_payload["deployment_record_id"] != failed_record_id + retry_sync = synced_client.post("/api/releases/tasks/sync-deployment-statuses") + assert retry_sync.status_code == 200 assert len(dispatch_calls) == 2 assert "deploy_scope" not in dispatch_calls[1]["inputs"]