From 483e8da3cef32bea60edfcf23a472a04227cc338 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Tue, 22 Sep 2026 12:27:08 -0700 Subject: [PATCH 1/2] fix: never restore over or destroy an operator-pinned aurora cname An aurora-restore deploy only skipped restore while its status was BUILT or READY. Any other status (a superseded run left QUEUED, a failed rollout, a teardown) fell through to a tag lookup and full restore that replaced a hand-set cname with a new cluster endpoint. A cname that is not an RDS endpoint can only have been set by an operator, since Lifecycle stores only endpoints returned by AWS. Such pinned cnames now skip restore regardless of status, and both targeted teardown and full build deletion skip the CLI destroy for them. --- src/server/lib/__tests__/cli.test.ts | 33 +++++++++++++ src/server/lib/cli.ts | 15 ++++++ src/server/services/__tests__/deploy.test.ts | 47 ++++++++++++++++++- .../services/__tests__/deployCleanup.test.ts | 27 +++++++++++ src/server/services/deploy.ts | 12 ++++- src/server/services/deployCleanup.ts | 7 ++- 6 files changed, 138 insertions(+), 3 deletions(-) diff --git a/src/server/lib/__tests__/cli.test.ts b/src/server/lib/__tests__/cli.test.ts index a9107d4b..03c68228 100644 --- a/src/server/lib/__tests__/cli.test.ts +++ b/src/server/lib/__tests__/cli.test.ts @@ -92,6 +92,7 @@ import { deleteBuild, deleteDeploy, deployBuild, + isPinnedCname, waitForCodefresh, } from '../cli'; @@ -477,6 +478,18 @@ describe('generic CLI deploy lifecycle', () => { expect(mockLoggerInfo).toHaveBeenCalledWith('CLI: deleted'); }); + test('deleteBuild leaves a pinned aurora database in place', async () => { + const pinned = createCliDeploy(DeployTypes.AURORA_RESTORE, { cname: 'database-rw.example.test' }); + const withGraphFetched = jest.fn().mockResolvedValue([pinned]); + const where = jest.fn(() => ({ withGraphFetched })); + mockDeployQuery.mockReturnValue({ where }); + + await deleteBuild({ id: 42, uuid: 'build-uuid' } as any); + + expect(mockShellPromise).not.toHaveBeenCalled(); + expect(mockLoggerInfo).not.toHaveBeenCalledWith('CLI: deleting'); + }); + test('codefresh deploy and destroy omit optional triggers and destroy preserves undefined CLI output', async () => { const deploy = createDeploy({ env: { ENABLED: true, EMPTY: null }, @@ -499,3 +512,23 @@ describe('generic CLI deploy lifecycle', () => { expect(mockUpdateLogContext).toHaveBeenCalledWith({ buildUuid: 'build-uuid' }); }); }); + +describe('isPinnedCname', () => { + test.each([ + [null, false], + [undefined, false], + ['', false], + [' ', false], + ['app-db-env.cluster-abc123.us-west-2.rds.amazonaws.com', false], + ['app-db-env.abc123.us-west-2.rds.amazonaws.com', false], + ['APP-DB-ENV.CLUSTER-ABC123.US-WEST-2.RDS.AMAZONAWS.COM.', false], + [' app-db-env.cluster-abc123.us-west-2.rds.amazonaws.com ', false], + ['app-db-env.cluster-abc123.cn-north-1.rds.amazonaws.com.cn', false], + ['app-db-rw.example.com', true], + ['app-db-rw.example.com.', true], + ['rds.amazonaws.com.example.com', true], + ['app-db.cluster-abc123.us-west-2.rds.amazonaws.com.example.com', true], + ])('%p → %p', (cname, expected) => { + expect(isPinnedCname(cname)).toBe(expected); + }); +}); diff --git a/src/server/lib/cli.ts b/src/server/lib/cli.ts index aa785c05..8a77ec06 100644 --- a/src/server/lib/cli.ts +++ b/src/server/lib/cli.ts @@ -205,6 +205,20 @@ export async function waitForCodefresh(id: string) { } } +const RDS_HOST = /\.rds\.amazonaws\.com(\.cn)?$/; + +/** + * Whether an aurora-restore deploy's cname was pinned by an operator rather than written by Lifecycle. + * + * Lifecycle only ever stores RDS endpoints returned by AWS, so any other host (typically a DNS record + * fronting an externally managed database) was set by hand. Pinned cnames must never be restored over + * or destroyed. + */ +export function isPinnedCname(cname: string | null | undefined): boolean { + const host = cname?.trim().toLowerCase().replace(/\.$/, ''); + return !!host && !RDS_HOST.test(host); +} + /** * Deletes CLI based services for this build * @param build the build to delete CLI services from @@ -223,6 +237,7 @@ export async function deleteBuild(build: Build) { deploys ?.filter((d) => { const serviceType: DeployTypes = d.deployable.type; + if (serviceType === DeployTypes.AURORA_RESTORE && isPinnedCname(d.cname)) return false; return CLIDeployTypes.has(serviceType) && d.active; }) .map(async (deploy) => { diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index 95fa386e..d7ecd4d0 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -129,6 +129,7 @@ jest.mock('server/lib/cli', () => ({ cliDeploy: (...args: any[]) => mockCliDeploy(...args), codefreshDeploy: (...args: any[]) => mockCodefreshDeploy(...args), waitForCodefresh: (...args: any[]) => mockWaitForCodefresh(...args), + isPinnedCname: jest.requireActual('server/lib/cli').isPinnedCname, })); describe('DeployService - shouldTriggerGithubDeployment', () => { @@ -2051,7 +2052,7 @@ describe('DeployService uncovered public behavior', () => { id: 1, uuid: 'database-env', status: DeployStatus.BUILT, - cname: 'database.example.test', + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', build: { uuid: 'env' }, deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, reload: jest.fn().mockResolvedValue(undefined), @@ -2064,6 +2065,50 @@ describe('DeployService uncovered public behavior', () => { expect(mockCliDeploy).not.toHaveBeenCalled(); }); + test.each([DeployStatus.QUEUED, DeployStatus.DEPLOY_FAILED, DeployStatus.ERROR, DeployStatus.TORN_DOWN])( + 'deployAurora keeps a pinned cname from %s without consulting AWS or running restore', + async (status) => { + const { service, deployPatch } = serviceHarness(); + const deploy: any = { + id: 1, + uuid: 'database-env', + status, + cname: 'database-rw.example.test', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockTaggingGetResources).not.toHaveBeenCalled(); + expect(mockCliDeploy).not.toHaveBeenCalled(); + expect(deployPatch).toHaveBeenCalledTimes(1); + expect(deployPatch).toHaveBeenCalledWith({ status: DeployStatus.BUILT }); + } + ); + + test('deployAurora leaves a ready pinned cname untouched', async () => { + const { service, deployPatch } = serviceHarness(); + const deploy: any = { + id: 1, + uuid: 'database-env', + status: DeployStatus.READY, + cname: 'database-rw.example.test', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockTaggingGetResources).not.toHaveBeenCalled(); + expect(mockCliDeploy).not.toHaveBeenCalled(); + expect(deployPatch).not.toHaveBeenCalled(); + }); + test('deployAurora adopts an existing cluster endpoint without running restore', async () => { const { service, deployPatch } = serviceHarness(); mockTaggingGetResources.mockResolvedValue({ diff --git a/src/server/services/__tests__/deployCleanup.test.ts b/src/server/services/__tests__/deployCleanup.test.ts index 1f10dbe8..fd858f37 100644 --- a/src/server/services/__tests__/deployCleanup.test.ts +++ b/src/server/services/__tests__/deployCleanup.test.ts @@ -42,6 +42,7 @@ jest.mock('server/lib/shell', () => ({ jest.mock('server/lib/cli', () => ({ codefreshDestroy: (...args: any[]) => mockCodefreshDestroy(...args), deleteDeploy: (...args: any[]) => mockDeleteDeploy(...args), + isPinnedCname: jest.requireActual('server/lib/cli').isPinnedCname, })); jest.mock('server/lib/metrics', () => ({ @@ -442,6 +443,32 @@ describe('DeployCleanupService', () => { ); }); + test('infra cleanup keeps a pinned aurora database and its cname', async () => { + const deploy = createDeploy({ + cname: 'database-rw.example.test', + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE, serviceDisksYaml: null }, + }); + const service = createService(); + + await expect(service.cleanupDeploy(deploy, { mode: 'infra' })).resolves.toBe(true); + + expect(mockDeleteDeploy).not.toHaveBeenCalled(); + expect(deploy.patch).toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.TORN_DOWN })); + expect(deploy.patch).not.toHaveBeenCalledWith(expect.objectContaining({ cname: expect.anything() })); + }); + + test('infra cleanup destroys a Lifecycle-restored aurora database', async () => { + const deploy = createDeploy({ + cname: 'database-build-1.cluster-abc.us-west-2.rds.amazonaws.com', + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE, serviceDisksYaml: null }, + }); + const service = createService(); + + await expect(service.cleanupDeploy(deploy, { mode: 'infra' })).resolves.toBe(true); + + expect(mockDeleteDeploy).toHaveBeenCalledWith(deploy); + }); + test('enqueueCleanup queues infra cleanup jobs', async () => { const queueAdd = jest.fn().mockResolvedValue(undefined); const service = createService({}, queueAdd); diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index 3deccd71..cfce55d9 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -262,7 +262,17 @@ export default class DeployService extends BaseService { throw new Error('Aurora restore deployable is missing.'); } - if ((deploy.status === DeployStatus.BUILT || deploy.status === DeployStatus.READY) && deploy.cname) { + const isBuilt = deploy.status === DeployStatus.BUILT || deploy.status === DeployStatus.READY; + + if (cli.isPinnedCname(deploy.cname)) { + getLogger().info('Aurora: skipped reason=pinned'); + if (!isBuilt) { + await this.patchDeployForRun(deploy, runUUID, { status: DeployStatus.BUILT }); + } + return true; + } + + if (isBuilt && deploy.cname) { getLogger().info('Aurora: skipped reason=alreadyBuilt'); return true; } diff --git a/src/server/services/deployCleanup.ts b/src/server/services/deployCleanup.ts index 9bef4760..0204a145 100644 --- a/src/server/services/deployCleanup.ts +++ b/src/server/services/deployCleanup.ts @@ -19,7 +19,7 @@ import { Deploy } from 'server/models'; import { shellPromise } from 'server/lib/shell'; import { extractContextForQueue, getLogger, withLogContext } from 'server/lib/logger'; import { CLIDeployTypes, DeployStatus, DeployTypes } from 'shared/constants'; -import { codefreshDestroy, deleteDeploy } from 'server/lib/cli'; +import { codefreshDestroy, deleteDeploy, isPinnedCname } from 'server/lib/cli'; import Metrics from 'server/lib/metrics'; import BaseService from './_service'; import { parseSecretRefsFromEnv } from 'server/lib/secretRefs'; @@ -389,6 +389,11 @@ export default class DeployCleanupService extends BaseService { return null; } + if (deployType === DeployTypes.AURORA_RESTORE && isPinnedCname(deploy.cname)) { + getLogger().info('Deploy cleanup: cli-destroy skipped reason=pinned'); + return null; + } + return { name: 'cli-destroy', resourceType: deployType, From 8c9737fb0bf643d39ce84ccfed7cb00de21d6ff0 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Tue, 22 Sep 2026 12:46:31 -0700 Subject: [PATCH 2/2] test: cover pinned aurora cname from built and building statuses --- src/server/services/__tests__/deploy.test.ts | 54 +++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index d7ecd4d0..d8d210ea 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -2065,8 +2065,35 @@ describe('DeployService uncovered public behavior', () => { expect(mockCliDeploy).not.toHaveBeenCalled(); }); - test.each([DeployStatus.QUEUED, DeployStatus.DEPLOY_FAILED, DeployStatus.ERROR, DeployStatus.TORN_DOWN])( - 'deployAurora keeps a pinned cname from %s without consulting AWS or running restore', + test.each([ + DeployStatus.QUEUED, + DeployStatus.BUILDING, + DeployStatus.DEPLOY_FAILED, + DeployStatus.ERROR, + DeployStatus.TORN_DOWN, + ])('deployAurora keeps a pinned cname from %s without consulting AWS or running restore', async (status) => { + const { service, deployPatch } = serviceHarness(); + const deploy: any = { + id: 1, + uuid: 'database-env', + status, + cname: 'database-rw.example.test', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockTaggingGetResources).not.toHaveBeenCalled(); + expect(mockCliDeploy).not.toHaveBeenCalled(); + expect(deployPatch).toHaveBeenCalledTimes(1); + expect(deployPatch).toHaveBeenCalledWith({ status: DeployStatus.BUILT }); + }); + + test.each([DeployStatus.BUILT, DeployStatus.READY])( + 'deployAurora leaves a %s pinned cname untouched', async (status) => { const { service, deployPatch } = serviceHarness(); const deploy: any = { @@ -2084,31 +2111,10 @@ describe('DeployService uncovered public behavior', () => { expect(mockTaggingGetResources).not.toHaveBeenCalled(); expect(mockCliDeploy).not.toHaveBeenCalled(); - expect(deployPatch).toHaveBeenCalledTimes(1); - expect(deployPatch).toHaveBeenCalledWith({ status: DeployStatus.BUILT }); + expect(deployPatch).not.toHaveBeenCalled(); } ); - test('deployAurora leaves a ready pinned cname untouched', async () => { - const { service, deployPatch } = serviceHarness(); - const deploy: any = { - id: 1, - uuid: 'database-env', - status: DeployStatus.READY, - cname: 'database-rw.example.test', - build: { uuid: 'env' }, - deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - - await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); - - expect(mockTaggingGetResources).not.toHaveBeenCalled(); - expect(mockCliDeploy).not.toHaveBeenCalled(); - expect(deployPatch).not.toHaveBeenCalled(); - }); - test('deployAurora adopts an existing cluster endpoint without running restore', async () => { const { service, deployPatch } = serviceHarness(); mockTaggingGetResources.mockResolvedValue({