diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e84ec72..8ddd139 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,7 +90,7 @@ TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:5433/hedwig_test npm Without `TEST_DATABASE_URL`, the database suite is skipped. In CI it is required, and the `test` job provides a Postgres service. -Test files run in parallel and would share the same database, so keep database tests in `test/email-checker.test.ts` or run them serially. Import `test/helpers/test-env.ts` first in any database test file. +Every database test file shares the same database and clears it between tests, so `npm test` runs test files one at a time (`--test-concurrency=1`). Import `test/helpers/test-env.ts` first in any database test file. ## Pull Request Checklist diff --git a/app/api/internal/lists/[id]/contacts/route.ts b/app/api/internal/lists/[id]/contacts/route.ts index 1a4625c..f6864fa 100644 --- a/app/api/internal/lists/[id]/contacts/route.ts +++ b/app/api/internal/lists/[id]/contacts/route.ts @@ -5,6 +5,7 @@ import { contacts, lists } from '@/lib/db/schema' import { eq, ilike, and, count, SQL } from 'drizzle-orm' import { createContactSchema } from '@/lib/validations/contacts' import { auditFromSession, logAudit } from '@/lib/audit' +import { isUniqueViolation } from '@/lib/db/errors' import { getQueue, JOBS } from '@/lib/queue' import { logger } from '@/lib/logger' import { enqueueContactEmailVerification } from '@/lib/email-verify' @@ -112,8 +113,7 @@ export async function POST( return NextResponse.json(created, { status: 201 }) } catch (e: unknown) { - const msg = e instanceof Error ? e.message : '' - if (msg.includes('unique') || msg.includes('duplicate')) { + if (isUniqueViolation(e)) { return NextResponse.json({ error: 'A contact with this email already exists in this list' }, { status: 409 }) } throw e diff --git a/app/api/v1/lists/[listId]/contacts/[id]/route.ts b/app/api/v1/lists/[listId]/contacts/[id]/route.ts index 9cc443c..87a00ff 100644 --- a/app/api/v1/lists/[listId]/contacts/[id]/route.ts +++ b/app/api/v1/lists/[listId]/contacts/[id]/route.ts @@ -5,6 +5,7 @@ import { eq, and } from 'drizzle-orm' import { withApiAuth } from '@/lib/api-auth' import { updateContactSchema } from '@/lib/validations/contacts' import { auditFromApiKey, logAudit } from '@/lib/audit' +import { isUniqueViolation } from '@/lib/db/errors' export async function PUT( req: NextRequest, @@ -33,11 +34,24 @@ export async function PUT( if (parsed.data.metadata !== undefined) updateData.metadata = parsed.data.metadata if (parsed.data.status !== undefined) updateData.status = parsed.data.status - const [updated] = await db - .update(contacts) - .set(updateData) - .where(and(eq(contacts.id, params.id), eq(contacts.listId, params.listId))) - .returning() + let updated: typeof contacts.$inferSelect | undefined + try { + const rows = await db + .update(contacts) + .set(updateData) + .where(and(eq(contacts.id, params.id), eq(contacts.listId, params.listId))) + .returning() + updated = rows[0] + } catch (err) { + // Changing the email to one another contact in this list already has. + if (isUniqueViolation(err)) { + return NextResponse.json( + { error: 'A contact with this email already exists in this list', data: null, meta: {} }, + { status: 409 }, + ) + } + throw err + } if (!updated) { return NextResponse.json({ error: 'Contact not found', data: null, meta: {} }, { status: 404 }) diff --git a/app/api/v1/lists/[listId]/contacts/route.ts b/app/api/v1/lists/[listId]/contacts/route.ts index be037bf..1b7711d 100644 --- a/app/api/v1/lists/[listId]/contacts/route.ts +++ b/app/api/v1/lists/[listId]/contacts/route.ts @@ -86,6 +86,8 @@ export async function POST( } const requireDoubleOptIn = list.requireDoubleOptIn + // A contact already in this list is a conflict, not a server error. Integrations + // retry on 5xx, so answering a duplicate with 500 makes them retry forever. const [created] = await db .insert(contacts) .values({ @@ -98,26 +100,54 @@ export async function POST( ? { status: 'pending', confirmationToken: randomUUID() } : {}), }) + .onConflictDoNothing({ target: [contacts.listId, contacts.email] }) .returning() - if (requireDoubleOptIn && created) { - try { + if (!created) { + const [existing] = await db + .select({ id: contacts.id, email: contacts.email, status: contacts.status }) + .from(contacts) + .where(and(eq(contacts.listId, params.listId), eq(contacts.email, parsed.data.email))) + return NextResponse.json( + { + data: existing ?? null, + meta: {}, + error: 'A contact with this email already exists in this list', + }, + { status: 409 }, + ) + } + + // The contact exists from here on, so nothing below may fail the request. A 5xx + // would make the caller retry, and every retry would then be a duplicate. + if (requireDoubleOptIn) { + await runSideEffect('enqueue confirmation job', created.id, async () => { const queue = await getQueue() await queue.send(JOBS.SEND_CONFIRMATION, { contactId: created.id }) - } catch (err) { - logger.error({ err, contactId: created.id }, 'Failed to enqueue confirmation job') - } + }) } - await enqueueContactEmailVerification([created.id]) + await runSideEffect('enqueue email verification', created.id, () => + enqueueContactEmailVerification([created.id]), + ) - await logAudit( - auditFromApiKey(req, auth), - 'contact.create', - { type: 'contact', id: created.id }, - { listId: params.listId, email: created.email, requireDoubleOptIn }, + await runSideEffect('write audit log', created.id, () => + logAudit( + auditFromApiKey(req, auth), + 'contact.create', + { type: 'contact', id: created.id }, + { listId: params.listId, email: created.email, requireDoubleOptIn }, + ), ) return NextResponse.json({ data: created, meta: {}, error: null }, { status: 201 }) }) } + +async function runSideEffect(label: string, contactId: string, fn: () => Promise) { + try { + await fn() + } catch (err) { + logger.error({ err, contactId }, `Failed to ${label} after creating contact`) + } +} diff --git a/lib/api-auth.ts b/lib/api-auth.ts index ab8a1fa..1121783 100644 --- a/lib/api-auth.ts +++ b/lib/api-auth.ts @@ -4,6 +4,7 @@ import { apiKeys } from '@/lib/db/schema' import { eq } from 'drizzle-orm' import bcrypt from 'bcryptjs' import { consumeApiKeyToken, type RateLimitResult } from '@/lib/rate-limit' +import { logger } from '@/lib/logger' export interface ApiAuthContext { keyId: string @@ -44,10 +45,27 @@ function v1ErrorResponse(status: number, error: string, headers?: Record Promise, +): Promise { + try { + return await authenticateAndRun(req, handler) + } catch (err) { + logger.error( + { err, method: req.method, path: req.nextUrl.pathname }, + 'Unhandled error in v1 API route', + ) + return v1ErrorResponse(500, 'Internal server error') + } +} + +async function authenticateAndRun( + req: NextRequest, + handler: (auth: ApiAuthContext) => Promise, ): Promise { const auth = await authenticateApiKey(req) if (!auth) { diff --git a/lib/db/errors.ts b/lib/db/errors.ts new file mode 100644 index 0000000..0e5f8ac --- /dev/null +++ b/lib/db/errors.ts @@ -0,0 +1,16 @@ +/** Postgres SQLSTATE for a unique constraint violation. */ +export const PG_UNIQUE_VIOLATION = '23505' + +/** + * True when err is a Postgres unique violation. drizzle-orm wraps driver errors in a + * DrizzleQueryError whose message is the failed SQL, so the SQLSTATE is on `cause`. + * Walks a few levels of `cause` to find it. + */ +export function isUniqueViolation(err: unknown): boolean { + let current: unknown = err + for (let depth = 0; depth < 5 && current && typeof current === 'object'; depth++) { + if ((current as { code?: unknown }).code === PG_UNIQUE_VIOLATION) return true + current = (current as { cause?: unknown }).cause + } + return false +} diff --git a/package.json b/package.json index 40d7bbf..7004ecc 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "next build", "start": "next start", "lint": "next lint", - "test": "tsx --test test/*.test.ts", + "test": "tsx --test --test-concurrency=1 test/*.test.ts", "db:generate": "drizzle-kit generate:pg", "db:migrate": "tsx --env-file=.env.local lib/db/migrate.ts", "db:migrate:prod": "tsx lib/db/migrate.ts", diff --git a/test/email-checker.test.ts b/test/email-checker.test.ts index 3dd9c37..12aa693 100644 --- a/test/email-checker.test.ts +++ b/test/email-checker.test.ts @@ -7,8 +7,8 @@ * * TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/hedwig_test npm test * - * Node runs test files in parallel, and every file would share this database, so - * keep database tests in this file (or run them serially) to avoid clobbering. + * Every database test file shares this database, so `npm test` runs test files + * one at a time to avoid clobbering. */ // Must be the first import: it points DATABASE_URL at the test database before lib/db loads. import { skipWithoutDatabase } from './helpers/test-env' diff --git a/test/v1-contacts.test.ts b/test/v1-contacts.test.ts new file mode 100644 index 0000000..3998142 --- /dev/null +++ b/test/v1-contacts.test.ts @@ -0,0 +1,220 @@ +/** + * Integration tests for the v1 contact write routes. + * + * These run against a real Postgres and DELETE DATA in it. They only run when + * TEST_DATABASE_URL is set, and never read DATABASE_URL. `npm test` runs test files + * one at a time, since every file shares this database. + * + * TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/hedwig_test npm test + */ +// Must be the first import: it points DATABASE_URL at the test database before lib/db loads. +import { skipWithoutDatabase } from './helpers/test-env' +import { after, before, beforeEach, describe, it } from 'node:test' +import assert from 'node:assert/strict' +import * as orm from 'drizzle-orm' +import bcrypt from 'bcryptjs' +import { NextRequest } from 'next/server' +import * as dbModule from '@/lib/db' +import * as schema from '@/lib/db/schema' +import * as queue from '@/lib/queue' +import { runMigrationsWithLock } from '@/lib/db/run-migrations' +import * as v1Contacts from '@/app/api/v1/lists/[listId]/contacts/route' +import * as v1Contact from '@/app/api/v1/lists/[listId]/contacts/[id]/route' +import * as internalContacts from '@/app/api/internal/lists/[id]/contacts/route' + +// Static imports keep a single instance of lib/db and lib/queue (see email-checker.test.ts). +const m = { db: dbModule, schema, queue, orm, v1Contacts, v1Contact, internalContacts } + +const API_KEY = 'test-api-key-for-v1-contacts' +const DUPLICATE_ERROR = 'A contact with this email already exists in this list' + +async function createList(name: string, requireDoubleOptIn = false) { + const [list] = await m.db.db.insert(m.schema.lists).values({ name, requireDoubleOptIn }).returning() + return list +} + +function jsonRequest(url: string, method: string, body: unknown) { + return new NextRequest(url, { + method, + headers: { authorization: `Bearer ${API_KEY}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) +} + +async function postContact(listId: string, body: unknown) { + const req = jsonRequest(`http://localhost/api/v1/lists/${listId}/contacts`, 'POST', body) + const res = await m.v1Contacts.POST(req, { params: { listId } }) + return { status: res.status, body: await res.json() } +} + +async function contactsIn(listId: string) { + const { contacts } = m.schema + return m.db.db.select().from(contacts).where(m.orm.eq(contacts.listId, listId)) +} + +async function jobCount(name: string) { + const result = await m.db.db.execute( + m.orm.sql`select count(*)::int as n from pgboss.job where name = ${name}`, + ) + return (result.rows[0] as { n: number }).n +} + +async function auditCount(action: string) { + const { auditLogs } = m.schema + const rows = await m.db.db.select().from(auditLogs).where(m.orm.eq(auditLogs.action, action)) + return rows.length +} + +describe('v1 contacts API', { skip: skipWithoutDatabase }, () => { + before(async () => { + await runMigrationsWithLock() + // Start pg-boss once so the job table and queues exist before the first count. + await m.queue.getQueue() + }) + + beforeEach(async () => { + const { sql } = m.orm + await m.db.db.execute(sql`delete from contacts; delete from lists; delete from audit_logs; delete from api_keys`) + await m.db.db.execute(sql`delete from pgboss.job`) + await m.db.db.insert(m.schema.apiKeys).values({ + name: 'test', + keyHash: bcrypt.hashSync(API_KEY, 4), + rateLimitPerMinute: 10_000, + rateLimitTokens: 10_000, + }) + }) + + after(async () => { + await (await m.queue.getQueue()).stop({ graceful: false }).catch(() => {}) + await m.db.pool.end() + }) + + describe('POST /api/v1/lists/:listId/contacts', () => { + it('creates a new contact with 201 and queues verification and an audit entry', async () => { + const list = await createList('Signups') + + const res = await postContact(list.id, { email: 'new@x.com', firstName: 'Ada' }) + + assert.equal(res.status, 201) + assert.equal(res.body.error, null) + assert.deepEqual(res.body.meta, {}) + assert.equal(res.body.data.email, 'new@x.com') + assert.equal(res.body.data.firstName, 'Ada') + assert.equal(res.body.data.status, 'active') + + const rows = await contactsIn(list.id) + assert.deepEqual(rows.map((r) => r.id), [res.body.data.id]) + assert.equal(await jobCount(m.queue.JOBS.VERIFY_CONTACT_EMAIL), 1) + assert.equal(await auditCount('contact.create'), 1) + }) + + it('answers a duplicate with a 409 envelope carrying the existing contact, without re-queueing jobs', async () => { + const list = await createList('Double opt-in', true) + const first = await postContact(list.id, { email: 'dup@x.com' }) + assert.equal(first.status, 201) + assert.equal(first.body.data.status, 'pending') + assert.equal(await jobCount(m.queue.JOBS.SEND_CONFIRMATION), 1) + assert.equal(await jobCount(m.queue.JOBS.VERIFY_CONTACT_EMAIL), 1) + + // The integration retries until it sees a 2xx or 409, so repeats must stay 409. + for (let i = 0; i < 2; i++) { + const again = await postContact(list.id, { email: 'dup@x.com', firstName: 'Changed' }) + assert.equal(again.status, 409) + assert.deepEqual(again.body, { + data: { id: first.body.data.id, email: 'dup@x.com', status: 'pending' }, + meta: {}, + error: DUPLICATE_ERROR, + }) + } + + const rows = await contactsIn(list.id) + assert.equal(rows.length, 1) + assert.equal(rows[0].firstName, null) + assert.equal(await jobCount(m.queue.JOBS.SEND_CONFIRMATION), 1) + assert.equal(await jobCount(m.queue.JOBS.VERIFY_CONTACT_EMAIL), 1) + assert.equal(await auditCount('contact.create'), 1) + }) + + it('still answers 201 when queueing jobs and writing the audit log fail', async (t) => { + const list = await createList('Double opt-in', true) + const boss = await m.queue.getQueue() + t.mock.method(boss, 'send', async () => { + throw new Error('queue down') + }) + t.mock.method(boss, 'insert', async () => { + throw new Error('queue down') + }) + const { sql } = m.orm + await m.db.db.execute(sql`alter table audit_logs rename to audit_logs_unavailable`) + try { + const res = await postContact(list.id, { email: 'flaky@x.com' }) + assert.equal(res.status, 201) + assert.equal(res.body.error, null) + assert.equal(res.body.data.email, 'flaky@x.com') + } finally { + await m.db.db.execute(sql`alter table audit_logs_unavailable rename to audit_logs`) + } + assert.equal((await contactsIn(list.id)).length, 1) + }) + + it('answers an unexpected error with a JSON 500 envelope that leaks no details', async () => { + // Not a uuid, so Postgres rejects the list lookup. + const res = await postContact('not-a-uuid', { email: 'x@x.com' }) + assert.equal(res.status, 500) + assert.deepEqual(res.body, { data: null, meta: {}, error: 'Internal server error' }) + }) + + it('keeps 400 and 404 responses unchanged', async () => { + const list = await createList('Signups') + const invalid = await postContact(list.id, { email: 'not an email' }) + assert.equal(invalid.status, 400) + assert.equal(invalid.body.error, 'Validation failed') + + const missing = await postContact('00000000-0000-0000-0000-000000000000', { email: 'x@x.com' }) + assert.equal(missing.status, 404) + assert.deepEqual(missing.body, { data: null, meta: {}, error: 'List not found' }) + }) + }) + + describe('PUT /api/v1/lists/:listId/contacts/:id', () => { + it('answers 409 when the new email belongs to another contact in the list', async () => { + const list = await createList('Signups') + await m.db.db.insert(m.schema.contacts).values({ listId: list.id, email: 'taken@x.com' }) + const [mover] = await m.db.db + .insert(m.schema.contacts) + .values({ listId: list.id, email: 'mover@x.com' }) + .returning() + + const req = jsonRequest( + `http://localhost/api/v1/lists/${list.id}/contacts/${mover.id}`, + 'PUT', + { email: 'taken@x.com' }, + ) + const res = await m.v1Contact.PUT(req, { params: { listId: list.id, id: mover.id } }) + + assert.equal(res.status, 409) + assert.deepEqual(await res.json(), { data: null, meta: {}, error: DUPLICATE_ERROR }) + const rows = await contactsIn(list.id) + assert.deepEqual(rows.map((r) => r.email).sort(), ['mover@x.com', 'taken@x.com']) + }) + }) + + describe('POST /api/internal/lists/:id/contacts', () => { + // Regression: drizzle-orm wraps driver errors and puts the SQL in the message, so + // matching 'unique' in the message never fired and duplicates became a 500. + it('answers a duplicate with 409', async () => { + const list = await createList('Signups') + await m.db.db.insert(m.schema.contacts).values({ listId: list.id, email: 'dup@x.com' }) + + const req = new NextRequest(`http://localhost/api/internal/lists/${list.id}/contacts`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ email: 'dup@x.com' }), + }) + const res = await m.internalContacts.POST(req, { params: { id: list.id } }) + + assert.equal(res.status, 409) + assert.deepEqual(await res.json(), { error: DUPLICATE_ERROR }) + }) + }) +})