From d34f40a36de5a185e7c4b6674d3150542e3323d4 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:15 -0700 Subject: [PATCH 01/32] test(kv): a getter of a stored value is read once, and its first answer is stored, as IndexedDB's put reads it --- tests/kv/valueCodec.spec.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/kv/valueCodec.spec.ts b/tests/kv/valueCodec.spec.ts index e3dde62..d216731 100644 --- a/tests/kv/valueCodec.spec.ts +++ b/tests/kv/valueCodec.spec.ts @@ -169,4 +169,32 @@ describe('the SQL key-value store value text', () => { expect(text.includes('\u0000')).toBe(false); expect(roundTrip({ nul: 'a\u0000b' })).toEqual({ nul: 'a\u0000b' }); }); + + // IndexedDB's put reads each property of a value once, through the + // structured clone. The SQL store wrote its text from a second read of the + // value, after the clone that refuses a proxy had read it once. + it('reads a getter once and keeps the answer the structured clone took', () => { + let reads = 0; + const value = { + get count(): number { + reads += 1; + return reads; + }, + }; + + expect(roundTrip(value)).toStrictEqual({ count: 1 }); + expect(reads).toBe(1); + }); + + it('stores a value whose getter answers a function on a second read, as the structured clone took it', () => { + let reads = 0; + const value = { + get field(): unknown { + reads += 1; + return reads === 1 ? 'first' : () => 'second'; + }, + }; + + expect(roundTrip(value)).toStrictEqual({ field: 'first' }); + }); }); From 4f4ebf1253be32dd1b4ab1b1ef681895db9a5d6b Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:15 -0700 Subject: [PATCH 02/32] fix(kv): write a value's text from the copy structuredClone returns, so each property is read once and the value copied once --- src/features/kv/valueCodec.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/features/kv/valueCodec.ts b/src/features/kv/valueCodec.ts index 9426851..30151d5 100644 --- a/src/features/kv/valueCodec.ts +++ b/src/features/kv/valueCodec.ts @@ -160,12 +160,12 @@ export function encodeValue(value: unknown): string { // The runtime's own structured clone runs first, for what no inspection can // tell apart from what it copies: a proxy answers every check below as the // object it wraps would (an array proxy is an array to Array.isArray), and the - // structured clone, as IndexedDB's put, refuses it. The text is still written - // from the value itself, so what comes back does not change. + // structured clone, as IndexedDB's put, refuses it. The text is written from + // the copy the clone returns, which read each property of the value once, as + // IndexedDB's put reads it: a getter that answers differently on a second + // read is stored with its first answer, and the value is copied once. const runtimeClone = (globalThis as unknown as { structuredClone?: (input: unknown) => unknown }).structuredClone; - if (typeof runtimeClone === 'function') { - runtimeClone.call(globalThis, value); - } + const source = typeof runtimeClone === 'function' ? runtimeClone.call(globalThis, value) : value; /** Each object met so far, by the order it was met in, which the decoder repeats. */ const ids = new Map(); @@ -287,7 +287,7 @@ export function encodeValue(value: unknown): string { return node; }; - return JSON.stringify([FORMAT, encode(value)]); + return JSON.stringify([FORMAT, encode(source)]); } /** Whether `object` is an error: one made by an error constructor, as the structured clone tells them. */ From 0c3b5449c03376dcb0df7329ef67887b2da5d613 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:15 -0700 Subject: [PATCH 03/32] test(indexeddb): PRAGMA assignments through get() and all() with a comment between their words, or a quoted name with no space before it --- tests/indexedDbAdapter.durability.spec.ts | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/indexedDbAdapter.durability.spec.ts b/tests/indexedDbAdapter.durability.spec.ts index ab4eadc..a30d82d 100644 --- a/tests/indexedDbAdapter.durability.spec.ts +++ b/tests/indexedDbAdapter.durability.spec.ts @@ -149,6 +149,23 @@ describe('IndexedDbAdapter durability', () => { expect(await storedPragma('commit-pragma-header', 'user_version')).toBe(4); }); + // SQLite reads a comment as whitespace between two tokens, and a quoted + // name needs no space before it (sqlite.org/lang_comment.html). + it.each([ + ['a block comment before the name', 'get', 'PRAGMA /* note */ user_version = 5', 'user_version', 5], + ['a block comment before the =', 'get', 'PRAGMA user_version /* note */ = 6', 'user_version', 6], + ['a line comment before the =', 'all', 'PRAGMA user_version -- note\n= 7', 'user_version', 7], + ['comments around the schema dot', 'all', 'PRAGMA main/* a */./* b */application_id = 8', 'application_id', 8], + ['a quoted name with no space before it', 'get', 'PRAGMA"user_version"=9', 'user_version', 9], + ] as const)('stores a PRAGMA assignment with %s before it resolves', async (_name, method, sql, pragma, value) => { + const dbName = `commit-pragma-comment-${value}`; + const adapter = await openWithItems(dbName); + + await (method === 'get' ? adapter.get(sql) : adapter.all(sql)); + + expect(await storedPragma(dbName, pragma)).toBe(value); + }); + it('writes nothing to IndexedDB for PRAGMA reads and connection settings through get() and all()', async () => { const adapter = await openWithItems('commit-pragma-reads'); const puts = vi.spyOn(IDBObjectStore.prototype, 'put'); @@ -161,6 +178,7 @@ describe('IndexedDbAdapter durability', () => { expect(await adapter.get('PRAGMA user_version')).toEqual({ user_version: 0 }); await adapter.get('PRAGMA foreign_keys = ON'); await adapter.get('PRAGMA cache_size = 500'); + expect(await adapter.get('PRAGMA /* = */ user_version -- = 3')).toEqual({ user_version: 0 }); expect(puts).not.toHaveBeenCalled(); } finally { @@ -325,6 +343,16 @@ describe('IndexedDbAdapter durability', () => { expect(await reopened.get('PRAGMA user_version')).toEqual({ user_version: 7 }); }); + it('keeps PRAGMA user_version set through get() with a comment before its = across close() and open()', async () => { + const adapter = await openWithItems('interval-pragma-comment', 'interval'); + await adapter.flush(); + await adapter.get('PRAGMA user_version /* note */ = 7'); + await adapter.close(); + + const reopened = await openAdapter('interval-pragma-comment', 'interval'); + expect(await reopened.get('PRAGMA user_version')).toEqual({ user_version: 7 }); + }); + describe('flush()', () => { it('stores the writes made so far, with no tick', async () => { const adapter = await openWithItems('flush-writes', 'interval'); From 750a6073b87a5dc397419fa6e626047b4e098a1c Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:15 -0700 Subject: [PATCH 04/32] fix(indexeddb): read a PRAGMA statement's head as SQLite's tokenizer does, comments as whitespace and a quoted name needing no space --- src/adapters/indexedDbAdapter.ts | 77 +++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/src/adapters/indexedDbAdapter.ts b/src/adapters/indexedDbAdapter.ts index ad61ccd..119761b 100644 --- a/src/adapters/indexedDbAdapter.ts +++ b/src/adapters/indexedDbAdapter.ts @@ -296,24 +296,81 @@ const IMAGE_PRAGMAS_WHEN_SET = new Set(['user_version', 'application_id', 'schem /** PRAGMAs that change the image in any form: incremental_vacuum frees pages, optimize may run ANALYZE. */ const IMAGE_PRAGMAS_ALWAYS = new Set(['incremental_vacuum', 'optimize']); -/** An identifier as SQLite reads one: bare, or in double quotes, brackets or backticks. */ -const SQL_IDENTIFIER = '(?:"(?:[^"]|"")*"|\\[[^\\]]*\\]|`(?:[^`]|``)*`|[A-Za-z_][A-Za-z0-9_$]*)'; +/** A token of a statement's head: a name, bare or quoted, or one character of punctuation. */ +interface HeadToken { + text: string; + name: boolean; +} + +/** The characters a bare name of SQLite's goes on with. */ +const NAME_PART = /[A-Za-z0-9_$]/; -/** `PRAGMA [schema.]name`, then `=` or `(` when the statement gives the pragma a value. */ -const PRAGMA_STATEMENT = new RegExp(`^PRAGMA\\s+(?:${SQL_IDENTIFIER}\\s*\\.\\s*)?(${SQL_IDENTIFIER})\\s*(=|\\()?`, 'i'); +/** + * The first `count` tokens of a statement as SQLite's tokenizer reads them: a + * comment (`-- ...` to the end of its line, `/* ... *\/`) or whitespace + * separates two tokens and is no token itself, and a quoted name (in double + * quotes, brackets or backticks) needs no space before it. + */ +const headTokens = (statement: string, count: number): HeadToken[] => { + const tokens: HeadToken[] = []; + let at = 0; + while (at < statement.length && tokens.length < count) { + const char = statement[at]; + if (/\s/.test(char)) { + at += 1; + } else if (statement.startsWith('--', at)) { + const end = statement.indexOf('\n', at + 2); + at = end < 0 ? statement.length : end + 1; + } else if (statement.startsWith('/*', at)) { + const end = statement.indexOf('*/', at + 2); + at = end < 0 ? statement.length : end + 2; + } else if (char === '"' || char === '`') { + // A quote character inside the name is written twice. + let text = ''; + let end = at + 1; + while (end < statement.length) { + if (statement[end] === char) { + if (statement[end + 1] !== char) break; + end += 1; + } + text += statement[end]; + end += 1; + } + tokens.push({ text, name: true }); + at = end + 1; + } else if (char === '[') { + const end = statement.indexOf(']', at + 1); + tokens.push({ text: statement.slice(at + 1, end < 0 ? statement.length : end), name: true }); + at = end < 0 ? statement.length : end + 1; + } else if (/[A-Za-z_]/.test(char)) { + let end = at + 1; + while (end < statement.length && NAME_PART.test(statement[end])) end += 1; + tokens.push({ text: statement.slice(at, end), name: true }); + at = end; + } else { + tokens.push({ text: char, name: false }); + at += 1; + } + } + return tokens; +}; /** * Whether a PRAGMA statement changes the database image although no row * changes: an assignment to a header field or a file setting, or a pragma that * rewrites pages. Reads (`PRAGMA table_info(t)`, `PRAGMA user_version`) and - * connection settings (`PRAGMA foreign_keys = ON`) do not. + * connection settings (`PRAGMA foreign_keys = ON`) do not. The statement is + * read as SQLite reads it: `PRAGMA [schema.]name`, then `=` or `(` when it + * gives the pragma a value, with any comment between two of them. */ const pragmaChangesImage = (statement: string): boolean => { - const match = PRAGMA_STATEMENT.exec(pastLeadingComments(statement)); - if (!match) return false; - const quoted = /^["[`]/.test(match[1]); - const name = (quoted ? match[1].slice(1, -1) : match[1]).toLowerCase(); - return IMAGE_PRAGMAS_ALWAYS.has(name) || (match[2] !== undefined && IMAGE_PRAGMAS_WHEN_SET.has(name)); + const tokens = headTokens(statement, 5); + if (tokens[0]?.text.toUpperCase() !== 'PRAGMA' || tokens[1]?.name !== true) return false; + const qualified = tokens[2]?.text === '.' && tokens[2].name === false && tokens[3]?.name === true; + const name = (qualified ? tokens[3] : tokens[1]).text.toLowerCase(); + const next = tokens[qualified ? 4 : 2]; + const setsValue = next !== undefined && next.name === false && (next.text === '=' || next.text === '('); + return IMAGE_PRAGMAS_ALWAYS.has(name) || (setsValue && IMAGE_PRAGMAS_WHEN_SET.has(name)); }; /** From a54909166260de362bc0fb0da763257a5c3e7aa4 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:22 -0700 Subject: [PATCH 05/32] test(resolver): a postgres candidate tried after an IndexedDB candidate that named a segment, on a pool of the caller and on a server --- tests/postgres.segment.spec.ts | 29 ++++++++++++++++++++++++++++- tests/segment.spec.ts | 23 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/postgres.segment.spec.ts b/tests/postgres.segment.spec.ts index f2b1536..35c9cb7 100644 --- a/tests/postgres.segment.spec.ts +++ b/tests/postgres.segment.spec.ts @@ -1,6 +1,13 @@ import { describe, expect, it, afterAll, afterEach, beforeAll, vi } from 'vitest'; import pg from 'pg'; -import { PostgresAdapter, createDatabase, createPostgresAdapter, exportData, type StorageAdapter } from '../src/index.js'; +import { + PostgresAdapter, + createDatabase, + createPostgresAdapter, + exportData, + resolveStorageAdapter, + type StorageAdapter, +} from '../src/index.js'; // A segment on PostgreSQL is a schema of its own: every connection of the // adapter's pool starts with search_path set to the segment, then public, so the @@ -437,6 +444,26 @@ describe.skipIf(!POSTGRES_TEST_URL)('PostgreSQL segments', () => { expect(await currentSchema(adapter)).toBe(segment); }); + // The segment an IndexedDB candidate's options name is the browser's: a + // server tried after it keeps the schemas its own options choose. + it('opens a postgres candidate tried after an IndexedDB candidate that asked for a segment outside it, and makes no schema', async () => { + const segment = unique('seg'); + schemas.push(segment); + // Node has no indexedDB, so the IndexedDB candidate fails. + const adapter = await resolveStorageAdapter({ + priority: ['indexeddb', 'postgres'], + indexedDb: { segment }, + postgres: { connectionString: url }, + quiet: true, + }); + opened.push(adapter); + + expect(adapter.kind).toBe('postgres'); + expect(await currentSchema(adapter)).not.toBe(segment); + const found = await admin.get<{ name: string }>('SELECT nspname AS name FROM pg_namespace WHERE nspname = ?', [segment]); + expect(found ?? null).toBeNull(); + }); + it('refuses a segment with a pool of the caller, whose connections it cannot start', async () => { const pool = new pg.Pool({ connectionString: url }); try { diff --git a/tests/segment.spec.ts b/tests/segment.spec.ts index f0aedb0..60b704f 100644 --- a/tests/segment.spec.ts +++ b/tests/segment.spec.ts @@ -416,6 +416,29 @@ describe('a segment through createDatabase and the resolver', () => { expect(adapter.kind).toBe('postgres'); }); + // A postgres candidate takes no segment another backend's options named, + // tried after it or before: the Postgres adapter refuses a segment on a pool + // of the caller's, so the resolution threw. + it('opens a pool of the caller tried after an IndexedDB candidate that asked for a segment, without that segment', async () => { + // Node has no indexedDB, so the IndexedDB candidate fails. + const pool = { + query: async () => ({ rows: [] }), + connect: async () => { + throw new Error('not used: the adapter takes up a pool of the caller without connecting'); + }, + end: async () => undefined, + }; + const adapter = await resolveStorageAdapter({ + priority: ['indexeddb', 'postgres'], + postgres: { pool }, + indexedDb: { segment: 'agentos' }, + quiet: true, + }); + adapters.push(adapter); + + expect(adapter.kind).toBe('postgres'); + }); + // The Postgres adapter refuses a segment on a pool its caller made. Without // strict, the resolver falls through from a postgres candidate that fails, // so the pair would open a local SQLite file in the segment instead. From 9a6769aad0f69852f6f123459ad245fadc992c29 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:22 -0700 Subject: [PATCH 06/32] fix(resolver): a postgres candidate opens in its own or the top-level segment alone; only local candidates take the segment another candidate's options named --- src/core/resolver.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/core/resolver.ts b/src/core/resolver.ts index d751f1b..71dc172 100644 --- a/src/core/resolver.ts +++ b/src/core/resolver.ts @@ -117,11 +117,12 @@ export interface StorageResolutionOptions { * the segment. Takes the place of a `segment` in `postgres`, `indexedDb` or * `capacitor`. * - * Without it, a candidate whose options name no segment of their own + * Without it, a local candidate whose options name no segment of their own * (better-sqlite3 and sql.js have none), tried after a candidate whose * options named one and failed, opens in that candidate's segment: a * resolution that asked for a segment never falls through to the database - * every user of the file or the browser shares. + * every user of the file or the browser shares. A postgres candidate opens + * in the segment of its own options alone. */ segment?: string; } @@ -355,14 +356,17 @@ export const resolveStorageAdapter = async (options: StorageResolutionOptions = const quiet = options.quiet === true; /** * The segment of the first candidate tried whose own options named one. A - * candidate after it with no segment of its own opens in it, so a candidate - * that asked for a segment and failed never falls through to the database - * every user of the file or the browser shares. + * local candidate after it (better-sqlite3, sql.js, IndexedDB, Capacitor) + * with no segment of its own opens in it, so a candidate that asked for a + * segment and failed never falls through to the database every user of the + * file or the browser shares. A postgres candidate takes no segment it did + * not name: its server's schemas are its own options' to choose, and a pool + * of the caller's cannot run in one. */ let askedFor: string | undefined; for (const candidate of candidates) { - const chosen = segment ?? candidate.ownSegment ?? askedFor; + const chosen = segment ?? candidate.ownSegment ?? (candidate.name === 'postgres' ? undefined : askedFor); askedFor ??= candidate.ownSegment; try { const adapter = await candidate.build(chosen); From ecb8919b364b1fd3ed911bfb1904cfa2aab4bae5 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:22 -0700 Subject: [PATCH 07/32] test(exporter): pg_dump reaches what the pool reaches: pg's reading of its connection string, PGHOST and the other defaults, a password function, TLS options, keyword=value passwords and passwords that cannot leave the command line --- tests/features.spec.ts | 12 +- tests/postgres.exporter.spec.ts | 69 ++++- tests/postgresExporter.args.spec.ts | 381 +++++++++++++++++++++++++++- 3 files changed, 444 insertions(+), 18 deletions(-) diff --git a/tests/features.spec.ts b/tests/features.spec.ts index f3c7628..c9d8945 100644 --- a/tests/features.spec.ts +++ b/tests/features.spec.ts @@ -40,7 +40,7 @@ describe('createStorageFeatures', () => { expect(features.fts).toBeInstanceOf(SqliteFts4); }); - it('returns PostgresDialect + PostgresFts for postgres adapter', () => { + it('returns PostgresDialect + PostgresFts for postgres adapter', async () => { const features = createStorageFeatures( mockAdapter('postgres', { options: { connectionString: 'postgresql://user:pass@localhost:5432/testdb' }, @@ -49,10 +49,10 @@ describe('createStorageFeatures', () => { expect(features.dialect).toBeInstanceOf(PostgresDialect); expect(features.fts).toBeInstanceOf(PostgresFts); expect(features.exporter).toBeInstanceOf(PostgresExporter); - // The exporter may name the adapter's database when an export runs. - const target = (features.exporter as unknown as { connectionString?: string | (() => string | undefined) })[ - 'connectionString' - ]; - expect(typeof target === 'function' ? target() : target).toBe('postgresql://user:pass@localhost:5432/testdb'); + // The exporter names the adapter's database when an export runs, as pg reads it. + const target = ( + features.exporter as unknown as { connectionString?: string | (() => Promise) } + )['connectionString']; + expect(typeof target === 'function' ? await target() : target).toBe('postgresql://user:pass@localhost:5432/testdb'); }); }); diff --git a/tests/postgres.exporter.spec.ts b/tests/postgres.exporter.spec.ts index a63abc5..6d6d602 100644 --- a/tests/postgres.exporter.spec.ts +++ b/tests/postgres.exporter.spec.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, afterAll, afterEach, beforeAll } from 'vitest'; +import { describe, expect, it, afterAll, afterEach, beforeAll, vi } from 'vitest'; import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; @@ -50,11 +50,20 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', }); afterEach(async () => { + vi.unstubAllEnvs(); for (const adapter of opened.splice(0)) { await adapter.close().catch(() => undefined); } }); + /** A database of its own for one spec, dropped after the suite. */ + const newDatabase = async (): Promise => { + const database = unique('export'); + databases.push(database); + await admin.exec(`CREATE DATABASE ${database}`); + return database; + }; + afterAll(async () => { for (const schema of schemas) { await admin.exec(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`).catch(() => undefined); @@ -111,6 +120,64 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', } }); + // pg calls a pool's password function for each connection it makes; the + // export asks it too, and pg_dump logs in with its answer. CI's server asks + // a TCP connection for its password (scram-sha-256), and PGPASSWORD is empty. + it('exports through a pool of the caller whose password is a function, which it asks when the export runs', async () => { + const database = await newDatabase(); + const server = new URL(url); + vi.stubEnv('PGPASSWORD', ''); + let asked = 0; + const pool = new pg.Pool({ + host: server.hostname, + port: Number(server.port || 5432), + database, + user: decodeURIComponent(server.username), + password: async () => { + asked += 1; + return decodeURIComponent(server.password); + }, + }); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE function_items (id INTEGER PRIMARY KEY)'); + const before = asked; + + const file = path.join(dir, `${database}-function.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.function_items']); + expect(asked).toBe(before + 1); + } finally { + await adapter.close(); + await pool.end(); + } + }); + + // pg reads a pool's connection string as a WHATWG URL, libpq by its own + // grammar: given this string as it was, pg_dump read the user part up to the + // @ in the query and the host y. + it('exports through a pool whose connection string libpq reads apart from pg, where pg reaches', async () => { + const database = await newDatabase(); + const server = new URL(url); + vi.stubEnv('PGUSER', decodeURIComponent(server.username)); + vi.stubEnv('PGPASSWORD', decodeURIComponent(server.password)); + vi.stubEnv('PGDATABASE', database); + const pool = new pg.Pool({ connectionString: `postgresql://${server.host}?application_name=x@y` }); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE parsed_items (id INTEGER PRIMARY KEY)'); + + const file = path.join(dir, `${database}-parsed.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.parsed_items']); + } finally { + await adapter.close(); + await pool.end(); + } + }); + // open({ connectionString }) gives the adapter a database its options do not // name: the exporter reads the pool the adapter runs on when an export runs. it('exports an adapter given its connection string at open(), from features made before it opened', async () => { diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index fd82b52..f281eab 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, beforeEach, vi } from 'vitest'; +import { describe, expect, it, afterEach, beforeEach, vi } from 'vitest'; // PostgresExporter runs pg_dump. These specs read what it hands pg_dump: the // arguments, each passed as it is with no shell between, and the environment. @@ -46,6 +46,16 @@ const postgresAdapter = (options: Record): StorageAdapter => const dbnameOf = (args: string[]): string | undefined => args.find((arg) => arg.startsWith('--dbname='))?.slice('--dbname='.length); +/** What an export refused with, or null when it ran. */ +const refusal = (run: Promise): Promise => + run.then( + () => null, + (error: Error) => error, + ); + +/** The variable pg reads the operating system's user name from. */ +const USER_VARIABLE = process.platform === 'win32' ? 'USERNAME' : 'USER'; + describe('PostgresExporter', () => { beforeEach(() => { calls.length = 0; @@ -121,12 +131,13 @@ describe('PostgresExporter', () => { expect(calls[0].env.PGPASSWORD).toBe(password); }); - it('leaves the password to the environment when the connection string holds none, or one libpq cannot decode', async () => { + it('leaves the password to the environment when the connection string holds none', async () => { for (const connectionString of [ 'postgresql://app@db.example/app', 'postgresql://app:@db.example/app', - 'postgresql://app:s3cr%zz@db.example/app', 'host=db.example dbname=app user=app', + // pg_dump takes a string with no = that is not a URI for a database name. + 'app', ]) { calls.length = 0; await new PostgresExporter(connectionString).exportToBytes(); @@ -136,6 +147,84 @@ describe('PostgresExporter', () => { } }); + // A password that cannot be decoded cannot go to PGPASSWORD, and the URI that + // holds it showed it in the process list for as long as pg_dump ran. + it.each([ + ['a % not followed by two hex digits', 'postgresql://app:s3cr%zz@db.example/app'], + ['a % at the end of the password', 'postgresql://app:s3cr%@db.example/app'], + ['bytes that are not UTF-8', 'postgresql://app:s3cr%ff@db.example/app'], + ['a password parameter', 'postgresql://app@db.example/app?password=s3cr%zz'], + ])('refuses a password it cannot decode, with %s, before pg_dump starts', async (_name, connectionString) => { + const failure = await refusal(new PostgresExporter(connectionString).exportToBytes()); + + expect(failure?.message).toMatch(/password.*not valid percent-encoding/); + expect(failure?.message).not.toContain('s3cr'); + expect(calls).toHaveLength(0); + }); + + // libpq refuses %00, and Node's own refusal of a NUL in an argument or an + // environment variable repeats the value, password and all. + it.each([ + ['a password that decodes to a NUL', 'postgresql://app:s3cr%00t@db.example/app'], + ['a connection string that holds a NUL', 'host=db.example password=s3cr\u0000t'], + ])('refuses %s before pg_dump starts', async (_name, connectionString) => { + const failure = await refusal(new PostgresExporter(connectionString).exportToBytes()); + + expect(failure?.message).toMatch(/NUL/); + expect(failure?.message).not.toContain('s3cr'); + expect(calls).toHaveLength(0); + }); + + // libpq takes everything before the first @ that comes before any / as the + // user part, a ? included (fe-connect.c, conninfo_uri_parse_options): pg_dump + // reaches other.example as the user db.example?application_name=x, from the + // string as given and from the string with its password taken out alike. + it('reads a ? before the @ as libpq does, so the password libpq reads stays off the command line', async () => { + await new PostgresExporter('postgresql://db.example?application_name=x:s3cret@other.example').exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://db.example?application_name=x@other.example'); + expect(calls[0].env.PGPASSWORD).toBe('s3cret'); + }); + + // libpq reads keyword=value settings too: a value runs to whitespace, or sits + // in single quotes, and a backslash takes the character after it as it is. + it.each([ + ['a bare value', 'host=db.example dbname=app user=app password=s3cret', 'host=db.example dbname=app user=app', 's3cret'], + [ + 'a quoted value with a space, a quote and a backslash', + "host=db.example password='s3 cr\\'e\\\\t' dbname=app", + 'host=db.example dbname=app', + "s3 cr'e\\t", + ], + ['a backslash before a space in a bare value', 'password=s3cr\\ et host=db.example', 'host=db.example', 's3cr et'], + ['spaces around the equals sign', 'password = s3cret host=db.example', 'host=db.example', 's3cret'], + ['two passwords, of which libpq takes the later', 'password=s3cr-first host=db.example password=s3cret', 'host=db.example', 's3cret'], + ])('keeps the password of keyword=value settings off the command line, with %s', async (_name, connectionString, dbname, password) => { + await new PostgresExporter(connectionString).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + expect(calls[0].args.join(' ')).not.toContain('s3cr'); + expect(calls[0].env.PGPASSWORD).toBe(password); + }); + + it.each([ + ['an unterminated quoted value', "host=db.example password='s3cret", /unterminated/], + ['a setting with no =', 'host=db.example password=s3cr et', /no "="/], + ])('refuses keyword=value settings libpq refuses, %s, before pg_dump starts', async (_name, connectionString, said) => { + const failure = await refusal(new PostgresExporter(connectionString).exportToBytes()); + + expect(failure?.message).toMatch(said); + expect(failure?.message).not.toContain('s3cr'); + expect(calls).toHaveLength(0); + }); + + it('takes the connection string from a function that resolves it when the export runs', async () => { + await new PostgresExporter(async () => 'postgresql://app:s3cret@db.example/app').exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example/app'); + expect(calls[0].env.PGPASSWORD).toBe('s3cret'); + }); + it('writes a scheme in upper case in lower case, which libpq alone reads, when the string holds no password', async () => { await new PostgresExporter('Postgres://app@db.example/app').exportToBytes(); @@ -147,10 +236,7 @@ describe('PostgresExporter', () => { outcome.error = Object.assign(new Error('Command failed: pg_dump --dbname=postgresql://app:hunter2@db/app'), { code: 1 }); outcome.stderr = 'pg_dump: error: connection to server failed: password authentication failed'; - const failure = await new PostgresExporter('postgresql://app:hunter2@db/app').exportToBytes().then( - () => null, - (error: Error) => error, - ); + const failure = await refusal(new PostgresExporter('postgresql://app:hunter2@db/app').exportToBytes()); expect(failure?.message).toMatch(/password authentication failed/); expect(failure?.message).not.toContain('hunter2'); @@ -168,8 +254,21 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { calls.length = 0; outcome.error = null; outcome.stderr = ''; + // pg reads these where the settings leave a value out, and takes the user + // name from the operating system's; each spec that needs one sets it. + for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGSSLMODE', 'USER', 'USERNAME']) { + vi.stubEnv(name, ''); + } + }); + + afterEach(() => { + vi.unstubAllEnvs(); }); + /** The features exporter of an adapter on a pool of the caller made with these settings. */ + const poolExporter = (settings: Record) => + createStorageFeatures(postgresAdapter({ pool: { options: settings } })).exporter; + it("exports a segmented adapter's segment alone", async () => { const { exporter } = createStorageFeatures( postgresAdapter({ connectionString: 'postgresql://app@db.example/app', segment: 'agentos' }), @@ -200,13 +299,273 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); }); - it('says what the pool lacks when a pool of the caller names no database', async () => { - const { exporter } = createStorageFeatures(postgresAdapter({ pool: { options: {} } })); + it('says what the pool lacks when a pool of the caller keeps no settings', async () => { + const { exporter } = createStorageFeatures(postgresAdapter({ pool: {} })); await expect(exporter.exportToBytes()).rejects.toThrow(/pool its caller made/); expect(calls).toHaveLength(0); }); + // pg reads PGHOST, PGPORT, PGUSER and PGDATABASE where the settings leave + // them out, then takes localhost, 5432, the operating system's user name and + // the user's name for the database (pg lib/connection-parameters.js). The + // export reaches the server pg reaches. + it('reaches the server the environment names for a pool of the caller made without a host or a port', async () => { + vi.stubEnv('PGHOST', 'env-host.example'); + vi.stubEnv('PGPORT', '6543'); + + await poolExporter({ database: 'app', user: 'app' }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@env-host.example:6543/app'); + }); + + it('takes the user and the database the environment names, and the user name for the database, as pg does', async () => { + vi.stubEnv('PGUSER', 'env-user'); + vi.stubEnv('PGDATABASE', 'env-db'); + await poolExporter({ host: 'db.example' }).exportToBytes(); + expect(dbnameOf(calls[0].args)).toBe('postgresql://env-user@db.example:5432/env-db'); + + calls.length = 0; + vi.stubEnv('PGUSER', ''); + vi.stubEnv('PGDATABASE', ''); + vi.stubEnv(USER_VARIABLE, 'os-user'); + await poolExporter({ host: 'db.example' }).exportToBytes(); + expect(dbnameOf(calls[0].args)).toBe('postgresql://os-user@db.example:5432/os-user'); + }); + + it('reaches the database pg reaches for a pool of the caller made with no settings', async () => { + vi.stubEnv(USER_VARIABLE, 'os-user'); + + await poolExporter({}).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://os-user@localhost:5432/os-user'); + }); + + it("keeps the adapter's own localhost and 5432 over PGHOST and PGPORT, as the pool it makes does", async () => { + vi.stubEnv('PGHOST', 'env-host.example'); + vi.stubEnv('PGPORT', '6543'); + + await createStorageFeatures(postgresAdapter({ database: 'app', user: 'app' })).exporter.exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@localhost:5432/app'); + }); + + // pg calls a password function for each connection it makes. The export + // calls it when it runs, and hands its answer to pg_dump in PGPASSWORD. + it('hands pg_dump the answer of the password function of a pool of the caller, asked when each export runs', async () => { + let asked = 0; + const seen: unknown[] = []; + const exporter = poolExporter({ + host: 'db.example', + database: 'app', + user: 'app', + password: async (parameters: unknown) => { + seen.push(parameters); + asked += 1; + return `s3cret-token-${asked}`; + }, + }); + + await exporter.exportToBytes(); + await exporter.exportToBytes(); + + expect(calls.map((call) => call.env.PGPASSWORD)).toEqual(['s3cret-token-1', 's3cret-token-2']); + expect(calls.map((call) => call.args.join(' ')).join(' ')).not.toContain('s3cr'); + expect(seen[0]).toMatchObject({ host: 'db.example', port: 5432, user: 'app', database: 'app' }); + }); + + // pg itself connects with no password then; pg_dump, given none, reads + // PGPASSWORD and ~/.pgpass as libpq does. + it('gives pg_dump no password when the password function answers undefined', async () => { + await poolExporter({ host: 'db.example', database: 'app', password: () => undefined }).exportToBytes(); + + expect(calls[0].env.PGPASSWORD).toBe(process.env.PGPASSWORD); + }); + + it('refuses when the password function fails or answers with no string, before pg_dump starts', async () => { + for (const password of [ + async () => { + throw new Error('token service down'); + }, + async () => 42, + ]) { + const failure = await refusal(poolExporter({ host: 'db.example', database: 'app', password }).exportToBytes()); + + expect(failure?.message).toMatch(/password function/); + } + expect(calls).toHaveLength(0); + }); + + // pg reads a connection string over the pool's other connection settings, + // and the password it reads from one that holds none is empty, so a + // password function beside it is never called. + it('calls no password function beside a connection string, as pg calls none', async () => { + const password = vi.fn(async () => 's3cret-token'); + + await poolExporter({ connectionString: 'postgresql://app@db.example/app', password }).exportToBytes(); + + expect(password).not.toHaveBeenCalled(); + expect(calls[0].env.PGPASSWORD).toBe(process.env.PGPASSWORD); + }); + + it.each([ + ['ssl: true', true], + ['rejectUnauthorized: false', { rejectUnauthorized: false }], + ['an empty ssl object', {}], + ])('asks pg_dump for TLS where the pool uses it, with %s', async (_name, ssl) => { + await poolExporter({ host: 'db.example', database: 'app', user: 'app', ssl }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=require'); + }); + + it('keeps a PGSSLMODE that checks the server, where the pool asks for TLS', async () => { + vi.stubEnv('PGSSLMODE', 'verify-full'); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', ssl: true }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app'); + expect(calls[0].env.PGSSLMODE).toBe('verify-full'); + }); + + // pg_dump reads TLS material from files alone, and sslmode=require in their + // place connected without the client certificate or checked no CA. + it.each([ + ['a CA', { ca: ['CA ONE', Buffer.from('CA TWO')] }, /ssl settings ca cannot reach pg_dump/], + ['a CA beside rejectUnauthorized: false', { ca: 'CA PEM', rejectUnauthorized: false }, /ssl settings ca cannot/], + ['a client certificate and a key', { cert: Buffer.from('CERT PEM'), key: 'KEY PEM' }, /ssl settings cert, key cannot/], + [ + 'a pfx, its passphrase and a server name', + { pfx: Buffer.from('x'), passphrase: 's3cret', servername: 'db.internal' }, + /ssl settings pfx, passphrase, servername cannot/, + ], + ])("refuses a pool's ssl with %s, naming the settings, before pg_dump starts", async (_name, ssl, said) => { + const failure = await refusal(poolExporter({ host: 'db.example', database: 'app', ssl }).exportToBytes()); + + expect(failure?.message).toMatch(said); + expect(failure?.message).toMatch(/sslrootcert/); + expect(failure?.message).not.toContain('s3cr'); + expect(calls).toHaveLength(0); + }); + + it('sees a key that pg-pool hides from enumeration', async () => { + const ssl: Record = { key: 'KEY PEM' }; + Object.defineProperty(ssl, 'key', { enumerable: false }); + + const failure = await refusal(poolExporter({ host: 'db.example', database: 'app', ssl }).exportToBytes()); + + expect(failure?.message).toMatch(/ssl settings key cannot/); + }); + + // pg makes no TLS connection when its ssl is false, or PGSSLMODE is a value + // it does not know; libpq would read PGSSLMODE, refuse pg's no-verify and an + // unknown value, and insist on TLS for require. + it.each([ + ["pg's no-verify beside ssl: false", 'no-verify', false], + ['require beside ssl: false', 'require', false], + ['a value neither knows, with no ssl', 'sometimes', undefined], + ])('connects without TLS where pg does, with %s', async (_name, sslmode, ssl) => { + vi.stubEnv('PGSSLMODE', sslmode); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', ssl }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=disable'); + }); + + it("reads PGSSLMODE as pg does where the pool sets no ssl: pg's no-verify is libpq's require", async () => { + vi.stubEnv('PGSSLMODE', 'no-verify'); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app' }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=require'); + }); + + it("applies the pool's ssl beside a connection string that sets no TLS of its own, as pg does", async () => { + await poolExporter({ connectionString: 'postgresql://app@db.example/app', ssl: { rejectUnauthorized: false } }).exportToBytes(); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=require'); + + calls.length = 0; + await poolExporter({ + connectionString: 'postgresql://app@db.example/app?sslmode=verify-full', + ssl: { ca: 'CA PEM' }, + }).exportToBytes(); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full'); + }); + + // pg reads a pool's connection string as a WHATWG URL (pg-connection-string), + // and libpq by its own grammar. The export hands pg_dump the string written + // so that libpq reads the parts pg reads. + it.each([ + ['an @ in the password, where pg reads to the last @ of the authority', 'postgresql://app:p@ss@db.example/app', 'postgresql://app@db.example:5432/app', 'p@ss'], + [ + 'an @ in a parameter of a string with no path, where libpq would read the host after it', + 'postgresql://db.example?application_name=x@y', + 'postgresql://app@db.example:5432/app?application_name=x%40y', + undefined, + ], + ['no host, where pg takes localhost', 'postgresql:///app', 'postgresql://app@localhost:5432/app', undefined], + ['credentials and no host', 'postgresql://app:s3cret@/app', 'postgresql://app@localhost:5432/app', 's3cret'], + [ + 'a host parameter and no host', + 'postgresql:///app?host=/var/run/postgresql', + 'postgresql://app@%2Fvar%2Frun%2Fpostgresql:5432/app', + undefined, + ], + ['a fragment, which pg drops', 'postgresql://app@db.example/app#note', 'postgresql://app@db.example:5432/app', undefined], + ['a % that starts no escape, which pg reads as itself', 'postgresql://app:50%off@db.example/app', 'postgresql://app@db.example:5432/app', '50%off'], + ["pg's sslmode=no-verify, libpq's require", 'postgresql://app@db.example/app?sslmode=no-verify', 'postgresql://app@db.example:5432/app?sslmode=require', undefined], + ["pg's ssl=1", 'postgresql://app@db.example/app?ssl=1', 'postgresql://app@db.example:5432/app?sslmode=require', undefined], + ["pg's ssl=0", 'postgresql://app@db.example/app?ssl=0', 'postgresql://app@db.example:5432/app?sslmode=disable', undefined], + [ + "pg's uselibpqcompat and statement_timeout, which libpq does not know", + 'postgresql://app@db.example/app?uselibpqcompat=true&statement_timeout=5000&application_name=reports', + 'postgresql://app@db.example:5432/app?application_name=reports', + undefined, + ], + [ + 'a certificate file, which turns pg\'s TLS on', + 'postgresql://app@db.example/app?sslrootcert=/etc/ssl/ca.pem', + 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=require', + undefined, + ], + // pg decodes the URL's user name only where no user parameter is given. + [ + 'a user parameter beside a user name that is not valid UTF-8, which pg never decodes', + 'postgresql://ba%ffd@db.example/app?user=app', + 'postgresql://app@db.example:5432/app', + undefined, + ], + // libpq reads these from the connection string alone, pg_dump's command + // line, and pg reads none of them. + [ + 'secrets pg does not read', + 'postgresql://app@db.example/app?sslpassword=s3cr1&oauth_client_secret=s3cr2&scram_client_key=s3cr3&application_name=reports', + 'postgresql://app@db.example:5432/app?application_name=reports', + undefined, + ], + ])('reads the connection string of a pool as pg does, with %s', async (_name, connectionString, dbname, password) => { + vi.stubEnv('PGUSER', 'app'); + + await poolExporter({ connectionString }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + expect(calls[0].env.PGPASSWORD).toBe(password ?? process.env.PGPASSWORD); + }); + + it('reaches the host PGHOST names for a connection string with no host, as pg does', async () => { + vi.stubEnv('PGHOST', 'env-host.example'); + + await poolExporter({ connectionString: 'postgresql://app@/app' }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@env-host.example:5432/app'); + }); + + it('passes keyword=value settings of a pool as they are, for libpq to read', async () => { + await poolExporter({ connectionString: 'host=db.example dbname=app user=app password=s3cret' }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('host=db.example dbname=app user=app'); + expect(calls[0].env.PGPASSWORD).toBe('s3cret'); + }); + it('writes a Unix-socket directory as libpq reads a host in a URI', async () => { const { exporter } = createStorageFeatures(postgresAdapter({ host: '/var/run/postgresql', database: 'app', user: 'app' })); @@ -236,7 +595,7 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { await exporter.exportToBytes(); - expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example/app'); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app'); expect(calls[0].env.PGPASSWORD).toBe('s3cret'); expect(calls[0].args).toContain('--schema="agentos"'); }); @@ -248,7 +607,7 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { await createStorageFeatures(adapter).exporter.exportToBytes(); - expect(dbnameOf(calls[0].args)).toBe('postgresql://app@opened.example/app'); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@opened.example:5432/app'); }); it('says what the adapter lacks when nothing names its database', async () => { From 9b0656264829bef92094b647f3f64b2c16b84343 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:22 -0700 Subject: [PATCH 08/32] fix(exporter): hand pg_dump a libpq URI of the connection pg makes, ask a password function when the export runs, refuse TLS material pg_dump cannot take, and keep every password off the command line or refuse the export --- src/core/contracts/features.ts | 75 +++--- src/exporters/PostgresExporter.ts | 166 +++++++++++-- src/exporters/pgConnection.ts | 382 ++++++++++++++++++++++++++++++ 3 files changed, 555 insertions(+), 68 deletions(-) create mode 100644 src/exporters/pgConnection.ts diff --git a/src/core/contracts/features.ts b/src/core/contracts/features.ts index 2c087fa..64e6933 100644 --- a/src/core/contracts/features.ts +++ b/src/core/contracts/features.ts @@ -12,57 +12,47 @@ import { NodeBlobCodec } from '../../codecs/NodeBlobCodec.js'; import { BrowserBlobCodec } from '../../codecs/BrowserBlobCodec.js'; import { SqliteFileExporter } from '../../exporters/SqliteFileExporter.js'; import { PostgresExporter, type PostgresExporterOptions } from '../../exporters/PostgresExporter.js'; +import { libpqConnectionStringOf, type PgConnectionSettings } from '../../exporters/pgConnection.js'; -/** Connection settings, as the Postgres adapter's options and a pg Pool's `options` hold them. */ -interface PostgresConnectionLike { - connectionString?: string; - host?: string; - port?: number | string; - database?: string; - user?: string; - /** pg also takes a function here, which a connection string cannot hold. */ - password?: unknown; - ssl?: boolean | object; -} - -interface PostgresAdapterOptionsLike extends PostgresConnectionLike { +interface PostgresAdapterOptionsLike extends PgConnectionSettings { segment?: string; /** A pool of the caller's; a pg Pool keeps the settings it was made with as `options`. */ - pool?: { options?: PostgresConnectionLike } | null; + pool?: { options?: PgConnectionSettings } | null; } /** What the exporter reads of a Postgres adapter. */ interface PostgresAdapterLike { options?: PostgresAdapterOptionsLike; /** The pool the adapter runs on: the pool it made when it opened, or its caller's; null while it is closed. */ - getPool?: () => { options?: PostgresConnectionLike } | null; + getPool?: () => { options?: PgConnectionSettings } | null; } -/** A connection string pg_dump reaches the database of these settings with. */ -function connectionStringOf(settings: PostgresConnectionLike | undefined): string | undefined { - if (!settings) return undefined; - if (settings.connectionString) return settings.connectionString; - if (!settings.database) return undefined; - - // libpq percent-decodes each part of a URI, so a host that is a Unix-socket - // directory goes in encoded: postgresql://%2Fvar%2Frun%2Fpostgresql:5432/app. - const host = encodeURIComponent(settings.host ?? 'localhost'); - const port = settings.port ?? 5432; - const user = settings.user ? encodeURIComponent(settings.user) : ''; - // A password pg reads from a function is left to PGPASSWORD or ~/.pgpass. - const password = typeof settings.password === 'string' && settings.password ? `:${encodeURIComponent(settings.password)}` : ''; - // A password with no user name goes in too: libpq takes the user name from PGUSER or the operating system. - const auth = user || password ? `${user}${password}@` : ''; - const sslQuery = settings.ssl ? '?sslmode=require' : ''; - return `postgresql://${auth}${host}:${port}/${encodeURIComponent(settings.database)}${sslQuery}`; -} +/** + * The settings of the pool a Postgres adapter makes when it opens, as its `poolConfig` makes them: the connection + * string, or the host and port (localhost and 5432 when left out), database, user and password. Undefined when the + * options name no server, so an adapter waiting for `open({ connectionString })` exports nothing before it opens. + */ +const ownPoolSettings = (options: PostgresAdapterOptionsLike | undefined): PgConnectionSettings | undefined => { + if (!options) return undefined; + if (options.connectionString) return { connectionString: options.connectionString, ssl: options.ssl }; + if (!options.host && !options.database && !options.user) return undefined; + return { + host: options.host ?? 'localhost', + port: options.port ?? 5432, + database: options.database, + user: options.user, + password: options.password, + ssl: options.ssl, + }; +}; /** - * The exporter of a Postgres adapter. It names the adapter's database when an - * export runs: the pool the adapter runs on names the database it reaches, - * the connection string `open()` was given included; before the adapter - * opens, its own settings name it, or those of the pool its caller gave it. - * A segmented adapter exports its segment's schema alone. + * The exporter of a Postgres adapter. When an export runs it reads the + * connection of the pool the adapter runs on, as pg reads it (the connection + * string `open()` was given included, and a password function of the pool's, + * which it calls), and hands pg_dump a libpq URI of it; before the adapter + * opens, the settings of the pool it will make, or of the pool its caller gave + * it. A segmented adapter exports its segment's schema alone. */ function postgresExporterOf(adapter: StorageAdapter): PostgresExporter { const postgres = adapter as StorageAdapter & PostgresAdapterLike; @@ -70,14 +60,15 @@ function postgresExporterOf(adapter: StorageAdapter): PostgresExporter { const exportOptions: PostgresExporterOptions = { ...(options?.segment ? { schema: options.segment } : {}), missingConnection: options?.pool - ? 'PostgresExporter: the adapter runs on a pool its caller made, and the pool names no connection string or ' + - 'database for pg_dump to reach. Make the pool with a connectionString, or with host, database, user and password.' + ? 'PostgresExporter: the adapter runs on a pool its caller made, and the pool keeps no connection settings ' + + 'for pg_dump to reach. Make the pool with a connectionString, or with host, database, user and password.' : 'PostgresExporter: the adapter names no database for pg_dump to reach. Make it with a connectionString, or ' + 'with host, database and user, or export once open() has been given a connection string.', }; - const connectionString = (): string | undefined => { + const connectionString = async (): Promise => { const pool = typeof postgres.getPool === 'function' ? postgres.getPool() : null; - return connectionStringOf(pool?.options) ?? connectionStringOf(options) ?? connectionStringOf(options?.pool?.options); + const settings = pool?.options ?? ownPoolSettings(options) ?? options?.pool?.options; + return settings ? libpqConnectionStringOf(settings) : undefined; }; return new PostgresExporter(connectionString, exportOptions); } diff --git a/src/exporters/PostgresExporter.ts b/src/exporters/PostgresExporter.ts index 199bcf6..be8c7d8 100644 --- a/src/exporters/PostgresExporter.ts +++ b/src/exporters/PostgresExporter.ts @@ -48,24 +48,42 @@ const decodePart = (part: string): string | undefined => { } }; +/** An export refused before pg_dump starts. It never holds the connection string, which can hold a password. */ +const refusal = (why: string): Error => new Error(`PostgresExporter: ${why}`); + /** - * The database pg_dump connects to, with the password taken out of the - * command line, where the process list shows it. A `postgres://` or - * `postgresql://` URI is read as libpq reads one: the password of its user - * part, or of its `password` parameter, which libpq takes over the user - * part's, goes to PGPASSWORD, which pg_dump reads. A connection string of - * another form, one that holds no password, or one whose password is not valid - * percent-encoding is passed as it is, its scheme in lower case. + * The password of a URI, decoded for PGPASSWORD. libpq refuses `%00`, and a password that is not valid + * percent-encoded UTF-8 cannot become an environment variable: either would otherwise stay in the URI on pg_dump's + * command line, where the process list shows it. */ -const connectionOf = (connectionString: string): { dbname: string; env: Record } => { - const found = URI_SCHEME.exec(connectionString)?.[0]; - if (found === undefined) return { dbname: connectionString, env: {} }; +const uriPassword = (encoded: string): string => { + const password = decodePart(encoded); + if (password === undefined) { + throw refusal( + 'the password in the connection string is not valid percent-encoding, so it cannot reach pg_dump through ' + + 'PGPASSWORD. Percent-encode it, as encodeURIComponent does.', + ); + } + if (password.includes('\u0000')) { + throw refusal('the password in the connection string decodes to a NUL character, which libpq refuses.'); + } + return password; +}; + +/** + * The database pg_dump connects to, with the password of a `postgres://` or `postgresql://` URI taken out of the + * command line, where the process list shows it. The URI is read as libpq reads one: the password of its user part, + * or of its `password` parameter, which libpq takes over the user part's, goes to PGPASSWORD, which pg_dump reads. + */ +const uriConnectionOf = (connectionString: string, found: string): { dbname: string; env: Record } => { const scheme = found.toLowerCase(); const uri = scheme + connectionString.slice(found.length); let rest = uri.slice(scheme.length); let user = ''; let encoded: string | undefined; + // libpq looks for the user part's @ up to the first / alone, so a ? before the @ belongs to the user part + // (fe-connect.c, conninfo_uri_parse_options), and it is read so here. const userPart = URI_USER_PART.exec(rest); if (userPart) { const colon = userPart[1].indexOf(':'); @@ -90,9 +108,100 @@ const connectionOf = (connectionString: string): { dbname: string; env: Record 0 ? `?${kept.join('&')}` : ''); } - const password = encoded === undefined ? undefined : decodePart(encoded); - if (password === undefined) return { dbname: uri, env: {} }; - return { dbname: `${scheme}${user ? `${user}@` : ''}${rest}`, env: { PGPASSWORD: password } }; + if (encoded === undefined) return { dbname: uri, env: {} }; + return { dbname: `${scheme}${user ? `${user}@` : ''}${rest}`, env: { PGPASSWORD: uriPassword(encoded) } }; +}; + +/** Whitespace as libpq's conninfo parser reads it (C's isspace). */ +const isSpace = (char: string | undefined): boolean => char !== undefined && /[ \t\n\v\f\r]/.test(char); + +/** + * The settings of a keyword=value connection string, with each `password` setting taken out for PGPASSWORD, read as + * libpq reads them (fe-connect.c, conninfo_parse): a keyword runs to `=` or whitespace, a value runs to whitespace + * or sits in single quotes, a backslash takes the character after it as it is, and the later of two passwords wins. + */ +const keywordConnectionOf = (connectionString: string): { dbname: string; env: Record } => { + const text = connectionString; + const passwordSpans: Array<[number, number]> = []; + let password: string | undefined; + let at = 0; + while (at < text.length) { + if (isSpace(text[at])) { + at += 1; + continue; + } + const start = at; + while (at < text.length && text[at] !== '=' && !isSpace(text[at])) at += 1; + const keyword = text.slice(start, at); + while (isSpace(text[at])) at += 1; + if (text[at] !== '=') { + throw refusal('the connection string is not keyword=value settings libpq reads: a setting has no "=".'); + } + at += 1; + while (isSpace(text[at])) at += 1; + let value = ''; + if (text[at] === "'") { + at += 1; + let closed = false; + while (at < text.length) { + const char = text[at]; + at += 1; + if (char === '\\') { + if (at < text.length) value += text[at]; + at += 1; + } else if (char === "'") { + closed = true; + break; + } else { + value += char; + } + } + if (!closed) throw refusal('the connection string holds an unterminated quoted value, which libpq refuses.'); + } else { + while (at < text.length && !isSpace(text[at])) { + const char = text[at]; + at += 1; + if (char === '\\') { + if (at < text.length) value += text[at]; + at += 1; + } else { + value += char; + } + } + } + if (keyword === 'password') { + password = value; + passwordSpans.push([start, Math.min(at, text.length)]); + } + } + if (passwordSpans.length === 0) return { dbname: connectionString, env: {} }; + let kept = ''; + let from = 0; + for (const [start, end] of passwordSpans) { + kept += text.slice(from, start); + from = end; + } + kept += text.slice(from); + return { dbname: kept.trim(), env: password ? { PGPASSWORD: password } : {} }; +}; + +/** + * The database pg_dump connects to, with the password taken out of the command line, where the process list shows + * it, for PGPASSWORD. A `postgres://` or `postgresql://` URI is read as libpq reads one, and so are keyword=value + * settings (`host=... password=...`). Any other string (a database name) is passed as it is. + * + * @throws {Error} When the string holds a NUL character, which neither libpq nor a process argument takes, or a + * password pg_dump could not be given through PGPASSWORD, or settings libpq refuses. + */ +const connectionOf = (connectionString: string): { dbname: string; env: Record } => { + if (connectionString.includes('\u0000')) { + throw refusal('the connection string holds a NUL character, which pg_dump cannot take.'); + } + const found = URI_SCHEME.exec(connectionString)?.[0]; + if (found !== undefined) return uriConnectionOf(connectionString, found); + // libpq reads a string with an = as keyword=value settings, and any other as a database name. + if (connectionString.includes('=')) return keywordConnectionOf(connectionString); + return { dbname: connectionString, env: {} }; }; /** @@ -101,13 +210,17 @@ const connectionOf = (connectionString: string): { dbname: string; env: Record string | undefined), + private readonly connectionString?: string | (() => string | undefined | Promise), options: PostgresExporterOptions = {}, ) { this.options = options; @@ -164,7 +278,7 @@ export class PostgresExporter implements IDatabaseExporter { /** Runs pg_dump, into the file at `outputPath` when one is given and to its output otherwise. */ private async dump(outputPath?: string): Promise { const connectionString = - typeof this.connectionString === 'function' ? this.connectionString() : this.connectionString; + typeof this.connectionString === 'function' ? await this.connectionString() : this.connectionString; if (!connectionString) { throw new Error(this.options.missingConnection ?? 'PostgresExporter requires a connection string for pg_dump.'); } diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts new file mode 100644 index 0000000..426623e --- /dev/null +++ b/src/exporters/pgConnection.ts @@ -0,0 +1,382 @@ +/** + * @fileoverview The connection a pg pool makes, written as a libpq connection URI, so that `pg_dump` reaches the + * server, the database and the user the adapter's pool reaches. + * + * pg reads a connection string with the WHATWG URL parser (pg-connection-string's `parse`), and fills what the + * settings leave out from `PGUSER`, `PGDATABASE`, `PGPORT` and `PGHOST`, then the operating system's user, the + * user's name, 5432 and localhost (pg `lib/connection-parameters.js`). libpq reads a URI by its own grammar + * (`fe-connect.c`, `conninfo_uri_parse_options`): it ends the user part at the first `@` that comes before any + * `/`, and it refuses a query parameter it does not know. So `postgresql://db.example?application_name=x@y` names + * the host `db.example` to pg and `y` to libpq. The URI written here holds pg's reading: the user, the host, the + * port and the database spelled out and percent-encoded, and the parameters libpq knows, pg's own spellings of + * TLS translated. + * @module sql-storage-adapter/exporters/pgConnection + */ + +/** Connection settings, as the Postgres adapter's options and a pg `Pool`'s `options` hold them. */ +export interface PgConnectionSettings { + connectionString?: string; + host?: string; + port?: number | string; + database?: string; + user?: string; + /** A string, or a function pg calls for each connection it makes, which may answer a promise. */ + password?: unknown; + /** pg's `ssl`: `true`, `false`, or Node TLS options. */ + ssl?: unknown; +} + +/** The connection pg makes from a set of settings. */ +interface PgConnection { + host: string; + port: string; + user?: string; + database?: string; + /** A string, or a function pg calls; nothing where `PGPASSWORD` and `~/.pgpass` answer. */ + password?: unknown; + /** The libpq parameters of the connection string, decoded, in their order. */ + parameters: Array<[string, string]>; +} + +type Environment = Record; + +/** The environment pg reads its defaults from, where there is one. */ +const environment = (): Environment => + typeof process !== 'undefined' && process.env ? (process.env as Environment) : {}; + +/** + * The parameters libpq takes in a connection URI (PostgreSQL 18, `fe-connect.c`, `PQconninfoOptions`), less the + * ones the URI spells out (`user`, `password`, `host`, `port`, `dbname`) and the ones pg does not read (`service`, + * `hostaddr`). libpq refuses a URI with any other, so pg's own (`statement_timeout`, `uselibpqcompat`) and names + * neither knows are left out. So are `sslpassword`, `oauth_client_secret`, `scram_client_key` and + * `scram_server_key`: libpq takes them from the connection string alone, which is pg_dump's command line, and pg + * reads none of them, so the connection pg makes needs none. + */ +const LIBPQ_PARAMETERS = new Set([ + 'passfile', + 'channel_binding', + 'connect_timeout', + 'client_encoding', + 'options', + 'application_name', + 'fallback_application_name', + 'keepalives', + 'keepalives_idle', + 'keepalives_interval', + 'keepalives_count', + 'tcp_user_timeout', + 'sslmode', + 'sslnegotiation', + 'sslcompression', + 'sslcert', + 'sslkey', + 'sslcertmode', + 'sslrootcert', + 'sslcrl', + 'sslcrldir', + 'sslsni', + 'requirepeer', + 'require_auth', + 'min_protocol_version', + 'max_protocol_version', + 'ssl_min_protocol_version', + 'ssl_max_protocol_version', + 'gssencmode', + 'krbsrvname', + 'gsslib', + 'gssdelegation', + 'replication', + 'target_session_attrs', + 'load_balance_hosts', + 'oauth_issuer', + 'oauth_client_id', + 'oauth_scope', + 'sslkeylogfile', +]); + +/** The connection string parameters that set TLS, after which pg reads none of the settings' `ssl`. */ +const TLS_PARAMETERS = ['ssl', 'sslmode', 'sslcert', 'sslkey', 'sslrootcert']; + +/** The `sslmode` values that make libpq use TLS, which a pool's `ssl` asks for. */ +const TLS_SSLMODES = new Set(['require', 'verify-ca', 'verify-full']); + +/** The `sslmode` values with which libpq connects without TLS to a server that offers none, as pg does with no `ssl`. */ +const PLAIN_SSLMODES = new Set(['disable', 'allow', 'prefer']); + +/** A URI scheme, which pg reads a connection string as a URL for. */ +const URL_SCHEME = /^[A-Za-z][A-Za-z0-9+.-]*:/; + +/** pg's `ssl` from `PGSSLMODE` where the settings give none (pg `lib/connection-parameters.js`). */ +const sslFromEnvironment = (sslmode: string | undefined): unknown => { + switch (sslmode) { + case 'prefer': + case 'require': + case 'verify-ca': + case 'verify-full': + return true; + case 'no-verify': + return { rejectUnauthorized: false }; + default: + return false; + } +}; + +/** + * What a connection string says, as pg-connection-string's `parse` reads it. pg assigns each part `parse` sets over + * the settings, an empty one too, and keeps the settings' value of a part it does not set. + */ +interface ParsedConnectionString { + user?: string; + password?: string; + host: string; + port?: string; + database?: string; + parameters: Array<[string, string]>; +} + +/** Percent-decoding that answers undefined where pg's decoding throws. */ +const decoded = (decode: (text: string) => string, text: string): string | undefined => { + try { + return decode(text); + } catch { + return undefined; + } +}; + +/** + * A connection string as pg-connection-string 2.x's `parse` reads it: a string that starts with `/` is a socket + * directory and a database name; anything else goes through the WHATWG URL parser, percent-encoded first where it + * holds a space or a `%` that starts no escape, with a stand-in host where a user part comes before an empty host. + * A query parameter `user`, `password`, `host` or `port` comes before the URL's own. + */ +const parseAsPg = (connectionString: string): ParsedConnectionString | undefined => { + if (connectionString.startsWith('/')) { + const [host, database] = connectionString.split(' '); + return { host, database, parameters: [] }; + } + let text = connectionString; + if (/ |%[^a-f0-9]|%[a-f0-9][^a-f0-9]/i.test(text)) { + text = encodeURI(text).replace(/%25(\d\d)/g, '%$1'); + } + const urlOf = (candidate: string): URL | undefined => { + try { + return new URL(candidate, 'postgres://base'); + } catch { + return undefined; + } + }; + let url = urlOf(text); + let standInHost = false; + if (url === undefined) { + url = urlOf(text.replace('@/', '@___DUMMY___/')); + standInHost = true; + } + if (url === undefined) return undefined; + const parameters: Array<[string, string]> = []; + url.searchParams.forEach((value, key) => { + parameters.push([key, value]); + }); + const parameter = (name: string): string | undefined => { + const found = parameters.filter(([key]) => key === name); + return found.length > 0 ? found[found.length - 1][1] : undefined; + }; + // pg decodes the URL's user name and password only where no parameter gives them, so a parameter stands in for + // one that is not valid percent-encoding. + const user = parameter('user') || decoded(decodeURIComponent, url.username); + const secret = parameter('password') || decoded(decodeURIComponent, url.password); + if (user === undefined || secret === undefined) return undefined; + + if (url.protocol === 'socket:') { + const host = decoded(decodeURI, url.pathname); + if (host === undefined) return undefined; + const port = parameter('port'); + return { + user, + password: secret, + host, + ...(port === undefined ? {} : { port }), + database: url.searchParams.get('db') ?? undefined, + parameters, + }; + } + + const hostname = (standInHost ? '' : url.hostname).replace(/^\[(.+)\]$/, '$1'); + let pathname = url.pathname; + let host = parameter('host'); + if (!host) { + const fromUrl = decoded(decodeURIComponent, hostname); + if (fromUrl === undefined) return undefined; + host = fromUrl; + } else if (hostname && /^%2f/i.test(hostname)) { + pathname = hostname + pathname; + } + const port = parameter('port') || url.port; + const path = pathname.slice(1); + const database = path ? decoded(decodeURI, path) : undefined; + if (path && database === undefined) return undefined; + return { user, password: secret, host, port, database, parameters }; +}; + +/** + * The TLS options of a pool's `ssl` that `sslmode=require` cannot carry to pg_dump, by name. Own properties are read + * whether they are enumerable or not: pg-pool hides `ssl.key` from enumeration. + */ +const tlsSettingsBeyondSslmode = (ssl: unknown): string[] => + ssl !== null && typeof ssl === 'object' + ? Object.getOwnPropertyNames(ssl).filter((name) => { + if (name === 'rejectUnauthorized') return false; + const value = (ssl as Record)[name]; + return value !== undefined && value !== null; + }) + : []; + +/** + * Reads connection settings as pg reads them for each connection of a pool: a connection string's parts over the + * settings', then the environment, then pg's defaults. Undefined where pg cannot read the connection string. + * + * @throws {Error} When the settings ask for TLS options libpq takes only from files. + */ +const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnection | undefined => { + let user = settings.user; + let password = settings.password; + let host = settings.host; + let port = settings.port === undefined ? undefined : String(settings.port); + let database = settings.database; + const parameters: Array<[string, string]> = []; + let ownTls = false; + + if (settings.connectionString) { + const parsed = parseAsPg(settings.connectionString); + if (parsed === undefined) return undefined; + // pg assigns each part the string sets over the settings, an empty part too. + host = parsed.host; + database = parsed.database; + if ('user' in parsed) user = parsed.user; + if ('password' in parsed) password = parsed.password; + if ('port' in parsed) port = parsed.port; + const last = (name: string): string | undefined => + parsed.parameters.filter(([key]) => key === name).pop()?.[1]; + const sslParameter = last('ssl'); + const sslmode = last('sslmode'); + // pg takes its TLS from the string, and not from the settings' ssl, when the string has an ssl + // parameter, a sslmode, sslcert, sslkey or sslrootcert, or sslnegotiation=direct. + ownTls = + sslParameter !== undefined || + TLS_PARAMETERS.some((name) => name !== 'ssl' && Boolean(last(name))) || + last('sslnegotiation') === 'direct'; + for (const [key, value] of parsed.parameters) { + if (key === 'sslmode') { + // pg's no-verify is libpq's require; an empty sslmode is none, to pg too. + if (value) parameters.push([key, value === 'no-verify' ? 'require' : value]); + } else if (LIBPQ_PARAMETERS.has(key)) { + parameters.push([key, value]); + } + } + if (!sslmode) { + if (sslParameter !== undefined) { + // pg reads ssl=0 and an empty ssl as no TLS and any other value as TLS; libpq knows only ssl=true. + parameters.push(['sslmode', sslParameter === '0' || sslParameter === '' ? 'disable' : 'require']); + } else if (ownTls && !TLS_SSLMODES.has(env.PGSSLMODE ?? '')) { + // A certificate file or direct negotiation turns pg's TLS on, which libpq's default sslmode leaves optional. + parameters.push(['sslmode', 'require']); + } + } + } + + const osUser = typeof process !== 'undefined' && process.platform === 'win32' ? env.USERNAME : env.USER; + const resolvedUser = user || env.PGUSER || osUser || undefined; + const connection: PgConnection = { + user: resolvedUser, + database: database || env.PGDATABASE || resolvedUser, + host: host || env.PGHOST || 'localhost', + port: port || env.PGPORT || '5432', + password: password || undefined, + parameters, + }; + + if (!ownTls) { + const ssl = settings.ssl !== undefined ? settings.ssl : sslFromEnvironment(env.PGSSLMODE); + if (ssl) { + const beyond = tlsSettingsBeyondSslmode(ssl); + if (beyond.length > 0) { + throw new Error( + `PostgresExporter: the pool's ssl settings ${beyond.join(', ')} cannot reach pg_dump, which reads TLS ` + + 'material from files alone. Give the pool a connectionString whose sslrootcert, sslcert and sslkey ' + + 'parameters name the files, or set PGSSLROOTCERT, PGSSLCERT and PGSSLKEY and give the pool ssl: true.', + ); + } + // libpq's require encrypts and checks no certificate; a PGSSLMODE that checks one is kept. + if (!TLS_SSLMODES.has(env.PGSSLMODE ?? '')) { + connection.parameters.push(['sslmode', 'require']); + } + } else if (env.PGSSLMODE && !PLAIN_SSLMODES.has(env.PGSSLMODE)) { + // pg connects without TLS here, its ssl being false or PGSSLMODE a value it does not know. libpq would read + // PGSSLMODE: it refuses pg's no-verify, and require or verify-* insist on TLS where pg makes none. + connection.parameters.push(['sslmode', 'disable']); + } + } + return connection; +}; + +/** + * The password pg sends for a connection: the settings' string, or the answer of the settings' function, which pg + * calls with the connection's parameters. Undefined gives pg_dump no password, so libpq reads `PGPASSWORD` and + * `~/.pgpass`; pg, given undefined by a function, connects with no password and reads neither. + * + * @throws {Error} When the function fails or answers something other than a string or undefined. + */ +const passwordOf = async (connection: PgConnection): Promise => { + const { password } = connection; + if (typeof password !== 'function') { + return typeof password === 'string' ? password : undefined; + } + let answer: unknown; + try { + answer = await (password as (parameters: object) => unknown)({ + user: connection.user, + database: connection.database, + host: connection.host, + port: Number(connection.port), + }); + } catch (error) { + // The function's own error is kept as the cause, out of the message, which a log may show. + const failure: Error & { cause?: unknown } = new Error( + 'PostgresExporter: the password function of the pool failed, so pg_dump has no password.', + ); + failure.cause = error; + throw failure; + } + if (answer === undefined) return undefined; + if (typeof answer !== 'string') { + throw new Error('PostgresExporter: the password function of the pool answered no string, which pg refuses too.'); + } + return answer; +}; + +/** + * A libpq connection URI that reaches what a pg pool with these settings reaches, with its password in the user + * part, where `PostgresExporter` takes it out for `PGPASSWORD`. A connection string pg cannot read as a URL (a + * keyword=value string) is given back as it is. Undefined when the settings name no database pg would reach. + * + * @throws {Error} When the settings ask for TLS that pg_dump cannot take, or the password function fails. + */ +export const libpqConnectionStringOf = async (settings: PgConnectionSettings): Promise => { + const { connectionString } = settings; + if (connectionString && !connectionString.startsWith('/') && !URL_SCHEME.test(connectionString)) { + return connectionString; + } + const env = environment(); + const connection = readAsPg(settings, env); + if (connection === undefined) return connectionString; + if (!connection.database) return undefined; + + const password = await passwordOf(connection); + const user = connection.user ? encodeURIComponent(connection.user) : ''; + const secret = password ? `:${encodeURIComponent(password)}` : ''; + const auth = user || secret ? `${user}${secret}@` : ''; + const query = connection.parameters.map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`); + return ( + `postgresql://${auth}${encodeURIComponent(connection.host)}:${encodeURIComponent(connection.port)}` + + `/${encodeURIComponent(connection.database)}${query.length > 0 ? `?${query.join('&')}` : ''}` + ); +}; From 436810c5a96dbfc23ff56e14c3c5547ff1cf81f4 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:22 -0700 Subject: [PATCH 09/32] docs: the exporter reads the pool as pg does, its TLS and password rules, the segment a postgres candidate takes, PRAGMA comments and the key-value clone --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7a4af17..f204c20 100644 --- a/README.md +++ b/README.md @@ -346,7 +346,7 @@ const agentos = await createDatabase({ url: process.env.DATABASE_URL, segment: ' A segment name is lower-case letters, digits and underscores, starts with a letter or an underscore, and has at most 63 characters; it is not `public`, `information_schema` or a name beginning with `pg_`. A name outside that is refused before any adapter is tried, whether it is given to `createDatabase()` or `resolveStorageAdapter()` or in their `postgres`, `indexedDb` or `capacitor` options. `segment` goes on `createDatabase()`, `resolveStorageAdapter()` and each adapter's own options, and `segmentName()` and `segmentFilePath()` give the name a segment's database or file has. -A `segment` on `createDatabase()` or `resolveStorageAdapter()` applies to every adapter they try. A segment in the options of one adapter (`postgres: { segment }`) applies to that adapter, and to each adapter tried after it whose options name none: without `strict`, a PostgreSQL that cannot be reached falls through to a local file in the same segment (`app-agentos.sqlite3`), never to the file every user shares. An adapter tried before it is left as it is. +A `segment` on `createDatabase()` or `resolveStorageAdapter()` applies to every adapter they try. A segment in the options of one adapter (`postgres: { segment }`) applies to that adapter, and to each local adapter (better-sqlite3, sql.js, IndexedDB, Capacitor) tried after it whose options name none: without `strict`, a PostgreSQL that cannot be reached falls through to a local file in the same segment (`app-agentos.sqlite3`), never to the file every user shares. A PostgreSQL adapter opens in the segment of its own options or of `createDatabase()` and `resolveStorageAdapter()` alone: a segment in `indexedDb` or `capacitor` names a browser's or a device's database, and a server tried after it keeps the schemas its own options choose, a pool of your own included. An adapter tried before it is left as it is. On PostgreSQL: @@ -631,7 +631,7 @@ await adapter.flush(); // stores ev - ✅ Several tabs on one database (`crossTab`) - ✅ A database of its own per segment (`segment`) -**Durability.** With `durability: 'interval'`, the default, the auto-save tick stores the writes made since the last save every `saveIntervalMs`, and a write can be lost when the page goes before the next tick. With `durability: 'commit'`, `run()`, `exec()`, a write through `all()` or `get()` (`INSERT ... RETURNING`, or a PRAGMA that sets `user_version`, `application_id`, `schema_version`, `page_size`, `auto_vacuum` or `encoding`), `transaction()` and `importDatabase()` resolve once their write is in IndexedDB, committed there (the IndexedDB transaction's `complete` event, not the put request's success), and reject when IndexedDB refuses or aborts it; the write stays in the database and the next save stores it, unless `crossTab`'s `rollbackRefusedSaves` gives it up (below). Each save writes the whole database image, so batch many writes in one `transaction()`. While a transaction begun with `BEGIN` is open nothing can be saved: its statements resolve at once, and its rows are stored when its `COMMIT` resolves. `flush()` stores every write made so far in either mode: it waits for an open `transaction()`, and rejects while a transaction begun with `BEGIN` is open, which sql.js's export would end. +**Durability.** With `durability: 'interval'`, the default, the auto-save tick stores the writes made since the last save every `saveIntervalMs`, and a write can be lost when the page goes before the next tick. With `durability: 'commit'`, `run()`, `exec()`, a write through `all()` or `get()` (`INSERT ... RETURNING`, or a PRAGMA that sets `user_version`, `application_id`, `schema_version`, `page_size`, `auto_vacuum` or `encoding`, read as SQLite reads it, a comment between two of its words included), `transaction()` and `importDatabase()` resolve once their write is in IndexedDB, committed there (the IndexedDB transaction's `complete` event, not the put request's success), and reject when IndexedDB refuses or aborts it; the write stays in the database and the next save stores it, unless `crossTab`'s `rollbackRefusedSaves` gives it up (below). Each save writes the whole database image, so batch many writes in one `transaction()`. While a transaction begun with `BEGIN` is open nothing can be saved: its statements resolve at once, and its rows are stored when its `COMMIT` resolves. `flush()` stores every write made so far in either mode: it waits for an open `transaction()`, and rejects while a transaction begun with `BEGIN` is open, which sql.js's export would end. **Several tabs.** Each tab holds the whole database in memory and writes it back whole, so without coordination the tab that saves last overwrites the other's writes. With `crossTab`, every statement and transaction runs under a [Web Lock](https://developer.mozilla.org/en-US/docs/Web/API/Web_Locks_API) on the database, on the newest stored image (a tab reloads it when a version stamp stored beside the image says another tab has saved), and is stored before the lock is let go: `durability` is `'commit'` whatever it says, and a tab reads what another tab stored. A transaction holds the other tabs' statements until it commits or rolls back. A transaction begun with `BEGIN` through `run()` or `exec()` cannot be held across tabs: it is rolled back and refused, so use `transaction()`. Web Locks need a secure context and are in Chrome 69, Firefox 96 and Safari 15.4 and later (on WebKit the feature is on by default and off in Lockdown Mode). Where there are none, the version stamp alone keeps both tabs' writes: a statement that finds another tab saved first runs again on that tab's image, and a `transaction()` another tab saved over is refused with `IndexedDbConflictError`, storing nothing of it, for its caller to run again. The adapter says once when it runs without Web Locks. A tab running a version of this package before 0.7 writes no stamp, so a tab with `crossTab` does not see its saves. @@ -698,7 +698,7 @@ const page = await openIndexedDbStore({ dbName: 'sayshere', storeName: 'kv' }); `KeyValueStore` has `get`, `set`, `delete`, `keys(prefix)` and `deletePrefix(prefix)`, and `memoryOnly`, true when nothing outlives the page or the process. `namespaced(store, prefix)` scopes a store to a prefix: end each prefix with a character no id holds (`ann/`, `anna/`), since the keys under `anna` are among those under `ann`. - `openIndexedDbStore({ dbName, storeName, version })` keeps values in one IndexedDB object store with out-of-line keys. It opens the database at `version` (1 by default) and creates the object store only when the open upgrades the database, so a database another page made at that version is read in place. A browser that refuses IndexedDB gets the memory store, and `memoryOnly` says so. This is the store AgentOS's `src/core/storage/browser` held, moved unchanged: the same database, object store, version and keys. -- `createSqlKeyValueStore(adapter, { table })` keeps values in a table it makes when it is missing, with a `key` and a `value` column. Each call is one statement on the adapter, and through a transaction's handle it joins the transaction. Keys are listed in the order of their UTF-16 code units, as IndexedDB orders them, and a prefix is read literally (`%`, `_` and `*` are characters). Each value is kept as text and comes back as the structured clone, which IndexedDB applies to every value it stores, gives it back: `undefined`, `-0`, `NaN`, bigints, `Date`, `RegExp`, `Map`, `Set`, `ArrayBuffer`, typed arrays and `DataView`, boxed primitives, errors with their message, stack and cause, objects reached twice and cycles. A value the structured clone refuses (a function, a symbol, a `WeakMap`, a promise, a `Proxy`) is refused with a `DataCloneError` before anything is written, as are the host objects text cannot hold (`Blob`, `File`, `CryptoKey`). The runtime's own `structuredClone` checks each value first, since only it tells a proxy apart from the object the proxy wraps. +- `createSqlKeyValueStore(adapter, { table })` keeps values in a table it makes when it is missing, with a `key` and a `value` column. Each call is one statement on the adapter, and through a transaction's handle it joins the transaction. Keys are listed in the order of their UTF-16 code units, as IndexedDB orders them, and a prefix is read literally (`%`, `_` and `*` are characters). Each value is kept as text and comes back as the structured clone, which IndexedDB applies to every value it stores, gives it back: `undefined`, `-0`, `NaN`, bigints, `Date`, `RegExp`, `Map`, `Set`, `ArrayBuffer`, typed arrays and `DataView`, boxed primitives, errors with their message, stack and cause, objects reached twice and cycles. A value the structured clone refuses (a function, a symbol, a `WeakMap`, a promise, a `Proxy`) is refused with a `DataCloneError` before anything is written, as are the host objects text cannot hold (`Blob`, `File`, `CryptoKey`). The text is written from the runtime's own `structuredClone` of the value, which alone tells a proxy apart from the object the proxy wraps, and which reads each property once, as IndexedDB's put does: a getter is read once, and its first answer is what is stored. - `migrateIndexedDbKeyValue({ source, adapter, into })` moves one namespace of an old IndexedDB key-value store (the one `openIndexedDbStore` opens) into a `KeyValueStore` over an adapter, behind a marker, its old writers locked out by the database's version. It opens the old database at `source.lockVersion`, above the `fromVersion` its writers open it at, so their open fails from then on and `openIndexedDbStore` answers from memory; writes the records under `source.prefix`, without it, into the store `into(trx)` makes, with the marker `{ phase: 'copied' }` under `MIGRATION_MARKER`, in one transaction of the adapter; flushes an adapter that answers `flush()`, so one that stores on an interval holds the copy; deletes the old records; and marks the move `done`. It answers `migrated`, `already`, `none` (nothing under the prefix) or `blocked` (a tab holds the old version open, and nothing moved: run it again once that tab is closed). A reload resumes from the committed side, and two moves at once copy once. It refuses a store that keeps nothing past the page (`memoryOnly`: an adapter without `persistence`) before it opens the old database, so the old records are never deleted while their one copy is in memory. A call made while an earlier one waits on an old tab waits behind it, since IndexedDB runs a database's opens in order. - `MemoryKeyValueStore` holds values for the page alone, copied in and out with `structuredClone`. @@ -949,7 +949,11 @@ await exporter.exportToFile('/backups/snapshot.db'); const bytes = await exporter.exportToBytes(); ``` -On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool` that was made with them, or one given its connection string at `open()`. The exporter names the database when an export runs, from the pool the adapter runs on, so it exports the database the adapter opened. A segmented adapter's export holds its segment's schema alone (`--schema`). `pg_dump` runs with no shell between, and the password of a `postgres://` or `postgresql://` connection string, in its user part or its `password` parameter, reaches it through `PGPASSWORD`, not its command line. `new PostgresExporter(connectionString, { schema })` exports one schema of any database; `connectionString` may be a function that gives it when an export runs. +On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export, as pg calls it for each connection, and its answer goes to `PGPASSWORD`; a function that fails or answers anything but a string or `undefined` refuses the export. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). + +TLS follows the pool: `ssl: true`, `{}` or `{ rejectUnauthorized }` becomes `sslmode=require`, which encrypts and checks no certificate, unless `PGSSLMODE` asks for `require`, `verify-ca` or `verify-full`; a pool that connects without TLS gets `sslmode=disable` where `PGSSLMODE` would make libpq insist on TLS or refuse the connection (pg's `no-verify`). `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. + +`pg_dump` runs with no shell between, and the password of the connection string reaches it through `PGPASSWORD`, not its command line: a `postgres://` or `postgresql://` URI's, in its user part or its `password` parameter, and the `password` of keyword=value settings (`host=db.example password='a b'`), each read as libpq reads it. A connection string that holds a NUL character, or a password that is not valid percent-encoding or decodes to a NUL, is refused before `pg_dump` starts, without the string in the message. libpq reads `sslpassword` and `oauth_client_secret` from the connection string alone, so a string handed to `new PostgresExporter()` with them keeps them in `--dbname`; the features exporter leaves them out, since pg reads neither. `new PostgresExporter(connectionString, { schema })` exports one schema of any database; `connectionString` may be a function that gives it, or a promise of it, when an export runs. ## CI, Releases, and Badges From b2e841d90f66c7eb13f05e070da9ce0fe7e84997 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 15:20:22 -0700 Subject: [PATCH 10/32] chore(release): 0.7.16 --- CHANGELOG.md | 22 ++++++++++++++++++++++ package.json | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b814ae..06d0b44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## [0.7.16] - 2026-10-17 + +### Security +- `PostgresExporter` refuses, before `pg_dump` starts, a connection string whose password it cannot move to `PGPASSWORD`: a URI password that is not valid percent-encoding (`%zz`, a `%` at its end, bytes that are not UTF-8) or that decodes to a NUL, and a connection string that holds a NUL. 0.7.15 passed such a URI whole, its password in `--dbname`, where the process list shows it. The refusal never holds the connection string. +- `PostgresExporter` moves the `password` of keyword=value settings (`host=db.example password='a b'`) to `PGPASSWORD` too, read as libpq reads them: a value runs to whitespace or sits in single quotes, a backslash takes the character after it, and the later of two passwords wins. Settings libpq refuses (an unterminated quote, a setting with no `=`) are refused before `pg_dump` starts. 0.7.15 passed them whole on the command line. +- The exporter of a PostgreSQL adapter leaves `sslpassword`, `oauth_client_secret` and the SCRAM keys of a pool's connection string off `pg_dump`'s command line, where libpq takes them alone: pg reads none of them. + +### Changed +- `new PostgresExporter(connectionString)` takes a function that gives the connection string, or a promise of it, when an export runs. + +### Fixed +- `createStorageFeatures(adapter).exporter` on PostgreSQL reads the connection of the adapter's pool as pg reads it, and hands `pg_dump` a libpq URI of it, so `pg_dump` reaches the server, the database and the user the pool reaches: + - A connection string pg and libpq read apart: `postgresql://db.example?application_name=x@y` names the host `db.example` to pg and `y` to libpq, and `pg_dump` connected to `y`. An `@` in a URL's password, a host left out, `ssl=1`, `sslmode=no-verify` and pg's own parameters are written as libpq reads them. + - A pool made without a host, a port, a user or a database: `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user and the user's name for the database, as pg fills them. 0.7.15 wrote localhost and 5432 for a pool that named no host or port, and refused one that named no database. The adapter's own settings keep the localhost and 5432 of the pool it makes. + - A pool whose password is a function: the function is called when the export runs, and its answer goes to `PGPASSWORD`. A function that fails, or answers anything but a string or `undefined`, refuses the export. 0.7.15 left the password out, so `pg_dump` could not log in. + - A pool whose `ssl` holds `ca`, `cert`, `key`, `pfx` or another TLS option is refused, naming the options and what to do instead: `pg_dump` reads TLS material from files alone. 0.7.15 wrote `sslmode=require` in its place, so `pg_dump` connected without the client certificate, or checked the server against no certificate authority. + - A pool that connects without TLS gets `sslmode=disable` where `PGSSLMODE` would make libpq insist on TLS or refuse the connection (pg's `no-verify`). +- `resolveStorageAdapter()` and `createDatabase()` open a PostgreSQL candidate in the segment of its own options or the top-level one alone. Tried after an IndexedDB or Capacitor candidate whose options named a segment, it took that segment: on a pool of the caller's the adapter threw, and on a connection string the server got a schema only the browser's options asked for. A local candidate still opens in it. +- IndexedDB adapter: a PRAGMA assignment through `all()` or `get()` with a comment between its words (`PRAGMA user_version /* note */ = 7`), or with a quoted name and no space before it (`PRAGMA"user_version"=9`), is saved; the statement is read as SQLite reads it. 0.7.15 found no assignment in it, so it was stored neither with `durability: 'commit'` nor by `flush()` or `close()`. +- `createSqlKeyValueStore()` writes a value's text from the copy the runtime's `structuredClone` returns, so each property is read once, as IndexedDB's put reads it, and the value is copied once. 0.7.15 read each property a second time for the text: a getter's second answer was stored, and a value whose getter answered a function the second time was refused. +- `S3StorageProvider.list()` follows `NextContinuationToken` to the last page (#64). It read the first page alone, at most 1,000 keys. + ## [0.7.15] - 2026-10-17 ### Added diff --git a/package.json b/package.json index 3799c81..46077dc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@framers/sql-storage-adapter", - "version": "0.7.15", + "version": "0.7.16", "description": "Robust cross-platform SQL storage abstraction with automatic fallbacks and runtime detection", "type": "module", "main": "dist/cjs/index.js", From bd204aeb3520ec904363305a70093a101e63d2f6 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 16:03:19 -0700 Subject: [PATCH 11/32] test(exporter): empty passwords keep the environment's off pg_dump, TLS follows pg's, and pg_dump never asks for a password - keyword=value settings with an empty password, and a pool whose password function answers none, hand pg_dump an empty PGPASSWORD while the environment holds another (libpq reads PGPASSWORD only for a password left out; pg reads none after a function's empty answer) - a pool that checks no certificate gets sslmode=require over a PGSSLMODE of verify-ca or verify-full; a pool that chose no TLS gets sslmode=disable over prefer or allow; an empty PGSSLMODE, which libpq refuses, gives way to disable; ssl=0 beside a certificate file and an sslmode libpq does not know are TLS, as pg reads them - pg's own ConnectionParameters names the host, port, user and database of the URI, a database parameter and an empty host parameter included; on CI's server the export reaches the database and host pg reaches for both, and is refused like the pool where the password function answers none - pg_dump runs with --no-password --- tests/postgres.exporter.spec.ts | 75 ++++++++++ tests/postgresExporter.args.spec.ts | 209 +++++++++++++++++++++++++++- tests/postgresExporter.spec.ts | 9 +- 3 files changed, 284 insertions(+), 9 deletions(-) diff --git a/tests/postgres.exporter.spec.ts b/tests/postgres.exporter.spec.ts index 6d6d602..c4cc8ad 100644 --- a/tests/postgres.exporter.spec.ts +++ b/tests/postgres.exporter.spec.ts @@ -178,6 +178,81 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', } }); + // pg-connection-string sets the database from the URL's path alone, and + // drops a database parameter (index.js:66-67): pg reaches PGDATABASE. The + // export reaches the database pg reaches, not the one the parameter names. + it("exports the database pg reaches for a connection string whose query names another, which pg drops", async () => { + const database = await newDatabase(); + const server = new URL(url); + vi.stubEnv('PGDATABASE', database); + const pool = new pg.Pool({ + connectionString: `postgresql://${server.username}:${server.password}@${server.host}/?database=${server.pathname.slice(1)}`, + }); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE query_items (id INTEGER PRIMARY KEY)'); + + const file = path.join(dir, `${database}-query.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.query_items']); + } finally { + await adapter.close(); + await pool.end(); + } + }); + + // pg reads an empty host parameter as none and keeps the URL's host + // (pg-connection-string index.js:54-56), never PGHOST. + it('exports through a pool whose connection string has an empty host parameter, where pg reaches', async () => { + const database = await newDatabase(); + const server = new URL(url); + vi.stubEnv('PGHOST', 'unreachable.invalid'); + const pool = new pg.Pool({ + connectionString: `postgresql://${server.username}:${server.password}@${server.host}/${database}?host=`, + }); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE host_items (id INTEGER PRIMARY KEY)'); + + const file = path.join(dir, `${database}-host.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.host_items']); + } finally { + await adapter.close(); + await pool.end(); + } + }); + + // pg connects with no password where a pool's password function answers + // none (lib/client.js, _getPassword), which CI's server refuses over TCP. + // pg_dump gets an empty PGPASSWORD and is refused alike: with PGPASSWORD + // left as it was, it logged in with the environment's password, which the + // pool never sends. With no password and --no-password it fails at once. + it("logs pg_dump in with no password where the pool's password function answers none, as the pool does", async () => { + const database = await newDatabase(); + const server = new URL(url); + vi.stubEnv('PGPASSWORD', decodeURIComponent(server.password)); + const pool = new pg.Pool({ + host: server.hostname, + port: Number(server.port || 5432), + database, + user: decodeURIComponent(server.username), + password: async () => undefined, + }); + const adapter = createPostgresAdapter({ pool }); + try { + await expect(pool.query('SELECT 1')).rejects.toThrow(); + + const file = path.join(dir, `${database}-no-password.dump`); + await expect(createStorageFeatures(adapter).exporter.exportToFile(file)).rejects.toThrow(/pg_dump failed.*password/s); + } finally { + await adapter.close(); + await pool.end(); + } + }); + // open({ connectionString }) gives the adapter a database its options do not // name: the exporter reads the pool the adapter runs on when an export runs. it('exports an adapter given its connection string at open(), from features made before it opened', async () => { diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index f281eab..08e5398 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -26,6 +26,9 @@ vi.mock('node:child_process', () => ({ import { PostgresExporter } from '../src/exporters/PostgresExporter.js'; import { createStorageFeatures } from '../src/core/contracts/features.js'; import type { StorageAdapter } from '../src/core/contracts/index.js'; +// pg's own reading of a pool's settings (pg exports ./lib/*). +// @ts-expect-error pg ships no declarations for its lib files. +import ConnectionParameters from 'pg/lib/connection-parameters.js'; /** An adapter of kind postgres with these options, as createStorageFeatures reads one. */ const postgresAdapter = (options: Record): StorageAdapter => @@ -56,6 +59,18 @@ const refusal = (run: Promise): Promise => /** The variable pg reads the operating system's user name from. */ const USER_VARIABLE = process.platform === 'win32' ? 'USERNAME' : 'USER'; +/** host, port, user and database of a URI the features exporter wrote, decoded. */ +const partsOf = (uri: string): { user: string; host: string; port: number; database: string } => { + const match = /^postgresql:\/\/(?:([^:@]*)(?::[^@]*)?@)?([^:/?]*):(\d+)\/([^?]*)(?:\?.*)?$/.exec(uri); + if (!match) throw new Error(`not a URI the features exporter writes: ${uri}`); + return { + user: decodeURIComponent(match[1] ?? ''), + host: decodeURIComponent(match[2]), + port: Number(match[3]), + database: decodeURIComponent(match[4]), + }; +}; + describe('PostgresExporter', () => { beforeEach(() => { calls.length = 0; @@ -63,6 +78,22 @@ describe('PostgresExporter', () => { outcome.stderr = ''; }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + + // Without a password, to a server that asks for one, pg_dump asks for it on + // the terminal, or with no terminal on its standard input, which execFile + // leaves open (pg_backup_db.c, ConnectDatabaseAhx; sprompt.c): the export + // waited there for good. + it('never lets pg_dump ask for a password: it runs with --no-password', async () => { + await new PostgresExporter('postgresql://app@db.example/app').exportToBytes(); + await new PostgresExporter('host=db.example dbname=app').exportToFile('/backups/app.dump'); + + expect(calls).toHaveLength(2); + for (const call of calls) expect(call.args).toContain('--no-password'); + }); + it('runs pg_dump with the schema it was given, in the custom format', async () => { const exporter = new PostgresExporter('postgresql://app@db.example:5432/app', { schema: 'agentos' }); @@ -207,6 +238,25 @@ describe('PostgresExporter', () => { expect(calls[0].env.PGPASSWORD).toBe(password); }); + // libpq reads PGPASSWORD only for a password the string leaves out + // (fe-connect.c, conninfo_add_defaults): an empty password given in the + // string is the password, and the environment's is never sent. An empty + // PGPASSWORD is read the same way. + it.each([ + ['an empty quoted password', "host=db.example password='' dbname=app", 'host=db.example dbname=app'], + ['an empty password at the end', 'host=db.example dbname=app password=', 'host=db.example dbname=app'], + ['an empty password after a password', "password=s3cret host=db.example password=''", 'host=db.example'], + ['an empty password parameter of a URI', 'postgresql://app@db.example/app?password=', 'postgresql://app@db.example/app'], + ])("hands pg_dump an empty PGPASSWORD, not the environment's, for %s", async (_name, connectionString, dbname) => { + vi.stubEnv('PGPASSWORD', 'ambient-secret'); + + await new PostgresExporter(connectionString).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + expect(calls[0].args.join(' ')).not.toContain('s3cr'); + expect(calls[0].env.PGPASSWORD).toBe(''); + }); + it.each([ ['an unterminated quoted value', "host=db.example password='s3cret", /unterminated/], ['a setting with no =', 'host=db.example password=s3cr et', /no "="/], @@ -256,8 +306,9 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { outcome.stderr = ''; // pg reads these where the settings leave a value out, and takes the user // name from the operating system's; each spec that needs one sets it. - for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGSSLMODE', 'USER', 'USERNAME']) { - vi.stubEnv(name, ''); + // They are removed, not emptied: libpq refuses an empty PGSSLMODE. + for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD', 'PGSSLMODE', 'USER', 'USERNAME']) { + vi.stubEnv(name, undefined); } }); @@ -374,12 +425,30 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(seen[0]).toMatchObject({ host: 'db.example', port: 5432, user: 'app', database: 'app' }); }); - // pg itself connects with no password then; pg_dump, given none, reads - // PGPASSWORD and ~/.pgpass as libpq does. - it('gives pg_dump no password when the password function answers undefined', async () => { - await poolExporter({ host: 'db.example', database: 'app', password: () => undefined }).exportToBytes(); + // pg connects with no password where the password function answers none, + // and reads no PGPASSWORD then (lib/client.js, _getPassword). pg_dump gets + // an empty PGPASSWORD, so it sends the environment's no more than pg does. + it.each([ + ['undefined', undefined], + ['an empty string', ''], + ])("hands pg_dump an empty PGPASSWORD, not the environment's, when the password function answers %s", async (_name, answer) => { + vi.stubEnv('PGPASSWORD', 'ambient-secret'); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', password: () => answer }).exportToBytes(); - expect(calls[0].env.PGPASSWORD).toBe(process.env.PGPASSWORD); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app'); + expect(calls[0].env.PGPASSWORD).toBe(''); + }); + + // A password the settings leave out, or give empty, pg reads from + // PGPASSWORD, then ~/.pgpass, as libpq does. + it('leaves PGPASSWORD to pg_dump where the pool gives no password, as pg reads it then', async () => { + vi.stubEnv('PGPASSWORD', 'env-secret'); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', password: '' }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app'); + expect(calls[0].env.PGPASSWORD).toBe('env-secret'); }); it('refuses when the password function fails or answers with no string, before pg_dump starts', async () => { @@ -471,6 +540,63 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=disable'); }); + // pg reads PGSSLMODE only where neither the settings nor the connection + // string set TLS (lib/connection-parameters.js). A pool that checks no + // certificate connects to a server whose certificate libpq's verify-ca or + // verify-full would refuse, so pg_dump gets require over them. + it.each([ + ['rejectUnauthorized: false, beside PGSSLMODE verify-full', { ssl: { rejectUnauthorized: false } }, 'verify-full'], + ['rejectUnauthorized: false, beside PGSSLMODE verify-ca', { ssl: { rejectUnauthorized: false } }, 'verify-ca'], + [ + "pg's sslmode=no-verify, beside PGSSLMODE verify-full", + { connectionString: 'postgresql://app@db.example/app?sslmode=no-verify' }, + 'verify-full', + ], + [ + 'sslmode=prefer with uselibpqcompat=true, which checks no certificate, beside PGSSLMODE verify-full', + { connectionString: 'postgresql://app@db.example/app?uselibpqcompat=true&sslmode=prefer' }, + 'verify-full', + ], + ])('asks pg_dump for TLS that checks no certificate where the pool checks none, with %s', async (_name, settings, sslmode) => { + vi.stubEnv('PGSSLMODE', sslmode); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', ...settings }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=require'); + }); + + // The pool's own choice of no TLS is pg's whatever PGSSLMODE says, where + // libpq's prefer or allow would use TLS with a server that offers it. + it.each([ + ['ssl: false beside PGSSLMODE prefer', { ssl: false }, 'prefer'], + ['ssl: false beside PGSSLMODE allow', { ssl: false }, 'allow'], + ['ssl: false and no PGSSLMODE', { ssl: false }, undefined], + ["the connection string's ssl=0 beside PGSSLMODE prefer", { connectionString: 'postgresql://app@db.example/app?ssl=0' }, 'prefer'], + ])('connects pg_dump without TLS where the pool chose none, with %s', async (_name, settings, sslmode) => { + if (sslmode !== undefined) vi.stubEnv('PGSSLMODE', sslmode); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', ...settings }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=disable'); + }); + + // Where pg uses no TLS by default (no ssl in the settings or the string, and + // no PGSSLMODE pg reads as TLS), libpq's default, prefer, and a PGSSLMODE of + // disable or allow connect where pg connects, and stay. pg reads an empty + // PGSSLMODE as no TLS, and libpq refuses it. + it.each([ + ['no PGSSLMODE', undefined, ''], + ['PGSSLMODE disable', 'disable', ''], + ['PGSSLMODE allow', 'allow', ''], + ['an empty PGSSLMODE', '', '?sslmode=disable'], + ])('leaves TLS to libpq where pg uses none by default, with %s, unless libpq refuses it', async (_name, sslmode, query) => { + if (sslmode !== undefined) vi.stubEnv('PGSSLMODE', sslmode); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app' }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(`postgresql://app@db.example:5432/app${query}`); + }); + it("reads PGSSLMODE as pg does where the pool sets no ssl: pg's no-verify is libpq's require", async () => { vi.stubEnv('PGSSLMODE', 'no-verify'); @@ -527,6 +653,21 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=require', undefined, ], + // pg-connection-string turns TLS on for a certificate file over ssl=0 + // (index.js:77-79). + [ + "pg's ssl=0 beside a certificate file, which turns pg's TLS on again", + 'postgresql://app@db.example/app?ssl=0&sslrootcert=/etc/ssl/ca.pem', + 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=require', + undefined, + ], + // pg uses TLS for any sslmode but disable; libpq refuses one it does not know. + [ + 'an sslmode libpq does not know, with which pg uses TLS', + 'postgresql://app@db.example/app?sslmode=sometimes', + 'postgresql://app@db.example:5432/app?sslmode=require', + undefined, + ], // pg decodes the URL's user name only where no user parameter is given. [ 'a user parameter beside a user name that is not valid UTF-8, which pg never decodes', @@ -559,6 +700,60 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(dbnameOf(calls[0].args)).toBe('postgresql://app@env-host.example:5432/app'); }); + // pg-connection-string 2.14.1 copies the query's parameters, then sets the + // database from the URL's path alone, null without one (index.js:40-42, + // 66-67): pg reads no database parameter, and reaches PGDATABASE, then the + // user's name. It reads an empty host parameter as none and keeps the URL's + // host (index.js:54-56), whatever PGHOST says. + it.each([ + ['a database parameter and no path', 'postgresql://db.example?database=reports', 'postgresql://app@db.example:5432/env-db'], + ['a database parameter and an empty path', 'postgresql://app@db.example/?database=reports', 'postgresql://app@db.example:5432/env-db'], + ['an empty host parameter', 'postgresql://app@url-host.example/app?host=', 'postgresql://app@url-host.example:5432/app'], + ])('reaches the database and the host pg reaches, with %s', async (_name, connectionString, dbname) => { + vi.stubEnv('PGUSER', 'app'); + vi.stubEnv('PGDATABASE', 'env-db'); + vi.stubEnv('PGHOST', 'env-host.example'); + + await poolExporter({ connectionString }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + }); + + // pg's own reading of a connection string, its ConnectionParameters given + // the same string and environment, names the host, the port, the user and + // the database the URI pg_dump gets names. + it.each([ + ['postgresql://db.example?database=reports'], + ['postgresql://app@db.example/?database=reports'], + ['postgresql://app@url-host.example/app?host='], + ['postgresql://app@/app?host='], + ['postgresql://db.example?application_name=x@y'], + ['postgresql://app:p@ss@db.example/app'], + ['postgresql:///app?host=/var/run/postgresql'], + ['postgresql://[::1]:6543/app'], + ['postgresql://app@db.example:5433/app?port=6000'], + ['postgresql://app@db.example:5433/app?port='], + ['postgresql://app@db.example/app?user=other'], + ['postgresql://%61pp@db.example/r%C3%A9ports'], + ['socket:/var/run/postgresql?db=app'], + ['/var/run/postgresql app'], + ])("names the host, the port, the user and the database pg's own reading names, for %s", async (connectionString) => { + vi.stubEnv('PGUSER', 'env-user'); + vi.stubEnv('PGDATABASE', 'env-db'); + vi.stubEnv('PGHOST', 'env-host.example'); + vi.stubEnv('PGPORT', '6432'); + const pg = new ConnectionParameters({ connectionString }); + + await poolExporter({ connectionString }).exportToBytes(); + + expect(partsOf(dbnameOf(calls[0].args) as string)).toEqual({ + user: pg.user, + host: pg.host, + port: pg.port, + database: pg.database, + }); + }); + it('passes keyword=value settings of a pool as they are, for libpq to read', async () => { await poolExporter({ connectionString: 'host=db.example dbname=app user=app password=s3cret' }).exportToBytes(); diff --git a/tests/postgresExporter.spec.ts b/tests/postgresExporter.spec.ts index 0092c8e..c62672a 100644 --- a/tests/postgresExporter.spec.ts +++ b/tests/postgresExporter.spec.ts @@ -16,7 +16,12 @@ describe('pgDumpInvocation', () => { it('passes the path and the connection string as single arguments and the password in PGPASSWORD alone', () => { const { file, args, env } = pgDumpInvocation('postgresql://dump_user:p%40ss%20word@db.example:5432/app?sslmode=require', HOSTILE_PATH); expect(file).toBe('pg_dump'); - expect(args).toEqual(['--dbname=postgresql://dump_user@db.example:5432/app?sslmode=require', '--format=custom', `--file=${HOSTILE_PATH}`]); + expect(args).toEqual([ + '--dbname=postgresql://dump_user@db.example:5432/app?sslmode=require', + '--format=custom', + '--no-password', + `--file=${HOSTILE_PATH}`, + ]); expect(env.PGPASSWORD).toBe('p@ss word'); for (const arg of args) { expect(arg).not.toContain('p@ss'); @@ -28,7 +33,7 @@ describe('pgDumpInvocation', () => { const before = process.env.PGPASSWORD; expect(pgDumpInvocation('postgres://reader@db.example/app').env.PGPASSWORD).toBe(before); const keyword = 'host=db.example dbname=app user=reader'; - expect(pgDumpInvocation(keyword).args).toEqual([`--dbname=${keyword}`, '--format=custom']); + expect(pgDumpInvocation(keyword).args).toEqual([`--dbname=${keyword}`, '--format=custom', '--no-password']); }); }); From 0dbf7e6b4c3209b85d52c749989182f8712a1686 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 16:03:26 -0700 Subject: [PATCH 12/32] fix(exporter): an empty password stays empty for pg_dump, TLS follows pg's reading, and pg_dump runs with --no-password - keyword=value settings: an empty password goes to PGPASSWORD empty, as libpq reads an empty setting, so the environment's PGPASSWORD is never sent in its place - a pool's password function that answers no password: the URI holds an empty password parameter, which becomes an empty PGPASSWORD, since pg then connects with none and reads no PGPASSWORD (lib/client.js, _getPassword) - TLS is read as pg reads it: the connection string's ssl as pg-connection-string makes it (a certificate file over ssl=0, no-verify and uselibpqcompat's prefer and require checking no certificate), else the settings', else PGSSLMODE. TLS that checks no certificate is require over any PGSSLMODE; no TLS the pool chose is disable; where no TLS is pg's default, libpq's default stays unless PGSSLMODE is one libpq refuses; an sslmode libpq does not know is require - pg_dump runs with --no-password: with no password for a server that asks, it asked on the terminal or waited on its standard input, which execFile leaves open - README and CHANGELOG 0.7.16 say so, and that a database parameter and an empty host parameter are read as pg reads them --- CHANGELOG.md | 8 +- README.md | 6 +- src/exporters/PostgresExporter.ts | 14 ++- src/exporters/pgConnection.ts | 194 +++++++++++++++++++----------- 4 files changed, 142 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06d0b44..98361f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Security - `PostgresExporter` refuses, before `pg_dump` starts, a connection string whose password it cannot move to `PGPASSWORD`: a URI password that is not valid percent-encoding (`%zz`, a `%` at its end, bytes that are not UTF-8) or that decodes to a NUL, and a connection string that holds a NUL. 0.7.15 passed such a URI whole, its password in `--dbname`, where the process list shows it. The refusal never holds the connection string. -- `PostgresExporter` moves the `password` of keyword=value settings (`host=db.example password='a b'`) to `PGPASSWORD` too, read as libpq reads them: a value runs to whitespace or sits in single quotes, a backslash takes the character after it, and the later of two passwords wins. Settings libpq refuses (an unterminated quote, a setting with no `=`) are refused before `pg_dump` starts. 0.7.15 passed them whole on the command line. +- `PostgresExporter` moves the `password` of keyword=value settings (`host=db.example password='a b'`) to `PGPASSWORD` too, read as libpq reads them: a value runs to whitespace or sits in single quotes, a backslash takes the character after it, and the later of two passwords wins. An empty password goes to `PGPASSWORD` empty, as libpq reads an empty setting, so the environment's `PGPASSWORD` is never sent in its place. Settings libpq refuses (an unterminated quote, a setting with no `=`) are refused before `pg_dump` starts. 0.7.15 passed them whole on the command line. - The exporter of a PostgreSQL adapter leaves `sslpassword`, `oauth_client_secret` and the SCRAM keys of a pool's connection string off `pg_dump`'s command line, where libpq takes them alone: pg reads none of them. ### Changed @@ -12,12 +12,14 @@ - `createStorageFeatures(adapter).exporter` on PostgreSQL reads the connection of the adapter's pool as pg reads it, and hands `pg_dump` a libpq URI of it, so `pg_dump` reaches the server, the database and the user the pool reaches: - A connection string pg and libpq read apart: `postgresql://db.example?application_name=x@y` names the host `db.example` to pg and `y` to libpq, and `pg_dump` connected to `y`. An `@` in a URL's password, a host left out, `ssl=1`, `sslmode=no-verify` and pg's own parameters are written as libpq reads them. - A pool made without a host, a port, a user or a database: `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user and the user's name for the database, as pg fills them. 0.7.15 wrote localhost and 5432 for a pool that named no host or port, and refused one that named no database. The adapter's own settings keep the localhost and 5432 of the pool it makes. - - A pool whose password is a function: the function is called when the export runs, and its answer goes to `PGPASSWORD`. A function that fails, or answers anything but a string or `undefined`, refuses the export. 0.7.15 left the password out, so `pg_dump` could not log in. + - A pool whose password is a function: the function is called when the export runs, and its answer goes to `PGPASSWORD`. A function that answers no password gives `pg_dump` an empty `PGPASSWORD`: pg then connects with no password and reads no `PGPASSWORD`, so the environment's is not sent. A function that fails, or answers anything but a string or `undefined`, refuses the export. 0.7.15 left the function's password out, so `pg_dump` logged in with `PGPASSWORD`, or not at all. - A pool whose `ssl` holds `ca`, `cert`, `key`, `pfx` or another TLS option is refused, naming the options and what to do instead: `pg_dump` reads TLS material from files alone. 0.7.15 wrote `sslmode=require` in its place, so `pg_dump` connected without the client certificate, or checked the server against no certificate authority. - - A pool that connects without TLS gets `sslmode=disable` where `PGSSLMODE` would make libpq insist on TLS or refuse the connection (pg's `no-verify`). + - TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`: a `PGSSLMODE` of `verify-ca` or `verify-full` refused the certificate pg accepts. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, where libpq's `prefer` or `allow` would use TLS. Where no TLS is pg's default, libpq's default stays, and an empty `PGSSLMODE` or one libpq does not know, which pg reads as no TLS and libpq refuses, gives way to `sslmode=disable`. A certificate file in the connection string turns TLS on over `ssl=0`, and an `sslmode` libpq does not know means TLS, as pg reads them. + - pg takes the database from the URL's path alone and drops a `database` parameter, and reads an empty `host` parameter as none; the URI holds that reading (pg-connection-string, `index.js`). libpq's own variables, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT`), configure `pg_dump` as they configure psql. - `resolveStorageAdapter()` and `createDatabase()` open a PostgreSQL candidate in the segment of its own options or the top-level one alone. Tried after an IndexedDB or Capacitor candidate whose options named a segment, it took that segment: on a pool of the caller's the adapter threw, and on a connection string the server got a schema only the browser's options asked for. A local candidate still opens in it. - IndexedDB adapter: a PRAGMA assignment through `all()` or `get()` with a comment between its words (`PRAGMA user_version /* note */ = 7`), or with a quoted name and no space before it (`PRAGMA"user_version"=9`), is saved; the statement is read as SQLite reads it. 0.7.15 found no assignment in it, so it was stored neither with `durability: 'commit'` nor by `flush()` or `close()`. - `createSqlKeyValueStore()` writes a value's text from the copy the runtime's `structuredClone` returns, so each property is read once, as IndexedDB's put reads it, and the value is copied once. 0.7.15 read each property a second time for the text: a getter's second answer was stored, and a value whose getter answered a function the second time was refused. +- `PostgresExporter` runs `pg_dump` with `--no-password`. Without a password, to a server that asks for one, `pg_dump` asked for it on the terminal, or with no terminal read its standard input, which the export leaves open, and the export never ended. - `S3StorageProvider.list()` follows `NextContinuationToken` to the last page (#64). It read the first page alone, at most 1,000 keys. ## [0.7.15] - 2026-10-17 diff --git a/README.md b/README.md index f204c20..bf57641 100644 --- a/README.md +++ b/README.md @@ -949,11 +949,11 @@ await exporter.exportToFile('/backups/snapshot.db'); const bytes = await exporter.exportToBytes(); ``` -On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export, as pg calls it for each connection, and its answer goes to `PGPASSWORD`; a function that fails or answers anything but a string or `undefined` refuses the export. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). +On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export, as pg calls it for each connection, and its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, and an empty `host` parameter leaves the URL's host. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). -TLS follows the pool: `ssl: true`, `{}` or `{ rejectUnauthorized }` becomes `sslmode=require`, which encrypts and checks no certificate, unless `PGSSLMODE` asks for `require`, `verify-ca` or `verify-full`; a pool that connects without TLS gets `sslmode=disable` where `PGSSLMODE` would make libpq insist on TLS or refuse the connection (pg's `no-verify`). `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. +TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts. A pool that checks one (`ssl: true`, `{}`, a certificate file in the connection string) gets the connection string's `verify-ca` or `verify-full`, else a `PGSSLMODE` of `require`, `verify-ca` or `verify-full`, else `sslmode=require`, which encrypts and checks no certificate: libpq checks one against files alone. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. -`pg_dump` runs with no shell between, and the password of the connection string reaches it through `PGPASSWORD`, not its command line: a `postgres://` or `postgresql://` URI's, in its user part or its `password` parameter, and the `password` of keyword=value settings (`host=db.example password='a b'`), each read as libpq reads it. A connection string that holds a NUL character, or a password that is not valid percent-encoding or decodes to a NUL, is refused before `pg_dump` starts, without the string in the message. libpq reads `sslpassword` and `oauth_client_secret` from the connection string alone, so a string handed to `new PostgresExporter()` with them keeps them in `--dbname`; the features exporter leaves them out, since pg reads neither. `new PostgresExporter(connectionString, { schema })` exports one schema of any database; `connectionString` may be a function that gives it, or a promise of it, when an export runs. +`pg_dump` runs with no shell between and with `--no-password`, so it never waits for a password on a terminal or on its standard input, and the password of the connection string reaches it through `PGPASSWORD`, not its command line: a `postgres://` or `postgresql://` URI's, in its user part or its `password` parameter, and the `password` of keyword=value settings (`host=db.example password='a b'`), each read as libpq reads it. An empty password given in the string goes to `PGPASSWORD` empty, so the environment's is never sent in its place. A connection string that holds a NUL character, or a password that is not valid percent-encoding or decodes to a NUL, is refused before `pg_dump` starts, without the string in the message. libpq reads `sslpassword` and `oauth_client_secret` from the connection string alone, so a string handed to `new PostgresExporter()` with them keeps them in `--dbname`; the features exporter leaves them out, since pg reads neither. `new PostgresExporter(connectionString, { schema })` exports one schema of any database; `connectionString` may be a function that gives it, or a promise of it, when an export runs. ## CI, Releases, and Badges diff --git a/src/exporters/PostgresExporter.ts b/src/exporters/PostgresExporter.ts index be8c7d8..5a5902d 100644 --- a/src/exporters/PostgresExporter.ts +++ b/src/exporters/PostgresExporter.ts @@ -119,6 +119,8 @@ const isSpace = (char: string | undefined): boolean => char !== undefined && /[ * The settings of a keyword=value connection string, with each `password` setting taken out for PGPASSWORD, read as * libpq reads them (fe-connect.c, conninfo_parse): a keyword runs to `=` or whitespace, a value runs to whitespace * or sits in single quotes, a backslash takes the character after it as it is, and the later of two passwords wins. + * An empty password goes to PGPASSWORD empty: libpq reads the environment's PGPASSWORD only for a password the + * settings leave out (conninfo_add_defaults), and reads an empty PGPASSWORD as it reads an empty setting. */ const keywordConnectionOf = (connectionString: string): { dbname: string; env: Record } => { const text = connectionString; @@ -182,7 +184,7 @@ const keywordConnectionOf = (connectionString: string): { dbname: string; env: R from = end; } kept += text.slice(from); - return { dbname: kept.trim(), env: password ? { PGPASSWORD: password } : {} }; + return { dbname: kept.trim(), env: password === undefined ? {} : { PGPASSWORD: password } }; }; /** @@ -215,7 +217,11 @@ const connectionOf = (connectionString: string): { dbname: string; env: Record = {}, ): PgDumpInvocation { const { dbname, env } = connectionOf(connectionString); - const args = [`--dbname=${dbname}`, '--format=custom']; + // Without a password, to a server that asks for one, pg_dump would ask for it on the terminal, or with no + // terminal on its standard input, and wait there (pg_backup_db.c, ConnectDatabaseAhx; sprompt.c). + const args = [`--dbname=${dbname}`, '--format=custom', '--no-password']; if (options.schema !== undefined) { // In double quotes pg_dump reads the name exactly: no case folding, and no pattern characters. args.push(`--schema="${options.schema.replace(/"/g, '""')}"`); diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts index 426623e..67a1e89 100644 --- a/src/exporters/pgConnection.ts +++ b/src/exporters/pgConnection.ts @@ -1,15 +1,18 @@ /** * @fileoverview The connection a pg pool makes, written as a libpq connection URI, so that `pg_dump` reaches the - * server, the database and the user the adapter's pool reaches. + * server, the database and the user the adapter's pool reaches, with the password and the TLS pg uses. * * pg reads a connection string with the WHATWG URL parser (pg-connection-string's `parse`), and fills what the - * settings leave out from `PGUSER`, `PGDATABASE`, `PGPORT` and `PGHOST`, then the operating system's user, the - * user's name, 5432 and localhost (pg `lib/connection-parameters.js`). libpq reads a URI by its own grammar - * (`fe-connect.c`, `conninfo_uri_parse_options`): it ends the user part at the first `@` that comes before any - * `/`, and it refuses a query parameter it does not know. So `postgresql://db.example?application_name=x@y` names - * the host `db.example` to pg and `y` to libpq. The URI written here holds pg's reading: the user, the host, the - * port and the database spelled out and percent-encoded, and the parameters libpq knows, pg's own spellings of - * TLS translated. + * settings leave out from `PGUSER`, `PGDATABASE`, `PGPORT`, `PGHOST`, `PGPASSWORD` and `PGSSLMODE`, then the + * operating system's user, the user's name, 5432 and localhost (pg `lib/connection-parameters.js`). libpq reads a + * URI by its own grammar (`fe-connect.c`, `conninfo_uri_parse_options`): it ends the user part at the first `@` + * that comes before any `/`, and it refuses a query parameter it does not know. So + * `postgresql://db.example?application_name=x@y` names the host `db.example` to pg and `y` to libpq. The URI + * written here holds pg's reading: the user, the host, the port and the database spelled out and percent-encoded, + * the parameters libpq knows, and an `sslmode` wherever libpq, left to itself, would use TLS differently from pg. + * + * The variables pg reads are read here as pg reads them. libpq's own, which pg does not read (`PGHOSTADDR`, + * `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure pg_dump as they configure psql. * @module sql-storage-adapter/exporters/pgConnection */ @@ -46,11 +49,11 @@ const environment = (): Environment => /** * The parameters libpq takes in a connection URI (PostgreSQL 18, `fe-connect.c`, `PQconninfoOptions`), less the - * ones the URI spells out (`user`, `password`, `host`, `port`, `dbname`) and the ones pg does not read (`service`, - * `hostaddr`). libpq refuses a URI with any other, so pg's own (`statement_timeout`, `uselibpqcompat`) and names - * neither knows are left out. So are `sslpassword`, `oauth_client_secret`, `scram_client_key` and - * `scram_server_key`: libpq takes them from the connection string alone, which is pg_dump's command line, and pg - * reads none of them, so the connection pg makes needs none. + * ones the URI spells out (`user`, `password`, `host`, `port`, `dbname`), `sslmode`, which is written for the TLS + * pg uses, and the ones pg does not read (`service`, `hostaddr`). libpq refuses a URI with any other, so pg's own + * (`statement_timeout`, `uselibpqcompat`) and names neither knows are left out. So are `sslpassword`, + * `oauth_client_secret`, `scram_client_key` and `scram_server_key`: libpq takes them from the connection string + * alone, which is pg_dump's command line, and pg reads none of them, so the connection pg makes needs none. */ const LIBPQ_PARAMETERS = new Set([ 'passfile', @@ -65,7 +68,6 @@ const LIBPQ_PARAMETERS = new Set([ 'keepalives_interval', 'keepalives_count', 'tcp_user_timeout', - 'sslmode', 'sslnegotiation', 'sslcompression', 'sslcert', @@ -94,10 +96,10 @@ const LIBPQ_PARAMETERS = new Set([ 'sslkeylogfile', ]); -/** The connection string parameters that set TLS, after which pg reads none of the settings' `ssl`. */ -const TLS_PARAMETERS = ['ssl', 'sslmode', 'sslcert', 'sslkey', 'sslrootcert']; - -/** The `sslmode` values that make libpq use TLS, which a pool's `ssl` asks for. */ +/** + * The `sslmode` values with which libpq always uses TLS. A pool that checks the server's certificate keeps one that + * `PGSSLMODE` names. + */ const TLS_SSLMODES = new Set(['require', 'verify-ca', 'verify-full']); /** The `sslmode` values with which libpq connects without TLS to a server that offers none, as pg does with no `ssl`. */ @@ -106,7 +108,7 @@ const PLAIN_SSLMODES = new Set(['disable', 'allow', 'prefer']); /** A URI scheme, which pg reads a connection string as a URL for. */ const URL_SCHEME = /^[A-Za-z][A-Za-z0-9+.-]*:/; -/** pg's `ssl` from `PGSSLMODE` where the settings give none (pg `lib/connection-parameters.js`). */ +/** pg's `ssl` from `PGSSLMODE`, which pg reads where neither the settings nor the connection string set TLS. */ const sslFromEnvironment = (sslmode: string | undefined): unknown => { switch (sslmode) { case 'prefer': @@ -121,6 +123,70 @@ const sslFromEnvironment = (sslmode: string | undefined): unknown => { } }; +/** + * The `ssl` pg-connection-string 2.14.1's `parse` makes of a connection string's parameters (index.js:69-158), the + * last value of a repeated parameter counting, or undefined where it makes none, so that pg reads the settings' + * `ssl`, then `PGSSLMODE`. A certificate file or an `sslmode` turns TLS on over `ssl=0`; `sslmode=disable` turns it + * off; `sslmode=no-verify`, and with `uselibpqcompat=true` `sslmode=prefer`, or `require` with no root certificate, + * check no certificate. + */ +const sslOfParameters = (last: (name: string) => string | undefined): unknown => { + let ssl: unknown = last('ssl'); + if (ssl === 'true' || ssl === '1') ssl = true; + if (ssl === '0') ssl = false; + const sslmode = last('sslmode'); + if (last('sslcert') || last('sslkey') || last('sslrootcert') || sslmode) ssl = {}; + if (last('sslnegotiation') === 'direct' && ssl === undefined) ssl = true; + if (sslmode === 'disable') return false; + const compatible = last('uselibpqcompat') === 'true'; + const checksNone = compatible + ? sslmode === 'prefer' || (sslmode === 'require' && !last('sslrootcert')) + : sslmode === 'no-verify'; + return checksNone ? { rejectUnauthorized: false } : ssl; +}; + +/** How a connection pg makes uses TLS: not at all, without checking the server's certificate, or checking it. */ +type PgTls = 'none' | 'unchecked' | 'checked'; + +/** + * How pg uses TLS with an `ssl`, which `lib/connection-parameters.js` reads `'true'` and `'no-verify'` in, and + * Node's TLS checks the certificate with unless `rejectUnauthorized` is false. + */ +const tlsOf = (ssl: unknown): PgTls => { + const value = ssl === 'true' ? true : ssl === 'no-verify' ? { rejectUnauthorized: false } : ssl; + if (!value) return 'none'; + return typeof value === 'object' && (value as { rejectUnauthorized?: unknown }).rejectUnauthorized === false + ? 'unchecked' + : 'checked'; +}; + +/** + * The `sslmode` pg_dump gets so that libpq uses TLS as pg does, or undefined where libpq left to itself does. + * - No TLS: `disable` where the pool's settings or its connection string chose it (pg reads no `PGSSLMODE` then, + * and libpq's `prefer` or `allow` would use TLS with a server that offers it). Where no TLS is pg's default, + * libpq's own default (`prefer`) and a `PGSSLMODE` of `disable` or `allow` connect where pg connects, and stay; + * an empty `PGSSLMODE`, or one libpq does not know, which libpq refuses, gives way to `disable`. + * - TLS that checks no certificate: `require`, over any `PGSSLMODE`, since `verify-ca` or `verify-full` would + * refuse the certificate pg accepts. + * - TLS that checks the certificate: the connection string's `verify-ca` or `verify-full`, else any other `sslmode` + * it names as `require`, else a `PGSSLMODE` of `require`, `verify-ca` or `verify-full`, else `require`, which + * encrypts and checks no certificate: libpq checks one against files alone, where Node checks against its own + * certificate authorities. + */ +const sslmodeFor = ( + tls: PgTls, + chosen: boolean, + ofString: string | undefined, + pgsslmode: string | undefined, +): string | undefined => { + if (tls === 'none') { + return chosen || (pgsslmode !== undefined && !PLAIN_SSLMODES.has(pgsslmode)) ? 'disable' : undefined; + } + if (tls === 'unchecked') return 'require'; + if (ofString === 'verify-ca' || ofString === 'verify-full') return ofString; + return ofString === undefined && TLS_SSLMODES.has(pgsslmode ?? '') ? undefined : 'require'; +}; + /** * What a connection string says, as pg-connection-string's `parse` reads it. pg assigns each part `parse` sets over * the settings, an empty one too, and keeps the settings' value of a part it does not set. @@ -243,7 +309,11 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio let port = settings.port === undefined ? undefined : String(settings.port); let database = settings.database; const parameters: Array<[string, string]> = []; - let ownTls = false; + // pg's ssl: the connection string's where its parameters make one, else the settings', else PGSSLMODE's + // (lib/connection-parameters.js reads PGSSLMODE only where the merged settings hold no ssl). + let ssl: unknown = settings.ssl; + let source: 'string' | 'settings' | 'environment' = settings.ssl !== undefined ? 'settings' : 'environment'; + let sslmodeOfString: string | undefined; if (settings.connectionString) { const parsed = parseAsPg(settings.connectionString); @@ -256,36 +326,36 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio if ('port' in parsed) port = parsed.port; const last = (name: string): string | undefined => parsed.parameters.filter(([key]) => key === name).pop()?.[1]; - const sslParameter = last('ssl'); - const sslmode = last('sslmode'); - // pg takes its TLS from the string, and not from the settings' ssl, when the string has an ssl - // parameter, a sslmode, sslcert, sslkey or sslrootcert, or sslnegotiation=direct. - ownTls = - sslParameter !== undefined || - TLS_PARAMETERS.some((name) => name !== 'ssl' && Boolean(last(name))) || - last('sslnegotiation') === 'direct'; + const fromString = sslOfParameters(last); + if (fromString !== undefined) { + ssl = fromString; + source = 'string'; + // An empty sslmode is none, to pg too. + sslmodeOfString = last('sslmode') || undefined; + } for (const [key, value] of parsed.parameters) { - if (key === 'sslmode') { - // pg's no-verify is libpq's require; an empty sslmode is none, to pg too. - if (value) parameters.push([key, value === 'no-verify' ? 'require' : value]); - } else if (LIBPQ_PARAMETERS.has(key)) { - parameters.push([key, value]); - } + if (LIBPQ_PARAMETERS.has(key)) parameters.push([key, value]); } - if (!sslmode) { - if (sslParameter !== undefined) { - // pg reads ssl=0 and an empty ssl as no TLS and any other value as TLS; libpq knows only ssl=true. - parameters.push(['sslmode', sslParameter === '0' || sslParameter === '' ? 'disable' : 'require']); - } else if (ownTls && !TLS_SSLMODES.has(env.PGSSLMODE ?? '')) { - // A certificate file or direct negotiation turns pg's TLS on, which libpq's default sslmode leaves optional. - parameters.push(['sslmode', 'require']); - } + } + if (source === 'environment') ssl = sslFromEnvironment(env.PGSSLMODE); + + const tls = tlsOf(ssl); + if (source === 'settings' && tls !== 'none') { + const beyond = tlsSettingsBeyondSslmode(ssl); + if (beyond.length > 0) { + throw new Error( + `PostgresExporter: the pool's ssl settings ${beyond.join(', ')} cannot reach pg_dump, which reads TLS ` + + 'material from files alone. Give the pool a connectionString whose sslrootcert, sslcert and sslkey ' + + 'parameters name the files, or set PGSSLROOTCERT, PGSSLCERT and PGSSLKEY and give the pool ssl: true.', + ); } } + const sslmode = sslmodeFor(tls, source !== 'environment', sslmodeOfString, env.PGSSLMODE); + if (sslmode !== undefined) parameters.push(['sslmode', sslmode]); const osUser = typeof process !== 'undefined' && process.platform === 'win32' ? env.USERNAME : env.USER; const resolvedUser = user || env.PGUSER || osUser || undefined; - const connection: PgConnection = { + return { user: resolvedUser, database: database || env.PGDATABASE || resolvedUser, host: host || env.PGHOST || 'localhost', @@ -293,35 +363,11 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio password: password || undefined, parameters, }; - - if (!ownTls) { - const ssl = settings.ssl !== undefined ? settings.ssl : sslFromEnvironment(env.PGSSLMODE); - if (ssl) { - const beyond = tlsSettingsBeyondSslmode(ssl); - if (beyond.length > 0) { - throw new Error( - `PostgresExporter: the pool's ssl settings ${beyond.join(', ')} cannot reach pg_dump, which reads TLS ` + - 'material from files alone. Give the pool a connectionString whose sslrootcert, sslcert and sslkey ' + - 'parameters name the files, or set PGSSLROOTCERT, PGSSLCERT and PGSSLKEY and give the pool ssl: true.', - ); - } - // libpq's require encrypts and checks no certificate; a PGSSLMODE that checks one is kept. - if (!TLS_SSLMODES.has(env.PGSSLMODE ?? '')) { - connection.parameters.push(['sslmode', 'require']); - } - } else if (env.PGSSLMODE && !PLAIN_SSLMODES.has(env.PGSSLMODE)) { - // pg connects without TLS here, its ssl being false or PGSSLMODE a value it does not know. libpq would read - // PGSSLMODE: it refuses pg's no-verify, and require or verify-* insist on TLS where pg makes none. - connection.parameters.push(['sslmode', 'disable']); - } - } - return connection; }; /** * The password pg sends for a connection: the settings' string, or the answer of the settings' function, which pg - * calls with the connection's parameters. Undefined gives pg_dump no password, so libpq reads `PGPASSWORD` and - * `~/.pgpass`; pg, given undefined by a function, connects with no password and reads neither. + * calls with the connection's parameters. Undefined where neither gives one. * * @throws {Error} When the function fails or answers something other than a string or undefined. */ @@ -355,8 +401,12 @@ const passwordOf = async (connection: PgConnection): Promise /** * A libpq connection URI that reaches what a pg pool with these settings reaches, with its password in the user - * part, where `PostgresExporter` takes it out for `PGPASSWORD`. A connection string pg cannot read as a URL (a - * keyword=value string) is given back as it is. Undefined when the settings name no database pg would reach. + * part, where `PostgresExporter` takes it out for `PGPASSWORD`. Where the pool has no password, the URI holds none + * and libpq reads `PGPASSWORD`, then `~/.pgpass`, as pg does. Where its password function answers none, pg connects + * with no password and reads no `PGPASSWORD` (`lib/client.js`, `_getPassword`): the URI holds an empty `password` + * parameter, which becomes an empty `PGPASSWORD`, so libpq sends the environment's no more than pg does. A + * connection string pg cannot read as a URL (a keyword=value string) is given back as it is. Undefined when the + * settings name no database pg would reach. * * @throws {Error} When the settings ask for TLS that pg_dump cannot take, or the password function fails. */ @@ -371,10 +421,12 @@ export const libpqConnectionStringOf = async (settings: PgConnectionSettings): P if (!connection.database) return undefined; const password = await passwordOf(connection); + const parameters = [...connection.parameters]; + if (typeof connection.password === 'function' && !password) parameters.push(['password', '']); const user = connection.user ? encodeURIComponent(connection.user) : ''; const secret = password ? `:${encodeURIComponent(password)}` : ''; const auth = user || secret ? `${user}${secret}@` : ''; - const query = connection.parameters.map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`); + const query = parameters.map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`); return ( `postgresql://${auth}${encodeURIComponent(connection.host)}:${encodeURIComponent(connection.port)}` + `/${encodeURIComponent(connection.database)}${query.length > 0 ? `?${query.join('&')}` : ''}` From 661e8c6b24de7ab0d740d429b6bbc544d23e6ed4 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 16:10:27 -0700 Subject: [PATCH 13/32] test(exporter): the no-password spec never opens a pool connection, which the server would hold half logged in pg-pool drops a client whose connection failed without ending it, and pg leaves its socket open, so the server kept that login waiting until authentication_timeout. DROP DATABASE ... WITH (FORCE) in the suite's afterAll waited on it (ProcSignalBarrier), and a CREATE INDEX CONCURRENTLY in another suite waited on DROP DATABASE's transaction. The export alone shows pg_dump refused with no password. --- tests/postgres.exporter.spec.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/postgres.exporter.spec.ts b/tests/postgres.exporter.spec.ts index c4cc8ad..1e41752 100644 --- a/tests/postgres.exporter.spec.ts +++ b/tests/postgres.exporter.spec.ts @@ -230,6 +230,10 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', // pg_dump gets an empty PGPASSWORD and is refused alike: with PGPASSWORD // left as it was, it logged in with the environment's password, which the // pool never sends. With no password and --no-password it fails at once. + // The pool itself never connects here: pg-pool keeps no client whose + // connection failed and never ends it, so the server would hold that + // half-done login until authentication_timeout, and DROP DATABASE waits on + // it (ProcSignalBarrier). it("logs pg_dump in with no password where the pool's password function answers none, as the pool does", async () => { const database = await newDatabase(); const server = new URL(url); @@ -243,8 +247,6 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', }); const adapter = createPostgresAdapter({ pool }); try { - await expect(pool.query('SELECT 1')).rejects.toThrow(); - const file = path.join(dir, `${database}-no-password.dump`); await expect(createStorageFeatures(adapter).exporter.exportToFile(file)).rejects.toThrow(/pg_dump failed.*password/s); } finally { From 234d89c8492ce894375cd1bd09b79cd84bac960f Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 16:19:51 -0700 Subject: [PATCH 14/32] chore(release): 0.7.17 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 46077dc..b3db96e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@framers/sql-storage-adapter", - "version": "0.7.16", + "version": "0.7.17", "description": "Robust cross-platform SQL storage abstraction with automatic fallbacks and runtime detection", "type": "module", "main": "dist/cjs/index.js", From 38bfc6699c2e283783592c505d987376cc99d3d7 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 20:03:39 -0700 Subject: [PATCH 15/32] test(exporter): a pool's password function is asked as pg asks it, on a client of the pool, with no arguments before pg 8.19.0 and the client's ConnectionParameters from 8.19.0 --- tests/postgres.exporter.spec.ts | 49 +++++++++++++ tests/postgresExporter.args.spec.ts | 109 ++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) diff --git a/tests/postgres.exporter.spec.ts b/tests/postgres.exporter.spec.ts index 1e41752..b5d0cbf 100644 --- a/tests/postgres.exporter.spec.ts +++ b/tests/postgres.exporter.spec.ts @@ -154,6 +154,55 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', } }); + // pg calls the function as a method of the client it connects, with no + // arguments before 8.19.0 and from 8.19.0 with the client's + // ConnectionParameters (lib/client.js, _getPassword). The export asks it as + // the pool's own pg asked it, whichever pg is installed. + it("asks a pool's password function as the pool's pg asks it: on a client, with that pg's arguments", async () => { + const database = await newDatabase(); + const server = new URL(url); + vi.stubEnv('PGPASSWORD', ''); + const asked: Array<{ self: unknown; args: unknown[] }> = []; + const pool = new pg.Pool({ + host: server.hostname, + port: Number(server.port || 5432), + database, + user: decodeURIComponent(server.username), + application_name: 'exporter-spec', + password: async function (this: unknown, ...args: unknown[]) { + asked.push({ self: this, args }); + return decodeURIComponent(server.password); + }, + }); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE asked_items (id INTEGER PRIMARY KEY)'); + const before = asked.length; + expect(before).toBeGreaterThan(0); + + const file = path.join(dir, `${database}-asked.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.asked_items']); + expect(asked).toHaveLength(before + 1); + const [byPool] = asked; + const byExport = asked[before]; + expect(byPool.self).toBeInstanceOf(pg.Client); + expect(byExport.self).toBeInstanceOf(pg.Client); + for (const field of ['user', 'database', 'host', 'port'] as const) { + expect((byExport.self as Record)[field]).toEqual((byPool.self as Record)[field]); + } + expect(byExport.args).toHaveLength(byPool.args.length); + expect(byExport.args).toEqual(byPool.args); + if (byPool.args.length > 0) { + expect(byExport.args[0]).toBeInstanceOf((byPool.args[0] as object).constructor); + } + } finally { + await adapter.close(); + await pool.end(); + } + }); + // pg reads a pool's connection string as a WHATWG URL, libpq by its own // grammar: given this string as it was, pg_dump read the user part up to the // @ in the query and the host y. diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index 08e5398..9d07461 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -23,6 +23,19 @@ vi.mock('node:child_process', () => ({ }, })); +// How a pool's password function is called follows the version of the pg this +// library loads. A spec that sets pgRead.as reads pg as that; the others read +// the pg installed. +const { pgRead } = vi.hoisted(() => ({ + pgRead: { as: null as null | { version?: string; Client?: unknown } }, +})); + +vi.mock('../src/exporters/pgModule.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, loadPg: async () => pgRead.as ?? actual.loadPg() }; +}); + +import { Client as PgClient } from 'pg'; import { PostgresExporter } from '../src/exporters/PostgresExporter.js'; import { createStorageFeatures } from '../src/core/contracts/features.js'; import type { StorageAdapter } from '../src/core/contracts/index.js'; @@ -310,6 +323,7 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD', 'PGSSLMODE', 'USER', 'USERNAME']) { vi.stubEnv(name, undefined); } + pgRead.as = null; }); afterEach(() => { @@ -477,6 +491,101 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(calls[0].env.PGPASSWORD).toBe(process.env.PGPASSWORD); }); + // pg calls a pool's password function as a method of the client it is + // connecting (pg-pool newClient: new this.Client(this.options)), with no + // arguments before 8.19.0 and from 8.19.0 with the client's + // ConnectionParameters (node-postgres #3602; lib/client.js, _getPassword). + it.each(['8.13.1', '8.18.0'])( + 'calls the password function with no arguments, on a client of the pool, where pg is %s, as that pg calls it', + async (version) => { + pgRead.as = { version, Client: PgClient }; + const seen: Array<{ self: unknown; args: unknown[] }> = []; + const exporter = poolExporter({ + host: 'db.example', + database: 'app', + user: 'app', + // A function written for pg before 8.19.0, which gives it no argument: + // it reads the client it is called on, and an argument names another + // secret, as when other code calls it. + password: function (this: { user?: string } | undefined, ...args: unknown[]) { + seen.push({ self: this, args }); + return args.length === 0 && this?.user === 'app' ? 'pool-secret' : 'another-secret'; + }, + }); + + await exporter.exportToBytes(); + + expect(seen).toHaveLength(1); + expect(seen[0].args).toEqual([]); + expect(seen[0].self).toBeInstanceOf(PgClient); + expect(seen[0].self).toMatchObject({ user: 'app', database: 'app', host: 'db.example', port: 5432 }); + expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); + }, + ); + + it.each(['8.19.0', '8.23.1'])( + "calls the password function with the client's ConnectionParameters, on that client, where pg is %s, as that pg calls it", + async (version) => { + pgRead.as = { version, Client: PgClient }; + const seen: Array<{ self: unknown; args: unknown[] }> = []; + const settings = { + host: 'db.example', + database: 'app', + user: 'app', + application_name: 'reports', + statement_timeout: 5000, + password: function (this: unknown, ...args: unknown[]) { + seen.push({ self: this, args }); + return 'pool-secret'; + }, + }; + + await poolExporter(settings).exportToBytes(); + + expect(seen).toHaveLength(1); + expect(seen[0].self).toBeInstanceOf(PgClient); + expect(seen[0].args).toHaveLength(1); + expect(seen[0].args[0]).toBe((seen[0].self as { connectionParameters: unknown }).connectionParameters); + expect(seen[0].args[0]).toBeInstanceOf(ConnectionParameters); + expect(seen[0].args[0]).toEqual(new ConnectionParameters(settings)); + expect(seen[0].args[0]).toMatchObject({ host: 'db.example', port: 5432, user: 'app', application_name: 'reports' }); + expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); + }, + ); + + // pg calls a stream function of the settings to open a client's socket, as + // a connector does (stream: () => tls.connect(...)): the client the export + // makes for the password function never opens one. + it("never calls the pool's stream function when it asks the password function", async () => { + pgRead.as = { version: '8.23.1', Client: PgClient }; + const stream = vi.fn(() => { + throw new Error('the export opened a socket'); + }); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', stream, password: () => 'pool-secret' }).exportToBytes(); + + expect(stream).not.toHaveBeenCalled(); + expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); + }); + + it("calls the password function with the connection's user, database, host and port where pg cannot be read", async () => { + pgRead.as = {}; + const seen: unknown[][] = []; + + await poolExporter({ + host: 'db.example', + database: 'app', + user: 'app', + password: (...args: unknown[]) => { + seen.push(args); + return 'pool-secret'; + }, + }).exportToBytes(); + + expect(seen).toEqual([[{ user: 'app', database: 'app', host: 'db.example', port: 5432 }]]); + expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); + }); + it.each([ ['ssl: true', true], ['rejectUnauthorized: false', { rejectUnauthorized: false }], From b89a9a2f9a12e18cc40850c5d2f9bb78852906c5 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 20:07:29 -0700 Subject: [PATCH 16/32] fix(exporter): ask a pool's password function as pg asks it: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and the client's ConnectionParameters from 8.19.0, without calling the settings' stream function --- CHANGELOG.md | 2 +- README.md | 2 +- src/exporters/pgConnection.ts | 65 ++++++++++++++++++++++++++++++----- src/exporters/pgModule.ts | 55 +++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 src/exporters/pgModule.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4901318..1de087f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - `createStorageFeatures(adapter).exporter` on PostgreSQL reads the connection of the adapter's pool as pg reads it, and hands `pg_dump` a libpq URI of it, so `pg_dump` reaches the server, the database and the user the pool reaches: - A connection string pg and libpq read apart: `postgresql://db.example?application_name=x@y` names the host `db.example` to pg and `y` to libpq, and `pg_dump` connected to `y`. An `@` in a URL's password, a host left out, `ssl=1`, `sslmode=no-verify` and pg's own parameters are written as libpq reads them. - A pool made without a host, a port, a user or a database: `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user and the user's name for the database, as pg fills them. 0.7.16 wrote localhost and 5432 for a pool that named no host or port, and refused one that named no database. The adapter's own settings keep the localhost and 5432 of the pool it makes. - - A pool whose password is a function: the function is called when the export runs, and its answer goes to `PGPASSWORD`. A function that answers no password gives `pg_dump` an empty `PGPASSWORD`: pg then connects with no password and reads no `PGPASSWORD`, so the environment's is not sent. A function that fails, or answers anything but a string or `undefined`, refuses the export. 0.7.16 left the function's password out, so `pg_dump` logged in with `PGPASSWORD`, or not at all. + - A pool whose password is a function: the function is called when the export runs, as pg calls it for each connection: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters` (node-postgres #3602), as the pg this library loads calls it. The client is never connected and is made without the settings' `stream`. Its answer goes to `PGPASSWORD`. A function that answers no password gives `pg_dump` an empty `PGPASSWORD`: pg then connects with no password and reads no `PGPASSWORD`, so the environment's is not sent. A function that fails, or answers anything but a string or `undefined`, refuses the export. 0.7.16 left the function's password out, so `pg_dump` logged in with `PGPASSWORD`, or not at all. - A pool whose `ssl` holds `ca`, `cert`, `key`, `pfx` or another TLS option is refused, naming the options and what to do instead: `pg_dump` reads TLS material from files alone. 0.7.16 wrote `sslmode=require` in its place, so `pg_dump` connected without the client certificate, or checked the server against no certificate authority. - TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`: a `PGSSLMODE` of `verify-ca` or `verify-full` refused the certificate pg accepts. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, where libpq's `prefer` or `allow` would use TLS. Where no TLS is pg's default, libpq's default stays, and an empty `PGSSLMODE` or one libpq does not know, which pg reads as no TLS and libpq refuses, gives way to `sslmode=disable`. A certificate file in the connection string turns TLS on over `ssl=0`, and an `sslmode` libpq does not know means TLS, as pg reads them. - pg takes the database from the URL's path alone and drops a `database` parameter, and reads an empty `host` parameter as none; the URI holds that reading (pg-connection-string, `index.js`). libpq's own variables, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT`), configure `pg_dump` as they configure psql. diff --git a/README.md b/README.md index bf57641..e34e2cb 100644 --- a/README.md +++ b/README.md @@ -949,7 +949,7 @@ await exporter.exportToFile('/backups/snapshot.db'); const bytes = await exporter.exportToBytes(); ``` -On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export, as pg calls it for each connection, and its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, and an empty `host` parameter leaves the URL's host. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). +On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export as pg calls it for each connection: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0, and from 8.19.0 with the client's `ConnectionParameters`. The client is of the pg this library loads, so a pool made with another copy of pg, or with a `Client` of its own, is asked as this library's pg asks; it is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, and an empty `host` parameter leaves the URL's host. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts. A pool that checks one (`ssl: true`, `{}`, a certificate file in the connection string) gets the connection string's `verify-ca` or `verify-full`, else a `PGSSLMODE` of `require`, `verify-ca` or `verify-full`, else `sslmode=require`, which encrypts and checks no certificate: libpq checks one against files alone. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts index 67a1e89..d19397e 100644 --- a/src/exporters/pgConnection.ts +++ b/src/exporters/pgConnection.ts @@ -16,6 +16,9 @@ * @module sql-storage-adapter/exporters/pgConnection */ +import { getStorageLogger } from '../shared/logger.js'; +import { loadPg } from './pgModule.js'; + /** Connection settings, as the Postgres adapter's options and a pg `Pool`'s `options` hold them. */ export interface PgConnectionSettings { connectionString?: string; @@ -365,25 +368,69 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio }; }; +/** Whether a pg of this version calls a password function with no arguments, as every pg before 8.19.0 does. */ +const callsPasswordWithNoArguments = (version: string): boolean => { + const match = /^(\d+)\.(\d+)\./.exec(version); + if (!match) return false; + const major = Number(match[1]); + return major < 8 || (major === 8 && Number(match[2]) < 19); +}; + +/** Whether the warning that the exporter could not read pg has been written; it is written once. */ +let toldPgUnread = false; + +/** + * The receiver and the arguments pg calls a pool's password function with for each connection the pool makes: a + * client made of the pool's settings (pg-pool `newClient`: `new this.Client(this.options)`), which pg calls the + * function as a method of, with no arguments before pg 8.19.0 and from 8.19.0 with the client's + * `ConnectionParameters` (node-postgres #3602; `lib/client.js` `_getPassword`: + * `this.password(this.connectionParameters)`). The client is of the pg this library loads, made without the + * settings' `stream`, a function pg calls to open the socket, and never connected. Where that pg cannot be read, the + * function gets no receiver and the connection's user, database, host and port, as pg 8.19.0 and later give it + * theirs, or nothing where the version read is older. + */ +const passwordCallOf = async ( + connection: PgConnection, + settings: PgConnectionSettings, +): Promise<{ self: unknown; args: unknown[] }> => { + const pg = await loadPg(); + const noArguments = pg.version !== undefined && callsPasswordWithNoArguments(pg.version); + if (pg.version !== undefined && pg.Client !== undefined) { + // pg-pool keeps the settings' password from enumeration, so it is copied by name. + const client = new pg.Client({ ...settings, password: settings.password, stream: undefined }); + return { self: client, args: noArguments ? [] : [client.connectionParameters] }; + } + if (!toldPgUnread) { + toldPgUnread = true; + getStorageLogger().warn( + { scope: 'PostgresExporter', pgVersion: pg.version ?? null, pgClient: pg.Client !== undefined }, + 'PostgresExporter could not read the pg this library loads, so it calls a pool\'s password function on no ' + + `client, with ${noArguments ? 'no arguments' : "the connection's user, database, host and port"}.`, + ); + } + return { + self: undefined, + args: noArguments + ? [] + : [{ user: connection.user, database: connection.database, host: connection.host, port: Number(connection.port) }], + }; +}; + /** * The password pg sends for a connection: the settings' string, or the answer of the settings' function, which pg - * calls with the connection's parameters. Undefined where neither gives one. + * calls on the connecting client, with its `ConnectionParameters` from pg 8.19.0. Undefined where neither gives one. * * @throws {Error} When the function fails or answers something other than a string or undefined. */ -const passwordOf = async (connection: PgConnection): Promise => { +const passwordOf = async (connection: PgConnection, settings: PgConnectionSettings): Promise => { const { password } = connection; if (typeof password !== 'function') { return typeof password === 'string' ? password : undefined; } + const { self, args } = await passwordCallOf(connection, settings); let answer: unknown; try { - answer = await (password as (parameters: object) => unknown)({ - user: connection.user, - database: connection.database, - host: connection.host, - port: Number(connection.port), - }); + answer = await (password as (...parameters: unknown[]) => unknown).apply(self, args); } catch (error) { // The function's own error is kept as the cause, out of the message, which a log may show. const failure: Error & { cause?: unknown } = new Error( @@ -420,7 +467,7 @@ export const libpqConnectionStringOf = async (settings: PgConnectionSettings): P if (connection === undefined) return connectionString; if (!connection.database) return undefined; - const password = await passwordOf(connection); + const password = await passwordOf(connection, settings); const parameters = [...connection.parameters]; if (typeof connection.password === 'function' && !password) parameters.push(['password', '']); const user = connection.user ? encodeURIComponent(connection.user) : ''; diff --git a/src/exporters/pgModule.ts b/src/exporters/pgModule.ts new file mode 100644 index 0000000..277b552 --- /dev/null +++ b/src/exporters/pgModule.ts @@ -0,0 +1,55 @@ +/** + * @fileoverview The pg this library loads, as the PostgreSQL exporter reads it to call a pool's password function + * as pg does: its version, which decides the function's arguments, and its `Client`, on which pg calls it. + * @module sql-storage-adapter/exporters/pgModule + */ + +/** pg's `Client`, as far as the exporter uses one: it is made of a pool's settings and never connected. */ +export type PgClientClass = new (config: object) => { connectionParameters?: unknown }; + +/** The pg this library loads. A part that cannot be read is undefined. */ +export interface LoadedPg { + /** The version in pg's `package.json`. */ + version?: string; + Client?: PgClientClass; +} + +let loading: Promise | undefined; + +/** + * Reads the pg that a `require` from this module resolves, which is the pg the Postgres adapter's `import('pg')` + * loads. Its `package.json` is found from the file `pg` resolves to, up through the directories above it, and not + * as `pg/package.json`, which the exports map of pg 8.15.0 to 8.16.0 does not export. Read once. + */ +export const loadPg = (): Promise => { + loading ??= readPg(); + return loading; +}; + +const readPg = async (): Promise => { + const loaded: LoadedPg = {}; + try { + const [{ createRequire }, fs, path] = await Promise.all([ + import('node:module'), + import('node:fs'), + import('node:path'), + ]); + const require = createRequire(import.meta.url); + for (let dir = path.dirname(require.resolve('pg')); ; dir = path.dirname(dir)) { + const file = path.join(dir, 'package.json'); + if (fs.existsSync(file)) { + const pkg = JSON.parse(fs.readFileSync(file, 'utf8')) as { name?: unknown; version?: unknown }; + if (pkg.name === 'pg') { + if (typeof pkg.version === 'string') loaded.version = pkg.version; + break; + } + } + if (path.dirname(dir) === dir) break; + } + const pg = require('pg') as { Client?: unknown }; + if (typeof pg.Client === 'function') loaded.Client = pg.Client as PgClientClass; + } catch { + // What was read stays; the exporter says what it does without the rest. + } + return loaded; +}; From fceaed5bf5349a49d2f33d568a5280b437973280 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 20:10:27 -0700 Subject: [PATCH 17/32] docs(exporter): how a pool's password function is called where the exporter cannot read the pg it loads --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e34e2cb..149edfe 100644 --- a/README.md +++ b/README.md @@ -949,7 +949,7 @@ await exporter.exportToFile('/backups/snapshot.db'); const bytes = await exporter.exportToBytes(); ``` -On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export as pg calls it for each connection: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0, and from 8.19.0 with the client's `ConnectionParameters`. The client is of the pg this library loads, so a pool made with another copy of pg, or with a `Client` of its own, is asked as this library's pg asks; it is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, and an empty `host` parameter leaves the URL's host. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). +On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export as pg calls it for each connection: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0, and from 8.19.0 with the client's `ConnectionParameters`. The client is of the pg this library loads, so a pool made with another copy of pg, or with a `Client` of its own, is asked as this library's pg asks; it is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. Where it cannot read that pg's version or its `Client` (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, and an empty `host` parameter leaves the URL's host. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts. A pool that checks one (`ssl: true`, `{}`, a certificate file in the connection string) gets the connection string's `verify-ca` or `verify-full`, else a `PGSSLMODE` of `require`, `verify-ca` or `verify-full`, else `sslmode=require`, which encrypts and checks no certificate: libpq checks one against files alone. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. From 9609a6bae219dac6c7ab4155ceb2e2e2172e4e59 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 20:33:00 -0700 Subject: [PATCH 18/32] test(exporter): pg_dump checks a pool's server as pg does, with verify-full and the system's authorities or the string's root certificate, on a CI server that offers TLS with a certificate no public authority signed --- .github/workflows/ci.yml | 31 ++++++++ tests/postgres.exporter.tls.spec.ts | 100 ++++++++++++++++++++++++++ tests/postgresExporter.args.spec.ts | 105 +++++++++++++++++++++++----- 3 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 tests/postgres.exporter.tls.spec.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fae7be7..c412534 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,37 @@ jobs: - name: Install dependencies run: npm install + # A PostgreSQL that offers TLS with a certificate made for this run, for + # localhost, which no public certificate authority signed, so that + # tests/postgres.exporter.tls.spec.ts can show pg_dump checking a server + # as pg does. The spec reads POSTGRES_TLS_TEST_URL and + # POSTGRES_TLS_ROOT_CERT. The server reads its key only when the + # postgres user (uid 999 in the image) owns it with mode 0600. + - name: Start a PostgreSQL that offers TLS + run: | + dir="$RUNNER_TEMP/pg-tls" + mkdir -p "$dir" + openssl req -new -x509 -days 2 -nodes -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" \ + -keyout "$dir/server.key" -out "$dir/server.crt" + chmod 644 "$dir/server.crt" + chmod 600 "$dir/server.key" + sudo chown 999:999 "$dir/server.key" + docker run -d --name pg-tls -p 5433:5432 \ + -e POSTGRES_USER=adapter -e POSTGRES_PASSWORD=adapter -e POSTGRES_DB=adapter_test \ + -v "$dir:/tls:ro" postgres:16 \ + -c ssl=on -c ssl_cert_file=/tls/server.crt -c ssl_key_file=/tls/server.key + for i in $(seq 1 90); do + if docker exec pg-tls pg_isready -U adapter -d adapter_test -h localhost >/dev/null 2>&1; then + echo "POSTGRES_TLS_TEST_URL=postgresql://adapter:adapter@localhost:5433/adapter_test" >> "$GITHUB_ENV" + echo "POSTGRES_TLS_ROOT_CERT=$dir/server.crt" >> "$GITHUB_ENV" + exit 0 + fi + sleep 1 + done + docker logs pg-tls + exit 1 + - name: Lint run: npm run lint || true diff --git a/tests/postgres.exporter.tls.spec.ts b/tests/postgres.exporter.tls.spec.ts new file mode 100644 index 0000000..094c61b --- /dev/null +++ b/tests/postgres.exporter.tls.spec.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, afterAll, beforeAll } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import pg from 'pg'; +import { createPostgresAdapter, createStorageFeatures } from '../src/index.js'; + +// CI starts a PostgreSQL that offers TLS with a certificate made for the run, +// for localhost, which no public certificate authority signed: it reads +// POSTGRES_TLS_TEST_URL, and POSTGRES_TLS_ROOT_CERT names the certificate. +// pg's TLS is Node's, which checks the server's certificate and host name +// unless rejectUnauthorized is false. The export hands pg_dump libpq settings +// that check the server as pg does: libpq's require checks nothing without a +// root certificate file (fe-secure-openssl.c), so pg_dump exported from a +// server pg refuses. + +const TLS_URL = process.env.POSTGRES_TLS_TEST_URL; +const ROOT_CERT = process.env.POSTGRES_TLS_ROOT_CERT; + +/** The tables a custom-format dump holds, as schema.table, from pg_restore's list of it. */ +const dumpedTables = (file: string): string[] => + execFileSync('pg_restore', ['--list', file], { encoding: 'utf8' }) + .split('\n') + .flatMap((line) => { + const match = /^\d+; \d+ \d+ TABLE (?!DATA )(\S+) (\S+) /.exec(line); + return match ? [`${match[1]}.${match[2]}`] : []; + }) + .sort(); + +describe.skipIf(!TLS_URL || !ROOT_CERT)('the PostgreSQL exporter, through pg_dump, on a server that offers TLS', () => { + const server = new URL(TLS_URL ?? 'postgresql://localhost'); + const settings = { + host: server.hostname, + port: Number(server.port || 5432), + database: server.pathname.slice(1), + user: decodeURIComponent(server.username), + password: decodeURIComponent(server.password), + }; + const table = `tls_items_${Math.random().toString(36).slice(2, 10)}`; + let dir: string; + let admin: pg.Pool; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pg-export-tls-')); + admin = new pg.Pool({ ...settings, ssl: { rejectUnauthorized: false } }); + await admin.query(`CREATE TABLE ${table} (id INTEGER PRIMARY KEY)`); + }); + + afterAll(async () => { + await admin.query(`DROP TABLE IF EXISTS ${table}`).catch(() => undefined); + await admin.end(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('refuses, as pg does, a server whose certificate no authority the pool trusts signed, with ssl: true', async () => { + const pool = new pg.Pool({ ...settings, ssl: true }); + try { + await expect(pool.query('SELECT 1')).rejects.toThrow(/self[- ]signed certificate/i); + + const file = path.join(dir, 'checked.dump'); + await expect(createStorageFeatures(createPostgresAdapter({ pool })).exporter.exportToFile(file)).rejects.toThrow( + /pg_dump failed.*certificate verify failed/s, + ); + } finally { + await pool.end(); + } + }); + + it('exports through a pool that checks no certificate, where pg connects, with rejectUnauthorized: false', async () => { + const pool = new pg.Pool({ ...settings, ssl: { rejectUnauthorized: false } }); + try { + await pool.query('SELECT 1'); + + const file = path.join(dir, 'unchecked.dump'); + await createStorageFeatures(createPostgresAdapter({ pool })).exporter.exportToFile(file); + + expect(dumpedTables(file)).toContain(`public.${table}`); + } finally { + await pool.end(); + } + }); + + // pg reads the root certificate the string names and checks the server and + // its host name against it; pg_dump gets the same file with verify-full. + it('exports through a pool whose connection string names the root certificate it checks the server against', async () => { + const connectionString = `${TLS_URL}${(TLS_URL as string).includes('?') ? '&' : '?'}sslrootcert=${encodeURIComponent(ROOT_CERT as string)}`; + const pool = new pg.Pool({ connectionString }); + try { + await pool.query('SELECT 1'); + + const file = path.join(dir, 'rooted.dump'); + await createStorageFeatures(createPostgresAdapter({ pool })).exporter.exportToFile(file); + + expect(dumpedTables(file)).toContain(`public.${table}`); + } finally { + await pool.end(); + } + }); +}); diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index 9d07461..264825f 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -320,7 +320,7 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { // pg reads these where the settings leave a value out, and takes the user // name from the operating system's; each spec that needs one sets it. // They are removed, not emptied: libpq refuses an empty PGSSLMODE. - for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD', 'PGSSLMODE', 'USER', 'USERNAME']) { + for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD', 'PGSSLMODE', 'PGSSLROOTCERT', 'USER', 'USERNAME']) { vi.stubEnv(name, undefined); } pgRead.as = null; @@ -586,23 +586,58 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); }); + // pg's TLS is Node's, which checks the server's certificate against Node's + // certificate authorities, and its host name, unless rejectUnauthorized is + // false. libpq checks a certificate with verify-ca or verify-full alone, the + // host name with verify-full alone, and against a root certificate file or + // the system's (sslrootcert=system, libpq 16 and later); its require checks + // nothing without a root certificate file (fe-secure-openssl.c). it.each([ ['ssl: true', true], - ['rejectUnauthorized: false', { rejectUnauthorized: false }], ['an empty ssl object', {}], - ])('asks pg_dump for TLS where the pool uses it, with %s', async (_name, ssl) => { + ['rejectUnauthorized: true', { rejectUnauthorized: true }], + ])("has pg_dump check the server's certificate and host name, as pg does, with %s", async (_name, ssl) => { await poolExporter({ host: 'db.example', database: 'app', user: 'app', ssl }).exportToBytes(); - expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=require'); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system'); }); - it('keeps a PGSSLMODE that checks the server, where the pool asks for TLS', async () => { - vi.stubEnv('PGSSLMODE', 'verify-full'); + // libpq reads PGSSLMODE where the URI names no sslmode; pg reads none once + // the settings set ssl. + it.each(['verify-full', 'verify-ca', 'require', 'prefer'])( + 'has pg_dump check the server as pg does over a PGSSLMODE of %s, where the pool asks for TLS', + async (sslmode) => { + vi.stubEnv('PGSSLMODE', sslmode); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app', ssl: true }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system'); + }, + ); + + // pg reads PGSSLMODE prefer, require, verify-ca and verify-full as ssl: true + // where the settings set no ssl (lib/connection-parameters.js, + // readSSLConfigFromEnvironment), and Node's TLS then checks the server. + it.each(['prefer', 'require', 'verify-ca', 'verify-full'])( + 'has pg_dump check the server as pg does for a PGSSLMODE of %s, where the pool sets no ssl', + async (sslmode) => { + vi.stubEnv('PGSSLMODE', sslmode); + + await poolExporter({ host: 'db.example', database: 'app', user: 'app' }).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system'); + }, + ); + + // PGSSLROOTCERT is libpq's alone: where it names a file, libpq checks the + // server against it, as psql does. + it('leaves the root certificate to PGSSLROOTCERT where it names one', async () => { + vi.stubEnv('PGSSLROOTCERT', '/etc/ssl/private-ca.pem'); await poolExporter({ host: 'db.example', database: 'app', user: 'app', ssl: true }).exportToBytes(); - expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app'); - expect(calls[0].env.PGSSLMODE).toBe('verify-full'); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full'); + expect(calls[0].env.PGSSLROOTCERT).toBe('/etc/ssl/private-ca.pem'); }); // pg_dump reads TLS material from files alone, and sslmode=require in their @@ -666,8 +701,9 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { { connectionString: 'postgresql://app@db.example/app?uselibpqcompat=true&sslmode=prefer' }, 'verify-full', ], + ['rejectUnauthorized: false, with no PGSSLMODE', { ssl: { rejectUnauthorized: false } }, undefined], ])('asks pg_dump for TLS that checks no certificate where the pool checks none, with %s', async (_name, settings, sslmode) => { - vi.stubEnv('PGSSLMODE', sslmode); + if (sslmode !== undefined) vi.stubEnv('PGSSLMODE', sslmode); await poolExporter({ host: 'db.example', database: 'app', user: 'app', ...settings }).exportToBytes(); @@ -718,12 +754,14 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { await poolExporter({ connectionString: 'postgresql://app@db.example/app', ssl: { rejectUnauthorized: false } }).exportToBytes(); expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=require'); + // The string's sslmode makes pg's ssl anew, without the settings' ca + // (pg-connection-string index.js:77-79): Node's own authorities check it. calls.length = 0; await poolExporter({ connectionString: 'postgresql://app@db.example/app?sslmode=verify-full', ssl: { ca: 'CA PEM' }, }).exportToBytes(); - expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full'); + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system'); }); // pg reads a pool's connection string as a WHATWG URL (pg-connection-string), @@ -748,7 +786,42 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { ['a fragment, which pg drops', 'postgresql://app@db.example/app#note', 'postgresql://app@db.example:5432/app', undefined], ['a % that starts no escape, which pg reads as itself', 'postgresql://app:50%off@db.example/app', 'postgresql://app@db.example:5432/app', '50%off'], ["pg's sslmode=no-verify, libpq's require", 'postgresql://app@db.example/app?sslmode=no-verify', 'postgresql://app@db.example:5432/app?sslmode=require', undefined], - ["pg's ssl=1", 'postgresql://app@db.example/app?ssl=1', 'postgresql://app@db.example:5432/app?sslmode=require', undefined], + ["pg's ssl=1", 'postgresql://app@db.example/app?ssl=1', 'postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system', undefined], + // Without uselibpqcompat=true, pg-connection-string makes ssl {} of + // require, verify-ca and verify-full alike (index.js:139-152), and Node's + // TLS checks the certificate and the host name. + [ + 'sslmode=require, which pg reads as a check of the server', + 'postgresql://app@db.example/app?sslmode=require', + 'postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system', + undefined, + ], + [ + 'sslmode=verify-ca, which pg reads as a check of the host name too', + 'postgresql://app@db.example/app?sslmode=verify-ca', + 'postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system', + undefined, + ], + // With uselibpqcompat=true, verify-ca, and require beside a root + // certificate, check the certificate and no host name (index.js:116-133). + [ + 'uselibpqcompat=true with verify-ca and a root certificate', + 'postgresql://app@db.example/app?uselibpqcompat=true&sslmode=verify-ca&sslrootcert=/etc/ssl/ca.pem', + 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=verify-ca', + undefined, + ], + [ + 'uselibpqcompat=true with require beside a root certificate', + 'postgresql://app@db.example/app?uselibpqcompat=true&sslmode=require&sslrootcert=/etc/ssl/ca.pem', + 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=verify-ca', + undefined, + ], + [ + 'uselibpqcompat=true with verify-full and a root certificate', + 'postgresql://app@db.example/app?uselibpqcompat=true&sslmode=verify-full&sslrootcert=/etc/ssl/ca.pem', + 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=verify-full', + undefined, + ], ["pg's ssl=0", 'postgresql://app@db.example/app?ssl=0', 'postgresql://app@db.example:5432/app?sslmode=disable', undefined], [ "pg's uselibpqcompat and statement_timeout, which libpq does not know", @@ -757,9 +830,9 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { undefined, ], [ - 'a certificate file, which turns pg\'s TLS on', + 'a certificate file, which turns pg\'s TLS on and is what it checks the server against', 'postgresql://app@db.example/app?sslrootcert=/etc/ssl/ca.pem', - 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=require', + 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=verify-full', undefined, ], // pg-connection-string turns TLS on for a certificate file over ssl=0 @@ -767,14 +840,14 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { [ "pg's ssl=0 beside a certificate file, which turns pg's TLS on again", 'postgresql://app@db.example/app?ssl=0&sslrootcert=/etc/ssl/ca.pem', - 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=require', + 'postgresql://app@db.example:5432/app?sslrootcert=%2Fetc%2Fssl%2Fca.pem&sslmode=verify-full', undefined, ], // pg uses TLS for any sslmode but disable; libpq refuses one it does not know. [ - 'an sslmode libpq does not know, with which pg uses TLS', + 'an sslmode libpq does not know, with which pg uses TLS that checks the server', 'postgresql://app@db.example/app?sslmode=sometimes', - 'postgresql://app@db.example:5432/app?sslmode=require', + 'postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system', undefined, ], // pg decodes the URL's user name only where no user parameter is given. From 681bdf4bf26ac9ddbec327d1ce4c16c7ab8d4bdf Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 20:57:11 -0700 Subject: [PATCH 19/32] fix(exporter): have pg_dump check the server as pg does: verify-full, or verify-ca where pg checks no host name, against the connection string's root certificate, else the file PGSSLROOTCERT names, else the system's authorities --- src/exporters/pgConnection.ts | 67 +++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts index d19397e..726be65 100644 --- a/src/exporters/pgConnection.ts +++ b/src/exporters/pgConnection.ts @@ -99,12 +99,6 @@ const LIBPQ_PARAMETERS = new Set([ 'sslkeylogfile', ]); -/** - * The `sslmode` values with which libpq always uses TLS. A pool that checks the server's certificate keeps one that - * `PGSSLMODE` names. - */ -const TLS_SSLMODES = new Set(['require', 'verify-ca', 'verify-full']); - /** The `sslmode` values with which libpq connects without TLS to a server that offers none, as pg does with no `ssl`. */ const PLAIN_SSLMODES = new Set(['disable', 'allow', 'prefer']); @@ -164,30 +158,37 @@ const tlsOf = (ssl: unknown): PgTls => { }; /** - * The `sslmode` pg_dump gets so that libpq uses TLS as pg does, or undefined where libpq left to itself does. - * - No TLS: `disable` where the pool's settings or its connection string chose it (pg reads no `PGSSLMODE` then, - * and libpq's `prefer` or `allow` would use TLS with a server that offers it). Where no TLS is pg's default, + * The libpq settings pg_dump gets so that libpq uses TLS as pg does and checks the server as pg does, none where + * libpq left to itself does. + * - No TLS: `sslmode=disable` where the pool's settings or its connection string chose it (pg reads no `PGSSLMODE` + * then, and libpq's `prefer` or `allow` would use TLS with a server that offers it). Where no TLS is pg's default, * libpq's own default (`prefer`) and a `PGSSLMODE` of `disable` or `allow` connect where pg connects, and stay; * an empty `PGSSLMODE`, or one libpq does not know, which libpq refuses, gives way to `disable`. - * - TLS that checks no certificate: `require`, over any `PGSSLMODE`, since `verify-ca` or `verify-full` would - * refuse the certificate pg accepts. - * - TLS that checks the certificate: the connection string's `verify-ca` or `verify-full`, else any other `sslmode` - * it names as `require`, else a `PGSSLMODE` of `require`, `verify-ca` or `verify-full`, else `require`, which - * encrypts and checks no certificate: libpq checks one against files alone, where Node checks against its own - * certificate authorities. + * - TLS that checks no certificate: `sslmode=require`, over any `PGSSLMODE`, since `verify-ca` or `verify-full` + * would refuse the certificate pg accepts. + * - TLS that checks the certificate: Node's TLS checks it against Node's certificate authorities, or the root + * certificate the connection string names, and checks the host name, unless pg reads the string as libpq does + * (`uselibpqcompat=true`) with `verify-ca`, or `require` beside a root certificate. libpq checks a certificate with + * `verify-ca` and `verify-full` alone, the host name with `verify-full` alone, and against a root certificate file + * or, with `sslrootcert=system` (libpq 16 and later), the system's certificate authorities; its `require` checks + * nothing without a root certificate file (fe-secure-openssl.c). So `sslmode=verify-full`, or `verify-ca` where + * pg checks no host name, over any `PGSSLMODE`, against the connection string's `sslrootcert`, else the file + * `PGSSLROOTCERT` names, else `sslrootcert=system`. */ -const sslmodeFor = ( +const tlsSettingsFor = ( tls: PgTls, chosen: boolean, - ofString: string | undefined, - pgsslmode: string | undefined, -): string | undefined => { + hostNameChecked: boolean, + rootCertificateNamed: boolean, + env: Environment, +): Array<[string, string]> => { if (tls === 'none') { - return chosen || (pgsslmode !== undefined && !PLAIN_SSLMODES.has(pgsslmode)) ? 'disable' : undefined; + const pgsslmode = env.PGSSLMODE; + return chosen || (pgsslmode !== undefined && !PLAIN_SSLMODES.has(pgsslmode)) ? [['sslmode', 'disable']] : []; } - if (tls === 'unchecked') return 'require'; - if (ofString === 'verify-ca' || ofString === 'verify-full') return ofString; - return ofString === undefined && TLS_SSLMODES.has(pgsslmode ?? '') ? undefined : 'require'; + if (tls === 'unchecked') return [['sslmode', 'require']]; + const sslmode: [string, string] = ['sslmode', hostNameChecked ? 'verify-full' : 'verify-ca']; + return rootCertificateNamed || env.PGSSLROOTCERT ? [sslmode] : [sslmode, ['sslrootcert', 'system']]; }; /** @@ -316,7 +317,10 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio // (lib/connection-parameters.js reads PGSSLMODE only where the merged settings hold no ssl). let ssl: unknown = settings.ssl; let source: 'string' | 'settings' | 'environment' = settings.ssl !== undefined ? 'settings' : 'environment'; - let sslmodeOfString: string | undefined; + // Node's TLS checks the host name unless pg reads the string as libpq does with verify-ca, or require beside a root + // certificate (pg-connection-string index.js:116-133); the root certificate pg checks against, the string names. + let hostNameChecked = true; + let rootCertificateNamed = false; if (settings.connectionString) { const parsed = parseAsPg(settings.connectionString); @@ -333,8 +337,12 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio if (fromString !== undefined) { ssl = fromString; source = 'string'; - // An empty sslmode is none, to pg too. - sslmodeOfString = last('sslmode') || undefined; + const sslmode = last('sslmode'); + rootCertificateNamed = Boolean(last('sslrootcert')); + hostNameChecked = !( + last('uselibpqcompat') === 'true' && + (sslmode === 'verify-ca' || (sslmode === 'require' && rootCertificateNamed)) + ); } for (const [key, value] of parsed.parameters) { if (LIBPQ_PARAMETERS.has(key)) parameters.push([key, value]); @@ -353,8 +361,7 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio ); } } - const sslmode = sslmodeFor(tls, source !== 'environment', sslmodeOfString, env.PGSSLMODE); - if (sslmode !== undefined) parameters.push(['sslmode', sslmode]); + parameters.push(...tlsSettingsFor(tls, source !== 'environment', hostNameChecked, rootCertificateNamed, env)); const osUser = typeof process !== 'undefined' && process.platform === 'win32' ? env.USERNAME : env.USER; const resolvedUser = user || env.PGUSER || osUser || undefined; @@ -404,8 +411,8 @@ const passwordCallOf = async ( toldPgUnread = true; getStorageLogger().warn( { scope: 'PostgresExporter', pgVersion: pg.version ?? null, pgClient: pg.Client !== undefined }, - 'PostgresExporter could not read the pg this library loads, so it calls a pool\'s password function on no ' + - `client, with ${noArguments ? 'no arguments' : "the connection's user, database, host and port"}.`, + "The pg this library loads could not be read, so a pool's password function is called on no client, with " + + `${noArguments ? 'no arguments' : "the connection's user, database, host and port"}.`, ); } return { From 7a70338219a24e85d784ce1ba559a50d0c4256f6 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 20:59:40 -0700 Subject: [PATCH 20/32] test(exporter): libpq's TLS checks follow pg's own reading of a connection string, verify-ca with uselibpqcompat=true and no root certificate is refused as pg refuses it, a libpq older than 16 gets words for sslrootcert=system, and a setting right after a quoted password reaches the host libpq reads --- tests/postgres.exporter.tls.spec.ts | 49 ++++++++++++ tests/postgresExporter.args.spec.ts | 113 +++++++++++++++++++++++++++- tests/postgresExporter.spec.ts | 25 ++++++ 3 files changed, 186 insertions(+), 1 deletion(-) diff --git a/tests/postgres.exporter.tls.spec.ts b/tests/postgres.exporter.tls.spec.ts index 094c61b..1115814 100644 --- a/tests/postgres.exporter.tls.spec.ts +++ b/tests/postgres.exporter.tls.spec.ts @@ -97,4 +97,53 @@ describe.skipIf(!TLS_URL || !ROOT_CERT)('the PostgreSQL exporter, through pg_dum await pool.end(); } }); + + /** The server's URL with another host, which reaches the same server, and these parameters. */ + const urlWith = (host: string, parameters: Record): string => { + const url = new URL(TLS_URL ?? 'postgresql://localhost'); + url.hostname = host; + for (const [name, value] of Object.entries(parameters)) url.searchParams.set(name, value); + return url.toString(); + }; + + // The certificate names localhost and 127.0.0.1, and 127.0.0.2 reaches the + // same server. Node's TLS checks the host name, an IP address too, against + // the certificate, and libpq's verify-full does (fe-secure-common.c, + // pq_verify_peer_name_matches_certificate). + it('refuses, as pg does, a host the certificate does not name, though its root certificate signed it', async () => { + const pool = new pg.Pool({ connectionString: urlWith('127.0.0.2', { sslrootcert: ROOT_CERT as string }) }); + try { + await expect(pool.query('SELECT 1')).rejects.toThrow(/does not match certificate's altnames/); + + const file = path.join(dir, 'other-host.dump'); + await expect(createStorageFeatures(createPostgresAdapter({ pool })).exporter.exportToFile(file)).rejects.toThrow( + /pg_dump failed.*does not match host name/s, + ); + } finally { + await pool.end(); + } + }); + + // pg reads verify-ca as libpq does with uselibpqcompat=true, and checks the + // certificate against the root certificate and no host name + // (pg-connection-string index.js:125-132); pg_dump gets verify-ca. + it('exports, where pg connects, from a host the certificate does not name with uselibpqcompat=true and verify-ca', async () => { + const pool = new pg.Pool({ + connectionString: urlWith('127.0.0.2', { + uselibpqcompat: 'true', + sslmode: 'verify-ca', + sslrootcert: ROOT_CERT as string, + }), + }); + try { + await pool.query('SELECT 1'); + + const file = path.join(dir, 'verify-ca.dump'); + await createStorageFeatures(createPostgresAdapter({ pool })).exporter.exportToFile(file); + + expect(dumpedTables(file)).toContain(`public.${table}`); + } finally { + await pool.end(); + } + }); }); diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index 264825f..5b6f8b6 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -1,4 +1,7 @@ -import { describe, expect, it, afterEach, beforeEach, vi } from 'vitest'; +import { describe, expect, it, afterAll, afterEach, beforeAll, beforeEach, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; // PostgresExporter runs pg_dump. These specs read what it hands pg_dump: the // arguments, each passed as it is with no shell between, and the environment. @@ -243,6 +246,17 @@ describe('PostgresExporter', () => { ['a backslash before a space in a bare value', 'password=s3cr\\ et host=db.example', 'host=db.example', 's3cr et'], ['spaces around the equals sign', 'password = s3cret host=db.example', 'host=db.example', 's3cret'], ['two passwords, of which libpq takes the later', 'password=s3cr-first host=db.example password=s3cret', 'host=db.example', 's3cret'], + // After a closing quote libpq reads the next keyword at once, with no + // space between, and a later value of a keyword replaces the earlier one + // (fe-connect.c, conninfo_parse and conninfo_storeval): libpq reaches + // other.example from the string as given and from the string with its + // password taken out alike. + [ + 'a setting right after a quoted password, with no space between', + "host=db.example password='s3cret'host=other.example dbname=app", + 'host=db.example host=other.example dbname=app', + 's3cret', + ], ])('keeps the password of keyword=value settings off the command line, with %s', async (_name, connectionString, dbname, password) => { await new PostgresExporter(connectionString).exportToBytes(); @@ -310,6 +324,26 @@ describe('PostgresExporter', () => { await expect(new PostgresExporter('postgresql://app@db/app').exportToBytes()).rejects.toThrow(/pg_dump was not found/); }); + + // libpq reads sslrootcert=system as the system's certificate authorities + // from PostgreSQL 16 on. An older libpq looks for a file named system, and + // without it refuses verify-full (REL_15_STABLE fe-secure-openssl.c, + // initialize_SSL), in these words. + it('says what to do where the libpq of pg_dump is older than 16 and looks for a file named system', async () => { + outcome.error = Object.assign(new Error('Command failed: pg_dump'), { code: 1 }); + outcome.stderr = + 'pg_dump: error: connection to server at "db.example" (192.0.2.10), port 5432 failed: ' + + 'root certificate file "system" does not exist\n' + + 'Either provide the file or change sslmode to disable server certificate verification.'; + + const failure = await refusal( + new PostgresExporter('postgresql://app@db.example/app?sslmode=verify-full&sslrootcert=system').exportToBytes(), + ); + + expect(failure?.message).toMatch(/root certificate file "system" does not exist/); + expect(failure?.message).toMatch(/PostgreSQL 16 or later/); + expect(failure?.message).toMatch(/sslrootcert parameter.*PGSSLROOTCERT/s); + }); }); describe('createStorageFeatures exporter on PostgreSQL', () => { @@ -640,6 +674,83 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(calls[0].env.PGSSLROOTCERT).toBe('/etc/ssl/private-ca.pem'); }); + // pg refuses sslmode=verify-ca read as libpq reads it (uselibpqcompat=true) + // where no root certificate names what to check the server against + // (pg-connection-string index.js:125-130), so the pool makes no connection; + // pg_dump, which PGSSLROOTCERT would give a root certificate, makes none + // either. + it('refuses, as pg does, sslmode=verify-ca with uselibpqcompat=true and no root certificate', async () => { + const connectionString = 'postgresql://app@db.example/app?uselibpqcompat=true&sslmode=verify-ca'; + expect(() => new ConnectionParameters({ connectionString })).toThrow(/verify-ca requires specifying a CA/); + vi.stubEnv('PGSSLROOTCERT', '/etc/ssl/private-ca.pem'); + + const failure = await refusal(poolExporter({ connectionString }).exportToBytes()); + + expect(failure?.message).toMatch(/sslmode=verify-ca.*sslrootcert/s); + expect(calls).toHaveLength(0); + }); + + // What libpq is asked to check follows pg's own reading of the string, the + // ssl of its ConnectionParameters: no TLS (disable), TLS that checks nothing + // (require), the certificate alone, where pg's checkServerIdentity checks + // nothing (verify-ca), or the certificate and the host name (verify-full), + // against the root certificate the string names, which pg reads, else the + // system's certificate authorities. + describe("asks libpq to check what pg's own reading of a connection string checks", () => { + let dir = ''; + let root = ''; + + beforeAll(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pg-tls-parity-')); + root = path.join(dir, 'root.pem'); + // pg reads the file's text, and leaves checking it to Node's TLS. + fs.writeFileSync(root, 'ROOT PEM'); + }); + + afterAll(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it.each([ + 'sslmode=disable', + 'sslmode=prefer', + 'sslmode=require', + 'sslmode=verify-ca', + 'sslmode=verify-full', + 'sslmode=no-verify', + 'ssl=true', + 'ssl=0', + 'sslrootcert=ROOT', + 'sslmode=require&sslrootcert=ROOT', + 'sslmode=no-verify&sslrootcert=ROOT', + 'uselibpqcompat=true&sslmode=disable', + 'uselibpqcompat=true&sslmode=prefer', + 'uselibpqcompat=true&sslmode=require', + 'uselibpqcompat=true&sslmode=require&sslrootcert=ROOT', + 'uselibpqcompat=true&sslmode=verify-ca&sslrootcert=ROOT', + 'uselibpqcompat=true&sslmode=verify-full', + 'uselibpqcompat=true&sslmode=verify-full&sslrootcert=ROOT', + ])('with %s', async (query) => { + const connectionString = `postgresql://app@db.example/app?${query.replace('ROOT', encodeURIComponent(root))}`; + const ssl = new ConnectionParameters({ connectionString }).ssl as + | boolean + | { rejectUnauthorized?: unknown; checkServerIdentity?: unknown }; + const named = query.includes('sslrootcert=ROOT') ? root : null; + const expected = !ssl + ? { sslmode: 'disable', sslrootcert: null } + : typeof ssl === 'object' && ssl.rejectUnauthorized === false + ? { sslmode: 'require', sslrootcert: named } + : typeof ssl === 'object' && typeof ssl.checkServerIdentity === 'function' + ? { sslmode: 'verify-ca', sslrootcert: named } + : { sslmode: 'verify-full', sslrootcert: named ?? 'system' }; + + await poolExporter({ connectionString }).exportToBytes(); + + const written = new URL(dbnameOf(calls[0].args) as string).searchParams; + expect({ sslmode: written.get('sslmode'), sslrootcert: written.get('sslrootcert') }).toEqual(expected); + }); + }); + // pg_dump reads TLS material from files alone, and sslmode=require in their // place connected without the client certificate or checked no CA. it.each([ diff --git a/tests/postgresExporter.spec.ts b/tests/postgresExporter.spec.ts index c62672a..b44f2df 100644 --- a/tests/postgresExporter.spec.ts +++ b/tests/postgresExporter.spec.ts @@ -126,4 +126,29 @@ describe.skipIf(!process.env.POSTGRES_TEST_URL || !hasPgDump())('PostgresExporte expect(fs.existsSync(path.join(process.cwd(), marker))).toBe(false); expect(fs.existsSync(path.join(process.cwd(), `${marker}-sub`))).toBe(false); }); + + // After a closing quote libpq reads the next keyword at once, with no space + // between, and a later host replaces the earlier one (fe-connect.c, + // conninfo_parse and conninfo_storeval). pg_dump given the settings as they + // are reaches the server the later host names, and the export, which takes + // the quoted password out for PGPASSWORD, reaches the same one. + it('reaches, as libpq reads them, the host of a setting right after a quoted password', async () => { + const url = new URL(databaseUrl); + const quoted = decodeURIComponent(url.password).replace(/[\\']/g, '\\$&'); + const settings = + `host=unreachable.invalid password='${quoted}'host=${url.hostname} port=${url.port || '5432'} ` + + `user=${decodeURIComponent(url.username)} dbname=${database}`; + + const schema = execFileSync('pg_dump', [`--dbname=${settings}`, '--schema-only', '--no-password', '--table=exported_rows'], { + encoding: 'utf8', + env: { ...process.env, PGPASSWORD: '' }, + }); + expect(schema).toContain('CREATE TABLE public.exported_rows'); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pg-dump-')); + dirs.push(dir); + const target = path.join(dir, 'after-quote.custom'); + await new PostgresExporter(settings).exportToFile(target); + expect(fs.readFileSync(target).toString('latin1')).toContain('CREATE TABLE public.exported_rows'); + }); }); From 9fa1d8bb5aae73156af7bbdde7ff43020427fb8c Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 21:04:56 -0700 Subject: [PATCH 21/32] fix(exporter): refuse verify-ca with uselibpqcompat=true and no root certificate, as pg does, and say what to do where pg_dump's libpq is older than 16 and reads sslrootcert=system as a file --- src/exporters/PostgresExporter.ts | 15 ++++++++++++++- src/exporters/pgConnection.ts | 20 ++++++++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/exporters/PostgresExporter.ts b/src/exporters/PostgresExporter.ts index 5a5902d..0017931 100644 --- a/src/exporters/PostgresExporter.ts +++ b/src/exporters/PostgresExporter.ts @@ -25,6 +25,18 @@ export interface PgDumpInvocation { /** The largest dump `exportToBytes()` holds in memory: 256 MiB. */ const MAX_DUMP_BYTES = 256 * 1024 * 1024; +/** + * What libpq older than PostgreSQL 16 says of `sslrootcert=system`: it looks for a file of that name + * (fe-secure-openssl.c, initialize_SSL), where libpq 16 and later read the system's certificate authorities. + */ +const NO_SYSTEM_ROOTS = /root certificate file "system" does not exist/; + +/** What to do where pg_dump's libpq reads `sslrootcert=system` as a file. */ +const SYSTEM_ROOTS_ADVICE = + " libpq reads sslrootcert=system as the system's certificate authorities from PostgreSQL 16 on, and an older " + + 'libpq looks for a file named system: run a pg_dump of PostgreSQL 16 or later, or name the root certificate file ' + + "in the connection string's sslrootcert parameter or in PGSSLROOTCERT."; + /** * The scheme of a connection URI, at the very start. libpq knows it in lower * case alone; pg, which the adapter connects with, reads it in any case. So it @@ -311,9 +323,10 @@ export class PostgresExporter implements IDatabaseExporter { // pg_dump's own words. The error's message repeats the command // line, which can hold a password, so it is left out. const said = stderr ? stderr.toString('utf8').trim() : ''; + const advice = NO_SYSTEM_ROOTS.test(said) ? SYSTEM_ROOTS_ADVICE : ''; reject( new Error( - `PostgresExporter: pg_dump failed${typeof code === 'number' ? ` (exit code ${code})` : ''}${said ? `: ${said}` : '.'}`, + `PostgresExporter: pg_dump failed${typeof code === 'number' ? ` (exit code ${code})` : ''}${said ? `: ${said}` : '.'}${advice}`, ), ); }); diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts index 726be65..83257cb 100644 --- a/src/exporters/pgConnection.ts +++ b/src/exporters/pgConnection.ts @@ -304,7 +304,8 @@ const tlsSettingsBeyondSslmode = (ssl: unknown): string[] => * Reads connection settings as pg reads them for each connection of a pool: a connection string's parts over the * settings', then the environment, then pg's defaults. Undefined where pg cannot read the connection string. * - * @throws {Error} When the settings ask for TLS options libpq takes only from files. + * @throws {Error} When the settings ask for TLS options libpq takes only from files, or the connection string for + * `verify-ca` read as libpq reads it with no root certificate, which pg refuses. */ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnection | undefined => { let user = settings.user; @@ -338,11 +339,17 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio ssl = fromString; source = 'string'; const sslmode = last('sslmode'); + const compatible = last('uselibpqcompat') === 'true'; rootCertificateNamed = Boolean(last('sslrootcert')); - hostNameChecked = !( - last('uselibpqcompat') === 'true' && - (sslmode === 'verify-ca' || (sslmode === 'require' && rootCertificateNamed)) - ); + // pg refuses verify-ca read as libpq reads it with no root certificate to check the server against + // (pg-connection-string index.js:125-130), so the pool makes no connection, and pg_dump makes none either. + if (compatible && sslmode === 'verify-ca' && !rootCertificateNamed) { + throw new Error( + 'PostgresExporter: the connection string asks for sslmode=verify-ca with uselibpqcompat=true and names no ' + + 'sslrootcert, which pg refuses, so pg_dump does not connect either.', + ); + } + hostNameChecked = !(compatible && (sslmode === 'verify-ca' || (sslmode === 'require' && rootCertificateNamed))); } for (const [key, value] of parsed.parameters) { if (LIBPQ_PARAMETERS.has(key)) parameters.push([key, value]); @@ -462,7 +469,8 @@ const passwordOf = async (connection: PgConnection, settings: PgConnectionSettin * connection string pg cannot read as a URL (a keyword=value string) is given back as it is. Undefined when the * settings name no database pg would reach. * - * @throws {Error} When the settings ask for TLS that pg_dump cannot take, or the password function fails. + * @throws {Error} When the settings ask for TLS that pg_dump cannot take or that pg refuses, or the password function + * fails. */ export const libpqConnectionStringOf = async (settings: PgConnectionSettings): Promise => { const { connectionString } = settings; From 4053b3fea5847eda333d8806a67fb2c18728be7f Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 21:04:56 -0700 Subject: [PATCH 22/32] docs(exporter): pg_dump checks the server as pg does: verify-full, or verify-ca where pg checks no host name, and the root certificate it checks against --- CHANGELOG.md | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1de087f..b269efb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - `PostgresExporter` refuses, before `pg_dump` starts, a connection string whose password it cannot move to `PGPASSWORD`: a URI password that is not valid percent-encoding (`%zz`, a `%` at its end, bytes that are not UTF-8) or that decodes to a NUL, and a connection string that holds a NUL. 0.7.16 passed such a URI whole, its password in `--dbname`, where the process list shows it. The refusal never holds the connection string. - `PostgresExporter` moves the `password` of keyword=value settings (`host=db.example password='a b'`) to `PGPASSWORD` too, read as libpq reads them: a value runs to whitespace or sits in single quotes, a backslash takes the character after it, and the later of two passwords wins. An empty password goes to `PGPASSWORD` empty, as libpq reads an empty setting, so the environment's `PGPASSWORD` is never sent in its place. Settings libpq refuses (an unterminated quote, a setting with no `=`) are refused before `pg_dump` starts. 0.7.16 passed them whole on the command line. - The exporter of a PostgreSQL adapter leaves `sslpassword`, `oauth_client_secret` and the SCRAM keys of a pool's connection string off `pg_dump`'s command line, where libpq takes them alone: pg reads none of them. +- The exporter of a PostgreSQL adapter has `pg_dump` check the server as the pool's pg does. pg's TLS is Node's, which checks the server's certificate and its host name, and 0.7.16 wrote `sslmode=require` for it, with which libpq checks neither without a root certificate file: an export connected to a server whose certificate the pool refuses. `pg_dump` gets `sslmode=verify-full`, or `verify-ca` where pg checks no host name (`uselibpqcompat=true` with `verify-ca`, or `require` beside a root certificate), against the root certificate file the connection string names, else the file `PGSSLROOTCERT` names, else the system's certificate authorities (`sslrootcert=system`, which libpq reads from PostgreSQL 16 on; with an older libpq the export fails, saying what to do). `uselibpqcompat=true` with `sslmode=verify-ca` and no root certificate is refused before `pg_dump` starts, as pg refuses it. ### Changed - `new PostgresExporter(connectionString)` takes a function that gives the connection string, or a promise of it, when an export runs. diff --git a/README.md b/README.md index 149edfe..750062e 100644 --- a/README.md +++ b/README.md @@ -951,7 +951,7 @@ const bytes = await exporter.exportToBytes(); On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export as pg calls it for each connection: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0, and from 8.19.0 with the client's `ConnectionParameters`. The client is of the pg this library loads, so a pool made with another copy of pg, or with a `Client` of its own, is asked as this library's pg asks; it is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. Where it cannot read that pg's version or its `Client` (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, and an empty `host` parameter leaves the URL's host. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). -TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts. A pool that checks one (`ssl: true`, `{}`, a certificate file in the connection string) gets the connection string's `verify-ca` or `verify-full`, else a `PGSSLMODE` of `require`, `verify-ca` or `verify-full`, else `sslmode=require`, which encrypts and checks no certificate: libpq checks one against files alone. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. +TLS follows pg's, and so does the check of the server. pg's TLS is Node's, which checks the server's certificate and its host name unless `rejectUnauthorized` is false. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`, and with `uselibpqcompat=true` `sslmode=prefer`, or `require` with no root certificate) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts; libpq's `require` still checks the certificate against a root certificate file it finds (the connection string's `sslrootcert`, `PGSSLROOTCERT`, `~/.postgresql/root.crt`), so there `pg_dump` can refuse a server pg reaches. A pool that checks the certificate (`ssl: true`, `{}`, `ssl=1`, a certificate file or any other `sslmode` but `disable` in the connection string, or a `PGSSLMODE` pg reads as TLS) gets `sslmode=verify-full` over any `PGSSLMODE`, and `sslmode=verify-ca` where pg checks no host name (`uselibpqcompat=true` with `verify-ca`, or `require` beside a root certificate). libpq checks the certificate against the root certificate file the connection string names, which pg reads too, else the file `PGSSLROOTCERT` names, else the system's certificate authorities (`sslrootcert=system`: OpenSSL's default locations, which `SSL_CERT_FILE` and `SSL_CERT_DIR` move), where Node checks against the authorities it trusts by default (the Mozilla list it bundles, and `NODE_EXTRA_CA_CERTS`): an authority one trusts and the other does not is named in the connection string's `sslrootcert` or in `PGSSLROOTCERT`. libpq reads `sslrootcert=system` from PostgreSQL 16 on; an older one looks for a file named `system`, and the export fails, saying what to do. `uselibpqcompat=true` with `sslmode=verify-ca` and no root certificate is refused before `pg_dump` starts, as pg refuses it. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. `pg_dump` runs with no shell between and with `--no-password`, so it never waits for a password on a terminal or on its standard input, and the password of the connection string reaches it through `PGPASSWORD`, not its command line: a `postgres://` or `postgresql://` URI's, in its user part or its `password` parameter, and the `password` of keyword=value settings (`host=db.example password='a b'`), each read as libpq reads it. An empty password given in the string goes to `PGPASSWORD` empty, so the environment's is never sent in its place. A connection string that holds a NUL character, or a password that is not valid percent-encoding or decodes to a NUL, is refused before `pg_dump` starts, without the string in the message. libpq reads `sslpassword` and `oauth_client_secret` from the connection string alone, so a string handed to `new PostgresExporter()` with them keeps them in `--dbname`; the features exporter leaves them out, since pg reads neither. `new PostgresExporter(connectionString, { schema })` exports one schema of any database; `connectionString` may be a function that gives it, or a promise of it, when an export runs. From 33ba55703a1e4911180a178520fbfb3e73097eff Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 21:07:43 -0700 Subject: [PATCH 23/32] docs(changelog): 0.7.17 releases the mutation sync (#65), which merged with no version of its own --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b269efb..0918a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## [0.7.17] - 2026-10-17 +### Added +- Mutation sync for named text rows (#65), exported from `./sync`: a push, pull and rebase protocol beside the cross-platform sync. `MutationServer` runs over a store port (`createSqlMutationStore` keeps it on the library's own tables); `MutationClient` keeps durable saves and a journal, rebases, joins, leaves and recovers across restore epochs; `createFetchMutationTransport`, `SyncLoop` with `attachSyncLoop`, `DrainGate`, the tab locks and `changeChannel` drive it. Nothing that exists changes. + ### Security - `PostgresExporter` refuses, before `pg_dump` starts, a connection string whose password it cannot move to `PGPASSWORD`: a URI password that is not valid percent-encoding (`%zz`, a `%` at its end, bytes that are not UTF-8) or that decodes to a NUL, and a connection string that holds a NUL. 0.7.16 passed such a URI whole, its password in `--dbname`, where the process list shows it. The refusal never holds the connection string. - `PostgresExporter` moves the `password` of keyword=value settings (`host=db.example password='a b'`) to `PGPASSWORD` too, read as libpq reads them: a value runs to whitespace or sits in single quotes, a backslash takes the character after it, and the later of two passwords wins. An empty password goes to `PGPASSWORD` empty, as libpq reads an empty setting, so the environment's `PGPASSWORD` is never sent in its place. Settings libpq refuses (an unterminated quote, a setting with no `=`) are refused before `pg_dump` starts. 0.7.16 passed them whole on the command line. From a2e91192953ea74bf8350c063036e8a2a96b13f2 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 21:35:14 -0700 Subject: [PATCH 24/32] test(indexeddb): a PRAGMA assignment whose schema or name is written in single quotes, which SQLite reads as a name there, is stored before get() or all() resolves --- tests/indexedDbAdapter.durability.spec.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/indexedDbAdapter.durability.spec.ts b/tests/indexedDbAdapter.durability.spec.ts index a30d82d..73bda4c 100644 --- a/tests/indexedDbAdapter.durability.spec.ts +++ b/tests/indexedDbAdapter.durability.spec.ts @@ -149,14 +149,18 @@ describe('IndexedDbAdapter durability', () => { expect(await storedPragma('commit-pragma-header', 'user_version')).toBe(4); }); - // SQLite reads a comment as whitespace between two tokens, and a quoted - // name needs no space before it (sqlite.org/lang_comment.html). + // SQLite reads a comment as whitespace between two tokens, a quoted name + // needs no space before it (sqlite.org/lang_comment.html), and a string in + // single quotes stands for a name where a PRAGMA's goes (parse.y: + // nm ::= STRING; cmd ::= PRAGMA nm dbnm EQ nmnum). it.each([ ['a block comment before the name', 'get', 'PRAGMA /* note */ user_version = 5', 'user_version', 5], ['a block comment before the =', 'get', 'PRAGMA user_version /* note */ = 6', 'user_version', 6], ['a line comment before the =', 'all', 'PRAGMA user_version -- note\n= 7', 'user_version', 7], ['comments around the schema dot', 'all', 'PRAGMA main/* a */./* b */application_id = 8', 'application_id', 8], ['a quoted name with no space before it', 'get', 'PRAGMA"user_version"=9', 'user_version', 9], + ['a name in single quotes', 'get', "PRAGMA 'user_version' = 10", 'user_version', 10], + ['a schema and a name in single quotes, with no space before them', 'all', "PRAGMA'main'.'application_id'=11", 'application_id', 11], ] as const)('stores a PRAGMA assignment with %s before it resolves', async (_name, method, sql, pragma, value) => { const dbName = `commit-pragma-comment-${value}`; const adapter = await openWithItems(dbName); @@ -179,6 +183,7 @@ describe('IndexedDbAdapter durability', () => { await adapter.get('PRAGMA foreign_keys = ON'); await adapter.get('PRAGMA cache_size = 500'); expect(await adapter.get('PRAGMA /* = */ user_version -- = 3')).toEqual({ user_version: 0 }); + expect(await adapter.get("PRAGMA 'user_version'")).toEqual({ user_version: 0 }); expect(puts).not.toHaveBeenCalled(); } finally { From 3593412ddbdeca9a2d021e4fb6375e7fd5d5ba96 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 21:45:57 -0700 Subject: [PATCH 25/32] chore(release): 0.7.18 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b3db96e..d42fd8f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@framers/sql-storage-adapter", - "version": "0.7.17", + "version": "0.7.18", "description": "Robust cross-platform SQL storage abstraction with automatic fallbacks and runtime detection", "type": "module", "main": "dist/cjs/index.js", From 75024e0d6cb2e6c987b6ece4ee61dee55c57cb2d Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 21:51:16 -0700 Subject: [PATCH 26/32] fix(indexeddb): read a PRAGMA's schema or name in single quotes as SQLite does, as a name, so its assignment is saved --- CHANGELOG.md | 2 +- README.md | 2 +- src/adapters/indexedDbAdapter.ts | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3326ff7..7408604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`: a `PGSSLMODE` of `verify-ca` or `verify-full` refused the certificate pg accepts. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, where libpq's `prefer` or `allow` would use TLS. Where no TLS is pg's default, libpq's default stays, and an empty `PGSSLMODE` or one libpq does not know, which pg reads as no TLS and libpq refuses, gives way to `sslmode=disable`. A certificate file in the connection string turns TLS on over `ssl=0`, and an `sslmode` libpq does not know means TLS, as pg reads them. - pg takes the database from the URL's path alone and drops a `database` parameter, and reads an empty `host` parameter as none; the URI holds that reading (pg-connection-string, `index.js`). libpq's own variables, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT`), configure `pg_dump` as they configure psql. - `resolveStorageAdapter()` and `createDatabase()` open a PostgreSQL candidate in the segment of its own options or the top-level one alone. Tried after an IndexedDB or Capacitor candidate whose options named a segment, it took that segment: on a pool of the caller's the adapter threw, and on a connection string the server got a schema only the browser's options asked for. A local candidate still opens in it. -- IndexedDB adapter: a PRAGMA assignment through `all()` or `get()` with a comment between its words (`PRAGMA user_version /* note */ = 7`), or with a quoted name and no space before it (`PRAGMA"user_version"=9`), is saved; the statement is read as SQLite reads it. 0.7.17 found no assignment in it, so it was stored neither with `durability: 'commit'` nor by `flush()` or `close()`. +- IndexedDB adapter: a PRAGMA assignment through `all()` or `get()` with a comment between its words (`PRAGMA user_version /* note */ = 7`), with a quoted name and no space before it (`PRAGMA"user_version"=9`), or with its schema or name in single quotes (`PRAGMA 'user_version' = 10`), which SQLite takes for a name there, is saved; the statement is read as SQLite reads it. 0.7.17 found no assignment in it, so it was stored neither with `durability: 'commit'` nor by `flush()` or `close()`. - `createSqlKeyValueStore()` writes a value's text from the copy the runtime's `structuredClone` returns, so each property is read once, as IndexedDB's put reads it, and the value is copied once. 0.7.17 read each property a second time for the text: a getter's second answer was stored, and a value whose getter answered a function the second time was refused. - `PostgresExporter` runs `pg_dump` with `--no-password`. Without a password, to a server that asks for one, `pg_dump` asked for it on the terminal, or with no terminal read its standard input, which the export leaves open, and the export never ended. diff --git a/README.md b/README.md index 57c92cb..3d610eb 100644 --- a/README.md +++ b/README.md @@ -631,7 +631,7 @@ await adapter.flush(); // stores ev - ✅ Several tabs on one database (`crossTab`) - ✅ A database of its own per segment (`segment`) -**Durability.** With `durability: 'interval'`, the default, the auto-save tick stores the writes made since the last save every `saveIntervalMs`, and a write can be lost when the page goes before the next tick. With `durability: 'commit'`, `run()`, `exec()`, a write through `all()` or `get()` (`INSERT ... RETURNING`, or a PRAGMA that sets `user_version`, `application_id`, `schema_version`, `page_size`, `auto_vacuum` or `encoding`, read as SQLite reads it, a comment between two of its words included), `transaction()` and `importDatabase()` resolve once their write is in IndexedDB, committed there (the IndexedDB transaction's `complete` event, not the put request's success), and reject when IndexedDB refuses or aborts it; the write stays in the database and the next save stores it, unless `crossTab`'s `rollbackRefusedSaves` gives it up (below). Each save writes the whole database image, so batch many writes in one `transaction()`. While a transaction begun with `BEGIN` is open nothing can be saved: its statements resolve at once, and its rows are stored when its `COMMIT` resolves. `flush()` stores every write made so far in either mode: it waits for an open `transaction()`, and rejects while a transaction begun with `BEGIN` is open, which sql.js's export would end. +**Durability.** With `durability: 'interval'`, the default, the auto-save tick stores the writes made since the last save every `saveIntervalMs`, and a write can be lost when the page goes before the next tick. With `durability: 'commit'`, `run()`, `exec()`, a write through `all()` or `get()` (`INSERT ... RETURNING`, or a PRAGMA that sets `user_version`, `application_id`, `schema_version`, `page_size`, `auto_vacuum` or `encoding`, read as SQLite reads it, a comment between two of its words or a name in single quotes included), `transaction()` and `importDatabase()` resolve once their write is in IndexedDB, committed there (the IndexedDB transaction's `complete` event, not the put request's success), and reject when IndexedDB refuses or aborts it; the write stays in the database and the next save stores it, unless `crossTab`'s `rollbackRefusedSaves` gives it up (below). Each save writes the whole database image, so batch many writes in one `transaction()`. While a transaction begun with `BEGIN` is open nothing can be saved: its statements resolve at once, and its rows are stored when its `COMMIT` resolves. `flush()` stores every write made so far in either mode: it waits for an open `transaction()`, and rejects while a transaction begun with `BEGIN` is open, which sql.js's export would end. **Several tabs.** Each tab holds the whole database in memory and writes it back whole, so without coordination the tab that saves last overwrites the other's writes. With `crossTab`, every statement and transaction runs under a [Web Lock](https://developer.mozilla.org/en-US/docs/Web/API/Web_Locks_API) on the database, on the newest stored image (a tab reloads it when a version stamp stored beside the image says another tab has saved), and is stored before the lock is let go: `durability` is `'commit'` whatever it says, and a tab reads what another tab stored. A transaction holds the other tabs' statements until it commits or rolls back. A transaction begun with `BEGIN` through `run()` or `exec()` cannot be held across tabs: it is rolled back and refused, so use `transaction()`. Web Locks need a secure context and are in Chrome 69, Firefox 96 and Safari 15.4 and later (on WebKit the feature is on by default and off in Lockdown Mode). Where there are none, the version stamp alone keeps both tabs' writes: a statement that finds another tab saved first runs again on that tab's image, and a `transaction()` another tab saved over is refused with `IndexedDbConflictError`, storing nothing of it, for its caller to run again. The adapter says once when it runs without Web Locks. A tab running a version of this package before 0.7 writes no stamp, so a tab with `crossTab` does not see its saves. diff --git a/src/adapters/indexedDbAdapter.ts b/src/adapters/indexedDbAdapter.ts index 119761b..95600c6 100644 --- a/src/adapters/indexedDbAdapter.ts +++ b/src/adapters/indexedDbAdapter.ts @@ -296,7 +296,7 @@ const IMAGE_PRAGMAS_WHEN_SET = new Set(['user_version', 'application_id', 'schem /** PRAGMAs that change the image in any form: incremental_vacuum frees pages, optimize may run ANALYZE. */ const IMAGE_PRAGMAS_ALWAYS = new Set(['incremental_vacuum', 'optimize']); -/** A token of a statement's head: a name, bare or quoted, or one character of punctuation. */ +/** A token of a statement's head: a name, bare or quoted (a string in single quotes included), or one character of punctuation. */ interface HeadToken { text: string; name: boolean; @@ -309,7 +309,11 @@ const NAME_PART = /[A-Za-z0-9_$]/; * The first `count` tokens of a statement as SQLite's tokenizer reads them: a * comment (`-- ...` to the end of its line, `/* ... *\/`) or whitespace * separates two tokens and is no token itself, and a quoted name (in double - * quotes, brackets or backticks) needs no space before it. + * quotes, brackets or backticks) needs no space before it. A string in single + * quotes is read as a name as well: SQLite's grammar takes one for a name where + * a PRAGMA's schema or name goes (parse.y, `nm ::= STRING`), and its tokenizer + * ends it as it ends a quoted name, at a quote not written twice (tokenize.c, + * `CC_QUOTE`). */ const headTokens = (statement: string, count: number): HeadToken[] => { const tokens: HeadToken[] = []; @@ -324,7 +328,7 @@ const headTokens = (statement: string, count: number): HeadToken[] => { } else if (statement.startsWith('/*', at)) { const end = statement.indexOf('*/', at + 2); at = end < 0 ? statement.length : end + 2; - } else if (char === '"' || char === '`') { + } else if (char === '"' || char === '`' || char === "'") { // A quote character inside the name is written twice. let text = ''; let end = at + 1; From 0cbd32f8a4d526978ea5340d08e720e51c36720b Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 22:39:46 -0700 Subject: [PATCH 27/32] test(exporter): a pool's password function is asked through the pool's own Client, the libpq parameters pg does not read are left out, the port is read with parseInt as pg reads it, and an IPv6 host reaches libpq whole On CI's server: a pool whose Client asks its password function with no arguments, as pg before 8.19.0 does; a pool connection string with a passfile parameter pg does not read, beside the password file PGPASSFILE names; and a pool whose port is 5432x, which pg reads as 5432. --- tests/fixtures/clientOfEarlierPg.ts | 41 +++++++ tests/postgres.exporter.spec.ts | 95 +++++++++++++++ tests/postgresExporter.args.spec.ts | 180 ++++++++++++++++++++++++++++ 3 files changed, 316 insertions(+) create mode 100644 tests/fixtures/clientOfEarlierPg.ts diff --git a/tests/fixtures/clientOfEarlierPg.ts b/tests/fixtures/clientOfEarlierPg.ts new file mode 100644 index 0000000..ec8073b --- /dev/null +++ b/tests/fixtures/clientOfEarlierPg.ts @@ -0,0 +1,41 @@ +import pg from 'pg'; + +/** + * A pg `Client` that asks a password function as pg before 8.19.0 asks it: as a + * method of the connecting client, with no arguments. Its `_getPassword` asks + * a function as pg 8.18.0's does (lib/client.js), where the installed pg + * (8.19.0 and later) passes the client's ConnectionParameters, and leaves a + * password that is no function as it is. It stands for the Client of a pool made + * with another copy of pg than the one this library loads: pg-pool makes each + * connection with the Client its options name, else its pg's (pg-pool 3.14.0, + * index.js:95 and 241), and keeps it as the pool's `Client`. + */ +export class ClientOfEarlierPg extends pg.Client { + // pg's own fields, which @types/pg does not declare. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _getPassword(this: any, cb: () => void): void { + const con = this.connection; + if (typeof this.password === 'function') { + this._Promise + .resolve() + .then(() => this.password()) + .then((pass: unknown) => { + if (pass !== undefined) { + if (typeof pass !== 'string') { + con.emit('error', new TypeError('Password must be a string')); + return; + } + this.connectionParameters.password = this.password = pass; + } else { + this.connectionParameters.password = this.password = null; + } + cb(); + }) + .catch((err: unknown) => { + con.emit('error', err); + }); + } else { + cb(); + } + } +} diff --git a/tests/postgres.exporter.spec.ts b/tests/postgres.exporter.spec.ts index b5d0cbf..e561f11 100644 --- a/tests/postgres.exporter.spec.ts +++ b/tests/postgres.exporter.spec.ts @@ -5,6 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import pg from 'pg'; import { createPostgresAdapter, createStorageFeatures, type StorageAdapter } from '../src/index.js'; +import { ClientOfEarlierPg } from './fixtures/clientOfEarlierPg.js'; // The feature exporter runs pg_dump against the PostgreSQL that // POSTGRES_TEST_URL names (CI's service); the runner has the PostgreSQL client @@ -304,6 +305,100 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', } }); + // pg-pool makes each connection with the Client its options name (pg-pool + // 3.14.0, index.js:95 and 241). This pool's Client asks the password + // function as pg before 8.19.0 does, with no arguments, where the pg this + // library loads passes the client's ConnectionParameters, and the function + // answers the server's password to its pool's call alone. PGPASSWORD is + // empty, so pg_dump logs in with the function's answer or not at all. + it("logs pg_dump in with the password the pool's own Client asks its password function for", async () => { + const database = await newDatabase(); + const server = new URL(url); + vi.stubEnv('PGPASSWORD', ''); + const pool = new pg.Pool({ + Client: ClientOfEarlierPg, + host: server.hostname, + port: Number(server.port || 5432), + database, + user: decodeURIComponent(server.username), + password: async function (this: unknown, ...args: unknown[]) { + return args.length === 0 && this instanceof ClientOfEarlierPg ? decodeURIComponent(server.password) : 'not-the-password'; + }, + } as pg.PoolConfig); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE client_items (id INTEGER PRIMARY KEY)'); + + const file = path.join(dir, `${database}-client.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.client_items']); + } finally { + await adapter.close(); + await pool.end(); + } + }); + + // pg reads no passfile parameter: with no password it reads the file + // PGPASSFILE names, else ~/.pgpass (pgpass 1.0.5, lib/helper.js:56-58), + // where libpq reads the passfile parameter first. Given the parameter, + // pg_dump read another file than the pool, and logged in with another + // password. + it('logs pg_dump in from the password file the pool reads, past a passfile parameter pg does not read', async () => { + const database = await newDatabase(); + const server = new URL(url); + // libpq and pgpass read a password file only when no one else may read it. + const passwordFile = (name: string, password: string): string => { + const file = path.join(dir, name); + fs.writeFileSync(file, `*:*:*:*:${password}\n`, { mode: 0o600 }); + return file; + }; + vi.stubEnv('PGPASSWORD', undefined); + vi.stubEnv('PGPASSFILE', passwordFile(`${database}-read.pgpass`, decodeURIComponent(server.password))); + const other = passwordFile(`${database}-other.pgpass`, 'not-the-password'); + const pool = new pg.Pool({ + connectionString: `postgresql://${server.username}@${server.host}/${database}?passfile=${encodeURIComponent(other)}`, + }); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE passfile_items (id INTEGER PRIMARY KEY)'); + + const file = path.join(dir, `${database}-passfile.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.passfile_items']); + } finally { + await adapter.close(); + await pool.end(); + } + }); + + // pg reads a pool's port with parseInt (lib/connection-parameters.js:70), + // so 5432x is 5432 to pg, and libpq refused it as no number. + it('exports through a pool whose port pg reads with parseInt, where pg reaches', async () => { + const database = await newDatabase(); + const server = new URL(url); + const pool = new pg.Pool({ + host: server.hostname, + port: `${server.port || 5432}x` as unknown as number, + database, + user: decodeURIComponent(server.username), + password: decodeURIComponent(server.password), + }); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE port_items (id INTEGER PRIMARY KEY)'); + + const file = path.join(dir, `${database}-port.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.port_items']); + } finally { + await adapter.close(); + await pool.end(); + } + }); + // open({ connectionString }) gives the adapter a database its options do not // name: the exporter reads the pool the adapter runs on when an export runs. it('exports an adapter given its connection string at open(), from features made before it opened', async () => { diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index 5b6f8b6..a232ef7 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -39,6 +39,7 @@ vi.mock('../src/exporters/pgModule.js', async (importOriginal) => { }); import { Client as PgClient } from 'pg'; +import { ClientOfEarlierPg } from './fixtures/clientOfEarlierPg.js'; import { PostgresExporter } from '../src/exporters/PostgresExporter.js'; import { createStorageFeatures } from '../src/core/contracts/features.js'; import type { StorageAdapter } from '../src/core/contracts/index.js'; @@ -620,6 +621,81 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); }); + // pg-pool makes each connection with the Client its options name, else its + // pg's, and keeps it as the pool's Client (pg-pool 3.14.0, index.js:95 and + // 241). A pool made with another copy of pg asks its password function as + // that copy asks it, so the export asks through the pool's own Client, + // whatever pg this library loads. + it("asks the password function as the pool's own Client asks it, where that Client's pg is earlier than the one this library loads", async () => { + pgRead.as = { version: '8.23.1', Client: PgClient }; + const seen: Array<{ self: unknown; args: unknown[] }> = []; + const settings = { + host: 'db.example', + database: 'app', + user: 'app', + // Written for the pool's pg, which gives it no argument. + password: function (this: unknown, ...args: unknown[]) { + seen.push({ self: this, args }); + return args.length === 0 && this instanceof ClientOfEarlierPg ? 'pool-secret' : 'another-secret'; + }, + }; + const { exporter } = createStorageFeatures(postgresAdapter({ pool: { options: settings, Client: ClientOfEarlierPg } })); + + await exporter.exportToBytes(); + + expect(seen).toHaveLength(1); + expect(seen[0].args).toEqual([]); + expect(seen[0].self).toBeInstanceOf(ClientOfEarlierPg); + expect(seen[0].self).toMatchObject({ user: 'app', database: 'app', host: 'db.example', port: 5432 }); + expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); + }); + + // The installed pg is 8.19.0 or later, which passes the client's + // ConnectionParameters. + it("asks the password function as the pool's own Client asks it, where this library loads an earlier pg", async () => { + pgRead.as = { version: '8.18.0', Client: ClientOfEarlierPg }; + const seen: Array<{ self: unknown; args: unknown[] }> = []; + const settings = { + host: 'db.example', + database: 'app', + user: 'app', + password: function (this: unknown, ...args: unknown[]) { + seen.push({ self: this, args }); + return 'pool-secret'; + }, + }; + const { exporter } = createStorageFeatures(postgresAdapter({ pool: { options: settings, Client: PgClient } })); + + await exporter.exportToBytes(); + + expect(seen).toHaveLength(1); + expect(seen[0].self).toBeInstanceOf(PgClient); + expect(seen[0].self).not.toBeInstanceOf(ClientOfEarlierPg); + expect(seen[0].args).toHaveLength(1); + expect(seen[0].args[0]).toBe((seen[0].self as { connectionParameters: unknown }).connectionParameters); + expect(seen[0].args[0]).toBeInstanceOf(ConnectionParameters); + expect(calls[0].env.PGPASSWORD).toBe('pool-secret'); + }); + + it("refuses, asking through the pool's own Client, a password function that fails or answers with no string", async () => { + const exporterWith = (password: () => unknown) => + createStorageFeatures( + postgresAdapter({ pool: { options: { host: 'db.example', database: 'app', user: 'app', password }, Client: PgClient } }), + ).exporter; + + const failed = await refusal( + exporterWith(async () => { + throw new Error('token service down'); + }).exportToBytes(), + ); + const numeric = await refusal(exporterWith(async () => 42).exportToBytes()); + + expect(failed?.message).toMatch(/password function of the pool failed/); + expect((failed as (Error & { cause?: unknown }) | null)?.cause).toMatchObject({ message: 'token service down' }); + expect(numeric?.message).toMatch(/password function of the pool answered no string/); + expect(calls).toHaveLength(0); + }); + // pg's TLS is Node's, which checks the server's certificate against Node's // certificate authorities, and its host name, unless rejectUnauthorized is // false. libpq checks a certificate with verify-ca or verify-full alone, the @@ -976,6 +1052,30 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { 'postgresql://app@db.example:5432/app?application_name=reports', undefined, ], + // pg reads options, application_name, fallback_application_name, + // client_encoding, replication and sslnegotiation of a connection string + // (lib/connection-parameters.js:63-124), and the certificate files + // (pg-connection-string index.js:88-100); libpq's other parameters it + // does not read. Given them, pg_dump read another password file + // (passfile), or refused a server, an authentication or a session the + // pool takes (target_session_attrs, require_auth, channel_binding, + // gssencmode, sslcrl). + [ + 'libpq parameters pg does not read, beside ones it reads', + 'postgresql://app@db.example/app?passfile=/etc/other.pgpass&application_name=reports&target_session_attrs=read-write' + + '&require_auth=scram-sha-256&channel_binding=require&gssencmode=require&connect_timeout=2&keepalives=0&sslsni=0' + + '&sslcrl=/etc/ssl/crl.pem&options=-c%20work_mem%3D64MB&client_encoding=UTF8&fallback_application_name=app' + + '&replication=database', + 'postgresql://app@db.example:5432/app?application_name=reports&options=-c%20work_mem%3D64MB&client_encoding=UTF8' + + '&fallback_application_name=app&replication=database', + undefined, + ], + [ + 'sslnegotiation=direct, which pg reads and which turns its TLS on', + 'postgresql://app@db.example/app?sslnegotiation=direct', + 'postgresql://app@db.example:5432/app?sslnegotiation=direct&sslmode=verify-full&sslrootcert=system', + undefined, + ], ])('reads the connection string of a pool as pg does, with %s', async (_name, connectionString, dbname, password) => { vi.stubEnv('PGUSER', 'app'); @@ -993,6 +1093,62 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(dbnameOf(calls[0].args)).toBe('postgresql://app@env-host.example:5432/app'); }); + // pg reads the port with parseInt (lib/connection-parameters.js:70), after + // its fallbacks: a setting that is empty or 0 gives way to PGPORT, then + // 5432 (val(), :9-23). A port with characters after its digits is that + // number to pg, and libpq refused it as no number. + it.each([ + [ + 'a port parameter with characters after its digits', + { connectionString: 'postgresql://app@db.example/app?port=6543x' }, + undefined, + 'postgresql://app@db.example:6543/app', + ], + [ + 'a port setting with characters after its digits', + { host: 'db.example', port: '6543x', database: 'app', user: 'app' }, + undefined, + 'postgresql://app@db.example:6543/app', + ], + [ + 'PGPORT with characters after its digits', + { host: 'db.example', database: 'app', user: 'app' }, + '6543x', + 'postgresql://app@db.example:6543/app', + ], + [ + 'a port setting of 0, which pg reads as none', + { host: 'db.example', port: 0, database: 'app', user: 'app' }, + '6432', + 'postgresql://app@db.example:6432/app', + ], + [ + 'a port setting with a space before its digits', + { host: 'db.example', port: ' 6543', database: 'app', user: 'app' }, + undefined, + 'postgresql://app@db.example:6543/app', + ], + ])('writes the port pg reads, with %s', async (_name, settings, pgport, dbname) => { + if (pgport !== undefined) vi.stubEnv('PGPORT', pgport); + + await poolExporter(settings).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + expect(Number(new URL(dbname).port)).toBe(new ConnectionParameters(settings).port); + }); + + // pg's parseInt reads no number in this port, and the pool makes no + // connection with it. + it('refuses a port pg reads no number in, before pg_dump starts', async () => { + const failure = await refusal( + poolExporter({ host: 'db.example', port: 'five', database: 'app', user: 'app' }).exportToBytes(), + ); + + expect(Number.isNaN(new ConnectionParameters({ port: 'five' }).port)).toBe(true); + expect(failure?.message).toMatch(/port/); + expect(calls).toHaveLength(0); + }); + // pg-connection-string 2.14.1 copies the query's parameters, then sets the // database from the URL's path alone, null without one (index.js:40-42, // 66-67): pg reads no database parameter, and reaches PGDATABASE, then the @@ -1026,6 +1182,7 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { ['postgresql://[::1]:6543/app'], ['postgresql://app@db.example:5433/app?port=6000'], ['postgresql://app@db.example:5433/app?port='], + ['postgresql://app@db.example/app?port=6543x'], ['postgresql://app@db.example/app?user=other'], ['postgresql://%61pp@db.example/r%C3%A9ports'], ['socket:/var/run/postgresql?db=app'], @@ -1062,6 +1219,29 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(dbnameOf(calls[0].args)).toBe('postgresql://app@%2Fvar%2Frun%2Fpostgresql:5432/app'); }); + // libpq reads a URI's host up to its first ':', '/', '?' or ',', unless it + // is in brackets, and percent-decodes it (fe-connect.c, REL_18_STABLE: + // conninfo_uri_parse_options, 6953-6964, and conninfo_storeval with + // uri_decode, 6995-6998). encodeURIComponent writes a ':' as %3A and a '%' + // as %25, so an IPv6 address, a zone included, reaches libpq whole. + it.each([ + ['an IPv6 host setting', { host: '::1', database: 'app', user: 'app' }, 'postgresql://app@%3A%3A1:5432/app'], + [ + 'an IPv6 host in brackets in a connection string', + { connectionString: 'postgresql://app@[::1]:6543/app' }, + 'postgresql://app@%3A%3A1:6543/app', + ], + [ + 'an IPv6 host with a zone', + { host: 'fe80::1%eth0', database: 'app', user: 'app' }, + 'postgresql://app@fe80%3A%3A1%25eth0:5432/app', + ], + ])('writes %s so that libpq reads the whole address as the host', async (_name, settings, dbname) => { + await poolExporter(settings).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + }); + it('gives PGPASSWORD a password set with no user name', async () => { const { exporter } = createStorageFeatures(postgresAdapter({ host: 'db.example', database: 'app', password: 's3cret' })); From f77bc9d9baead93295d1767e053cbfdc5aadbeea Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 22:42:27 -0700 Subject: [PATCH 28/32] fix(exporter): ask a pool's password function through the pool's own Client, leave out the libpq parameters pg does not read, and read the port with parseInt as pg does pg-pool keeps the Client it makes its connections with as the pool's Client, and that Client's own routine (_getPassword, _checkPgPass in pg 8.13.1 and before) asks the function on a client of the pool's settings with its pg's arguments, so a pool of another copy of pg is asked as its own pg asks. The URI keeps the parameters pg reads (options, application_name, fallback_application_name, client_encoding, replication, sslnegotiation and the certificate files); passfile, target_session_attrs, require_auth and the rest made pg_dump read another password file or refuse what the pool takes. A port of 5432x is 5432 and a port of 0 gives way to PGPORT, as pg reads them; a port pg reads no number in is refused. --- src/core/contracts/features.ts | 31 +++-- src/exporters/pgConnection.ts | 216 +++++++++++++++++++++++---------- 2 files changed, 176 insertions(+), 71 deletions(-) diff --git a/src/core/contracts/features.ts b/src/core/contracts/features.ts index 64e6933..bb89346 100644 --- a/src/core/contracts/features.ts +++ b/src/core/contracts/features.ts @@ -14,17 +14,26 @@ import { SqliteFileExporter } from '../../exporters/SqliteFileExporter.js'; import { PostgresExporter, type PostgresExporterOptions } from '../../exporters/PostgresExporter.js'; import { libpqConnectionStringOf, type PgConnectionSettings } from '../../exporters/pgConnection.js'; +/** + * A pg Pool as the exporter reads one: the settings it was made with (`options`) and the `Client` it makes its + * connections with, which pg-pool keeps (pg-pool 3.14.0, `index.js:95`). + */ +interface PgPoolLike { + options?: PgConnectionSettings; + Client?: unknown; +} + interface PostgresAdapterOptionsLike extends PgConnectionSettings { segment?: string; - /** A pool of the caller's; a pg Pool keeps the settings it was made with as `options`. */ - pool?: { options?: PgConnectionSettings } | null; + /** A pool of the caller's. */ + pool?: PgPoolLike | null; } /** What the exporter reads of a Postgres adapter. */ interface PostgresAdapterLike { options?: PostgresAdapterOptionsLike; /** The pool the adapter runs on: the pool it made when it opened, or its caller's; null while it is closed. */ - getPool?: () => { options?: PgConnectionSettings } | null; + getPool?: () => PgPoolLike | null; } /** @@ -50,9 +59,10 @@ const ownPoolSettings = (options: PostgresAdapterOptionsLike | undefined): PgCon * The exporter of a Postgres adapter. When an export runs it reads the * connection of the pool the adapter runs on, as pg reads it (the connection * string `open()` was given included, and a password function of the pool's, - * which it calls), and hands pg_dump a libpq URI of it; before the adapter - * opens, the settings of the pool it will make, or of the pool its caller gave - * it. A segmented adapter exports its segment's schema alone. + * which it asks through the pool's own Client), and hands pg_dump a libpq URI + * of it; before the adapter opens, the settings of the pool it will make, or of + * the pool its caller gave it. A segmented adapter exports its segment's schema + * alone. */ function postgresExporterOf(adapter: StorageAdapter): PostgresExporter { const postgres = adapter as StorageAdapter & PostgresAdapterLike; @@ -65,10 +75,15 @@ function postgresExporterOf(adapter: StorageAdapter): PostgresExporter { : 'PostgresExporter: the adapter names no database for pg_dump to reach. Make it with a connectionString, or ' + 'with host, database and user, or export once open() has been given a connection string.', }; + // A password function is asked through the Client of the pool whose settings are read; the pool the adapter will + // make is of the pg this library loads. const connectionString = async (): Promise => { const pool = typeof postgres.getPool === 'function' ? postgres.getPool() : null; - const settings = pool?.options ?? ownPoolSettings(options) ?? options?.pool?.options; - return settings ? libpqConnectionStringOf(settings) : undefined; + if (pool?.options) return libpqConnectionStringOf(pool.options, pool.Client); + const own = ownPoolSettings(options); + if (own) return libpqConnectionStringOf(own); + const given = options?.pool; + return given?.options ? libpqConnectionStringOf(given.options, given.Client) : undefined; }; return new PostgresExporter(connectionString, exportOptions); } diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts index 83257cb..a49070e 100644 --- a/src/exporters/pgConnection.ts +++ b/src/exporters/pgConnection.ts @@ -9,7 +9,7 @@ * that comes before any `/`, and it refuses a query parameter it does not know. So * `postgresql://db.example?application_name=x@y` names the host `db.example` to pg and `y` to libpq. The URI * written here holds pg's reading: the user, the host, the port and the database spelled out and percent-encoded, - * the parameters libpq knows, and an `sslmode` wherever libpq, left to itself, would use TLS differently from pg. + * the parameters pg reads that libpq takes, and the `sslmode` that has libpq use TLS and check the server as pg does. * * The variables pg reads are read here as pg reads them. libpq's own, which pg does not read (`PGHOSTADDR`, * `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure pg_dump as they configure psql. @@ -51,52 +51,28 @@ const environment = (): Environment => typeof process !== 'undefined' && process.env ? (process.env as Environment) : {}; /** - * The parameters libpq takes in a connection URI (PostgreSQL 18, `fe-connect.c`, `PQconninfoOptions`), less the - * ones the URI spells out (`user`, `password`, `host`, `port`, `dbname`), `sslmode`, which is written for the TLS - * pg uses, and the ones pg does not read (`service`, `hostaddr`). libpq refuses a URI with any other, so pg's own - * (`statement_timeout`, `uselibpqcompat`) and names neither knows are left out. So are `sslpassword`, - * `oauth_client_secret`, `scram_client_key` and `scram_server_key`: libpq takes them from the connection string - * alone, which is pg_dump's command line, and pg reads none of them, so the connection pg makes needs none. + * The parameters of a connection string that pg reads and that libpq takes in a connection URI (PostgreSQL 18, + * `fe-connect.c`, `PQconninfoOptions`): the settings pg sends when it connects (`options`, `application_name`, + * `fallback_application_name`, `client_encoding`, `replication`; pg 8.23.1 `lib/connection-parameters.js:63-124`), + * `sslnegotiation`, which pg reads from 8.22.0, and the certificate files pg-connection-string reads (`sslcert`, + * `sslkey`, `sslrootcert`; `index.js:88-100`). The URI spells out the user, the host, the port and the database, and + * `sslmode` is written for the TLS pg uses. libpq's other parameters pg does not read, and pg_dump given them would + * read another password file than the pool (`passfile`), or refuse a server, an authentication or a session the pool + * takes (`target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `sslcrl` and the rest), so they + * are left out; so are pg's own (`statement_timeout`, `uselibpqcompat`), which libpq refuses in a URI, and + * `sslpassword`, `oauth_client_secret`, `scram_client_key` and `scram_server_key`, which libpq takes from the + * connection string alone, pg_dump's command line. */ -const LIBPQ_PARAMETERS = new Set([ - 'passfile', - 'channel_binding', - 'connect_timeout', - 'client_encoding', +const PG_READ_PARAMETERS = new Set([ 'options', 'application_name', 'fallback_application_name', - 'keepalives', - 'keepalives_idle', - 'keepalives_interval', - 'keepalives_count', - 'tcp_user_timeout', + 'client_encoding', + 'replication', 'sslnegotiation', - 'sslcompression', 'sslcert', 'sslkey', - 'sslcertmode', 'sslrootcert', - 'sslcrl', - 'sslcrldir', - 'sslsni', - 'requirepeer', - 'require_auth', - 'min_protocol_version', - 'max_protocol_version', - 'ssl_min_protocol_version', - 'ssl_max_protocol_version', - 'gssencmode', - 'krbsrvname', - 'gsslib', - 'gssdelegation', - 'replication', - 'target_session_attrs', - 'load_balance_hosts', - 'oauth_issuer', - 'oauth_client_id', - 'oauth_scope', - 'sslkeylogfile', ]); /** The `sslmode` values with which libpq connects without TLS to a server that offers none, as pg does with no `ssl`. */ @@ -305,13 +281,14 @@ const tlsSettingsBeyondSslmode = (ssl: unknown): string[] => * settings', then the environment, then pg's defaults. Undefined where pg cannot read the connection string. * * @throws {Error} When the settings ask for TLS options libpq takes only from files, or the connection string for - * `verify-ca` read as libpq reads it with no root certificate, which pg refuses. + * `verify-ca` read as libpq reads it with no root certificate, or a port pg reads no number in, which pg refuses. */ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnection | undefined => { let user = settings.user; let password = settings.password; let host = settings.host; - let port = settings.port === undefined ? undefined : String(settings.port); + // As the settings give it: pg reads 0 and an empty port as none (its val() takes a falsy setting for none). + let port: unknown = settings.port; let database = settings.database; const parameters: Array<[string, string]> = []; // pg's ssl: the connection string's where its parameters make one, else the settings', else PGSSLMODE's @@ -352,7 +329,7 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio hostNameChecked = !(compatible && (sslmode === 'verify-ca' || (sslmode === 'require' && rootCertificateNamed))); } for (const [key, value] of parsed.parameters) { - if (LIBPQ_PARAMETERS.has(key)) parameters.push([key, value]); + if (PG_READ_PARAMETERS.has(key)) parameters.push([key, value]); } } if (source === 'environment') ssl = sslFromEnvironment(env.PGSSLMODE); @@ -370,13 +347,23 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio } parameters.push(...tlsSettingsFor(tls, source !== 'environment', hostNameChecked, rootCertificateNamed, env)); + // pg reads the port with parseInt, after the setting, PGPORT and 5432 (lib/connection-parameters.js:70), so + // `5432x` is 5432, which libpq would refuse as no number. + const portNumber = Number.parseInt(String(port || env.PGPORT || 5432), 10); + if (Number.isNaN(portNumber)) { + throw new Error( + "PostgresExporter: pg reads no number in the pool's port, so the pool makes no connection, and pg_dump makes " + + 'none either.', + ); + } + const osUser = typeof process !== 'undefined' && process.platform === 'win32' ? env.USERNAME : env.USER; const resolvedUser = user || env.PGUSER || osUser || undefined; return { user: resolvedUser, database: database || env.PGDATABASE || resolvedUser, host: host || env.PGHOST || 'localhost', - port: port || env.PGPORT || '5432', + port: String(portNumber), password: password || undefined, parameters, }; @@ -398,10 +385,10 @@ let toldPgUnread = false; * client made of the pool's settings (pg-pool `newClient`: `new this.Client(this.options)`), which pg calls the * function as a method of, with no arguments before pg 8.19.0 and from 8.19.0 with the client's * `ConnectionParameters` (node-postgres #3602; `lib/client.js` `_getPassword`: - * `this.password(this.connectionParameters)`). The client is of the pg this library loads, made without the - * settings' `stream`, a function pg calls to open the socket, and never connected. Where that pg cannot be read, the - * function gets no receiver and the connection's user, database, host and port, as pg 8.19.0 and later give it - * theirs, or nothing where the version read is older. + * `this.password(this.connectionParameters)`), where the pool names no `Client` that asks it itself. The client is of + * the pg this library loads, made without the settings' `stream`, a function pg calls to open the socket, and never + * connected. Where that pg cannot be read, the function gets no receiver and the connection's user, database, host + * and port, as pg 8.19.0 and later give it theirs, or nothing where the version read is older. */ const passwordCallOf = async ( connection: PgConnection, @@ -430,33 +417,132 @@ const passwordCallOf = async ( }; }; +/** pg's words for an answer of a password function that is not a string (`lib/client.js`, `_getPassword`). */ +const NOT_A_STRING = 'Password must be a string'; + +/** + * The refusal of an export whose password function failed, or answered no string, which pg reports as a + * `TypeError` in its own words. The function's own error is kept as the cause, out of the message, which a log may + * show. + */ +const passwordFailure = (error: unknown): Error => { + if (error instanceof TypeError && error.message === NOT_A_STRING) { + return new Error('PostgresExporter: the password function of the pool answered no string, which pg refuses too.'); + } + const failure: Error & { cause?: unknown } = new Error( + 'PostgresExporter: the password function of the pool failed, so pg_dump has no password.', + ); + failure.cause = error; + return failure; +}; + +/** The connection of a pg `Client`, whose `error` event reports what its password routine meets. */ +interface ClientConnection { + once(event: 'error', listener: (error: unknown) => void): unknown; + removeListener?(event: 'error', listener: (error: unknown) => void): unknown; +} + +/** A pg `Client` as the exporter uses one: its password, its connection and the routine that asks a function. */ +interface AskingClient { + password?: unknown; + connection?: unknown; + _getPassword?: (done: () => void) => void; + _checkPgPass?: (done: () => void) => void; +} + +const hasEvents = (value: unknown): value is ClientConnection => + value !== null && typeof value === 'object' && typeof (value as { once?: unknown }).once === 'function'; + +/** + * The name of the routine with which a pg `Client` class asks a password function for each connection: + * `_getPassword`, which pg 8.13.1 and earlier name `_checkPgPass`. Undefined for a class with neither, which is no + * pg `Client` (pg's native one among them). + */ +const askingRoutineOf = (PoolClient: unknown): '_getPassword' | '_checkPgPass' | undefined => { + const prototype = typeof PoolClient === 'function' ? (PoolClient as { prototype?: unknown }).prototype : undefined; + if (prototype === null || typeof prototype !== 'object') return undefined; + if (typeof (prototype as AskingClient)._getPassword === 'function') return '_getPassword'; + if (typeof (prototype as AskingClient)._checkPgPass === 'function') return '_checkPgPass'; + return undefined; +}; + +/** + * Asks a pool's password function through the pool's own `Client`, as that Client asks it for each connection the + * pool makes (pg-pool 3.14.0, `newClient`, `index.js:241`): the Client's routine runs on a client of the pool's + * settings, made without the settings' `stream` and `connection` and never connected, and calls the function on that + * client with the arguments its pg gives, none before pg 8.19.0 and the client's `ConnectionParameters` from 8.19.0. + * So a pool made with another copy of pg than the one this library loads is asked as its own pg asks. The answer is + * the client's password once the routine is done: a string, or undefined where the function answered none. + * Undefined where the Client has no such routine, or no connection whose errors it can be heard on. + * + * @throws {unknown} What the routine reports on the client's connection: the function's own error, or pg's + * `TypeError` for an answer that is not a string. + */ +const askThroughPoolClient = async ( + PoolClient: unknown, + settings: PgConnectionSettings, +): Promise<{ answer: string | undefined } | undefined> => { + const routine = askingRoutineOf(PoolClient); + if (routine === undefined) return undefined; + const client = new (PoolClient as new (config: object) => AskingClient)({ + ...settings, + // pg-pool keeps the settings' password from enumeration, so it is copied by name. + password: settings.password, + stream: undefined, + connection: undefined, + }); + const ask = client[routine]; + const { connection } = client; + if (typeof ask !== 'function' || !hasEvents(connection)) return undefined; + return new Promise((resolve, reject) => { + const failed = (error: unknown): void => reject(error); + connection.once('error', failed); + try { + ask.call(client, () => { + connection.removeListener?.('error', failed); + resolve({ answer: typeof client.password === 'string' ? client.password : undefined }); + }); + } catch (error) { + connection.removeListener?.('error', failed); + reject(error); + } + }); +}; + /** * The password pg sends for a connection: the settings' string, or the answer of the settings' function, which pg - * calls on the connecting client, with its `ConnectionParameters` from pg 8.19.0. Undefined where neither gives one. + * calls on the connecting client, with its `ConnectionParameters` from pg 8.19.0. The function is asked through the + * pool's own `Client` where the pool names one, as pg-pool's pools do, and otherwise as the pg this library loads + * asks it. Undefined where neither gives one. * * @throws {Error} When the function fails or answers something other than a string or undefined. */ -const passwordOf = async (connection: PgConnection, settings: PgConnectionSettings): Promise => { +const passwordOf = async ( + connection: PgConnection, + settings: PgConnectionSettings, + PoolClient?: unknown, +): Promise => { const { password } = connection; if (typeof password !== 'function') { return typeof password === 'string' ? password : undefined; } + let asked: { answer: string | undefined } | undefined; + try { + asked = await askThroughPoolClient(PoolClient, settings); + } catch (error) { + throw passwordFailure(error); + } + if (asked !== undefined) return asked.answer; + const { self, args } = await passwordCallOf(connection, settings); let answer: unknown; try { answer = await (password as (...parameters: unknown[]) => unknown).apply(self, args); } catch (error) { - // The function's own error is kept as the cause, out of the message, which a log may show. - const failure: Error & { cause?: unknown } = new Error( - 'PostgresExporter: the password function of the pool failed, so pg_dump has no password.', - ); - failure.cause = error; - throw failure; + throw passwordFailure(error); } if (answer === undefined) return undefined; - if (typeof answer !== 'string') { - throw new Error('PostgresExporter: the password function of the pool answered no string, which pg refuses too.'); - } + if (typeof answer !== 'string') throw passwordFailure(new TypeError(NOT_A_STRING)); return answer; }; @@ -467,12 +553,16 @@ const passwordOf = async (connection: PgConnection, settings: PgConnectionSettin * with no password and reads no `PGPASSWORD` (`lib/client.js`, `_getPassword`): the URI holds an empty `password` * parameter, which becomes an empty `PGPASSWORD`, so libpq sends the environment's no more than pg does. A * connection string pg cannot read as a URL (a keyword=value string) is given back as it is. Undefined when the - * settings name no database pg would reach. + * settings name no database pg would reach. `PoolClient` is the `Client` the pool makes its connections with, which + * pg-pool keeps as the pool's `Client`: a password function is asked through it. * - * @throws {Error} When the settings ask for TLS that pg_dump cannot take or that pg refuses, or the password function - * fails. + * @throws {Error} When the settings ask for TLS that pg_dump cannot take or that pg refuses, or a port pg reads no + * number in, or the password function fails. */ -export const libpqConnectionStringOf = async (settings: PgConnectionSettings): Promise => { +export const libpqConnectionStringOf = async ( + settings: PgConnectionSettings, + PoolClient?: unknown, +): Promise => { const { connectionString } = settings; if (connectionString && !connectionString.startsWith('/') && !URL_SCHEME.test(connectionString)) { return connectionString; @@ -482,7 +572,7 @@ export const libpqConnectionStringOf = async (settings: PgConnectionSettings): P if (connection === undefined) return connectionString; if (!connection.database) return undefined; - const password = await passwordOf(connection, settings); + const password = await passwordOf(connection, settings, PoolClient); const parameters = [...connection.parameters]; if (typeof connection.password === 'function' && !password) parameters.push(['password', '']); const user = connection.user ? encodeURIComponent(connection.user) : ''; From c8be1444e05ce33e768acc733c8dffe22b914659 Mon Sep 17 00:00:00 2001 From: jddunn Date: Sat, 17 Oct 2026 22:42:27 -0700 Subject: [PATCH 29/32] docs: the exporter asks a password function through the pool's own Client, keeps the parameters pg reads and reads the port as pg does; 0.7.18 releases the browser entry (#71) --- CHANGELOG.md | 7 ++++++- README.md | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7408604..ac5c135 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## [0.7.18] - 2026-10-18 +### Added +- `@framers/sql-storage-adapter/browser` (#71), which merged with no version of its own: what a page needs, in one entry. It holds the IndexedDB adapter with its durable commits and cross-tab coordination, the key-value module with the move of an old IndexedDB key-value store into it, the tab lock, and the mutation sync's client with its wire contract, transport, loop and change channel, and its modules import no Node built-in and no pg. + ### Security - `PostgresExporter` refuses, before `pg_dump` starts, a connection string whose password it cannot move to `PGPASSWORD`: a URI password that is not valid percent-encoding (`%zz`, a `%` at its end, bytes that are not UTF-8) or that decodes to a NUL, and a connection string that holds a NUL. 0.7.17 passed such a URI whole, its password in `--dbname`, where the process list shows it. The refusal never holds the connection string. - `PostgresExporter` moves the `password` of keyword=value settings (`host=db.example password='a b'`) to `PGPASSWORD` too, read as libpq reads them: a value runs to whitespace or sits in single quotes, a backslash takes the character after it, and the later of two passwords wins. An empty password goes to `PGPASSWORD` empty, as libpq reads an empty setting, so the environment's `PGPASSWORD` is never sent in its place. Settings libpq refuses (an unterminated quote, a setting with no `=`) are refused before `pg_dump` starts. 0.7.17 passed them whole on the command line. @@ -13,10 +16,12 @@ - `createStorageFeatures(adapter).exporter` on PostgreSQL reads the connection of the adapter's pool as pg reads it, and hands `pg_dump` a libpq URI of it, so `pg_dump` reaches the server, the database and the user the pool reaches: - A connection string pg and libpq read apart: `postgresql://db.example?application_name=x@y` names the host `db.example` to pg and `y` to libpq, and `pg_dump` connected to `y`. An `@` in a URL's password, a host left out, `ssl=1`, `sslmode=no-verify` and pg's own parameters are written as libpq reads them. - A pool made without a host, a port, a user or a database: `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user and the user's name for the database, as pg fills them. 0.7.17 wrote localhost and 5432 for a pool that named no host or port, and refused one that named no database. The adapter's own settings keep the localhost and 5432 of the pool it makes. - - A pool whose password is a function: the function is called when the export runs, as pg calls it for each connection: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters` (node-postgres #3602), as the pg this library loads calls it. The client is never connected and is made without the settings' `stream`. Its answer goes to `PGPASSWORD`. A function that answers no password gives `pg_dump` an empty `PGPASSWORD`: pg then connects with no password and reads no `PGPASSWORD`, so the environment's is not sent. A function that fails, or answers anything but a string or `undefined`, refuses the export. 0.7.17 left the function's password out, so `pg_dump` logged in with `PGPASSWORD`, or not at all. + - A pool whose password is a function: the function is asked when the export runs, as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls it as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters` (node-postgres #3602). So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks; a pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks. The client is never connected and is made without the settings' `stream`. Its answer goes to `PGPASSWORD`. A function that answers no password gives `pg_dump` an empty `PGPASSWORD`: pg then connects with no password and reads no `PGPASSWORD`, so the environment's is not sent. A function that fails, or answers anything but a string or `undefined`, refuses the export. 0.7.17 left the function's password out, so `pg_dump` logged in with `PGPASSWORD`, or not at all. - A pool whose `ssl` holds `ca`, `cert`, `key`, `pfx` or another TLS option is refused, naming the options and what to do instead: `pg_dump` reads TLS material from files alone. 0.7.17 wrote `sslmode=require` in its place, so `pg_dump` connected without the client certificate, or checked the server against no certificate authority. - TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`: a `PGSSLMODE` of `verify-ca` or `verify-full` refused the certificate pg accepts. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, where libpq's `prefer` or `allow` would use TLS. Where no TLS is pg's default, libpq's default stays, and an empty `PGSSLMODE` or one libpq does not know, which pg reads as no TLS and libpq refuses, gives way to `sslmode=disable`. A certificate file in the connection string turns TLS on over `ssl=0`, and an `sslmode` libpq does not know means TLS, as pg reads them. - pg takes the database from the URL's path alone and drops a `database` parameter, and reads an empty `host` parameter as none; the URI holds that reading (pg-connection-string, `index.js`). libpq's own variables, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT`), configure `pg_dump` as they configure psql. + - The port is read as pg reads it, with `parseInt` after the setting, `PGPORT` and 5432: a port of `5432x` is 5432, which libpq refused as no number, and a port of 0 gives way to `PGPORT`, where 0.7.17 wrote 0. A port pg reads no number in is refused before `pg_dump` starts, since the pool makes no connection with it. + - Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication`, `sslnegotiation` and the certificate files. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out: given them, `pg_dump` read another password file than the pool, or refused a server, an authentication or a session the pool takes. - `resolveStorageAdapter()` and `createDatabase()` open a PostgreSQL candidate in the segment of its own options or the top-level one alone. Tried after an IndexedDB or Capacitor candidate whose options named a segment, it took that segment: on a pool of the caller's the adapter threw, and on a connection string the server got a schema only the browser's options asked for. A local candidate still opens in it. - IndexedDB adapter: a PRAGMA assignment through `all()` or `get()` with a comment between its words (`PRAGMA user_version /* note */ = 7`), with a quoted name and no space before it (`PRAGMA"user_version"=9`), or with its schema or name in single quotes (`PRAGMA 'user_version' = 10`), which SQLite takes for a name there, is saved; the statement is read as SQLite reads it. 0.7.17 found no assignment in it, so it was stored neither with `durability: 'commit'` nor by `flush()` or `close()`. - `createSqlKeyValueStore()` writes a value's text from the copy the runtime's `structuredClone` returns, so each property is read once, as IndexedDB's put reads it, and the value is copied once. 0.7.17 read each property a second time for the text: a getter's second answer was stored, and a value whose getter answered a function the second time was refused. diff --git a/README.md b/README.md index 7d0a30d..dd98b72 100644 --- a/README.md +++ b/README.md @@ -950,7 +950,7 @@ await exporter.exportToFile('/backups/snapshot.db'); const bytes = await exporter.exportToBytes(); ``` -On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is called at each export as pg calls it for each connection: as a method of a client made of the pool's settings, with no arguments before pg 8.19.0, and from 8.19.0 with the client's `ConnectionParameters`. The client is of the pg this library loads, so a pool made with another copy of pg, or with a `Client` of its own, is asked as this library's pg asks; it is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. Where it cannot read that pg's version or its `Client` (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, and an empty `host` parameter leaves the URL's host. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). +On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is asked at each export as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls the function as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters`. So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks. That client is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. A pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks; where the exporter cannot read that pg's version or its `Client` either (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, an empty `host` parameter leaves the URL's host, and the port is read with `parseInt`, so `5432x` is 5432 and a port of 0 gives way to `PGPORT`; a port pg reads no number in is refused before `pg_dump` starts. Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication`, `sslnegotiation` and the certificate files. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out, so `pg_dump` reads the password file the pool reads and refuses no server, authentication or session the pool takes. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). TLS follows pg's, and so does the check of the server. pg's TLS is Node's, which checks the server's certificate and its host name unless `rejectUnauthorized` is false. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`, and with `uselibpqcompat=true` `sslmode=prefer`, or `require` with no root certificate) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts; libpq's `require` still checks the certificate against a root certificate file it finds (the connection string's `sslrootcert`, `PGSSLROOTCERT`, `~/.postgresql/root.crt`), so there `pg_dump` can refuse a server pg reaches. A pool that checks the certificate (`ssl: true`, `{}`, `ssl=1`, a certificate file or any other `sslmode` but `disable` in the connection string, or a `PGSSLMODE` pg reads as TLS) gets `sslmode=verify-full` over any `PGSSLMODE`, and `sslmode=verify-ca` where pg checks no host name (`uselibpqcompat=true` with `verify-ca`, or `require` beside a root certificate). libpq checks the certificate against the root certificate file the connection string names, which pg reads too, else the file `PGSSLROOTCERT` names, else the system's certificate authorities (`sslrootcert=system`: OpenSSL's default locations, which `SSL_CERT_FILE` and `SSL_CERT_DIR` move), where Node checks against the authorities it trusts by default (the Mozilla list it bundles, and `NODE_EXTRA_CA_CERTS`): an authority one trusts and the other does not is named in the connection string's `sslrootcert` or in `PGSSLROOTCERT`. libpq reads `sslrootcert=system` from PostgreSQL 16 on; an older one looks for a file named `system`, and the export fails, saying what to do. `uselibpqcompat=true` with `sslmode=verify-ca` and no root certificate is refused before `pg_dump` starts, as pg refuses it. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. From 78f4a3af96af4c0cf97de9f98561db8a45ad0ffa Mon Sep 17 00:00:00 2001 From: jddunn Date: Sun, 18 Oct 2026 02:24:59 -0700 Subject: [PATCH 30/32] test(exporter): pg_dump gets sslnegotiation only where the pool's own pg reads it, and TLS where its parse turns TLS on for direct pg reads sslnegotiation from 8.22.0, from the connection string, then the settings, then PGSSLNEGOTIATION, and with direct starts TLS at once. pg 8.13.1 to 8.21.x read none and open TLS with an SSLRequest, and pg-connection-string turns TLS on for sslnegotiation=direct from 2.14.0, a version their caret ranges admit from either side. A fixture Client stands for such a pool, with either parse. On CI's PostgreSQL 16, which offers no TLS, a pool whose pg reads nothing in sslnegotiation=direct connects; on CI's TLS server, a pool that opens TLS with an SSLRequest past it connects. Each export must reach what the pool reaches. --- .../clientOfPgBeforeDirectNegotiation.ts | 45 ++++++++ tests/postgres.exporter.spec.ts | 29 +++++ tests/postgres.exporter.tls.spec.ts | 22 ++++ tests/postgresExporter.args.spec.ts | 109 +++++++++++++++++- 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/clientOfPgBeforeDirectNegotiation.ts diff --git a/tests/fixtures/clientOfPgBeforeDirectNegotiation.ts b/tests/fixtures/clientOfPgBeforeDirectNegotiation.ts new file mode 100644 index 0000000..0e21f6d --- /dev/null +++ b/tests/fixtures/clientOfPgBeforeDirectNegotiation.ts @@ -0,0 +1,45 @@ +import pg from 'pg'; + +/** + * How the pg-connection-string under a pg before 8.22.0 reads `sslnegotiation=direct` in a connection string: + * 2.14.0 and later turn TLS on for it where the string sets no other TLS (index.js:81-86), and earlier ones read + * nothing in it. pg 8.13.1 to 8.21.x take pg-connection-string through a caret range (^2.7.0 to ^2.13.0) that admits + * versions either side of 2.14.0, so either reading is what an install of them may run. + */ +export type ParseOfDirect = 'turns-tls-on' | 'reads-nothing'; + +/** A connection string with its `sslnegotiation` parameters taken out. */ +const withoutSslNegotiation = (connectionString: string): string => { + const url = new URL(connectionString); + url.searchParams.delete('sslnegotiation'); + return url.toString(); +}; + +/** + * The `Client` of a pool made with a pg before 8.22.0, which reads no `sslnegotiation`: it keeps none among its + * ConnectionParameters and opens TLS with an SSLRequest whatever a connection string or the settings say (pg 8.21.0 + * `lib/client.js` and `lib/connection.js`), where the installed pg (8.22.0 and later) reads the parameter and with + * `direct` starts TLS at once. Whether `direct` turns TLS on is the parse's doing, chosen here. It stands for a pool + * of another copy of pg than the one this library loads: pg-pool makes each connection with the Client its options + * name (pg-pool 3.14.0, index.js:95 and 241) and keeps it as the pool's `Client`. + */ +export const clientOfPgBeforeDirectNegotiation = (parse: ParseOfDirect): typeof pg.Client => + class ClientOfPgBeforeDirectNegotiation extends pg.Client { + constructor(config?: string | pg.ClientConfig) { + const settings: pg.ClientConfig = typeof config === 'string' ? { connectionString: config } : { ...config }; + if (typeof config === 'object' && config !== null) { + // pg-pool keeps the settings' password from enumeration, so it is copied by name. + settings.password = config.password; + } + if (parse === 'reads-nothing' && settings.connectionString && !settings.connectionString.startsWith('/')) { + settings.connectionString = withoutSslNegotiation(settings.connectionString); + } + super(settings); + // pg's own fields, which @types/pg does not declare. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const self = this as any; + delete self.connectionParameters.sslnegotiation; + self.sslNegotiation = 'postgres'; + if (self.connection) self.connection.sslNegotiation = 'postgres'; + } + }; diff --git a/tests/postgres.exporter.spec.ts b/tests/postgres.exporter.spec.ts index e561f11..a4b4a7f 100644 --- a/tests/postgres.exporter.spec.ts +++ b/tests/postgres.exporter.spec.ts @@ -6,6 +6,7 @@ import path from 'node:path'; import pg from 'pg'; import { createPostgresAdapter, createStorageFeatures, type StorageAdapter } from '../src/index.js'; import { ClientOfEarlierPg } from './fixtures/clientOfEarlierPg.js'; +import { clientOfPgBeforeDirectNegotiation } from './fixtures/clientOfPgBeforeDirectNegotiation.js'; // The feature exporter runs pg_dump against the PostgreSQL that // POSTGRES_TEST_URL names (CI's service); the runner has the PostgreSQL client @@ -415,4 +416,32 @@ describe.skipIf(!POSTGRES_TEST_URL)('the PostgreSQL exporter, through pg_dump', expect(dumpedTables(file)).toEqual([`${segment}.opened_items`]); }); + + // A pg before 8.22.0 reads no sslnegotiation, and under it a + // pg-connection-string before 2.14.0 makes no TLS of sslnegotiation=direct, + // so the pool connects to CI's server, which offers no TLS. pg_dump was + // given sslnegotiation=direct with verify-full, which this server cannot + // meet, and which a libpq before 17 refuses as a parameter it does not know. + it('exports through a pool whose pg reads nothing in sslnegotiation=direct, where the pool connects without TLS', async () => { + const database = await newDatabase(); + const server = new URL(url); + server.pathname = `/${database}`; + server.searchParams.set('sslnegotiation', 'direct'); + const pool = new pg.Pool({ + connectionString: server.toString(), + Client: clientOfPgBeforeDirectNegotiation('reads-nothing'), + } as pg.PoolConfig); + const adapter = createPostgresAdapter({ pool }); + try { + await adapter.exec('CREATE TABLE negotiation_items (id INTEGER PRIMARY KEY)'); + + const file = path.join(dir, `${database}-negotiation.dump`); + await createStorageFeatures(adapter).exporter.exportToFile(file); + + expect(dumpedTables(file)).toEqual(['public.negotiation_items']); + } finally { + await adapter.close(); + await pool.end(); + } + }); }); diff --git a/tests/postgres.exporter.tls.spec.ts b/tests/postgres.exporter.tls.spec.ts index 1115814..07b65fa 100644 --- a/tests/postgres.exporter.tls.spec.ts +++ b/tests/postgres.exporter.tls.spec.ts @@ -5,6 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import pg from 'pg'; import { createPostgresAdapter, createStorageFeatures } from '../src/index.js'; +import { clientOfPgBeforeDirectNegotiation } from './fixtures/clientOfPgBeforeDirectNegotiation.js'; // CI starts a PostgreSQL that offers TLS with a certificate made for the run, // for localhost, which no public certificate authority signed: it reads @@ -146,4 +147,25 @@ describe.skipIf(!TLS_URL || !ROOT_CERT)('the PostgreSQL exporter, through pg_dum await pool.end(); } }); + + // A pg before 8.22.0 opens TLS with an SSLRequest whatever sslnegotiation + // says. pg_dump was given sslnegotiation=direct, which starts TLS at once: + // this PostgreSQL 16 takes no direct TLS (it came with PostgreSQL 17), and a + // libpq before 17 refuses the parameter as one it does not know. + it('exports through a pool whose pg opens TLS with an SSLRequest past sslnegotiation=direct, as pg before 8.22.0 does', async () => { + const pool = new pg.Pool({ + connectionString: urlWith('localhost', { sslrootcert: ROOT_CERT as string, sslnegotiation: 'direct' }), + Client: clientOfPgBeforeDirectNegotiation('turns-tls-on'), + } as pg.PoolConfig); + try { + await pool.query('SELECT 1'); + + const file = path.join(dir, 'negotiation.dump'); + await createStorageFeatures(createPostgresAdapter({ pool })).exporter.exportToFile(file); + + expect(dumpedTables(file)).toContain(`public.${table}`); + } finally { + await pool.end(); + } + }); }); diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index a232ef7..6a69967 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -40,8 +40,10 @@ vi.mock('../src/exporters/pgModule.js', async (importOriginal) => { import { Client as PgClient } from 'pg'; import { ClientOfEarlierPg } from './fixtures/clientOfEarlierPg.js'; +import { clientOfPgBeforeDirectNegotiation, type ParseOfDirect } from './fixtures/clientOfPgBeforeDirectNegotiation.js'; import { PostgresExporter } from '../src/exporters/PostgresExporter.js'; import { createStorageFeatures } from '../src/core/contracts/features.js'; +import { setStorageLogger } from '../src/shared/logger.js'; import type { StorageAdapter } from '../src/core/contracts/index.js'; // pg's own reading of a pool's settings (pg exports ./lib/*). // @ts-expect-error pg ships no declarations for its lib files. @@ -355,7 +357,7 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { // pg reads these where the settings leave a value out, and takes the user // name from the operating system's; each spec that needs one sets it. // They are removed, not emptied: libpq refuses an empty PGSSLMODE. - for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD', 'PGSSLMODE', 'PGSSLROOTCERT', 'USER', 'USERNAME']) { + for (const name of ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD', 'PGSSLMODE', 'PGSSLROOTCERT', 'PGSSLNEGOTIATION', 'USER', 'USERNAME']) { vi.stubEnv(name, undefined); } pgRead.as = null; @@ -1284,4 +1286,109 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { await expect(createStorageFeatures(adapter).exporter.exportToBytes()).rejects.toThrow(/names no database/); expect(calls).toHaveLength(0); }); + + // pg reads sslnegotiation from 8.22.0 (lib/connection-parameters.js:102-112): + // the connection string's over the settings', an empty one as none, then + // PGSSLNEGOTIATION; with direct it starts TLS at once instead of with an + // SSLRequest. pg 8.13.1 to 8.21.x read none and open TLS with an SSLRequest. + // pg-connection-string turns TLS on for sslnegotiation=direct from 2.14.0 + // (index.js:81-86), and the caret ranges of pg 8.13.1 to 8.21.x admit one + // from either side of it. pg_dump gets what the pool's own pg reads. + describe("sslnegotiation, as the pool's own pg reads it", () => { + /** The features exporter of an adapter on a pool of the caller whose connections are made with this Client. */ + const clientPoolExporter = (options: Record, Client: unknown) => + createStorageFeatures(postgresAdapter({ pool: { options, Client } })).exporter; + + it.each([ + [ + 'whose parse reads nothing in sslnegotiation=direct: no TLS, and no sslnegotiation', + 'reads-nothing', + 'postgresql://app@db.example/app?sslnegotiation=direct', + 'postgresql://app@db.example:5432/app', + ], + [ + 'whose parse turns TLS on for sslnegotiation=direct: the TLS, and no sslnegotiation', + 'turns-tls-on', + 'postgresql://app@db.example/app?sslnegotiation=direct', + 'postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system', + ], + [ + 'given sslnegotiation=postgres beside application_name: application_name alone', + 'turns-tls-on', + 'postgresql://app@db.example/app?sslnegotiation=postgres&application_name=reports', + 'postgresql://app@db.example:5432/app?application_name=reports', + ], + ])('gives pg_dump what a pool whose pg before 8.22.0 reads, %s', async (_name, parse, connectionString, dbname) => { + const Client = clientOfPgBeforeDirectNegotiation(parse as ParseOfDirect); + + await clientPoolExporter({ connectionString }, Client).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + // The pool's own reading: TLS where pg_dump is given it, and no sslnegotiation. + const { connectionParameters } = new Client({ connectionString }) as unknown as { + connectionParameters: { ssl?: unknown; sslnegotiation?: unknown }; + }; + expect(Boolean(connectionParameters.ssl)).toBe(dbname.includes('sslmode=verify-full')); + expect(connectionParameters.sslnegotiation).toBeUndefined(); + }); + + it('gives pg_dump the sslnegotiation of the settings of a pool whose pg reads it', async () => { + const settings = { host: 'db.example', database: 'app', user: 'app', ssl: true, sslnegotiation: 'direct' }; + + await poolExporter(settings).exportToBytes(); + + expect(new ConnectionParameters(settings).sslnegotiation).toBe('direct'); + expect(dbnameOf(calls[0].args)).toBe( + 'postgresql://app@db.example:5432/app?sslnegotiation=direct&sslmode=verify-full&sslrootcert=system', + ); + }); + + it('gives pg_dump no sslnegotiation of the settings of a pool whose pg before 8.22.0 reads none', async () => { + const options = { host: 'db.example', database: 'app', user: 'app', ssl: true, sslnegotiation: 'direct' }; + + await clientPoolExporter(options, clientOfPgBeforeDirectNegotiation('turns-tls-on')).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system'); + }); + + it.each([ + [ + "a connection string's, over the settings'", + { connectionString: 'postgresql://app@db.example/app?sslnegotiation=postgres', ssl: true, sslnegotiation: 'direct' }, + 'postgresql://app@db.example:5432/app?sslnegotiation=postgres&sslmode=verify-full&sslrootcert=system', + ], + [ + 'an empty one in a connection string, which pg reads as none', + { connectionString: 'postgresql://app@db.example/app?sslnegotiation=', ssl: true, sslnegotiation: 'direct' }, + 'postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system', + ], + ])('gives pg_dump the sslnegotiation pg reads, with %s', async (_name, settings, dbname) => { + await poolExporter(settings).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe(dbname); + const written = new URL(dbname).searchParams.get('sslnegotiation') ?? undefined; + expect(written).toBe(new ConnectionParameters(settings).sslnegotiation); + }); + + it('reads sslnegotiation as pg 8.22.0 and later do where pg cannot be read, and says so once', async () => { + pgRead.as = {}; + const warn = vi.fn(); + setStorageLogger({ info: () => undefined, warn, error: () => undefined }); + try { + for (let run = 0; run < 2; run += 1) { + await poolExporter({ connectionString: 'postgresql://app@db.example/app?sslnegotiation=direct' }).exportToBytes(); + } + } finally { + setStorageLogger(null); + } + + expect(calls.map((call) => dbnameOf(call.args))).toEqual([ + 'postgresql://app@db.example:5432/app?sslnegotiation=direct&sslmode=verify-full&sslrootcert=system', + 'postgresql://app@db.example:5432/app?sslnegotiation=direct&sslmode=verify-full&sslrootcert=system', + ]); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toMatchObject({ scope: 'PostgresExporter' }); + expect(warn.mock.calls[0][1]).toMatch(/sslnegotiation/); + }); + }); }); From b488a8535568080d229d686ae2a6c5462f8b44d1 Mon Sep 17 00:00:00 2001 From: jddunn Date: Wed, 21 Oct 2026 03:20:05 -0700 Subject: [PATCH 31/32] fix(exporter): pg_dump gets sslnegotiation as the pool's own pg reads it, and TLS for direct only where its parse turns TLS on pg reads sslnegotiation from 8.22.0 (the connection string's over the settings', an empty one as none, then PGSSLNEGOTIATION) and with direct starts TLS at once; pg 8.13.1 to 8.21.x read none and start TLS with an SSLRequest, and pg-connection-string turns TLS on for sslnegotiation=direct from 2.14.0 only. The exporter handed pg_dump the string's sslnegotiation and turned TLS on for direct whatever the pool's pg read, so an export failed where the pool connected. sslNegotiationOf() reads it on a client of the pool's Client (or the pg this library loads), never connected, and a probe client shows whether that Client's parse turns TLS on for direct; where neither can be read, the reading is pg 8.22.0's, which the storage logger says once. A value pg refuses refuses the export. The adapter's exporter runs pg_dump without PGSSLNEGOTIATION (PostgresExporterOptions.unsetEnvironment), which libpq 17 would read beside the URI, and a libpq older than 17 that refuses the parameter gets words saying what to do. The README and the changelog say so. --- CHANGELOG.md | 6 +- README.md | 6 +- src/core/contracts/features.ts | 2 + src/exporters/PostgresExporter.ts | 39 +++++++- src/exporters/pgConnection.ts | 142 ++++++++++++++++++++++++++-- tests/postgresExporter.args.spec.ts | 53 +++++++++++ 6 files changed, 229 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbbec9c..0153824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## [Unreleased] +### Added +- `PostgresExporterOptions.unsetEnvironment` names libpq variables left out of `pg_dump`'s environment. The exporter of a PostgreSQL adapter leaves out `PGSSLNEGOTIATION`. + ### Security - `PostgresExporter` refuses, before `pg_dump` starts, a connection string whose password it cannot move to `PGPASSWORD`: a URI password that is not valid percent-encoding (`%zz`, a `%` at its end, bytes that are not UTF-8) or that decodes to a NUL, and a connection string that holds a NUL. Versions through 0.8.2 passed such a URI whole, its password in `--dbname`, where the process list shows it. The refusal never holds the connection string. - `PostgresExporter` moves the `password` of keyword=value settings (`host=db.example password='a b'`) to `PGPASSWORD` too, read as libpq reads them: a value runs to whitespace or sits in single quotes, a backslash takes the character after it, and the later of two passwords wins. An empty password goes to `PGPASSWORD` empty, as libpq reads an empty setting, so the environment's `PGPASSWORD` is never sent in its place. Settings libpq refuses (an unterminated quote, a setting with no `=`) are refused before `pg_dump` starts. Versions through 0.8.2 passed them whole on the command line. @@ -18,7 +21,8 @@ - TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`: a `PGSSLMODE` of `verify-ca` or `verify-full` refused the certificate pg accepts. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, where libpq's `prefer` or `allow` would use TLS. Where no TLS is pg's default, libpq's default stays, and an empty `PGSSLMODE` or one libpq does not know, which pg reads as no TLS and libpq refuses, gives way to `sslmode=disable`. A certificate file in the connection string turns TLS on over `ssl=0`, and an `sslmode` libpq does not know means TLS, as pg reads them. - pg takes the database from the URL's path alone and drops a `database` parameter, and reads an empty `host` parameter as none; the URI holds that reading (pg-connection-string, `index.js`). libpq's own variables, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT`), configure `pg_dump` as they configure psql. - The port is read as pg reads it, with `parseInt` after the setting, `PGPORT` and 5432: a port of `5432x` is 5432, which libpq refused as no number, and a port of 0 gives way to `PGPORT`, where versions through 0.8.2 wrote 0. A port pg reads no number in is refused before `pg_dump` starts, since the pool makes no connection with it. - - Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication`, `sslnegotiation` and the certificate files. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out: given them, `pg_dump` read another password file than the pool, or refused a server, an authentication or a session the pool takes. + - Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication` and the certificate files. + - `sslnegotiation` reaches `pg_dump` as the pool's own pg reads it. pg reads it from 8.22.0 (the connection string's over the settings', an empty one as none, then `PGSSLNEGOTIATION`) and with `direct` starts TLS at once; pg 8.13.1 to 8.21.x read none and start TLS with an SSLRequest. The exporter reads it on a client of the pool's `Client`, never connected, so a pool whose pg reads none gives `pg_dump` none, and TLS for `direct` only where that pg's parse of the connection string turns TLS on for it (pg-connection-string 2.14.0 and later). `pg_dump` runs without `PGSSLNEGOTIATION`, which libpq 17 and later read: the URI holds what the pool's pg reads. A value pg refuses (one but `direct` or `postgres`, or `direct` without TLS) refuses the export, and where `pg_dump`'s libpq is older than 17, which does not know `sslnegotiation`, the error says to run a pg_dump of PostgreSQL 17 or later. Versions through 0.8.2 handed `pg_dump` the string's `sslnegotiation` and turned TLS on for `direct` whatever the pool's pg read, so an export failed against a server the pool reached: one that offers no TLS, or one older than PostgreSQL 17, which takes no direct TLS. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out: given them, `pg_dump` read another password file than the pool, or refused a server, an authentication or a session the pool takes. - `resolveStorageAdapter()` and `createDatabase()` open a PostgreSQL candidate in the segment of its own options or the top-level one alone. Tried after an IndexedDB or Capacitor candidate whose options named a segment, it took that segment: on a pool of the caller's the adapter threw, and on a connection string the server got a schema only the browser's options asked for. A local candidate still opens in it. - IndexedDB adapter: a PRAGMA assignment through `all()` or `get()` with a comment between its words (`PRAGMA user_version /* note */ = 7`), with a quoted name and no space before it (`PRAGMA"user_version"=9`), or with its schema or name in single quotes (`PRAGMA 'user_version' = 10`), which SQLite takes for a name there, is saved; the statement is read as SQLite reads it. Versions through 0.8.2 found no assignment in it, so it was stored neither with `durability: 'commit'` nor by `flush()` or `close()`. - `createSqlKeyValueStore()` writes a value's text from the copy the runtime's `structuredClone` returns, so each property is read once, as IndexedDB's put reads it, and the value is copied once. Versions through 0.8.2 read each property a second time for the text: a getter's second answer was stored, and a value whose getter answered a function the second time was refused. diff --git a/README.md b/README.md index 0fbf7e0..05bb27d 100644 --- a/README.md +++ b/README.md @@ -955,11 +955,11 @@ await exporter.exportToFile('/backups/snapshot.db'); const bytes = await exporter.exportToBytes(); ``` -On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is asked at each export as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls the function as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters`. So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks. That client is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. A pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks; where the exporter cannot read that pg's version or its `Client` either (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, an empty `host` parameter leaves the URL's host, and the port is read with `parseInt`, so `5432x` is 5432 and a port of 0 gives way to `PGPORT`; a port pg reads no number in is refused before `pg_dump` starts. Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication`, `sslnegotiation` and the certificate files. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out, so `pg_dump` reads the password file the pool reads and refuses no server, authentication or session the pool takes. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). +On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is asked at each export as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls the function as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters`. So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks. That client is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. A pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks; where the exporter cannot read that pg's version or its `Client` either (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, an empty `host` parameter leaves the URL's host, and the port is read with `parseInt`, so `5432x` is 5432 and a port of 0 gives way to `PGPORT`; a port pg reads no number in is refused before `pg_dump` starts. Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication` and the certificate files. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out, so `pg_dump` reads the password file the pool reads and refuses no server, authentication or session the pool takes. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). -TLS follows pg's, and so does the check of the server. pg's TLS is Node's, which checks the server's certificate and its host name unless `rejectUnauthorized` is false. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`, and with `uselibpqcompat=true` `sslmode=prefer`, or `require` with no root certificate) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts; libpq's `require` still checks the certificate against a root certificate file it finds (the connection string's `sslrootcert`, `PGSSLROOTCERT`, `~/.postgresql/root.crt`), so there `pg_dump` can refuse a server pg reaches. A pool that checks the certificate (`ssl: true`, `{}`, `ssl=1`, a certificate file or any other `sslmode` but `disable` in the connection string, or a `PGSSLMODE` pg reads as TLS) gets `sslmode=verify-full` over any `PGSSLMODE`, and `sslmode=verify-ca` where pg checks no host name (`uselibpqcompat=true` with `verify-ca`, or `require` beside a root certificate). libpq checks the certificate against the root certificate file the connection string names, which pg reads too, else the file `PGSSLROOTCERT` names, else the system's certificate authorities (`sslrootcert=system`: OpenSSL's default locations, which `SSL_CERT_FILE` and `SSL_CERT_DIR` move), where Node checks against the authorities it trusts by default (the Mozilla list it bundles, and `NODE_EXTRA_CA_CERTS`): an authority one trusts and the other does not is named in the connection string's `sslrootcert` or in `PGSSLROOTCERT`. libpq reads `sslrootcert=system` from PostgreSQL 16 on; an older one looks for a file named `system`, and the export fails, saying what to do. `uselibpqcompat=true` with `sslmode=verify-ca` and no root certificate is refused before `pg_dump` starts, as pg refuses it. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. +TLS follows pg's, and so does the check of the server. pg's TLS is Node's, which checks the server's certificate and its host name unless `rejectUnauthorized` is false. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`, and with `uselibpqcompat=true` `sslmode=prefer`, or `require` with no root certificate) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts; libpq's `require` still checks the certificate against a root certificate file it finds (the connection string's `sslrootcert`, `PGSSLROOTCERT`, `~/.postgresql/root.crt`), so there `pg_dump` can refuse a server pg reaches. A pool that checks the certificate (`ssl: true`, `{}`, `ssl=1`, a certificate file or any other `sslmode` but `disable` in the connection string, or a `PGSSLMODE` pg reads as TLS) gets `sslmode=verify-full` over any `PGSSLMODE`, and `sslmode=verify-ca` where pg checks no host name (`uselibpqcompat=true` with `verify-ca`, or `require` beside a root certificate). libpq checks the certificate against the root certificate file the connection string names, which pg reads too, else the file `PGSSLROOTCERT` names, else the system's certificate authorities (`sslrootcert=system`: OpenSSL's default locations, which `SSL_CERT_FILE` and `SSL_CERT_DIR` move), where Node checks against the authorities it trusts by default (the Mozilla list it bundles, and `NODE_EXTRA_CA_CERTS`): an authority one trusts and the other does not is named in the connection string's `sslrootcert` or in `PGSSLROOTCERT`. libpq reads `sslrootcert=system` from PostgreSQL 16 on; an older one looks for a file named `system`, and the export fails, saying what to do. `uselibpqcompat=true` with `sslmode=verify-ca` and no root certificate is refused before `pg_dump` starts, as pg refuses it. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. `sslnegotiation` follows the pool's own pg too. pg reads it from 8.22.0 (the connection string's over the settings', an empty one as none, then `PGSSLNEGOTIATION`) and with `direct` starts TLS at once, where pg 8.13.1 to 8.21.x read none and start TLS with an SSLRequest; the exporter reads it on a client of the pool's `Client`, never connected, so `pg_dump` gets the `sslnegotiation` that pg reads, none where it reads none, and TLS for `sslnegotiation=direct` only where that pg's parse turns TLS on for it (pg-connection-string 2.14.0 and later). `pg_dump` runs without `PGSSLNEGOTIATION`, which libpq 17 and later would read beside the URI. A value pg refuses (one but `direct` or `postgres`, or `direct` without TLS) refuses the export. libpq reads `sslnegotiation` from PostgreSQL 17 on; an older one refuses it as a parameter it does not know, and the export fails, saying what to do. -`pg_dump` runs with no shell between and with `--no-password`, so it never waits for a password on a terminal or on its standard input, and the password of the connection string reaches it through `PGPASSWORD`, not its command line: a `postgres://` or `postgresql://` URI's, in its user part or its `password` parameter, and the `password` of keyword=value settings (`host=db.example password='a b'`), each read as libpq reads it. An empty password given in the string goes to `PGPASSWORD` empty, so the environment's is never sent in its place. A connection string that holds a NUL character, or a password that is not valid percent-encoding or decodes to a NUL, is refused before `pg_dump` starts, without the string in the message. libpq reads `sslpassword` and `oauth_client_secret` from the connection string alone, so a string handed to `new PostgresExporter()` with them keeps them in `--dbname`; the features exporter leaves them out, since pg reads neither. `new PostgresExporter(connectionString, { schema })` exports one schema of any database; `connectionString` may be a function that gives it, or a promise of it, when an export runs. +`pg_dump` runs with no shell between and with `--no-password`, so it never waits for a password on a terminal or on its standard input, and the password of the connection string reaches it through `PGPASSWORD`, not its command line: a `postgres://` or `postgresql://` URI's, in its user part or its `password` parameter, and the `password` of keyword=value settings (`host=db.example password='a b'`), each read as libpq reads it. An empty password given in the string goes to `PGPASSWORD` empty, so the environment's is never sent in its place. A connection string that holds a NUL character, or a password that is not valid percent-encoding or decodes to a NUL, is refused before `pg_dump` starts, without the string in the message. libpq reads `sslpassword` and `oauth_client_secret` from the connection string alone, so a string handed to `new PostgresExporter()` with them keeps them in `--dbname`; the features exporter leaves them out, since pg reads neither. `new PostgresExporter(connectionString, { schema })` exports one schema of any database; `connectionString` may be a function that gives it, or a promise of it, when an export runs, and `unsetEnvironment` names libpq variables to leave out of `pg_dump`'s environment. ## CI, Releases, and Badges diff --git a/src/core/contracts/features.ts b/src/core/contracts/features.ts index bb89346..f34c0c9 100644 --- a/src/core/contracts/features.ts +++ b/src/core/contracts/features.ts @@ -69,6 +69,8 @@ function postgresExporterOf(adapter: StorageAdapter): PostgresExporter { const options = postgres.options; const exportOptions: PostgresExporterOptions = { ...(options?.segment ? { schema: options.segment } : {}), + // The URI holds the sslnegotiation the pool's pg reads, none where it reads none; libpq 17 would read this too. + unsetEnvironment: ['PGSSLNEGOTIATION'], missingConnection: options?.pool ? 'PostgresExporter: the adapter runs on a pool its caller made, and the pool keeps no connection settings ' + 'for pg_dump to reach. Make the pool with a connectionString, or with host, database, user and password.' diff --git a/src/exporters/PostgresExporter.ts b/src/exporters/PostgresExporter.ts index 0017931..2bef194 100644 --- a/src/exporters/PostgresExporter.ts +++ b/src/exporters/PostgresExporter.ts @@ -13,6 +13,14 @@ export interface PostgresExporterOptions { * pg_dump needs one. */ missingConnection?: string; + /** + * libpq variables left out of pg_dump's environment. The exporter of a + * PostgreSQL adapter leaves out `PGSSLNEGOTIATION`: the connection URI it + * hands pg_dump holds the `sslnegotiation` the pool's own pg reads, which is + * none for a pg before 8.22.0, where libpq 17 and later would read the + * variable and start TLS otherwise than the pool. Default: none. + */ + unsetEnvironment?: readonly string[]; } /** How `pg_dump` is started: the program, its arguments and its environment. */ @@ -37,6 +45,17 @@ const SYSTEM_ROOTS_ADVICE = 'libpq looks for a file named system: run a pg_dump of PostgreSQL 16 or later, or name the root certificate file ' + "in the connection string's sslrootcert parameter or in PGSSLROOTCERT."; +/** + * What libpq older than PostgreSQL 17 says of `sslnegotiation`, a URI parameter it does not know (fe-connect.c, + * conninfo_uri_parse_params); libpq 17 and later read it, and `PGSSLNEGOTIATION`. + */ +const NO_SSL_NEGOTIATION = /invalid URI query parameter: "sslnegotiation"/; + +/** What to do where pg_dump's libpq does not know `sslnegotiation`. */ +const SSL_NEGOTIATION_ADVICE = + ' The connection names an sslnegotiation, which libpq reads from PostgreSQL 17 on: run a pg_dump of PostgreSQL 17 ' + + 'or later.'; + /** * The scheme of a connection URI, at the very start. libpq knows it in lower * case alone; pg, which the adapter connects with, reads it in any case. So it @@ -233,7 +252,8 @@ const connectionOf = (connectionString: string): { dbname: string; env: Record = {}, + options: Pick = {}, ): PgDumpInvocation { const { dbname, env } = connectionOf(connectionString); // Without a password, to a server that asks for one, pg_dump would ask for it on the terminal, or with no @@ -254,7 +274,9 @@ export function pgDumpInvocation( args.push(`--schema="${options.schema.replace(/"/g, '""')}"`); } if (outputPath !== undefined) args.push(`--file=${outputPath}`); - return { file: 'pg_dump', args, env: { ...(typeof process !== 'undefined' ? process.env : {}), ...env } }; + const inherited: NodeJS.ProcessEnv = { ...(typeof process !== 'undefined' ? process.env : {}) }; + for (const name of options.unsetEnvironment ?? []) delete inherited[name]; + return { file: 'pg_dump', args, env: { ...inherited, ...env } }; } /** @@ -302,7 +324,10 @@ export class PostgresExporter implements IDatabaseExporter { if (!connectionString) { throw new Error(this.options.missingConnection ?? 'PostgresExporter requires a connection string for pg_dump.'); } - const { file, args, env } = pgDumpInvocation(connectionString, outputPath, { schema: this.options.schema }); + const { file, args, env } = pgDumpInvocation(connectionString, outputPath, { + schema: this.options.schema, + unsetEnvironment: this.options.unsetEnvironment, + }); const { execFile } = await import('node:child_process'); return new Promise((resolve, reject) => { @@ -323,7 +348,11 @@ export class PostgresExporter implements IDatabaseExporter { // pg_dump's own words. The error's message repeats the command // line, which can hold a password, so it is left out. const said = stderr ? stderr.toString('utf8').trim() : ''; - const advice = NO_SYSTEM_ROOTS.test(said) ? SYSTEM_ROOTS_ADVICE : ''; + const advice = NO_SYSTEM_ROOTS.test(said) + ? SYSTEM_ROOTS_ADVICE + : NO_SSL_NEGOTIATION.test(said) + ? SSL_NEGOTIATION_ADVICE + : ''; reject( new Error( `PostgresExporter: pg_dump failed${typeof code === 'number' ? ` (exit code ${code})` : ''}${said ? `: ${said}` : '.'}${advice}`, diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts index a49070e..92ea331 100644 --- a/src/exporters/pgConnection.ts +++ b/src/exporters/pgConnection.ts @@ -30,6 +30,8 @@ export interface PgConnectionSettings { password?: unknown; /** pg's `ssl`: `true`, `false`, or Node TLS options. */ ssl?: unknown; + /** pg's `sslnegotiation`, `direct` or `postgres`, which pg reads from 8.22.0. */ + sslnegotiation?: unknown; } /** The connection pg makes from a set of settings. */ @@ -54,9 +56,10 @@ const environment = (): Environment => * The parameters of a connection string that pg reads and that libpq takes in a connection URI (PostgreSQL 18, * `fe-connect.c`, `PQconninfoOptions`): the settings pg sends when it connects (`options`, `application_name`, * `fallback_application_name`, `client_encoding`, `replication`; pg 8.23.1 `lib/connection-parameters.js:63-124`), - * `sslnegotiation`, which pg reads from 8.22.0, and the certificate files pg-connection-string reads (`sslcert`, - * `sslkey`, `sslrootcert`; `index.js:88-100`). The URI spells out the user, the host, the port and the database, and - * `sslmode` is written for the TLS pg uses. libpq's other parameters pg does not read, and pg_dump given them would + * and the certificate files pg-connection-string reads (`sslcert`, `sslkey`, `sslrootcert`; `index.js:88-100`). The + * URI spells out the user, the host, the port and the database, `sslmode` is written for the TLS pg uses, and + * `sslnegotiation` is written as the pool's own pg reads it (`sslNegotiationOf`), not as the string gives it, since a + * pg before 8.22.0 reads none. libpq's other parameters pg does not read, and pg_dump given them would * read another password file than the pool (`passfile`), or refuse a server, an authentication or a session the pool * takes (`target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `sslcrl` and the rest), so they * are left out; so are pg's own (`statement_timeout`, `uselibpqcompat`), which libpq refuses in a URI, and @@ -69,7 +72,6 @@ const PG_READ_PARAMETERS = new Set([ 'fallback_application_name', 'client_encoding', 'replication', - 'sslnegotiation', 'sslcert', 'sslkey', 'sslrootcert', @@ -101,15 +103,17 @@ const sslFromEnvironment = (sslmode: string | undefined): unknown => { * last value of a repeated parameter counting, or undefined where it makes none, so that pg reads the settings' * `ssl`, then `PGSSLMODE`. A certificate file or an `sslmode` turns TLS on over `ssl=0`; `sslmode=disable` turns it * off; `sslmode=no-verify`, and with `uselibpqcompat=true` `sslmode=prefer`, or `require` with no root certificate, - * check no certificate. + * check no certificate. `sslnegotiation=direct` turns TLS on where the string sets no other TLS, as + * pg-connection-string does from 2.14.0 (index.js:81-86); an earlier one reads nothing in it, which + * `directTurnsTlsOn` false stands for. */ -const sslOfParameters = (last: (name: string) => string | undefined): unknown => { +const sslOfParameters = (last: (name: string) => string | undefined, directTurnsTlsOn: boolean): unknown => { let ssl: unknown = last('ssl'); if (ssl === 'true' || ssl === '1') ssl = true; if (ssl === '0') ssl = false; const sslmode = last('sslmode'); if (last('sslcert') || last('sslkey') || last('sslrootcert') || sslmode) ssl = {}; - if (last('sslnegotiation') === 'direct' && ssl === undefined) ssl = true; + if (directTurnsTlsOn && last('sslnegotiation') === 'direct' && ssl === undefined) ssl = true; if (sslmode === 'disable') return false; const compatible = last('uselibpqcompat') === 'true'; const checksNone = compatible @@ -276,6 +280,116 @@ const tlsSettingsBeyondSslmode = (ssl: unknown): string[] => }) : []; +/** + * How the pool's pg reads `sslnegotiation`, which says how its connections start TLS. pg reads it from 8.22.0 + * (`lib/connection-parameters.js:104-112`: the connection string's over the settings', an empty one as none, then + * `PGSSLNEGOTIATION`; a value but `direct` or `postgres`, or `direct` without TLS, it refuses) and with `direct` + * starts TLS at once; pg 8.13.1 to 8.21.x read none and start TLS with an SSLRequest. pg-connection-string turns TLS + * on for `sslnegotiation=direct` from 2.14.0 (`index.js:81-86`), and the caret ranges of pg before 8.22.0 admit a + * version either side of it. + */ +interface SslNegotiation { + /** Whether the pool's pg reads `sslnegotiation`. */ + read: boolean; + /** What it reads: `direct` or `postgres`, or undefined for none. */ + value?: string; + /** Whether its parse of a connection string turns TLS on for `sslnegotiation=direct`. */ + directTurnsTlsOn: boolean; +} + +/** Whether the warning that sslnegotiation was read without pg has been written; it is written once. */ +let toldNegotiationUnread = false; + +/** + * A connection string whose one TLS setting is `sslnegotiation=direct`, beside settings whose `ssl` is false, which + * the parse's TLS overrides and which keeps pg from reading `PGSSLMODE`: a client of it uses TLS exactly where its + * parse turns TLS on for `direct`. + */ +const DIRECT_PROBE = { connectionString: 'postgresql://probe@localhost/probe?sslnegotiation=direct', ssl: false }; + +/** The `ConnectionParameters` of a pg client, where it has them. */ +const connectionParametersOf = (client: unknown): Record | undefined => { + const parameters = (client as { connectionParameters?: unknown } | null | undefined)?.connectionParameters; + return parameters !== null && typeof parameters === 'object' ? (parameters as Record) : undefined; +}; + +/** + * The refusal of an export whose pool's pg refuses its `sslnegotiation`, so that the pool makes no connection. + * pg's two refusals of it are named; any other error a client of the settings meets is left out of the message, + * which a log may show and which could hold the connection string. + */ +const negotiationRefusal = (error: unknown): Error => { + const said = error instanceof Error ? error.message : ''; + const named = /sslnegotiation/i.test(said) && !/[@/]/.test(said) ? ` (${said})` : ''; + return new Error( + `PostgresExporter: the pool's pg refuses its settings${named}, so the pool makes no connection, and pg_dump ` + + 'makes none either.', + ); +}; + +/** + * Reads `sslnegotiation` as the pool's own pg reads it, on a client of the pool's settings made with the pool's + * `Client` where the pool names one, as pg-pool keeps it, and otherwise with the pg this library loads; the client is + * made without the settings' `stream` and `connection` and never connected. A second client, of `DIRECT_PROBE`, shows + * whether that Client's parse turns TLS on for `direct`. Where that Client cannot be read, the reading is pg 8.22.0's + * and later's, which a warning says once. Nothing is read where neither the connection string, the settings nor + * `PGSSLNEGOTIATION` name an sslnegotiation. + * + * @throws {Error} When the pool's pg refuses the settings: an sslnegotiation it does not know, or `direct` without + * TLS. + */ +const sslNegotiationOf = async ( + settings: PgConnectionSettings, + PoolClient: unknown, + env: Environment, +): Promise => { + const fromString = settings.connectionString + ? parseAsPg(settings.connectionString) + ?.parameters.filter(([key]) => key === 'sslnegotiation') + .pop()?.[1] + : undefined; + if (fromString === undefined && settings.sslnegotiation === undefined && !env.PGSSLNEGOTIATION) { + return { read: true, directTurnsTlsOn: true }; + } + const Client = typeof PoolClient === 'function' ? PoolClient : (await loadPg()).Client; + if (typeof Client === 'function') { + const Made = Client as new (config: object) => unknown; + let parameters: Record | undefined; + try { + parameters = connectionParametersOf( + // pg-pool keeps the settings' password from enumeration, so it is copied by name. + new Made({ ...settings, password: settings.password, stream: undefined, connection: undefined }), + ); + } catch (error) { + throw negotiationRefusal(error); + } + if (parameters !== undefined) { + let directTurnsTlsOn = false; + if (fromString === 'direct') { + try { + directTurnsTlsOn = Boolean(connectionParametersOf(new Made({ ...DIRECT_PROBE }))?.ssl); + } catch { + // A pg that reads direct without a parse that turns TLS on refuses the probe: its parse does not. + } + } + const read = 'sslnegotiation' in parameters; + const value = parameters.sslnegotiation; + return { read, value: read && typeof value === 'string' && value ? value : undefined, directTurnsTlsOn }; + } + } + if (!toldNegotiationUnread) { + toldNegotiationUnread = true; + getStorageLogger().warn( + { scope: 'PostgresExporter', pgClient: typeof Client === 'function' }, + "The pg the pool connects with could not be read, so sslnegotiation is read as pg 8.22.0 and later read it: " + + "the connection string's over the settings', then PGSSLNEGOTIATION.", + ); + } + const given = fromString !== undefined ? fromString : settings.sslnegotiation; + const value = typeof given === 'string' && given ? given : env.PGSSLNEGOTIATION || undefined; + return { read: true, value, directTurnsTlsOn: true }; +}; + /** * Reads connection settings as pg reads them for each connection of a pool: a connection string's parts over the * settings', then the environment, then pg's defaults. Undefined where pg cannot read the connection string. @@ -283,7 +397,11 @@ const tlsSettingsBeyondSslmode = (ssl: unknown): string[] => * @throws {Error} When the settings ask for TLS options libpq takes only from files, or the connection string for * `verify-ca` read as libpq reads it with no root certificate, or a port pg reads no number in, which pg refuses. */ -const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnection | undefined => { +const readAsPg = ( + settings: PgConnectionSettings, + env: Environment, + negotiation: SslNegotiation, +): PgConnection | undefined => { let user = settings.user; let password = settings.password; let host = settings.host; @@ -311,7 +429,7 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio if ('port' in parsed) port = parsed.port; const last = (name: string): string | undefined => parsed.parameters.filter(([key]) => key === name).pop()?.[1]; - const fromString = sslOfParameters(last); + const fromString = sslOfParameters(last, negotiation.directTurnsTlsOn); if (fromString !== undefined) { ssl = fromString; source = 'string'; @@ -332,6 +450,9 @@ const readAsPg = (settings: PgConnectionSettings, env: Environment): PgConnectio if (PG_READ_PARAMETERS.has(key)) parameters.push([key, value]); } } + // The sslnegotiation the pool's pg reads, which libpq takes from 17 on; an earlier libpq refuses the parameter, and + // PostgresExporter says so. + if (negotiation.read && negotiation.value) parameters.push(['sslnegotiation', negotiation.value]); if (source === 'environment') ssl = sslFromEnvironment(env.PGSSLMODE); const tls = tlsOf(ssl); @@ -568,7 +689,8 @@ export const libpqConnectionStringOf = async ( return connectionString; } const env = environment(); - const connection = readAsPg(settings, env); + if (connectionString && parseAsPg(connectionString) === undefined) return connectionString; + const connection = readAsPg(settings, env, await sslNegotiationOf(settings, PoolClient, env)); if (connection === undefined) return connectionString; if (!connection.database) return undefined; diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index 6a69967..4a5d467 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -347,6 +347,23 @@ describe('PostgresExporter', () => { expect(failure?.message).toMatch(/PostgreSQL 16 or later/); expect(failure?.message).toMatch(/sslrootcert parameter.*PGSSLROOTCERT/s); }); + + // libpq takes sslnegotiation from PostgreSQL 17 on. An older libpq refuses + // it as a URI parameter it does not know (REL_16_STABLE fe-connect.c, + // conninfo_uri_parse_params), in these words. + it('says what to do where the libpq of pg_dump is older than 17 and does not know sslnegotiation', async () => { + outcome.error = Object.assign(new Error('Command failed: pg_dump'), { code: 1 }); + outcome.stderr = 'pg_dump: error: invalid URI query parameter: "sslnegotiation"'; + + const failure = await refusal( + new PostgresExporter( + 'postgresql://app@db.example/app?sslnegotiation=direct&sslmode=verify-full&sslrootcert=system', + ).exportToBytes(), + ); + + expect(failure?.message).toMatch(/invalid URI query parameter: "sslnegotiation"/); + expect(failure?.message).toMatch(/PostgreSQL 17 or later/); + }); }); describe('createStorageFeatures exporter on PostgreSQL', () => { @@ -1390,5 +1407,41 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { expect(warn.mock.calls[0][0]).toMatchObject({ scope: 'PostgresExporter' }); expect(warn.mock.calls[0][1]).toMatch(/sslnegotiation/); }); + + // libpq 17 and later read PGSSLNEGOTIATION too. The URI holds the + // sslnegotiation the pool's pg reads, so pg_dump's environment drops the + // variable: a pool whose pg reads none would otherwise see pg_dump start + // TLS at once where the pool sends an SSLRequest. + it("leaves PGSSLNEGOTIATION out of pg_dump's environment where the pool's pg before 8.22.0 reads none", async () => { + vi.stubEnv('PGSSLNEGOTIATION', 'direct'); + const options = { connectionString: 'postgresql://app@db.example/app?sslmode=require' }; + + await clientPoolExporter(options, clientOfPgBeforeDirectNegotiation('turns-tls-on')).exportToBytes(); + + expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app?sslmode=verify-full&sslrootcert=system'); + expect(Object.keys(calls[0].env)).not.toContain('PGSSLNEGOTIATION'); + }); + + it("writes the PGSSLNEGOTIATION a pool's pg reads into the URI, and leaves the variable out of pg_dump's environment", async () => { + vi.stubEnv('PGSSLNEGOTIATION', 'direct'); + const settings = { connectionString: 'postgresql://app@db.example/app?sslmode=require' }; + + await poolExporter(settings).exportToBytes(); + + expect(new ConnectionParameters(settings).sslnegotiation).toBe('direct'); + expect(dbnameOf(calls[0].args)).toBe( + 'postgresql://app@db.example:5432/app?sslnegotiation=direct&sslmode=verify-full&sslrootcert=system', + ); + expect(Object.keys(calls[0].env)).not.toContain('PGSSLNEGOTIATION'); + }); + + it("refuses the export where the pool's pg refuses its sslnegotiation, and names it", async () => { + const failure = await refusal( + poolExporter({ host: 'db.example', database: 'app', user: 'app', ssl: true, sslnegotiation: 'fast' }).exportToBytes(), + ); + + expect(failure?.message).toMatch(/refuses its settings \(Invalid sslnegotiation value: "fast"/); + expect(calls).toHaveLength(0); + }); }); }); From 66aa17080457a9b69286e30da21c337592e9ba69 Mon Sep 17 00:00:00 2001 From: jddunn Date: Wed, 21 Oct 2026 03:35:16 -0700 Subject: [PATCH 32/32] fix(exporter): pg_dump reads no password file where the pool's password function answers none pg connects with no password then and reads neither PGPASSWORD nor a password file (lib/client.js, _getPassword). The URI held an empty password, which kept the environment's PGPASSWORD off pg_dump, but libpq given an empty password reads PGPASSFILE's file or ~/.pgpass (fe-connect.c, connectOptions2), so pg_dump could log in with a password the pool never sends. The URI now also names a passfile that does not exist, which libpq takes as none with no warning (passwordFromFile). --- CHANGELOG.md | 2 +- README.md | 2 +- src/exporters/pgConnection.ts | 18 +++++++++++++++--- tests/postgresExporter.args.spec.ts | 14 ++++++++++---- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0153824..3ff8bca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ - `createStorageFeatures(adapter).exporter` on PostgreSQL reads the connection of the adapter's pool as pg reads it, and hands `pg_dump` a libpq URI of it, so `pg_dump` reaches the server, the database and the user the pool reaches: - A connection string pg and libpq read apart: `postgresql://db.example?application_name=x@y` names the host `db.example` to pg and `y` to libpq, and `pg_dump` connected to `y`. An `@` in a URL's password, a host left out, `ssl=1`, `sslmode=no-verify` and pg's own parameters are written as libpq reads them. - A pool made without a host, a port, a user or a database: `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user and the user's name for the database, as pg fills them. Versions through 0.8.2 wrote localhost and 5432 for a pool that named no host or port, and refused one that named no database. The adapter's own settings keep the localhost and 5432 of the pool it makes. - - A pool whose password is a function: the function is asked when the export runs, as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls it as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters` (node-postgres #3602). So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks; a pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks. The client is never connected and is made without the settings' `stream`. Its answer goes to `PGPASSWORD`. A function that answers no password gives `pg_dump` an empty `PGPASSWORD`: pg then connects with no password and reads no `PGPASSWORD`, so the environment's is not sent. A function that fails, or answers anything but a string or `undefined`, refuses the export. Versions through 0.8.2 left the function's password out, so `pg_dump` logged in with `PGPASSWORD`, or not at all. + - A pool whose password is a function: the function is asked when the export runs, as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls it as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters` (node-postgres #3602). So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks; a pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks. The client is never connected and is made without the settings' `stream`. Its answer goes to `PGPASSWORD`. A function that answers no password gives `pg_dump` an empty `PGPASSWORD` and a `passfile` that does not exist: pg then connects with no password and reads neither `PGPASSWORD` nor a password file, so neither the environment's password nor `~/.pgpass`, which libpq reads for an empty password, is sent. A function that fails, or answers anything but a string or `undefined`, refuses the export. Versions through 0.8.2 left the function's password out, so `pg_dump` logged in with `PGPASSWORD`, or not at all. - A pool whose `ssl` holds `ca`, `cert`, `key`, `pfx` or another TLS option is refused, naming the options and what to do instead: `pg_dump` reads TLS material from files alone. Versions through 0.8.2 wrote `sslmode=require` in its place, so `pg_dump` connected without the client certificate, or checked the server against no certificate authority. - TLS follows pg's. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`) gets `sslmode=require` over any `PGSSLMODE`: a `PGSSLMODE` of `verify-ca` or `verify-full` refused the certificate pg accepts. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, where libpq's `prefer` or `allow` would use TLS. Where no TLS is pg's default, libpq's default stays, and an empty `PGSSLMODE` or one libpq does not know, which pg reads as no TLS and libpq refuses, gives way to `sslmode=disable`. A certificate file in the connection string turns TLS on over `ssl=0`, and an `sslmode` libpq does not know means TLS, as pg reads them. - pg takes the database from the URL's path alone and drops a `database` parameter, and reads an empty `host` parameter as none; the URI holds that reading (pg-connection-string, `index.js`). libpq's own variables, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT`), configure `pg_dump` as they configure psql. diff --git a/README.md b/README.md index 05bb27d..3b68a87 100644 --- a/README.md +++ b/README.md @@ -955,7 +955,7 @@ await exporter.exportToFile('/backups/snapshot.db'); const bytes = await exporter.exportToBytes(); ``` -On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is asked at each export as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls the function as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters`. So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks. That client is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. A pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks; where the exporter cannot read that pg's version or its `Client` either (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD`, since pg then connects with none and reads no `PGPASSWORD`, and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, an empty `host` parameter leaves the URL's host, and the port is read with `parseInt`, so `5432x` is 5432 and a port of 0 gives way to `PGPORT`; a port pg reads no number in is refused before `pg_dump` starts. Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication` and the certificate files. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out, so `pg_dump` reads the password file the pool reads and refuses no server, authentication or session the pool takes. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). +On Postgres, `features.exporter` runs `pg_dump` in its custom format, so it needs a Node runtime with `pg_dump` on the PATH, and an adapter whose database it can name: one made with a connection string, or with a host (a Unix-socket directory too), database and user, or with a `pg` `Pool`, or one given its connection string at `open()`. When an export runs, the exporter reads the connection of the pool the adapter runs on as pg reads it, and hands `pg_dump` a libpq URI that reaches the same server, database and user: a connection string's parts over the pool's other settings, then `PGHOST`, `PGPORT`, `PGUSER` and `PGDATABASE`, then localhost, 5432, the operating system's user, and the user's name for the database. A password function of the pool's is asked at each export as the pool asks it for each connection: through the `Client` the pool makes its connections with, which pg-pool keeps as the pool's `Client`, whose own routine calls the function as a method of a client made of the pool's settings, with no arguments before pg 8.19.0 and from 8.19.0 with the client's `ConnectionParameters`. So a pool made with another copy of pg than the one this library loads, or with a `Client` of its own, is asked as its own pg asks. That client is never connected, and it is made without the settings' `stream`, so a connector's socket function is not called. A pool whose `Client` has no such routine (pg's native one) is asked as the pg this library loads asks; where the exporter cannot read that pg's version or its `Client` either (a bundle that holds pg in itself), it calls the function on no client, with no arguments where the version it read is older than 8.19.0 and otherwise with the connection's user, database, host and port, and the storage logger says so once. Its answer goes to `PGPASSWORD`; a function that answers no password gives `pg_dump` an empty `PGPASSWORD` and a `passfile` that does not exist, since pg then connects with none and reads neither `PGPASSWORD` nor a password file, where libpq given an empty password reads `PGPASSFILE`'s file or `~/.pgpass`; and a function that fails or answers anything but a string or `undefined` refuses the export. A pool with no password leaves `PGPASSWORD`, then `~/.pgpass`, to `pg_dump`, as pg reads them. pg and libpq read some connection strings apart (an `@` in a parameter of a string with no path, a host left out), and the URI holds pg's reading: pg takes the database from the URL's path alone, so a `database` parameter is dropped, an empty `host` parameter leaves the URL's host, and the port is read with `parseInt`, so `5432x` is 5432 and a port of 0 gives way to `PGPORT`; a port pg reads no number in is refused before `pg_dump` starts. Of a connection string's parameters the URI holds the ones pg reads and libpq takes: `options`, `application_name`, `fallback_application_name`, `client_encoding`, `replication` and the certificate files. libpq's others, which pg does not read (`passfile`, `target_session_attrs`, `require_auth`, `channel_binding`, `gssencmode`, `connect_timeout`, `sslcrl` and the rest), are left out, so `pg_dump` reads the password file the pool reads and refuses no server, authentication or session the pool takes. The variables pg reads are read as pg reads them; libpq's own, which pg does not read (`PGHOSTADDR`, `PGSERVICE`, `PGREQUIRESSL`, `PGSSLROOTCERT` and the rest), configure `pg_dump` as they configure psql. Before the adapter opens, the exporter reads the settings of the pool the adapter will make, whose host and port default to localhost and 5432. A segmented adapter's export holds its segment's schema alone (`--schema`). TLS follows pg's, and so does the check of the server. pg's TLS is Node's, which checks the server's certificate and its host name unless `rejectUnauthorized` is false. A pool that checks no certificate (`ssl: { rejectUnauthorized: false }`, pg's `no-verify`, and with `uselibpqcompat=true` `sslmode=prefer`, or `require` with no root certificate) gets `sslmode=require` over any `PGSSLMODE`, since `verify-ca` or `verify-full` would refuse the certificate pg accepts; libpq's `require` still checks the certificate against a root certificate file it finds (the connection string's `sslrootcert`, `PGSSLROOTCERT`, `~/.postgresql/root.crt`), so there `pg_dump` can refuse a server pg reaches. A pool that checks the certificate (`ssl: true`, `{}`, `ssl=1`, a certificate file or any other `sslmode` but `disable` in the connection string, or a `PGSSLMODE` pg reads as TLS) gets `sslmode=verify-full` over any `PGSSLMODE`, and `sslmode=verify-ca` where pg checks no host name (`uselibpqcompat=true` with `verify-ca`, or `require` beside a root certificate). libpq checks the certificate against the root certificate file the connection string names, which pg reads too, else the file `PGSSLROOTCERT` names, else the system's certificate authorities (`sslrootcert=system`: OpenSSL's default locations, which `SSL_CERT_FILE` and `SSL_CERT_DIR` move), where Node checks against the authorities it trusts by default (the Mozilla list it bundles, and `NODE_EXTRA_CA_CERTS`): an authority one trusts and the other does not is named in the connection string's `sslrootcert` or in `PGSSLROOTCERT`. libpq reads `sslrootcert=system` from PostgreSQL 16 on; an older one looks for a file named `system`, and the export fails, saying what to do. `uselibpqcompat=true` with `sslmode=verify-ca` and no root certificate is refused before `pg_dump` starts, as pg refuses it. A pool that chose no TLS (`ssl: false`, `ssl=0`, `sslmode=disable`) gets `sslmode=disable`, whatever `PGSSLMODE` says. Where no TLS is pg's default, libpq's default (`prefer`, TLS where the server offers it) and a `PGSSLMODE` of `disable` or `allow` stay, and an empty `PGSSLMODE` or one libpq does not know, which libpq refuses, gives way to `sslmode=disable`. `pg_dump` reads certificates and keys from files alone, so an export from a pool whose `ssl` holds `ca`, `cert`, `key` or another TLS option is refused, naming them: give the pool a connection string whose `sslrootcert`, `sslcert` and `sslkey` parameters name the files, or set `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` and give the pool `ssl: true`. `sslnegotiation` follows the pool's own pg too. pg reads it from 8.22.0 (the connection string's over the settings', an empty one as none, then `PGSSLNEGOTIATION`) and with `direct` starts TLS at once, where pg 8.13.1 to 8.21.x read none and start TLS with an SSLRequest; the exporter reads it on a client of the pool's `Client`, never connected, so `pg_dump` gets the `sslnegotiation` that pg reads, none where it reads none, and TLS for `sslnegotiation=direct` only where that pg's parse turns TLS on for it (pg-connection-string 2.14.0 and later). `pg_dump` runs without `PGSSLNEGOTIATION`, which libpq 17 and later would read beside the URI. A value pg refuses (one but `direct` or `postgres`, or `direct` without TLS) refuses the export. libpq reads `sslnegotiation` from PostgreSQL 17 on; an older one refuses it as a parameter it does not know, and the export fails, saying what to do. diff --git a/src/exporters/pgConnection.ts b/src/exporters/pgConnection.ts index 92ea331..f45b5a2 100644 --- a/src/exporters/pgConnection.ts +++ b/src/exporters/pgConnection.ts @@ -490,6 +490,15 @@ const readAsPg = ( }; }; +/** + * A password file that does not exist, named where a pool's password function answers none. pg then connects with no + * password and reads no password file (`lib/client.js`, `_getPassword`), where libpq given an empty password reads its + * password file, `PGPASSFILE`'s or `~/.pgpass` (PostgreSQL 17 `fe-connect.c`, `connectOptions2`, line 1300), and takes + * a file it cannot stat as none, with no warning (`passwordFromFile`, line 7469). `/dev/null` would draw libpq's "not + * a plain file" warning. + */ +const NO_PASSWORD_FILE = '/nonexistent/sql-storage-adapter/no-password-file'; + /** Whether a pg of this version calls a password function with no arguments, as every pg before 8.19.0 does. */ const callsPasswordWithNoArguments = (version: string): boolean => { const match = /^(\d+)\.(\d+)\./.exec(version); @@ -671,8 +680,9 @@ const passwordOf = async ( * A libpq connection URI that reaches what a pg pool with these settings reaches, with its password in the user * part, where `PostgresExporter` takes it out for `PGPASSWORD`. Where the pool has no password, the URI holds none * and libpq reads `PGPASSWORD`, then `~/.pgpass`, as pg does. Where its password function answers none, pg connects - * with no password and reads no `PGPASSWORD` (`lib/client.js`, `_getPassword`): the URI holds an empty `password` - * parameter, which becomes an empty `PGPASSWORD`, so libpq sends the environment's no more than pg does. A + * with no password and reads neither `PGPASSWORD` nor a password file (`lib/client.js`, `_getPassword`): the URI holds + * an empty `password` parameter, which becomes an empty `PGPASSWORD`, so libpq sends the environment's no more than pg + * does, and a `passfile` that does not exist, so libpq reads no `~/.pgpass` either. A * connection string pg cannot read as a URL (a keyword=value string) is given back as it is. Undefined when the * settings name no database pg would reach. `PoolClient` is the `Client` the pool makes its connections with, which * pg-pool keeps as the pool's `Client`: a password function is asked through it. @@ -696,7 +706,9 @@ export const libpqConnectionStringOf = async ( const password = await passwordOf(connection, settings, PoolClient); const parameters = [...connection.parameters]; - if (typeof connection.password === 'function' && !password) parameters.push(['password', '']); + if (typeof connection.password === 'function' && !password) { + parameters.push(['password', ''], ['passfile', NO_PASSWORD_FILE]); + } const user = connection.user ? encodeURIComponent(connection.user) : ''; const secret = password ? `:${encodeURIComponent(password)}` : ''; const auth = user || secret ? `${user}${secret}@` : ''; diff --git a/tests/postgresExporter.args.spec.ts b/tests/postgresExporter.args.spec.ts index 4a5d467..35c3533 100644 --- a/tests/postgresExporter.args.spec.ts +++ b/tests/postgresExporter.args.spec.ts @@ -494,17 +494,23 @@ describe('createStorageFeatures exporter on PostgreSQL', () => { }); // pg connects with no password where the password function answers none, - // and reads no PGPASSWORD then (lib/client.js, _getPassword). pg_dump gets - // an empty PGPASSWORD, so it sends the environment's no more than pg does. + // and reads neither PGPASSWORD nor a password file then (lib/client.js, + // _getPassword). pg_dump gets an empty PGPASSWORD, so it sends the + // environment's no more than pg does, and a passfile that does not exist, + // since libpq reads PGPASSFILE's file or ~/.pgpass for an empty password + // (fe-connect.c, connectOptions2) and takes a file it cannot stat as none. it.each([ ['undefined', undefined], ['an empty string', ''], - ])("hands pg_dump an empty PGPASSWORD, not the environment's, when the password function answers %s", async (_name, answer) => { + ])("hands pg_dump an empty PGPASSWORD and no password file, not the environment's, when the password function answers %s", async (_name, answer) => { vi.stubEnv('PGPASSWORD', 'ambient-secret'); + vi.stubEnv('PGPASSFILE', '/home/app/.pgpass'); await poolExporter({ host: 'db.example', database: 'app', user: 'app', password: () => answer }).exportToBytes(); - expect(dbnameOf(calls[0].args)).toBe('postgresql://app@db.example:5432/app'); + expect(dbnameOf(calls[0].args)).toBe( + 'postgresql://app@db.example:5432/app?passfile=%2Fnonexistent%2Fsql-storage-adapter%2Fno-password-file', + ); expect(calls[0].env.PGPASSWORD).toBe(''); });