diff --git a/.github/workflows/connection.yml b/.github/workflows/connection.yml new file mode 100644 index 0000000..4bee05b --- /dev/null +++ b/.github/workflows/connection.yml @@ -0,0 +1,63 @@ +name: connection + +# Pluggable Connection CI for chdb-node. +# +# Two jobs: +# - contract: runs the in-repo Connection contract suite +# (test/v3/connection/contract.test.ts) against ChdbConnection. +# Fast, fully self-contained, no network. +# - parity: clones @clickhouse/client at the configured ref (see +# tests/clickhouse-js/skip_list.json#syncedAgainst), links +# this checkout into it, and runs the upstream integration +# suite with ChdbConnection injected. tests/clickhouse-js/ +# skip_list.json filters out the ~23 cases that are real +# chdb capability gaps (HTTP transport, auth/RBAC, server +# runtime). A new failure here is either a real new chdb +# gap (add to skip_list) or a bug in ChdbConnection. + +on: + pull_request: + branches: ["main"] + paths-ignore: ["**/*.md"] + push: + branches: ["main"] + paths-ignore: ["**/*.md"] + workflow_dispatch: {} + +concurrency: + group: connection-${{ github.ref }} + cancel-in-progress: true + +jobs: + contract: + runs-on: ubuntu-latest + env: + TZ: UTC + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + - name: Install patchelf + run: sudo apt-get update && sudo apt-get install -y patchelf + - run: npm install --ignore-scripts + - run: npm run libchdb + - run: npm run build + - run: npm run test:v3 -- test/v3/connection/ + + parity: + runs-on: ubuntu-latest + env: + TZ: UTC + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + - name: Install patchelf + run: sudo apt-get update && sudo apt-get install -y patchelf + - run: npm install --ignore-scripts + - run: npm run libchdb + - run: npm run build + - name: Cross-suite parity (clone @clickhouse/client, inject ChdbConnection, apply skip_list) + run: node tests/clickhouse-js/runner.mjs diff --git a/README.md b/README.md index 749bb34..9edba43 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,51 @@ Errors are typed (`ChdbSyntaxError`, `ChdbQueryError`, `ChdbConnectionError`, | AbortSignal / timeout | ✅ (single-shot is honest: rejects early; native runs to completion) | | Arrow **scan** (`registerArrowTable`, Arrow input) | ⏳ follow-up | | Arrow zero-copy (M2, `{ zeroCopy: true }`) | ⏳ follow-up | +| chDB ↔ `@clickhouse/client` integration (`chdb/connection`, **experimental**) | ✅ | + +### chDB ↔ `@clickhouse/client` integration (`chdb/connection`, experimental) + +> **Status**: this integration uses the experimental +> `createClient({ connection })` hook in `@clickhouse/client` +> ([clickhouse-js#879](https://github.com/ClickHouse/clickhouse-js/pull/879) +> merged; framing follow-up +> [#880](https://github.com/ClickHouse/clickhouse-js/pull/880) merged). +> Upstream considers this a deliberately narrow chDB-only hook — not a +> public plugin system — and the shape may change. We'll keep +> `chdb/connection` working against whatever the upstream hook evolves +> into. + +For users coming from `@clickhouse/client`, chdb-node ships a +**Connection** implementation under the `chdb/connection` subpath that +plugs into `@clickhouse/client`'s `createClient({ connection })` +injection point (tracking issue: +[clickhouse-js#865](https://github.com/ClickHouse/clickhouse-js/issues/865)). + +```ts +import { createChdbConnection } from 'chdb/connection' + +const conn = createChdbConnection({ path: ':memory:' }) +const r = await conn.query({ query: 'SELECT * FROM numbers(5)', format: 'JSONEachRow' }) +let body = '' +for await (const chunk of r.stream) body += Buffer.from(chunk).toString('utf8') +console.log(JSON.parse(`[${body.trim().split('\n').join(',')}]`)) +await conn.close() + +// chDB-specific escape hatches (raw ChdbResult, raw insert, session info) +conn.chdb.queryAsync('SELECT 1', { format: 'arrow' }) // bytes/text/json/toArrow +conn.chdb.session.path // bound on-disk path +``` + +See [docs/design/pluggable-connection.md](docs/design/pluggable-connection.md) +for the full design, the `Connection` interface, the `.chdb` extension +namespace, the `tests/clickhouse-js/skip_list.json` parity blacklist, +and the sync policy with `@clickhouse/client`. ### Design docs - [Layered API design](docs/design/architecture.md): the Layer 1 / Layer 2 / Layer 3 architecture, package shape, and intended user-facing surfaces. - [Layer 1 native binding reviewer guide](docs/design/layer1-native-binding.md): the PR #43 design and implementation map, organized by commit and review feedback. +- [chDB ↔ `@clickhouse/client` integration (experimental)](docs/design/pluggable-connection.md): the `chdb/connection` surface, the `Connection` interface chdb-node implements, the `.chdb` extension namespace, and the parity-test sync policy. ### Runtimes diff --git a/docs/design/pluggable-connection.md b/docs/design/pluggable-connection.md new file mode 100644 index 0000000..eb73b5d --- /dev/null +++ b/docs/design/pluggable-connection.md @@ -0,0 +1,225 @@ +# chDB ↔ `@clickhouse/client` integration (experimental) + +> **Status**: implemented in this branch (`feat/layer2-pluggable-backend`). +> Upstream tracking issue: +> [ClickHouse/clickhouse-js#865](https://github.com/ClickHouse/clickhouse-js/issues/865). +> Upstream injection point: +> [ClickHouse/clickhouse-js#879](https://github.com/ClickHouse/clickhouse-js/pull/879) +> (merged 2026-06-23 as `c2e28b9`); upstream framing: +> [ClickHouse/clickhouse-js#880](https://github.com/ClickHouse/clickhouse-js/pull/880) +> (merged 2026-06-23 as `274844f`). +> Parallel Python proposal: +> [ClickHouse/clickhouse-connect#809](https://github.com/ClickHouse/clickhouse-connect/issues/809). +> +> Upstream treats `createClient({ connection })` as a **deliberately +> narrow, internal experiment to unblock the chDB integration** — not +> the start of a public plugin system. The hook's shape may change. +> This document describes the chdb-side implementation that uses it +> today; it does NOT promise a stable third-party plugin contract. + +## What this delivers + +A `chdb/connection` subpath export that lets users plug in-process +chDB into `@clickhouse/client` with a one-line change at construction: + +```ts +import { createClient } from "@clickhouse/client"; +import { createChdbConnection } from "chdb/connection"; + +const client = createClient({ + connection: createChdbConnection({ path: ":memory:" }), +}); +``` + +All downstream code (`client.query` / `client.insert` / `client.exec` / +`client.command` / `client.ping` / `client.close`) runs unchanged +against the in-process backend. + +## Ownership split + +``` +┌──────────────────────────────────────┐ ┌──────────────────────────────────────┐ +│ @clickhouse/client (upstream) │ │ chdb-node (this repo) │ +│ │ │ │ +│ • Connection interface │ │ • ChdbConnection implements │ +│ (internal, experimental) │ │ Connection verbatim │ +│ • createClient({ connection }) │ ◄── │ • chdb/connection subpath export │ +│ one experimental option │ │ • tests/clickhouse-js/skip_list.json │ +│ │ │ (chdb-owned blacklist) │ +│ ❌ zero chdb code │ │ • tests/clickhouse-js/runner.mjs │ +│ ❌ zero chdb tests │ │ clones upstream, injects chdb, │ +│ ❌ zero chdb CI dependency │ │ applies skip_list │ +└──────────────────────────────────────┘ └──────────────────────────────────────┘ +``` + +The only upstream change is the `createClient({ connection })` +injection point (≈3 lines wrapping `NodeConfigImpl`). All chdb-side +logic (the adapter, the parity sync policy, the skip list, the CI +matrix) lives in chdb-node so the upstream codebase stays +backend-agnostic. + +## Public surface (this repo) + +`chdb/connection` exports: + +| name | purpose | +|---|---| +| `createChdbConnection(opts)` | factory; default `path: ':memory:'` (refcounted shared temp dir across same-process memory connections) | +| `ChdbConnection` | class implementing `Connection` | +| `ChdbExtension` | type for the `.chdb` namespace | +| `Connection`, `ConnQueryResult`, `ConnInsertParams`, … | RE-EXPORTED from `@clickhouse/client-common` (chdb-node does not define its own) | + +### Why re-export, not redefine + +ChdbConnection must match `@clickhouse/client-common`'s +`Connection` interface EXACTLY for the injection to +work. Defining our own "compatible" interface here would silently +drift. Instead we add `@clickhouse/client-common` as a devDependency, +re-export the real types, and let TypeScript prove conformance at +compile time. Bumping the upstream package forces a deliberate +decision in chdb-node about which contract version we conform to. + +### `.chdb` extension namespace + +`ChdbConnection.chdb` exposes chDB-specific operations that do NOT +belong on the cross-backend Connection contract: + +```ts +conn.chdb.session.path // bound on-disk path +conn.chdb.session.isTemp // process-managed temp dir? +conn.chdb.queryAsync(sql, opts) // native ChdbResult (bytes/text/json/toArrow) +conn.chdb.queryStream(sql, opts) // chunked streaming +conn.chdb.rawInsert(params) // Buffer-passthrough insert +conn.chdb.streamInsert(params) // backpressured streaming insert +``` + +The exposed set is intentionally limited to what chdb-node actually +supports today; aspirational items (`Python()` table function, UDF +registration) are NOT exposed because chdb-node does not implement +them yet. + +### `supportsZeroCopyStreaming` + +A ChdbConnection-only property (NOT part of the upstream Connection +contract) flagging whether the connection can surface result buffers +to JS without copying. **`false` today**: chdb-node copies each result +chunk into a JS Buffer (see `lib/chdb_node.cpp:678–680` "true zero-copy +is a later optimization"; `src/result.ts:56–58` "bytes are owned by +JS (copied off the engine)"). The flag will flip to `true` once +chdb-node lands the N-API external-ArrayBuffer Arrow path. Downstream +code that wants to make a routing decision today can read it as +`Boolean((conn as any).supportsZeroCopyStreaming)`. + +## Test sync policy + +Cross-suite parity (running `@clickhouse/client`'s integration suite +against `ChdbConnection`) is governed by `tests/clickhouse-js/`. The +upstream `@clickhouse/client` repo has NO chdb-related test +infrastructure — no markers, no skip-profile loader, no chdb test +runner. Everything lives here. + +### Which upstream ref we test against + +`tests/clickhouse-js/skip_list.json#syncedAgainst.ref` records the +`@clickhouse/client` ref this skip list is calibrated against: + +| `@clickhouse/client` state | Runner targets | +|---|---| +| `createClient({ connection })` PR open on personal fork | `ShawnChen-Sirius/clickhouse-js feat/pluggable-connection` | +| **PR merged to upstream (current state — #879)** | **`ClickHouse/clickhouse-js main`** | +| Released | the latest released tag | + +### When chdb-node re-runs parity + +**Only when `@clickhouse/client` cuts a new release.** Between +releases, the suite is frozen against the recorded ref; the +`skip_list.json` is stable. When a new release lands: + +1. Bump `syncedAgainst.ref` in `skip_list.json`. +2. Run `node tests/clickhouse-js/runner.mjs --refresh`. +3. For each NEW failure, decide: real capability gap (add to + `skipFiles`) vs ChdbConnection bug (fix in `src/connection/`). +4. Commit the updated `skip_list.json` and any ChdbConnection fixes. + +This means chdb-node does **not** participate in upstream CI on every +upstream commit — that would impose maintenance cost on +`@clickhouse/client` (and contradict the "minimal upstream +modification" principle). The skip_list is a snapshot, refreshed on +release boundaries. + +### Initial categories on the skip list + +The first `skip_list.json` (calibrated against `ClickHouse/clickhouse-js main` +post-#879) carries 25 file-level skips grouped as: + +- **HTTP transport** — no socket, no keep-alive, no agents, no + HTTP-level compression / headers / `/ping`. +- **Auth / RBAC / TLS** — chdb has no auth layer. +- **Server runtime / cluster** — `system.query_log`, multi-client + cluster scenarios. +- **HTTP session_id semantics** — chdb persistence is its + `Session.path`, not an HTTP session. +- **Abort divergence** — chdb's single-shot abort rejects early while + the underlying computation runs to completion (C ABI has no + interrupt); documented divergence. + +Each entry has a `reason` so reviewers can tell at a glance whether a +skip is a real chdb capability gap or an HTTP-only-feature mismatch. +When `@clickhouse/client` cuts a new release and a previously-passing +test starts failing under the runner, the triage rule is: fixable in +`ChdbConnection` → patch the adapter, real capability gap → add to +`skipFiles` / `skipTests` with a one-line reason. + +## Implementation notes + +### Memory model + +`createChdbConnection({ path: ':memory:' })` doesn't ask Layer 1 for +a fresh temp dir each time — that would fail libchdb's +single-active-data-directory rule. Instead chdb-node maintains a +process-shared, refcounted memory dir; all `:memory:` connections +share it. The multi-connection model from +[chdb-node#51](https://github.com/chdb-io/chdb-node/pull/51) lets N +native connections coexist on the same bound path, so this gives the +same shared-state semantics as clickhouse-js's `chdb://memory` while +keeping each ChdbConnection's query state independent. + +### Eager-buffered `query` / `exec` + +Both materialize via `Session.queryAsync` and surface the result as a +one-shot Readable. clickhouse-js's `Client.query` callers expect +errors at `await connection.query(...)`, not mid-stream — eager +buffering preserves that. The day chdb-node lands the N-API +external-ArrayBuffer reader, ChdbConnection will route streaming +through `queryStream`-equivalents and `supportsZeroCopyStreaming` will +flip to `true`. + +### `insert`: inline body + +clickhouse-js sends `INSERT INTO t (...) FORMAT X` with the body as a +SEPARATE `values` parameter. chDB's parser accepts the body inline +after the statement, so ChdbConnection materializes the values stream +into a string and concatenates with the query before executing as a +single statement. Callers that want chdb's zero-copy +Buffer-passthrough insert can reach for the `.chdb.rawInsert` escape +hatch. + +### Synthetic `query_id` / `response_headers` / `http_status_code` + +The Connection contract requires these on every result. chdb has no +server-side query_id, no HTTP layer; ChdbConnection synthesizes a +UUIDv4, returns `{}` for headers and `200` for status. Test code that +relies on `query_id` for log correlation should expect a unique value +but cannot map it back to any cross-process log entry. + +## What this design deliberately does NOT do + +- **No chdb-specific code in `@clickhouse/client`.** The only upstream + change is the one `connection?:` option on `createClient`. +- **No per-feature test markers in upstream tests.** Backends own + their own blacklists. +- **No HTTP loopback inside chdb-node.** Going through a wire format + would erase chdb's reason for existing (in-process, no marshalling). +- **No client-web support.** chdb is N-API; browsers cannot run it. A + future web-runtime embedded backend would repeat this pattern in + `client-web` symmetrically. diff --git a/package-lock.json b/package-lock.json index de287b5..ec0a1dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "chdb", - "version": "2.0.0", + "version": "3.1.0-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "chdb", - "version": "2.0.0", + "version": "3.1.0-rc.1", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { @@ -14,6 +14,7 @@ "node-gyp-build": "^4.6.0" }, "devDependencies": { + "@clickhouse/client-common": "^1.22.0", "@types/node": "^20.14.0", "apache-arrow": "^21.1.0", "chai": "^4.5.0", @@ -24,8 +25,87 @@ }, "engines": { "node": ">=18.0.0" + }, + "optionalDependencies": { + "@chdb/lib-darwin-arm64": "26.5.1-rc.1", + "@chdb/lib-darwin-x64": "26.5.1-rc.1", + "@chdb/lib-linux-arm64-gnu": "26.5.1-rc.1", + "@chdb/lib-linux-x64-gnu": "26.5.1-rc.1" + }, + "peerDependencies": { + "apache-arrow": ">=14" + }, + "peerDependenciesMeta": { + "apache-arrow": { + "optional": true + } } }, + "node_modules/@chdb/lib-darwin-arm64": { + "version": "26.5.1-rc.1", + "resolved": "https://registry.npmjs.org/@chdb/lib-darwin-arm64/-/lib-darwin-arm64-26.5.1-rc.1.tgz", + "integrity": "sha512-OsGlc7Y+xjG/C6tYHPrEXdzlC5vRAWHGa7CATTFpjq3McefwbGJqHNvnTFuMPObv1WK0zSDep1acsJOq9QSfWw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@chdb/lib-darwin-x64": { + "version": "26.5.1-rc.1", + "resolved": "https://registry.npmjs.org/@chdb/lib-darwin-x64/-/lib-darwin-x64-26.5.1-rc.1.tgz", + "integrity": "sha512-X6YzNT+eLudVAorAas7cAi+O48/8lgV990Yc5xuY22ydn3GI9ggUat93Ew9pGzy11Lx1JyrWp9PNvsKNTUWTgQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@chdb/lib-linux-arm64-gnu": { + "version": "26.5.1-rc.1", + "resolved": "https://registry.npmjs.org/@chdb/lib-linux-arm64-gnu/-/lib-linux-arm64-gnu-26.5.1-rc.1.tgz", + "integrity": "sha512-dxl6YAFhvBiN+xFbhHL15ibLm2oX5d0jOIf7yiw9Zypna64K763H2ixloWBx97tZWGlTLcHSR8DwM8wKDwjP0w==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@chdb/lib-linux-x64-gnu": { + "version": "26.5.1-rc.1", + "resolved": "https://registry.npmjs.org/@chdb/lib-linux-x64-gnu/-/lib-linux-x64-gnu-26.5.1-rc.1.tgz", + "integrity": "sha512-8g/P+cPX7vp1tLl3dFDDoEXuRZrevQqINfJX9DMk4uL/qPXhM5r8GDsaM8ObAai2o/L6g/cfS1/KHZ7PEl8Png==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@clickhouse/client-common": { + "version": "1.22.0", + "resolved": "https://registry.npmjs.org/@clickhouse/client-common/-/client-common-1.22.0.tgz", + "integrity": "sha512-MQgXRhoYXut6GhRrTJlub42bnPX7+5Vm+5gHNR0zZXU5+EwZKsBgMXiWXPOerAmQd3weGKm8hzoeZJCfU3Cw2w==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", diff --git a/package.json b/package.json index 615e3b4..0c6a255 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,11 @@ "import": "./index.mjs", "require": "./index.js" }, + "./connection": { + "types": "./dist/connection/index.d.ts", + "require": "./dist/connection/index.js", + "default": "./dist/connection/index.js" + }, "./package.json": "./package.json" }, "repository": { @@ -55,6 +60,7 @@ "vitest": "^2.0.0" }, "dependencies": { + "@clickhouse/client-common": "^1.22.0", "node-addon-api": "^6.1.0", "node-gyp-build": "^4.6.0" }, diff --git a/src/connection/chdb-connection.ts b/src/connection/chdb-connection.ts new file mode 100644 index 0000000..2d199b4 --- /dev/null +++ b/src/connection/chdb-connection.ts @@ -0,0 +1,571 @@ +/** + * ChdbConnection — concrete implementation of `@clickhouse/client-common`'s + * `Connection` contract on top of chdb-node's Layer 1 + * native session. + * + * import { createClient } from '@clickhouse/client' + * import { createChdbConnection } from 'chdb/connection' + * const client = createClient({ + * connection: createChdbConnection({ path: ':memory:' }), + * }) + * + * Method shapes (parameters, return types, error contract) match + * `@clickhouse/client-common` exactly; we do NOT define our own types. + * + * - `query` / `exec` MATERIALIZE the result via Session.queryAsync and + * surface it as a one-shot Readable. clickhouse-js's Client expects + * errors at `await connection.query(...)`, not mid-stream — eager + * buffering preserves that. A future native record-batch streaming + * path lands behind `supportsZeroCopyStreaming` (false today). + * - `insert` reconstructs `${params.query}\n${body}` and runs it as a + * single statement; chDB's parser accepts the inline `INSERT … FORMAT + * X\n` shape the clickhouse-js client emits. + * - `ping({ select: true|false })` runs `SELECT 1`; the HTTP `/ping` + * endpoint has no embedded analogue, so a successful SELECT is the + * strongest liveness signal we have. + * - `query_id` is synthesized as a UUIDv4 (chDB has no server-side + * query_id concept). + * - `response_headers` is `{}` and `http_status_code` is `200` (synthetic + * for the no-wire embedded path). + */ + +import { Readable } from "stream"; +import { randomUUID } from "crypto"; +import { mkdtempSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +import { parseError } from "@clickhouse/client-common"; +import type { + ChdbConnectionOptions, + ClickHouseSummary, + ConnBaseQueryParams, + ConnBaseResult, + ConnCommandResult, + ConnExecParams, + ConnExecResult, + ConnInsertParams, + ConnInsertResult, + ConnPingParams, + ConnPingResult, + ConnQueryResult, + Connection, +} from "./connection"; +import { makeChdbExtension, type ChdbExtension } from "./extension"; + +/** + * Wrap a chdb-node native Session call so any thrown error is reshaped + * into clickhouse-js's `ClickHouseError` (with the expected `.code` / + * `.type` / `.message` fields parsed out of chDB's `Code: N. + * DB::Exception: ... (TYPE)` format). Tests that assert on + * `{ code, type }` work uniformly across the HTTP and chdb backends. + */ +async function withClickHouseError(p: Promise): Promise { + try { + return await p; + } catch (err) { + const msg = + err instanceof Error && err.message ? err.message : String(err); + // parseError returns either ClickHouseError (when the input matches + // `Code: N. DB::Exception: ... (TYPE)`) or an Error wrapping the + // input verbatim. Either way we throw a typed value with the + // clickhouse-js public shape. + const mapped = parseError(msg); + // Preserve the chdb-side cause chain for debuggers that want it. + if (mapped !== err) (mapped as Error & { cause?: unknown }).cause = err; + throw mapped; + } +} + +// --------------------------------------------------------------------------- +// Layer 1 shape — pulled in via require so the same module instance is shared +// with the rest of chdb-node's runtime (see the createRequire note in +// index.mjs about the portable CJS bridge across Node/Deno/Bun). + +interface NativeChdbResult { + elapsed: number; + rowsRead: number; + bytesRead: number; + bytes(): Uint8Array; + text(): string; + json(): T; + toArrow?(): unknown; +} + +interface NativeSession { + path: string; + isTemp: boolean; + connection: unknown; + readonly open: boolean; + query(query: string, format?: string): string; + queryAsync( + query: string, + opts?: { format?: string; signal?: AbortSignal; timeout?: number }, + ): Promise; + queryBindAsync( + query: string, + params: object, + opts?: { format?: string; signal?: AbortSignal; timeout?: number }, + ): Promise; + queryStream( + query: string, + opts?: { format?: string; signal?: AbortSignal }, + ): unknown; + insert(params: object): Promise<{ + rowsWritten: number; + bytesRead?: number; + bytesWritten?: number; + bytesSent?: number; + elapsed: number; + }>; + close(): void; +} + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const rootRequire = require("../../index.js") as { + Session: new ( + path?: string, + opts?: { installSignalHandlers?: boolean }, + ) => NativeSession; + version(): { + chdb: string; + libchdb: string; + platform: string; + arch: string; + napi?: number; + }; +}; + +const { Session: SessionCtor, version } = rootRequire; + +// --------------------------------------------------------------------------- +// :memory: refcounted shared dir — libchdb binds exactly one data dir per +// process; every `:memory:` ChdbConnection grabs this single shared dir, and +// the multi-connection model lets N connections coexist on the same bound +// path. The dir is materialized on the first memory connection and removed +// when the last one closes — matching clickhouse-js's `chdb://memory` +// shared-state semantics. + +let memoryDir: string | null = null; +let memoryRefs = 0; + +function acquireMemoryDir(): string { + if (memoryDir === null) { + memoryDir = mkdtempSync(join(tmpdir(), "chdb-node-mem-")); + memoryRefs = 0; + } + memoryRefs++; + return memoryDir; +} + +function releaseMemoryDir(): void { + if (memoryDir === null) return; + if (--memoryRefs <= 0) { + const dir = memoryDir; + memoryDir = null; + memoryRefs = 0; + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + /* best-effort */ + } + } +} + +// --------------------------------------------------------------------------- +// Result-shape builders matching the @clickhouse/client-common Connection +// contract. + +function nsFromSeconds(elapsedSeconds: number | undefined): string { + return String(Math.max(0, Math.round((elapsedSeconds ?? 0) * 1e9))); +} + +function readSummary(opts: { + rowsRead: number; + bytesRead: number; + elapsed: number; +}): ClickHouseSummary { + const rr = String(opts.rowsRead || 0); + const rb = String(opts.bytesRead || 0); + return { + read_rows: rr, + read_bytes: rb, + written_rows: "0", + written_bytes: "0", + result_rows: rr, + result_bytes: rb, + total_rows_to_read: "0", + elapsed_ns: nsFromSeconds(opts.elapsed), + }; +} + +function writeSummary(opts: { + rowsWritten: number; + bytesWritten?: number; + bytesSent?: number; + elapsed: number; +}): ClickHouseSummary { + const rw = String(opts.rowsWritten || 0); + const wb = String(opts.bytesWritten ?? opts.bytesSent ?? 0); + return { + read_rows: rw, + read_bytes: wb, + written_rows: rw, + written_bytes: wb, + result_rows: "0", + result_bytes: "0", + total_rows_to_read: "0", + elapsed_ns: nsFromSeconds(opts.elapsed), + }; +} + +function baseResult(query_id: string): ConnBaseResult { + return { query_id, response_headers: {}, http_status_code: 200 }; +} + +/** + * Settings that exist only in the ClickHouse HTTP-server layer and have + * no counterpart in the embedded engine. clickhouse-js sets some of these + * by default (notably `wait_end_of_query`, which it uses to ensure the + * HTTP response body is fully written before the request settles); when + * the same Connection contract is satisfied by an in-process backend, + * those settings have no effect and emitting them as `SET` would raise + * an UNKNOWN_SETTING error. Drop them silently. + * + * Keep this set tight: only settings we know clickhouse-js emits and chdb + * provably rejects. Anything else should pass through so a real typo in + * caller-supplied `clickhouse_settings` still surfaces. + */ +const HTTP_ONLY_SETTINGS = new Set([ + "wait_end_of_query", + "send_progress_in_http_headers", + "http_response_buffer_size", + "http_headers_progress_interval_ms", + "http_send_progress_interval_ms", + "enable_http_compression", + "http_zlib_compression_level", + "http_response_compression_method", + "http_native_compression_disable_checksumming_on_decompress", +]); + +/** + * Settings clickhouse-js routes through `clickhouse_settings` for the HTTP + * backend that map onto chdb-side options instead of being SET on the + * engine. `default_format` is the canonical one: in HTTP it's a URL + * parameter telling the server which format to use when the SQL has no + * explicit FORMAT clause; in chdb the queryAsync `format` option does the + * same job. Drop it from the SET list and forward as the chdb format opt. + */ +const SETTING_TO_CHDB_OPTION: Record = { + default_format: "format", +}; + +/** Result of splitting a clickhouse_settings map into "SET-prefix" SQL and + * chdb-option overrides (currently just `format`). */ +interface SettingsSplit { + sql: string; + chdbOpts: { format?: string }; +} + +function applySettings( + sql: string, + settings?: Record, +): SettingsSplit { + const chdbOpts: { format?: string } = {}; + if (!settings) return { sql, chdbOpts }; + const stmts: string[] = []; + for (const [k, v] of Object.entries(settings)) { + if (v === undefined) continue; + if (HTTP_ONLY_SETTINGS.has(k)) continue; + if (SETTING_TO_CHDB_OPTION[k] === "format") { + chdbOpts.format = String(v); + continue; + } + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(k)) { + throw new Error(`Invalid setting name: ${JSON.stringify(k)}`); + } + if (typeof v === "number") stmts.push(`SET ${k} = ${v}`); + else if (typeof v === "boolean") stmts.push(`SET ${k} = ${v ? 1 : 0}`); + // ClickHouse SQL string literals escape `'` by DOUBLING it (`''`), not + // with a backslash (`\'`). Using backslash here would either silently + // truncate the value or produce malformed SQL on engines that don't + // accept backslash escapes — drift between adapter and engine. + else stmts.push(`SET ${k} = '${String(v).replace(/'/g, "''")}'`); + } + return { + sql: stmts.length === 0 ? sql : `${stmts.join("; ")}; ${sql}`, + chdbOpts, + }; +} + +function readableFromBuffer(buf: Buffer | Uint8Array): Readable { + const b = Buffer.isBuffer(buf) + ? buf + : Buffer.from(buf.buffer, buf.byteOffset, buf.byteLength); + return new Readable({ + read() { + this.push(b); + this.push(null); + }, + }); +} + +async function streamToString( + values: string | Readable | AsyncIterable, +): Promise { + if (typeof values === "string") return values; + let body = ""; + for await (const chunk of values as AsyncIterable) { + if (typeof chunk === "string") body += chunk; + else if (Buffer.isBuffer(chunk)) body += chunk.toString("utf8"); + else if (chunk instanceof Uint8Array) body += Buffer.from(chunk).toString("utf8"); + else throw new Error(`Unsupported insert chunk type: ${typeof chunk}`); + } + return body; +} + +// --------------------------------------------------------------------------- + +/** + * Concrete `Connection` backed by an in-process chDB Layer 1 + * Session. Created via {@link createChdbConnection}. + */ +export class ChdbConnection implements Connection { + /** Routing hint for callers that key off the in-process backend identity. + * Not part of the Connection contract — a ChdbConnection-only property. */ + readonly connectionName = "chdb"; + + /** False today: chdb-node copies each result chunk into a JS Buffer. + * Will flip to true once the N-API external-ArrayBuffer Arrow path + * lands and downstream code can plumb its streaming/DataFrame path + * directly to chDB's record-batch reader. */ + readonly supportsZeroCopyStreaming = false; + + readonly #session: NativeSession; + readonly #chdb: ChdbExtension; + readonly #ownsMemoryRef: boolean; + #closed = false; + + #serverVersion?: Promise; + #serverTimezone?: Promise; + + constructor(opts: ChdbConnectionOptions = {}) { + const isMemory = opts.path === ":memory:" || opts.path === undefined; + if (isMemory) { + const sharedDir = acquireMemoryDir(); + this.#ownsMemoryRef = true; + try { + this.#session = new SessionCtor(sharedDir); + } catch (e) { + releaseMemoryDir(); + throw e; + } + } else { + this.#ownsMemoryRef = false; + this.#session = new SessionCtor(opts.path); + } + this.#chdb = makeChdbExtension( + this.#session as Parameters[0], + ); + } + + /** chDB-specific raw escape hatches. See {@link ChdbExtension}. */ + get chdb(): ChdbExtension { + return this.#chdb; + } + + // ------------------------------------------------------------------------- + // Connection implementation + + /** + * Run the SQL via chdb's parameterized bind path when `query_params` is + * non-empty (mirrors clickhouse-js's `param_*` URL-binding HTTP path), + * otherwise the plain `queryAsync`. chdb-node's queryBindAsync handles the + * JS-to-engine value formatting; for top-level strings clickhouse-js's + * unquoted convention is honored by chdb's parameter binder. + */ + async #runSql( + sql: string, + params: ConnBaseQueryParams, + chdbOpts: { format?: string } = {}, + ): Promise { + const opts = { ...chdbOpts, signal: params.abort_signal } as const; + if (params.query_params && Object.keys(params.query_params).length > 0) { + return withClickHouseError( + this.#session.queryBindAsync(sql, params.query_params, opts), + ); + } + return withClickHouseError(this.#session.queryAsync(sql, opts)); + } + + async query( + params: ConnBaseQueryParams, + ): Promise> { + this.#assertOpen(); + const query_id = params.query_id ?? randomUUID(); + const { sql, chdbOpts } = applySettings( + params.query, + params.clickhouse_settings, + ); + const result = await this.#runSql(sql, params, chdbOpts); + return { + stream: readableFromBuffer(result.bytes()), + ...baseResult(query_id), + }; + } + + async exec( + params: ConnExecParams, + ): Promise> { + this.#assertOpen(); + const query_id = params.query_id ?? randomUUID(); + const applied = applySettings(params.query, params.clickhouse_settings); + let sql = applied.sql; + const chdbOpts = applied.chdbOpts; + if (params.values !== undefined) { + const body = await streamToString(params.values); + // Empty exec-with-values body short-circuit. clickhouse-js's exec + // callers may pass an empty / already-closed stream; chdb's parser + // rejects an INSERT without rows with "No data to insert", which + // contradicts the no-op-success semantics every caller expects. + if (body.trim().length === 0) { + return { + stream: readableFromBuffer(Buffer.alloc(0)), + ...baseResult(query_id), + summary: writeSummary({ + rowsWritten: 0, + bytesWritten: 0, + elapsed: 0, + }), + }; + } + sql = `${sql}\n${body}`; + } + const result = await this.#runSql(sql, params, chdbOpts); + return { + stream: readableFromBuffer(result.bytes()), + ...baseResult(query_id), + summary: readSummary(result), + }; + } + + async command(params: ConnBaseQueryParams): Promise { + this.#assertOpen(); + const query_id = params.query_id ?? randomUUID(); + const { sql, chdbOpts } = applySettings( + params.query, + params.clickhouse_settings, + ); + const result = await this.#runSql(sql, params, chdbOpts); + return { ...baseResult(query_id), summary: readSummary(result) }; + } + + async insert( + params: ConnInsertParams, + ): Promise { + this.#assertOpen(); + const query_id = params.query_id ?? randomUUID(); + // clickhouse-js sends `INSERT INTO t (...) FORMAT X` with the body as a + // SEPARATE `values` parameter. chDB accepts the body inline after the + // statement, so we materialize the stream/string and concatenate. The + // .chdb extension's rawInsert remains available for callers that want + // chdb-node's Buffer-passthrough zero-copy insert. + const body = await streamToString(params.values); + // Empty body short-circuit. clickhouse-js's exec/insert callers may + // pass an empty / already-closed stream when there's nothing to send; + // running `INSERT INTO t FORMAT X\n` against chdb raises + // "No data to insert", whereas a no-op success is the expected + // semantics. + if (body.trim().length === 0) { + return { + ...baseResult(query_id), + summary: writeSummary({ + rowsWritten: 0, + bytesWritten: 0, + elapsed: 0, + }), + }; + } + const { sql, chdbOpts } = applySettings( + `${params.query}\n${body}`, + params.clickhouse_settings, + ); + const result = await this.#runSql(sql, params, chdbOpts); + return { + ...baseResult(query_id), + summary: writeSummary({ + rowsWritten: result.rowsRead || 0, + bytesWritten: result.bytesRead, + elapsed: result.elapsed, + }), + }; + } + + async ping(_params: ConnPingParams): Promise { + if (this.#closed) { + return { success: false, error: new Error("ChdbConnection is closed") }; + } + try { + // Both `{ select: true }` and `{ select: false }` execute SELECT 1. + // The HTTP `/ping` endpoint has no embedded analogue, so a successful + // SELECT against the engine is the strongest liveness signal we have. + await withClickHouseError(this.#session.queryAsync("SELECT 1", { format: "CSV" })); + return { success: true }; + } catch (e) { + return { success: false, error: this.mapError(e) }; + } + } + + async close(): Promise { + if (this.#closed) return; + this.#closed = true; + this.#session.close(); + if (this.#ownsMemoryRef) releaseMemoryDir(); + } + + // ------------------------------------------------------------------------- + // Diagnostic helpers (not part of the Connection contract). + + get serverVersion(): Promise { + if (!this.#serverVersion) { + this.#serverVersion = Promise.resolve(version().libchdb); + } + return this.#serverVersion; + } + + get serverTimezone(): Promise { + if (!this.#serverTimezone) { + this.#serverTimezone = Promise.resolve("UTC"); + } + return this.#serverTimezone; + } + + // ------------------------------------------------------------------------- + + /** + * Coerce any thrown value into an `Error` instance. Mirrors + * `NodeBaseConnection.mapError` from clickhouse-js — useful for callers + * that catch from this connection and want a uniform Error shape. + * Not part of the cross-backend `Connection` interface. + */ + mapError(err: unknown): Error { + if (err instanceof Error) return err; + return new Error(String(err)); + } + + #assertOpen(): void { + if (this.#closed) throw new Error("ChdbConnection is closed"); + } +} + +/** + * Factory for {@link ChdbConnection}. Use `path: ':memory:'` (the default) + * for an ephemeral session whose temp dir is reference-counted across + * other `:memory:` connections in the same process; pass an absolute + * path to bind a persistent session. + */ +export function createChdbConnection( + opts: ChdbConnectionOptions = {}, +): ChdbConnection { + return new ChdbConnection(opts); +} diff --git a/src/connection/connection.ts b/src/connection/connection.ts new file mode 100644 index 0000000..daae023 --- /dev/null +++ b/src/connection/connection.ts @@ -0,0 +1,61 @@ +/** + * Pluggable Connection contract for chdb-node. + * + * chdb-node's `ChdbConnection` implements `@clickhouse/client-common`'s + * published `Connection` interface EXACTLY — same method + * signatures, same result shapes, same error semantics. That lets a user + * write the same `@clickhouse/client` code against either a remote + * ClickHouse server or in-process chDB with a one-line change at + * construction: + * + * import { createClient } from '@clickhouse/client' + * import { createChdbConnection } from 'chdb/connection' + * const client = createClient({ + * connection: createChdbConnection({ path: ':memory:' }), + * }) + * + * To keep that promise this module DOES NOT define its own Connection / + * ConnQueryResult / ConnInsertParams / … types. Instead it re-exports + * them verbatim from `@clickhouse/client-common`. Bumping that package + * forces a deliberate decision in chdb-node about which version of the + * contract we conform to. + * + * The chDB-specific options ({@link ChdbConnectionOptions}) and the raw + * escape-hatch surface (`.chdb`, declared in extension.ts) stay defined + * here in chdb-node — they have no counterpart in clickhouse-js's + * Connection contract. + */ + +export type { + Connection, + ConnBaseQueryParams, + ConnBaseResult, + ConnCommandResult, + ConnExecParams, + ConnExecResult, + ConnInsertParams, + ConnInsertResult, + ConnOperation, + ConnPingParams, + ConnPingResult, + ConnQueryResult, +} from "@clickhouse/client-common"; + +export type { + ClickHouseSummary, + WithClickHouseSummary, +} from "@clickhouse/client-common"; + +/** + * Options accepted by {@link createChdbConnection}. + */ +export interface ChdbConnectionOptions { + /** + * On-disk session path. `':memory:'` (the default) uses an ephemeral + * temp dir that is reference-counted across same-process `:memory:` + * connections, so two ChdbConnections both on `:memory:` see the same + * in-process data (matching the chDB single-engine-per-process model). + * A different on-disk path while one is live is rejected by Layer 1. + */ + path?: string; +} diff --git a/src/connection/extension.ts b/src/connection/extension.ts new file mode 100644 index 0000000..ec9b5b5 --- /dev/null +++ b/src/connection/extension.ts @@ -0,0 +1,131 @@ +/** + * `.chdb` extension namespace — chDB-specific escape hatches. + * + * These methods expose chdb-node's native surface that does NOT fit on the + * cross-backend {@link Connection} interface (which by design is shaped after + * `@clickhouse/client`'s public Connection contract). They are reachable as + * `connection.chdb.*` on a ChdbConnection instance, and let users: + * + * - bypass the materialize-then-Readable shape of `Connection.query` and + * work directly with the native single-shot ChdbResult (`bytes()`, + * `text()`, `json()`, `toArrow()` — including the apache-arrow path + * for `format: 'arrow'`); + * - run a real streaming query through `Session.queryStream` (chunked + * output in the chosen format, lazily fetched on the libuv thread); + * - do raw passthrough inserts where the V8 Buffer is handed to the + * native side zero-copy and chDB's multithreaded parser does the work + * (matches the byte-compat semantics of chdb-node's `Session.insert` + * raw form); + * - read session metadata (the on-disk path, `isTemp`). + * + * Why a separate namespace rather than additional methods on {@link Connection}? + * + * The Connection interface is the cross-backend contract: same shape will + * be implemented by future backends and consumed by `@clickhouse/client`'s + * eventual `connection` injection point. chDB-only capabilities do not + * belong on that contract — they live here, scoped under `.chdb`, so a + * higher-level client can statically narrow on the brand and surface + * `client.chdb.queryAsync` exclusively when the connection is a chdb one. + * + * The set of methods exposed here is intentionally LIMITED to what chdb-node + * actually supports today. Aspirational features (Python() table function, + * UDF registration) are not present because chdb-node does not expose them + * yet; they will be added when chdb-node lands the underlying native + * support. + */ + +import type { + ChdbResult as RootChdbResult, + ChdbQueryStream as RootChdbQueryStream, + QueryOptions as RootQueryOptions, + StreamOptions as RootStreamOptions, + RawInsertParams as RootRawInsertParams, + RawInsertSummary as RootRawInsertSummary, + StreamInsertParams as RootStreamInsertParams, + StreamInsertSummary as RootStreamInsertSummary, +} from '../../index' + +/** + * Read-only session metadata. Same data Layer 1's {@link Session} exposes — + * just surfaced under `.chdb.session` to make the connection-level access + * pattern obvious. + */ +export interface ChdbSessionInfo { + /** Filesystem path the underlying chDB session is bound to. */ + readonly path: string + /** True when the session is using a process-managed temp directory. */ + readonly isTemp: boolean +} + +/** + * The `.chdb` extension namespace. Reachable as + * `chdbConnection.chdb` on instances of {@link ChdbConnection}. + * + * Methods are thin wrappers around the corresponding Layer 1 Session + * methods, preserving their full option surface (signal, timeout, format, + * etc.) and their return shapes so existing chdb-node users can migrate + * to the pluggable Connection without losing access to the native API. + */ +export interface ChdbExtension { + /** Session metadata (path, isTemp). */ + readonly session: ChdbSessionInfo + + /** + * Single-shot async query — returns the native {@link ChdbResult} with + * `bytes() / text() / json() / toArrow()` views over the same buffer. + * Use this when you want the result as bytes or apache-arrow Table + * without going through {@link Connection.query}'s Readable shape. + */ + queryAsync(sql: string, opts?: RootQueryOptions): Promise + + /** + * True streaming query — returns a {@link ChdbQueryStream} that yields + * one {@link StreamChunk} per native fetch. The chosen format must be a + * streaming-friendly format (default: `'JSONEachRow'`). Only one active + * stream per session. + */ + queryStream(sql: string, opts?: RootStreamOptions): RootChdbQueryStream + + /** + * Raw passthrough insert: the payload Buffer is handed to the native + * side zero-copy, and chDB's multithreaded parser does all the parsing. + * The payload must already be in the declared FORMAT (whitelisted — + * see Layer 1's RawInsertFormat). + */ + rawInsert(params: RootRawInsertParams): Promise + + /** + * Backpressured streaming insert — the source is consumed pull-based, + * with at most one bounded chunk buffered, so a fast producer is + * throttled to the chDB write rate. Returns aggregate progress. + */ + streamInsert(params: RootStreamInsertParams): Promise +} + +/** + * Build a {@link ChdbExtension} bound to a Layer 1 Session. Internal to + * chdb-node; users obtain a ChdbExtension via `chdbConnection.chdb`. + */ +export function makeChdbExtension(session: { + path: string + isTemp: boolean + queryAsync: (sql: string, opts?: RootQueryOptions) => Promise + queryStream: (sql: string, opts?: RootStreamOptions) => RootChdbQueryStream + insert: (params: object) => Promise +}): ChdbExtension { + const sessionInfo: ChdbSessionInfo = Object.freeze({ + get path() { + return session.path + }, + get isTemp() { + return session.isTemp + }, + }) + return { + session: sessionInfo, + queryAsync: (sql, opts) => session.queryAsync(sql, opts), + queryStream: (sql, opts) => session.queryStream(sql, opts), + rawInsert: (params) => session.insert(params) as Promise, + streamInsert: (params) => session.insert(params) as Promise, + } +} diff --git a/src/connection/index.ts b/src/connection/index.ts new file mode 100644 index 0000000..432c912 --- /dev/null +++ b/src/connection/index.ts @@ -0,0 +1,40 @@ +/** + * `chdb/connection` — pluggable Connection surface for `@clickhouse/client`. + * + * import { createClient } from '@clickhouse/client' + * import { createChdbConnection } from 'chdb/connection' + * const client = createClient({ + * connection: createChdbConnection({ path: ':memory:' }), + * }) + * + * Public surface: + * - `createChdbConnection`, `ChdbConnection` — factory + class + * - `ChdbExtension`, `ChdbSessionInfo` — the `.chdb` raw escape hatches + * - `ChdbConnectionOptions` — constructor options + * - The `Connection` interface and its result types are + * RE-EXPORTED VERBATIM from `@clickhouse/client-common` (chdb-node + * does not define its own); see ./connection.ts. + */ + +export { createChdbConnection, ChdbConnection } from "./chdb-connection"; + +export type { ChdbConnectionOptions } from "./connection"; + +export type { + Connection, + ConnBaseQueryParams, + ConnBaseResult, + ConnCommandResult, + ConnExecParams, + ConnExecResult, + ConnInsertParams, + ConnInsertResult, + ConnOperation, + ConnPingParams, + ConnPingResult, + ConnQueryResult, + ClickHouseSummary, + WithClickHouseSummary, +} from "./connection"; + +export type { ChdbExtension, ChdbSessionInfo } from "./extension"; diff --git a/test/v3/connection/contract.test.ts b/test/v3/connection/contract.test.ts new file mode 100644 index 0000000..84402c5 --- /dev/null +++ b/test/v3/connection/contract.test.ts @@ -0,0 +1,175 @@ +/** + * Connection-contract suite for ChdbConnection. + * + * Pins the cross-backend invariants of the public `Connection` + * contract (the one re-exported from `@clickhouse/client-common`) that + * ChdbConnection must satisfy. The assertions here use the snake_case + * Connection-layer shape (`query_id`, `response_headers`, `http_status_code`, + * `summary`) — the Client layer above translates to camelCase before + * user-facing code sees it, but that's a layer above this. + * + * What this file does NOT test: + * - byte-level output parity against a real ClickHouse server + * → tests/clickhouse-js/runner.mjs + * - chDB-specific raw-channel behavior (queryAsync / queryStream / + * rawInsert) → existing test/v3/*.test.ts Layer 1 suite + */ + +import { Readable } from 'stream' +import { describe, it, expect } from 'vitest' +import { createChdbConnection, ChdbConnection } from '../../../src/connection' +import type { Connection } from '../../../src/connection' + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + +async function drainReadable(stream: Readable): Promise { + let body = '' + for await (const chunk of stream) { + body += Buffer.isBuffer(chunk) + ? chunk.toString('utf8') + : typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8') + } + return body +} + +describe('ChdbConnection — Connection contract', () => { + it('satisfies Connection assignment', () => { + // Compile-time assignment is the strongest check; if ChdbConnection ever + // drifts from the upstream interface this won't typecheck. + const conn: Connection = createChdbConnection({ path: ':memory:' }) + expect(conn.connectionName).toBe('chdb') + }) + + it('is also a ChdbConnection instance (for type narrowing on the brand)', () => { + const conn = createChdbConnection({ path: ':memory:' }) + expect(conn).toBeInstanceOf(ChdbConnection) + }) + + it('query() returns { stream, query_id, response_headers, http_status_code }', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + const r = await conn.query({ + query: 'SELECT 1 AS n FORMAT JSONEachRow', + }) + // Snake_case Connection-layer shape (matches @clickhouse/client-common). + expect(typeof r.query_id).toBe('string') + expect(r.query_id).toMatch(UUID_RE) + expect(r.response_headers).toEqual({}) + expect(r.http_status_code).toBe(200) + expect(await drainReadable(r.stream)).toContain('"n":1') + }) + + it('honors clickhouse_settings via SET-prefix (default_format → format opt)', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + // Use upstream's real settings field — default_format routes to chdb's + // queryAsync format option (HTTP backend uses it as a URL param). + const r = await conn.query({ + query: 'SELECT 1', + clickhouse_settings: { default_format: 'CSV' as never }, + }) + const body = await drainReadable(r.stream) + expect(body.trim()).toBe('1') + }) + + it('exec() returns query result shape PLUS summary', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + const r = await conn.exec({ query: 'SELECT 2 FORMAT CSV' }) + expect(typeof r.query_id).toBe('string') + expect(r.summary.read_rows).toMatch(/^\d+$/) + expect((await drainReadable(r.stream)).trim()).toBe('2') + }) + + it('command() executes DDL and returns base result + summary, no stream', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + const r = await conn.command({ query: 'CREATE TABLE t1 (x Int32) ENGINE=Memory' }) + expect(r.query_id).toMatch(UUID_RE) + expect(typeof r.summary.elapsed_ns).toBe('string') + // ConnCommandResult has no `stream` field — only base result + summary. + expect('stream' in r).toBe(false) + + await conn.command({ query: 'INSERT INTO t1 VALUES (42)' }) + const sel = await conn.query({ query: 'SELECT x FROM t1 FORMAT CSV' }) + expect((await drainReadable(sel.stream)).trim()).toBe('42') + }) + + it('insert() takes a pre-built INSERT query + values and returns summary', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + await conn.command({ query: 'CREATE TABLE t2 (id Int32, name String) ENGINE=Memory' }) + const payload = '{"id":1,"name":"a"}\n{"id":2,"name":"b"}\n' + // ConnInsertParams expects { query, values: string | Stream } — the + // Client layer above translates user-facing `{ table, format, values }` + // into the `INSERT INTO t2 FORMAT JSONEachRow` query string before + // this point. Pass values as `string` (the simplest valid form). + const r = await conn.insert({ + query: 'INSERT INTO t2 FORMAT JSONEachRow', + values: payload, + }) + expect(r.query_id).toMatch(UUID_RE) + // Summary is populated with the documented stringly-typed numeric fields. + // The exact written_rows value is engine-dependent (chdb's native + // INSERT path doesn't always report it); we assert SHAPE here and + // verify the data landed via a follow-up SELECT below. + expect(typeof r.summary.written_rows).toBe('string') + expect(typeof r.summary.elapsed_ns).toBe('string') + + const sel = await conn.query({ query: 'SELECT count() FROM t2 FORMAT CSV' }) + expect((await drainReadable(sel.stream)).trim()).toBe('2') + }) + + it('ping() returns ConnPingResult { success: true } when live', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + const r = await conn.ping({ select: false }) + expect(r.success).toBe(true) + }) + + it('ping() returns { success: false, error } after close (does not throw)', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + await conn.close() + const r = await conn.ping({ select: false }) + expect(r.success).toBe(false) + if (!r.success) expect(r.error).toBeInstanceOf(Error) + }) + + it('close() is idempotent', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + await conn.close() + await conn.close() // must not throw + }) + + it('mapError() coerces any thrown value into an Error', () => { + // Not on the Connection interface but mirrors clickhouse-js's + // NodeBaseConnection.mapError convention; useful for callers. + const conn = createChdbConnection({ path: ':memory:' }) + expect(conn.mapError(new TypeError('boom'))).toBeInstanceOf(Error) + expect(conn.mapError('non-error')).toBeInstanceOf(Error) + expect(conn.mapError({ msg: 'object' }).message).toContain('object Object') + }) + + it('.chdb extension surface is present with documented methods', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + expect(conn.chdb).toBeDefined() + expect(typeof conn.chdb.session.path).toBe('string') + expect(typeof conn.chdb.session.isTemp).toBe('boolean') + expect(typeof conn.chdb.queryAsync).toBe('function') + expect(typeof conn.chdb.queryStream).toBe('function') + expect(typeof conn.chdb.rawInsert).toBe('function') + expect(typeof conn.chdb.streamInsert).toBe('function') + + // Raw queryAsync surfaces native ChdbResult views (bytes/text/json) + const native = await conn.chdb.queryAsync('SELECT 1 AS n', { format: 'JSON' }) + expect(native.rowsRead).toBe(1) + const parsed = native.json<{ data: Array<{ n: number }> }>() + expect(parsed.data[0]?.n).toBe(1) + }) +}) + +describe('ChdbConnection — query() materialization invariant', () => { + it('rejects at await when the SQL fails (eager-buffered semantics)', async () => { + const conn = createChdbConnection({ path: ':memory:' }) + // BAD_FUNCTION(): the engine has no such function, so this must reject at + // await — not push an error chunk into the Readable. This pins the + // byte-compat semantics clickhouse-js expects of Connection.query today. + await expect( + conn.query({ query: 'SELECT BAD_FUNCTION()' }) + ).rejects.toThrow() + }) +}) diff --git a/tests/clickhouse-js/README.md b/tests/clickhouse-js/README.md new file mode 100644 index 0000000..9f4dcfd --- /dev/null +++ b/tests/clickhouse-js/README.md @@ -0,0 +1,97 @@ +# chdb-node × @clickhouse/client parity runner + +This directory carries everything chdb-node needs to verify that the +`@clickhouse/client` integration suite passes against `ChdbConnection`. +**All chdb-side parity logic lives here, in chdb-node's repo.** The +upstream `@clickhouse/client` repo carries zero chdb-specific code and +zero chdb CI dependency — the only upstream change is the +`createClient({ connection })` injection point. + +## What's in this directory + +| File | Purpose | +|---|---| +| `skip_list.json` | The AUTHORITATIVE blacklist of upstream tests chdb does not support. Maintained by the chdb team. | +| `runner.mjs` | Clones `@clickhouse/client` at the configured ref, links this chdb-node checkout into it, **patches the upstream `vitest.node.setup.ts` in-place** to wrap `globalThis.environmentSpecificCreateClient` with `createChdbConnection`, runs the integration suite with `skip_list.json` applied as `--exclude` patterns, then restores the patch in a `try/finally` block. | + +## Sync policy + +`skip_list.json#syncedAgainst.ref` records which `@clickhouse/client` +version chdb-node currently asserts parity against. The policy: + +| `@clickhouse/client` state | Runner targets | +|---|---| +| `createClient({ connection })` PR open on the personal fork | `ShawnChen-Sirius/clickhouse-js feat/pluggable-connection` | +| **PR merged to upstream (current state — [#879](https://github.com/ClickHouse/clickhouse-js/pull/879))** | **`ClickHouse/clickhouse-js main`** | +| Released | the latest released tag (e.g. `v1.24.0`) | + +**chdb-node only re-runs parity (and updates `skip_list.json`) when +`@clickhouse/client` cuts a new release.** During the rolling period +the runner stays pointed at whichever ref the policy table specifies; +that's a single line in `skip_list.json#syncedAgainst.ref`. + +## Running the runner + +```sh +node tests/clickhouse-js/runner.mjs # full run +node tests/clickhouse-js/runner.mjs --refresh # re-clone fresh +``` + +Environment overrides: + +```sh +CHDB_CLICKHOUSE_JS_REPO=https://github.com/.../clickhouse-js.git +CHDB_CLICKHOUSE_JS_REF=feat/pluggable-connection +CHDB_RUNNER_WORK_DIR=/tmp/chdb-runner/clickhouse-js +node tests/clickhouse-js/runner.mjs +``` + +## Adding / removing a skip entry + +When a new release lands and the runner surfaces a NEW failing test: + +1. **Real capability gap** (chdb does not support the feature): + add the file to `skipFiles` with a one-line `reason`. +2. **Bug in `ChdbConnection`** (chdb should support the case but the + adapter is wrong): fix in `src/connection/chdb-connection.ts`. +3. **Per-test divergence inside an otherwise-runnable file**: add the + full vitest node-id (`packages/.../foo.test.ts > describe > it`) to + `skipTests`. + +When chdb implements a previously-missing capability and the test now +passes, remove the entry from `skipFiles`. + +## Categories already in the skip list + +The initial `skip_list.json` was populated from a representative run +against `ClickHouse/clickhouse-js main` (post-[#879](https://github.com/ClickHouse/clickhouse-js/pull/879)). +The categories are: + +- **HTTP transport** (no socket, no headers, no compression, no + keep-alive, no agents): `node_socket_handling`, `node_keep_alive*`, + `node_max_open_connections`, `node_custom_http_agent`, + `node_compression`, `node_ping` (HTTP `/ping`), `node_summary` + (X-ClickHouse-Summary header), `node_logger_support`, + `multipart_params`, `request_compression`, `response_compression`, + `exception_header`. +- **Auth / RBAC** (chdb has no auth layer): `auth`, `role`, + `read_only_user`, `node_jwt_auth`, `tls`. +- **Server runtime / cluster topology**: `query_log`, `multiple_clients`. +- **HTTP session semantics**: `session`. +- **AbortSignal divergence**: `abort_request` (chdb's single-shot abort + rejects early while the underlying computation runs to completion; + documented divergence). +- **Default-timezone divergence** (per-test, in `skipTests`): chdb + resolves DateTime[64] values without an explicit TZ argument against + the host's system timezone, not UTC — libchdb captures the ICU TZ + at load time and `SET session_timezone='UTC'` does not influence the + output formatter. Round-trip is lossless at the instant level; only + the display string differs by the host's UTC offset. +- **Float64 output precision** (per-test): chdb prints 17 significant + digits for IEEE 754 round-trip; clickhouse-server prints 15. + +Each entry has a `reason`. When `@clickhouse/client` cuts a new release +and a previously-passing test starts failing under the runner, the +triage rule is: fixable in `ChdbConnection` → patch +`src/connection/chdb-connection.ts`, real capability gap → add to +`skipFiles` / `skipTests` with a one-line reason. diff --git a/tests/clickhouse-js/runner.mjs b/tests/clickhouse-js/runner.mjs new file mode 100644 index 0000000..4beb60d --- /dev/null +++ b/tests/clickhouse-js/runner.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +/** + * chdb-node × @clickhouse/client cross-suite parity runner. + * + * Clones @clickhouse/client at a configured ref, builds it, patches + * upstream's `vitest.node.setup.ts` in-place to wrap + * `globalThis.environmentSpecificCreateClient` with `createChdbConnection`, + * then runs the integration suite filtered by + * `tests/clickhouse-js/skip_list.json`. The setup patch is restored in a + * try/finally block so the upstream checkout isn't left dirty. + * + * Sync policy (encoded in skip_list.json's `syncedAgainst` block): + * + * stage of the upstream injection PR → ref this runner uses + * ----------------------------------------+----------------------- + * open on the personal fork → ShawnChen-Sirius/clickhouse-js + * feat/pluggable-connection + * merged to ClickHouse/clickhouse-js → ClickHouse/clickhouse-js main + * (current state — #879 is merged) + * released → the latest released tag + * + * Override at runtime: + * + * CHDB_CLICKHOUSE_JS_REPO=https://github.com/.../clickhouse-js.git + * CHDB_CLICKHOUSE_JS_REF=main + * + * Usage: + * + * node tests/clickhouse-js/runner.mjs — run the whole suite + * node tests/clickhouse-js/runner.mjs --refresh — re-clone (no cache) + */ + +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, "../.."); + +const SKIP = JSON.parse(readFileSync(join(HERE, "skip_list.json"), "utf8")); +const synced = SKIP.syncedAgainst || {}; + +const REPO = process.env.CHDB_CLICKHOUSE_JS_REPO + || (synced.ref?.startsWith("ShawnChen-Sirius/") + ? "https://github.com/ShawnChen-Sirius/clickhouse-js.git" + : "https://github.com/ClickHouse/clickhouse-js.git"); +const REF = process.env.CHDB_CLICKHOUSE_JS_REF + || synced.ref?.split(" ").pop() + || "main"; + +const WORK_DIR = process.env.CHDB_RUNNER_WORK_DIR + || join(REPO_ROOT, ".chdb-runner", "clickhouse-js"); +const REFRESH = process.argv.includes("--refresh"); + +function sh(cmd, args, opts = {}) { + console.log(`[runner] $ ${cmd} ${args.join(" ")}${opts.cwd ? ` (in ${opts.cwd})` : ""}`); + const r = spawnSync(cmd, args, { stdio: "inherit", ...opts }); + if (r.status !== 0) { + throw new Error(`${cmd} ${args.join(" ")} → exit ${r.status}`); + } +} + +function clone() { + if (REFRESH && existsSync(WORK_DIR)) { + rmSync(WORK_DIR, { recursive: true, force: true }); + } + if (existsSync(WORK_DIR)) { + // Reset any setup-file patches from a previous run, then update to REF. + // `git pull origin ` fails when REF is a tag (the sync policy + // points at the latest released tag, not a branch). `fetch + checkout` + // handles both branches and tags uniformly. + sh("git", ["-C", WORK_DIR, "reset", "--hard", "HEAD"]); + sh("git", ["-C", WORK_DIR, "fetch", "--tags", "origin"]); + sh("git", ["-C", WORK_DIR, "checkout", REF]); + return; + } + mkdirSync(dirname(WORK_DIR), { recursive: true }); + sh("git", ["clone", "--depth", "50", "--branch", REF, REPO, WORK_DIR]); +} + +function installAndBuild() { + sh("npm", ["install"], { cwd: WORK_DIR }); + sh("npm", ["--workspaces", "run", "build"], { cwd: WORK_DIR }); + // Link this chdb-node checkout as the `chdb` dependency so the injection + // setup can `import { createChdbConnection } from 'chdb/connection'`. + sh("npm", ["install", REPO_ROOT, "--save-dev"], { cwd: WORK_DIR }); +} + +// Sentinel comment block — used to identify our injected snippet so we can +// strip it back out even if a previous run crashed before restoration. +const CHDB_INJECTION_BEGIN = "// >>> chdb-runner injection — DO NOT EDIT >>>"; +const CHDB_INJECTION_END = "// <<< chdb-runner injection — DO NOT EDIT <<<"; + +/** + * Append the ChdbConnection wrapping snippet to upstream's + * `vitest.node.setup.ts` so vitest actually loads it (the setup file is + * already in upstream's `setupFiles` config, so appending to it is the + * one mechanism guaranteed to be picked up regardless of vitest version + * or CLI flag support). Returns a `{file, original}` snapshot for + * try/finally restoration. + */ +function injectSetup() { + const target = join(WORK_DIR, "vitest.node.setup.ts"); + if (!existsSync(target)) { + throw new Error( + `[runner] upstream vitest.node.setup.ts not found at ${target}; ` + + `injection cannot proceed`, + ); + } + const original = readFileSync(target, "utf8"); + // Strip any leftover injection from a prior crashed run, then append fresh. + const cleaned = stripInjection(original); + const SNIPPET = ` + +${CHDB_INJECTION_BEGIN} +// Wrap globalThis.environmentSpecificCreateClient so every createClient +// in the integration suite gets a ChdbConnection injected. Runs only +// when CH_TEST_BACKEND=chdb; the default code path stays unchanged. +// @ts-expect-error require() is fine in setup files +const __chdb = require("chdb/connection"); +// @ts-expect-error require() is fine in setup files +const __chdbCreate = require("@clickhouse/client").createClient; +if (process.env.CH_TEST_BACKEND === "chdb") { + // @ts-expect-error overriding the upstream global on purpose + globalThis.environmentSpecificCreateClient = (config) => { + const connection = __chdb.createChdbConnection({ path: ":memory:" }); + return __chdbCreate({ ...(config || {}), connection }); + }; + console.log("[chdb-runner] createClient wrapped with ChdbConnection"); +} +${CHDB_INJECTION_END} +`; + writeFileSync(target, cleaned + SNIPPET); + return { file: target, original }; +} + +function stripInjection(content) { + const start = content.indexOf(CHDB_INJECTION_BEGIN); + if (start === -1) return content; + const end = content.indexOf(CHDB_INJECTION_END); + if (end === -1) return content; // malformed; leave alone + // Trim the leading blank line(s) we added before BEGIN as well. + return content.slice(0, start).replace(/\n+$/, "\n") + content.slice(end + CHDB_INJECTION_END.length).replace(/^\n+/, ""); +} + +function restoreSetup(snapshot) { + if (!snapshot) return; + try { + writeFileSync(snapshot.file, snapshot.original); + } catch (e) { + console.warn(`[runner] failed to restore ${snapshot.file}: ${e.message}`); + } +} + +function buildSkipPattern() { + const files = (SKIP.skipFiles || []).map((entry) => + typeof entry === "string" ? entry : entry.file); + return files; +} + +/** + * For each `skipTests` entry, patch the relevant test file in place so the + * matching `it(""` becomes `it.skip(""` (and same for `test(`). + * Returns a list of `{ file, original }` snapshots so the caller can + * restore the originals after vitest finishes. vitest 2.x has no native + * test-id-level skip via CLI, so in-place patching is the simplest + * cross-file mechanism. + */ +function applyPerTestSkips() { + const entries = SKIP.skipTests || []; + if (entries.length === 0) return []; + const patches = []; + // Group entries by file so we make one read/write per file. + const byFile = new Map(); + for (const e of entries) { + if (!byFile.has(e.file)) byFile.set(e.file, []); + byFile.get(e.file).push(e); + } + for (const [relFile, list] of byFile) { + const abs = join(WORK_DIR, relFile); + if (!existsSync(abs)) { + console.warn(`[runner] skipTests file not found, ignoring: ${relFile}`); + continue; + } + const original = readFileSync(abs, "utf8"); + let patched = original; + let applied = 0; + for (const e of list) { + // The vitest test node-id is ` > > `. + // We only need the LAST segment (the `it` name) to locate the + // `it(""` call in the source. + const itName = e.test.split(" > ").pop(); + // Escape regex metacharacters in the it name. + const escaped = itName + .replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + // Match `it(` or `test(` followed by the string literal of itName. + const re = new RegExp( + String.raw`\b(it|test)\(\s*(['"\`])${escaped}\2`, + "g", + ); + const next = patched.replace(re, "$1.skip($2" + itName + "$2"); + if (next !== patched) { + applied += 1; + patched = next; + } else { + console.warn( + `[runner] skipTests: pattern not found in ${relFile}: ${itName}`, + ); + } + } + if (patched !== original) { + writeFileSync(abs, patched); + patches.push({ file: abs, original }); + console.log( + `[runner] applied ${applied} per-test skip(s) to ${relFile}`, + ); + } + } + return patches; +} + +function restorePerTestSkips(patches) { + for (const { file, original } of patches) { + try { + writeFileSync(file, original); + } catch (e) { + console.warn(`[runner] failed to restore ${file}: ${e.message}`); + } + } +} + +function runVitest() { + const setupSnap = injectSetup(); + const excludes = buildSkipPattern(); + const patches = applyPerTestSkips(); + try { + const args = [ + "run", + "test:node:integration", + "--", + "--config", + "vitest.node.config.ts", + ]; + for (const f of excludes) args.push("--exclude", f); + const env = { + ...process.env, + TZ: "UTC", + CH_TEST_BACKEND: "chdb", + }; + sh("npm", args, { cwd: WORK_DIR, env }); + } finally { + restorePerTestSkips(patches); + restoreSetup(setupSnap); + } +} + +console.log(`[runner] @clickhouse/client repo: ${REPO}`); +console.log(`[runner] ref: ${REF}`); +console.log(`[runner] work dir: ${WORK_DIR}`); +console.log(`[runner] skip_list entries: ${(SKIP.skipFiles || []).length} files`); +console.log(); + +clone(); +installAndBuild(); +runVitest(); diff --git a/tests/clickhouse-js/skip_list.json b/tests/clickhouse-js/skip_list.json new file mode 100644 index 0000000..856dd1f --- /dev/null +++ b/tests/clickhouse-js/skip_list.json @@ -0,0 +1,276 @@ +{ + "$schema_comment": "chdb-node skip list for @clickhouse/client's integration suite. AUTHORITATIVE blacklist applied by tests/clickhouse-js/runner.mjs when running @clickhouse/client's tests with ChdbConnection injected. Format: skipFiles is per-file (entire suite of tests in the file is excluded — for files where chdb does not implement the capability the file exercises); skipTests is per-case for cases inside an otherwise-runnable file (the runner patches the file in place to convert it(...) → it.skip(...) before vitest collects, and restores after). reasons documents WHY each entry is here. Maintained per @clickhouse/client release; chdb-node refreshes the list when upstream cuts a new version.", + "syncedAgainst": { + "ref": "ClickHouse/clickhouse-js main", + "policy": "While the createClient({ connection }) injection PR is open: ShawnChen-Sirius fork branch. After merge: ClickHouse/clickhouse-js main (CURRENT — PR #879 was merged 2026-06-23 as commit c2e28b9; PR #880 follow-up was merged 2026-06-23 as commit 274844f). After next release: latest released tag.", + "lastSyncedSha": "274844f", + "lastSyncedAt": "2026-06-24" + }, + "skipFiles": [ + { + "file": "packages/client-node/__tests__/integration/node_socket_handling.test.ts", + "reason": "HTTP transport plumbing: slow-server / dropped-connection / eager-socket-destroy scenarios exist only on the wire." + }, + { + "file": "packages/client-node/__tests__/integration/node_eager_socket_destroy.test.ts", + "reason": "HTTP socket-pool behavior; no analogue in-process." + }, + { + "file": "packages/client-node/__tests__/integration/node_keep_alive.test.ts", + "reason": "HTTP keep-alive; no socket pool in-process." + }, + { + "file": "packages/client-node/__tests__/integration/node_keep_alive_header.test.ts", + "reason": "HTTP keep-alive header inspection; no HTTP transport." + }, + { + "file": "packages/client-node/__tests__/integration/node_max_open_connections.test.ts", + "reason": "HTTP connection pool sizing; no socket pool in-process." + }, + { + "file": "packages/client-node/__tests__/integration/node_custom_http_agent.test.ts", + "reason": "HTTP agent injection; no HTTP transport in-process." + }, + { + "file": "packages/client-node/__tests__/integration/node_ping.test.ts", + "reason": "Tests the HTTP /ping endpoint; chdb's ping is SELECT 1 (already covered by the Connection contract suite in test/v3/connection/)." + }, + { + "file": "packages/client-node/__tests__/integration/node_compression.test.ts", + "reason": "HTTP request/response body compression; no wire to compress in-process." + }, + { + "file": "packages/client-node/__tests__/integration/node_logger_support.test.ts", + "reason": "HTTP-request logger surface; in-process backend has no HTTP request to log." + }, + { + "file": "packages/client-node/__tests__/integration/node_summary.test.ts", + "reason": "Asserts X-ClickHouse-Summary HTTP response header; chdb synthesizes summary fields but has no HTTP headers." + }, + { + "file": "packages/client-node/__tests__/integration/node_jwt_auth.test.ts", + "reason": "JWT auth; chdb has no auth layer." + }, + { + "file": "packages/client-node/__tests__/tls/tls.test.ts", + "reason": "TLS transport; no transport security layer in-process." + }, + { + "file": "packages/client-common/__tests__/integration/auth.test.ts", + "reason": "Username/password authentication; chdb has no auth layer." + }, + { + "file": "packages/client-common/__tests__/integration/role.test.ts", + "reason": "RBAC roles; chdb has no users/roles." + }, + { + "file": "packages/client-common/__tests__/integration/read_only_user.test.ts", + "reason": "RBAC read-only user enforcement; chdb has no users." + }, + { + "file": "packages/client-common/__tests__/integration/query_log.test.ts", + "reason": "Reads from system.query_log; chdb has no server runtime / query_log table." + }, + { + "file": "packages/client-common/__tests__/integration/multipart_params.test.ts", + "reason": "HTTP multipart body for parameter binding; no HTTP transport." + }, + { + "file": "packages/client-common/__tests__/integration/multiple_clients.test.ts", + "reason": "Multiple parallel clients against shared cluster topology; chdb is one in-process engine, not a cluster." + }, + { + "file": "packages/client-common/__tests__/integration/session.test.ts", + "reason": "HTTP session_id semantics; chdb has no HTTP session concept (persistence is the chdb Session.path)." + }, + { + "file": "packages/client-common/__tests__/integration/request_compression.test.ts", + "reason": "HTTP request-body compression; no wire to compress." + }, + { + "file": "packages/client-common/__tests__/integration/response_compression.test.ts", + "reason": "HTTP response-body compression; no wire to compress." + }, + { + "file": "packages/client-common/__tests__/integration/exception_header.test.ts", + "reason": "X-ClickHouse-Exception HTTP response header; chdb maps errors via JS exceptions, not HTTP headers." + }, + { + "file": "packages/client-common/__tests__/integration/abort_request.test.ts", + "reason": "AbortSignal cancellation semantics differ: chdb single-shot abort rejects early while the underlying computation runs to completion (the C ABI has no interrupt). Documented divergence." + }, + { + "file": "packages/client-node/__tests__/integration/node_stream_json_each_row_with_progress.test.ts", + "reason": "JSONEachRowWithProgress emits HTTP X-ClickHouse-Progress events interleaved with rows — chdb returns plain JSONEachRow output without the progress envelope." + }, + { + "file": "packages/client-common/__tests__/integration/clickhouse_settings.test.ts", + "reason": "Iterates the live system.settings catalog and tries to SET each one. chdb advertises a different set than the server (engine-only subset; some HTTP-specific entries don't exist; some chdb-specific entries are extras the test doesn't expect)." + }, + { + "file": "packages/client-node/__tests__/integration/node_abort_request.test.ts", + "reason": "Same AbortSignal divergence as `client-common/abort_request.test.ts` — chdb's single-shot abort rejects early while the underlying computation runs to completion (C ABI has no interrupt). The client-node variant additionally exercises HTTP wire-level cancel timing that has no analogue in-process." + }, + { + "file": "packages/client-node/__tests__/integration/node_stream_error_handling.test.ts", + "reason": "chdb's `query()` materializes results eagerly (one-shot Readable from a buffered result); errors surface at the awaited Promise rather than mid-stream. Tests here assert HTTP-style mid-stream error delivery which the embedded backend can't reproduce. Pinned by `test/v3/connection/contract.test.ts > query() materialization invariant`." + } + ], + "skipTests": [ + { + "file": "packages/client-common/__tests__/integration/insert.test.ts", + "test": "insert > inserts values using JSON format and get the response headers", + "reason": "Asserts response_headers['Content-Type']; chdb has no HTTP response headers (synthesized as {})." + }, + { + "file": "packages/client-common/__tests__/integration/exec_and_command.test.ts", + "test": "exec and command > should get the response headers with command", + "reason": "Asserts HTTP response headers; chdb has none." + }, + { + "file": "packages/client-common/__tests__/integration/exec_and_command.test.ts", + "test": "exec and command > should get the response headers with exec", + "reason": "Asserts HTTP response headers; chdb has none." + }, + { + "file": "packages/client-node/__tests__/integration/node_command.test.ts", + "test": "[Node.js] command > ignore error response > should not throw an error when ignore_error_response is true", + "reason": "ignore_error_response gates HTTP non-200 responses; chdb errors are typed JS throws, not HTTP responses." + }, + { + "file": "packages/client-node/__tests__/integration/node_command.test.ts", + "test": "[Node.js] command > ignore error response > should throw an error by default when ignore_error_response is not set", + "reason": "Same; HTTP-specific." + }, + { + "file": "packages/client-node/__tests__/integration/node_command.test.ts", + "test": "[Node.js] command > ignore error response > should throw an error when ignore_error_response is false", + "reason": "Same; HTTP-specific." + }, + { + "file": "packages/client-node/__tests__/integration/node_exec.test.ts", + "test": "[Node.js] exec > disabled stream decompression > should force decompress in case of an error", + "reason": "HTTP wire-level stream decompression; chdb has no compressed wire." + }, + { + "file": "packages/client-node/__tests__/integration/node_exec.test.ts", + "test": "[Node.js] exec > disabled stream decompression > should get a compressed response stream without decompressing it", + "reason": "HTTP wire-level stream decompression." + }, + { + "file": "packages/client-node/__tests__/integration/node_exec.test.ts", + "test": "[Node.js] exec > ignore error response > should get a compressed response stream if ignore_error_response is true and decompression is disabled", + "reason": "HTTP wire-level stream decompression + non-200 handling." + }, + { + "file": "packages/client-node/__tests__/integration/node_exec.test.ts", + "test": "[Node.js] exec > ignore error response > should get a decompressed response stream if ignore_error_response is true and default decompression config is passed", + "reason": "HTTP wire-level stream decompression + non-200 handling." + }, + { + "file": "packages/client-common/__tests__/integration/config.test.ts", + "test": "config > should set request timeout with \"request_timeout\" setting", + "reason": "HTTP socket-level request_timeout; no HTTP socket in-process." + }, + { + "file": "packages/client-common/__tests__/integration/config.test.ts", + "test": "config > should specify the default database name on creation", + "reason": "Uses `SELECT * FROM numbers LIMIT 2` (no parens) which only works against the server's `numbers` table-function shorthand; chdb requires `numbers(N)` argument form. Test isn't asserting chdb-supported behavior." + }, + { + "file": "packages/client-common/__tests__/integration/data_types.test.ts", + "test": "data types > should work with floating point types", + "reason": "Float64 output precision divergence: chdb emits 17 significant digits for full IEEE 754 round-trip safety (e.g. -0.11325939434432401), clickhouse-server emits 15 (-0.113259394344324). Same numeric value, different string representation. Round-trip through chdb is lossless; the test asserts string-equality of the original fixture." + }, + { + "file": "packages/client-common/__tests__/integration/data_types.test.ts", + "test": "data types > Dates > should work with JS Date objects", + "reason": "chdb resolves DateTime[64] without an explicit TZ argument against the system timezone (libchdb captures ICU at load time and `SET session_timezone='UTC'` does not influence the output formatter); clickhouse-server defaults to UTC in CI. Round-trip is lossless at the instant level — display string differs by the host's UTC offset. Same divergence underlies the other 4 datetime skipTests below." + }, + { + "file": "packages/client-common/__tests__/integration/date_time.test.ts", + "test": "DateTime > DateTime > should insert DateTime and get it back", + "reason": "Same chdb default-timezone divergence (DateTime values display in the host's local TZ, not UTC)." + }, + { + "file": "packages/client-common/__tests__/integration/date_time.test.ts", + "test": "DateTime > DateTime64(3) > should insert DateTime64(3) and get it back", + "reason": "Same chdb default-timezone divergence." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > Date(Time) > handles DateTime in a parameterized query", + "reason": "Same chdb default-timezone divergence." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > Date(Time) > handles DateTime64(3) in a parameterized query", + "reason": "Same chdb default-timezone divergence." + }, + { + "file": "packages/client-common/__tests__/integration/select.test.ts", + "test": "select > returns an error details provided by ClickHouse", + "reason": "Error-type parsing divergence: chdb's error string is `Code: 62. DB::Exception: Syntax error: failed at position 16 (JSON) (line 2, col 8): JSON. Expected end of query. (SYNTAX_ERROR)`. `parseError` in @clickhouse/client-common picks the first parenthesized token as `type`, so it extracts `JSON` instead of `SYNTAX_ERROR`. Same numeric code (62), same message content; only the parsed `type` field differs." + }, + { + "file": "packages/client-common/__tests__/integration/select.test.ts", + "test": "select > should get the response headers", + "reason": "Asserts HTTP response headers; chdb has none." + }, + { + "file": "packages/client-common/__tests__/integration/data_types.test.ts", + "test": "data types > Nested > should work with nested (input_format_import_nested_json = 1)", + "reason": "Nested type with the HTTP-side `input_format_import_nested_json=1` setting; chdb's nested-JSON input path has a different default behavior. Not a chdb engine gap per se (Nested in general works), but the specific JSON-input setting interaction differs from server." + }, + { + "file": "packages/client-node/__tests__/integration/node_streaming_e2e.test.ts", + "test": "[Node.js] streaming e2e > should stream a Parquet file", + "reason": "Streams a Parquet file into chdb via `INSERT FORMAT Parquet`; chdb's `ParquetV3BlockInputFormat` rejects this specific fixture with `apache::thrift::protocol::TProtocolException: invalid TType`. The server's older Parquet input format accepts it. Engine-version divergence in Parquet decoder." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > handles tuples in a parametrized query", + "reason": "chdb's `queryBindAsync` serializes Tuple parameter values using `{'field':value}` map-literal form. The engine's parser expects `(value1, value2, ...)` positional form. Real chdb-engine parameter-binding gap for compound types." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > handles arrays of tuples in a parametrized query", + "reason": "Same chdb parameter-binding gap for Tuple — applies to Array(Tuple(...))." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > handles maps with tuples in a parametrized query", + "reason": "Same chdb parameter-binding gap for Tuple — applies to Map(K, Tuple(...))." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > handles maps with nested arrays in a parametrized query", + "reason": "chdb parameter-binding for Map(K, Array(...)) generates a serialization the engine parser rejects." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > handles maps with nullable values in a parametrized query", + "reason": "chdb parameter-binding for Map(K, Nullable(V)) generates a serialization the engine parser rejects." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > NULL parameter binding > should work with nulls", + "reason": "chdb's `queryBindAsync` NULL substitution diverges from the server's `param_*` URL-binding semantics (server-side strict-NULL handling)." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > NULL parameter binding > should with an explicit undefined", + "reason": "Same chdb NULL/undefined parameter-binding divergence." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > Nested boolean types > handles boolean in a tuple", + "reason": "Boolean inside Tuple parameter binding hits the same Tuple-serialization divergence above." + }, + { + "file": "packages/client-common/__tests__/integration/select_query_binding.test.ts", + "test": "select with query binding > Nested boolean types > handles boolean in a mixed nested structure", + "reason": "Boolean inside nested Tuple/Map parameter binding — same root cause." + } + ] +} diff --git a/vitest.config.ts b/vitest.config.ts index d2acfd5..8d6f329 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -17,5 +17,20 @@ export default defineConfig({ fileParallelism: false, pool: 'forks', poolOptions: { forks: { singleFork: true } }, + server: { + deps: { + // The CJS entrypoint (./index.js) owns the process-wide session and + // pending-op registries. Test files and setup.ts import it + // (`../../index.js`); src/connection/* reaches it via + // `require('../../index.js')` after tsc. If vitest transforms the + // import copy, the entrypoint evaluates TWICE — the global afterEach + // safety net (setup.ts) then drains a DIFFERENT instance than the + // one ChdbConnection creates sessions on, sessions leak across files, + // and the next file fails with "only one active data directory per + // process". Externalizing every `…/index.js` import routes it through + // Node's require cache so every importer shares one instance. + external: [/\/index\.js$/], + }, + }, }, })