From 69d63ef981b9c30d80eec99b9fb14fca194ab132 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Thu, 12 Nov 2026 15:20:31 +1300 Subject: [PATCH 1/4] Add direct Arrow A1-A6 bindings on rc.3 --- .github/workflows/chdb-node-test.yml | 11 +- arrow.mjs | 12 + docs/arrow-rc3-conformance.md | 45 + docs/design/durable-control-plane.md | 8 +- index.d.ts | 29 + index.js | 282 +++++- index.mjs | 5 + lib/chdb_node.cpp | 968 +++++++++++++++++++- lib/chdb_node.h | 5 +- llms-full.txt | 38 +- package-lock.json | 8 +- package.json | 14 +- scripts/smoke-arrow-e2e.mjs | 48 + src/arrow-direct.ts | 113 +++ src/arrow-entry.ts | 30 + src/layer3/compiler/identifier.ts | 8 + src/layer3/execute/arrow-input.ts | 225 ++++- src/layer3/execute/arrow-insert.ts | 85 ++ src/layer3/execute/arrow-reader.ts | 122 +++ src/layer3/index.ts | 8 +- src/layer3/runtime.ts | 19 +- test/v3/arrow-direct.test.ts | 196 ++++ test/v3/arrow-entry.test.ts | 23 + test/v3/layer3/arrow-input.test.ts | 376 +++++++- test/v3/layer3/arrow-insert.test.ts | 95 ++ test/v3/layer3/arrow-reader.test.ts | 227 +++++ test/v3/layer3/arrow-worker-isolate.test.ts | 66 ++ update_libchdb.sh | 10 +- 28 files changed, 2955 insertions(+), 121 deletions(-) create mode 100644 arrow.mjs create mode 100644 docs/arrow-rc3-conformance.md create mode 100644 scripts/smoke-arrow-e2e.mjs create mode 100644 src/arrow-direct.ts create mode 100644 src/arrow-entry.ts create mode 100644 src/layer3/execute/arrow-insert.ts create mode 100644 src/layer3/execute/arrow-reader.ts create mode 100644 test/v3/arrow-direct.test.ts create mode 100644 test/v3/arrow-entry.test.ts create mode 100644 test/v3/layer3/arrow-insert.test.ts create mode 100644 test/v3/layer3/arrow-reader.test.ts create mode 100644 test/v3/layer3/arrow-worker-isolate.test.ts diff --git a/.github/workflows/chdb-node-test.yml b/.github/workflows/chdb-node-test.yml index fff5bb9..80d0e09 100644 --- a/.github/workflows/chdb-node-test.yml +++ b/.github/workflows/chdb-node-test.yml @@ -246,7 +246,8 @@ jobs: # subpackage from the registry — the cleanroom then tests the released # binary instead of the one just built, which is how a non-relocatable # binary (chdb-io/chdb-node#50) shipped past a green cleanroom. - export CHDB_LIB_VERSION="$(grep -E '^LIBCHDB_NPM_VERSION=' update_libchdb.sh | cut -d= -f2)" + CHDB_LIB_VERSION="$(grep -E '^LIBCHDB_NPM_VERSION=' update_libchdb.sh | cut -d= -f2)" + export CHDB_LIB_VERSION npm run build:platform MAIN_TGZ="$PWD/$(npm pack --silent)" SUB_DIR="$PWD/npm/@chdb/lib-linux-x64-gnu" @@ -257,7 +258,7 @@ jobs: run: | mkdir -p /tmp/cleanroom && cd /tmp/cleanroom npm init -y >/dev/null - npm install --no-save "$SUB_TGZ" "$MAIN_TGZ" + npm install --no-save "$SUB_TGZ" "$MAIN_TGZ" apache-arrow@21.1.0 # The exact binary that will be dlopen'd must be relocatable. Assert it # directly on the INSTALLED file — this also fails loudly (instead of a # cryptic dlopen error) if npm resolved the subpackage from the registry @@ -265,6 +266,7 @@ jobs: INSTALLED_NODE="node_modules/@chdb/lib-linux-x64-gnu/chdb_node.node" bash "$GITHUB_WORKSPACE/scripts/assert-relocatable.sh" "$INSTALLED_NODE" cp "$GITHUB_WORKSPACE/scripts/smoke-e2e.mjs" ./smoke-e2e.mjs + cp "$GITHUB_WORKSPACE/scripts/smoke-arrow-e2e.mjs" ./smoke-arrow-e2e.mjs # Relocation-faithful e2e: delete the build-machine libchdb.so so a # baked absolute/build-dir library reference (chdb-io/chdb-node#50) # can no longer be satisfied from the build tree. The installed package @@ -273,9 +275,12 @@ jobs: # passes on the build runner even when the binary is unshippable. rm -f "$GITHUB_WORKSPACE/libchdb.so" node smoke-e2e.mjs + node smoke-arrow-e2e.mjs - name: The same install has to run on Debian bookworm # The symbol assert reads the binary; this runs it. bookworm is older # than every GitHub runner image and is the base of the node:22 / # node:20 / node:lts images, so it is where users first meet a binary # that demands a newer libstdc++ or glibc than they have. - run: docker run --rm -v /tmp/cleanroom:/w -w /w node:22-bookworm node smoke-e2e.mjs + run: | + docker run --rm -v /tmp/cleanroom:/w -w /w node:22-bookworm node smoke-e2e.mjs + docker run --rm -v /tmp/cleanroom:/w -w /w node:22-bookworm node smoke-arrow-e2e.mjs diff --git a/arrow.mjs b/arrow.mjs new file mode 100644 index 0000000..99d48db --- /dev/null +++ b/arrow.mjs @@ -0,0 +1,12 @@ +// ESM bridge for the optional typed Arrow subpath. Its runtime only delegates +// to Session; apache-arrow is loaded when the result is converted. +import { createRequire } from 'node:module' +const mod = createRequire(import.meta.url)('./dist/arrow-entry.js') +export const queryArrow = mod.queryArrow +export const queryArrowBind = mod.queryArrowBind +export const queryArrowStream = mod.queryArrowStream +export const queryArrowStreamBind = mod.queryArrowStreamBind +export const registerArrowSource = mod.registerArrowSource +export const registerArrowReader = mod.registerArrowReader +export const insertArrow = mod.insertArrow +export const insertArrowReader = mod.insertArrowReader diff --git a/docs/arrow-rc3-conformance.md b/docs/arrow-rc3-conformance.md new file mode 100644 index 0000000..2cce54d --- /dev/null +++ b/docs/arrow-rc3-conformance.md @@ -0,0 +1,45 @@ +# Native Arrow A1–A6 against chdb-core v26.11.0-rc.3 + +The development baseline is the `macos-arm64-libchdb.tar.gz` asset from +`v26.11.0-rc.3` (SHA-256 `7b0ea049efcf75a851e744c7d74325d7a9b143227f9b72097e5df252564e3d58`). +The release tag is #323's merge commit `574f212fb5ee`. The manifest and +lockfile target `@chdb/lib-*` version `26.11.0-rc.3.1`, but those npm packages +have **not yet been published**. npm omitted their package metadata from the +lockfile; regenerate it after publication to capture `resolved` and `integrity`. +The local macOS arm64 tests used the official rc.3 +tarball with the previous ignored `libchdb.so` backed up under +`/tmp/chdb-node-rc3-verify/`. A cleanroom install from locally packed main and +arm64 subpackage tarballs passed `smoke-arrow-e2e.mjs`. The other three +platforms and registry-installed packages remain CI/release gates. + +The shared contract is `chdb-core/docs/NATIVE_ARROW_BINDING_CONTRACT.md`. In +this binding, **direct** means C Data/C Stream without Arrow IPC. Input and +output may copy buffers; no end-to-end zero-copy claim is made. +The optional `chdb/arrow` subpath supplies TypeScript `Table`/`RecordBatch` +return types without making the base `chdb` entry require `apache-arrow`. + +| Capability | Implemented now | Further validation / documented scope | +| --- | --- | --- | +| A1 SQL source | `registerArrowTable` accepts supported column descriptors, `RecordBatch`, and materialized multi-batch `Table`; C Stream callbacks use pinned JS buffers. `registerArrowSource` generates opaque process-unique names. Primitive, Timestamp, Decimal128/256, sliced buffers/bitmaps, and one-chunk Dictionary input are tested. | Nested types and multi-chunk dictionary input explicitly reject; more platform/isolate stress remains. | +| A2 reader source | `registerArrowReader` accepts sync/async RecordBatch iterables. The JS producer awaits capacity in a two-batch native queue; worker-thread `get_next` never calls JS. Empty schema, producer error, schema mismatch, cancellation, one-release-per-delivered-batch, one-shot behavior and worker-isolate teardown are tested. | ASan/RSS stress and platform matrix. Core may retain batches after pulling them (e.g. sorting); queue capacity counts only batches not yet handed to core. `close()` cancels native promptly even if a user producer never resolves `next()`; that producer must cooperate to release its own resources. | +| A3 typed result | `Session.queryArrow` calls `chdb_query_arrow_n`, copies C buffers before release, and constructs an `apache-arrow.Table`; `queryArrowBind` materializes the bound A4 stream. Empty schema and SQL errors tested. | Nested types and large-result memory stress. | +| A4 typed batches | `Session.queryArrowStream` pulls C Data per batch; `queryArrowStreamBind` uses `chdb_stream_query_arrow_with_params_n` with server-side binding. Early-break, in-flight cancel and bound values are tested. | Nested types, repeated cancellation and memory stress. | +| A5 bulk insert | `insertArrow` handles materialized batches; `insertArrowReader` uses A2's bounded source. Both map destination fields by name and use engine-side `INSERT … SELECT`. | Storage-side write ledger (current `rowsSubmitted` is only the input count), settings/mapping matrix and cancellation under write load. | +| A6 ABI/types | New direct Arrow paths verify the loaded `EngineVersion()` is at least `v26.11.0-rc.3` and reject older builds with `CHDB_ABI`. Basic numeric, Bool, Utf8, Binary, Timestamp, Decimal128/256 and LowCardinality Utf8/Binary dictionary output are mapped. `unsupportedAsBinary` is tested for an otherwise unsupported AggregateFunction; IPC `toArrow()` remains unchanged. | Four-platform build/runtime CI; publish all four rc.3 npm packages and regenerate the lockfile afterwards. Additional nested/type-option cases are outside this supported subset. | + +## Core ownership regression and release follow-up + +rc.2 predates `chdb-core#323`, which is included in rc.3. Core issues +[`#321`](https://github.com/chdb-io/chdb-core/issues/321) and +[`#322`](https://github.com/chdb-io/chdb-core/issues/322) already record its +`get_schema` release and failed array-registration wrapper leaks. Do not open +duplicate issues. The previously conditional registration-schema release test +now runs unconditionally on rc.3. An ASan/RSS *failed-registration leak budget* +is still a future stress gate, not a test claimed here. Node's own close/eviction +path unregisters from core before dropping pinned buffers. + +If another **new** rc.3 core defect appears: keep a small reproducer, file a +core issue before designing around its proposed fix, link the issue from a +clearly skipped/conditional test, and describe the required follow-up. Filing +an issue is a remote write and requires separate per-action confirmation; other +local implementation and unaffected tests can continue meanwhile. diff --git a/docs/design/durable-control-plane.md b/docs/design/durable-control-plane.md index e4efdc0..8d395b7 100644 --- a/docs/design/durable-control-plane.md +++ b/docs/design/durable-control-plane.md @@ -724,11 +724,9 @@ this package: 1. Publish `@chdb/lib-` packages built from a tree that binds the durable ABI. `optionalDependencies` and `update_libchdb.sh` already name - `26.11.0-rc.2.1`, and the release workflow derives the subpackage version - from the latter — but nothing has been tagged since, so that version is not - on npm at all. The newest published is `26.7.0-stable.1` (main `chdb@3.3.0` - pins it), which carries an addon from before this work and exports none of - the four entry points. + `26.11.0-rc.3.1`, and the release workflow derives the subpackage version + from the latter — but this version has not yet been published on npm. The + currently published rc.2.1 packages do not contain the rc.3 ownership fix. Two different failures follow, depending on what is installed. A fresh install resolves no subpackage — an unsatisfiable *optional* dependency is diff --git a/index.d.ts b/index.d.ts index f34be51..d5e49b6 100644 --- a/index.d.ts +++ b/index.d.ts @@ -240,6 +240,23 @@ export interface ChdbQueryStream extends AsyncIterable { cancel(): void; } +/** Incremental Arrow C Data reader; batches are apache-arrow RecordBatch objects. */ +export interface ChdbArrowReader extends AsyncIterable { + readonly closed: boolean; + /** Available after the first fetch, including for an empty result. */ + readonly schema: unknown | null; + cancel(): void; +} + +export interface DirectArrowOptions { + /** Degrade otherwise unmappable engine types to Arrow Binary. Default false. */ + unsupportedAsBinary?: boolean; + /** Export ClickHouse String as Arrow Utf8 (default) or Binary. */ + stringAsString?: boolean; + /** Preserve LowCardinality as a dictionary (Utf8/Binary values, Int32/UInt32 indices). */ + lowCardinalityAsDictionary?: boolean; +} + /** * Options for constructing a {@link Session}. */ @@ -348,6 +365,18 @@ export class Session { */ queryAsync(query: string, opts?: QueryOptions): Promise; + /** Direct Arrow C Data output (no IPC); requires the optional apache-arrow peer. */ + queryArrow(query: string, opts?: DirectArrowOptions): Promise; + + /** Materialize direct C Data batches with server-side {name:Type} parameters. */ + queryArrowBind(query: string, params: object, opts?: DirectArrowOptions): Promise; + + /** Pulls one typed Arrow batch per native fetch; cancel on partial consumption. */ + queryArrowStream(query: string, opts?: DirectArrowOptions): ChdbArrowReader; + + /** Incremental direct C Data result with server-side {name:Type} parameters. */ + queryArrowStreamBind(query: string, params: object, opts?: DirectArrowOptions): ChdbArrowReader; + /** * Executes a session-bound parameterized query asynchronously. */ diff --git a/index.js b/index.js index 1cc7f51..2a57ffe 100644 --- a/index.js +++ b/index.js @@ -14,6 +14,7 @@ const { ChdbTimeoutError, ChdbInsertError, ChdbStreamError, + ChdbBinaryVersionMismatchError, mapNativeError, } = require('./dist/errors.js'); @@ -35,6 +36,7 @@ function prepArrow(query, opts, defaultFormat) { } const { formatParamValue } = require('./dist/serialize.js'); const { ChdbResult } = require('./dist/result.js'); +const { arrowTableFromDirect } = require('./dist/arrow-direct.js'); const { buildInsertSQL } = require('./dist/insert.js'); const { buildRawInsertPrefix, isRawValues, isStreamValues } = require('./dist/rawInsert.js'); const { streamInsert } = require('./dist/streamInsert.js'); @@ -318,6 +320,24 @@ function asConnectionError(e) { return new ChdbConnectionError(message, { cause: e }); } +// The full native Arrow contract is first tested against chdb-core +// v26.11.0-rc.3 (including the #323 ownership fix). SELECT version() reports a different +// ClickHouse build number; EngineVersion is the release identity to compare. +function _assertArrowCore() { + const version = String(chdbNode.EngineVersion()); + const match = /^(\d+)\.(\d+)\.(\d+)(?:-rc\.(\d+))?$/.exec(version); + const current = match && match.slice(1, 4).map(Number); + const minimum = [26, 11, 0]; + const older = !current || current.some((part, i) => { + for (let j = 0; j < i; j++) if (current[j] !== minimum[j]) return false; + return part < minimum[i]; + }); + if (older || (current.every((v, i) => v === minimum[i]) && match[4] && Number(match[4]) < 3)) { + throw new ChdbBinaryVersionMismatchError( + `Native Arrow requires chdb-core >= v26.11.0-rc.3 (loaded ${version})`); + } +} + // One materialized chunk of a streaming result. class StreamChunk { constructor(bytes, numRows, format) { @@ -347,6 +367,7 @@ class ChdbQueryStream { this._signal = signal; this._closed = false; this._inflight = null; // the fetch currently on a libuv thread, if any + activeStreams.add(this); } get closed() { return this._closed; } @@ -370,12 +391,28 @@ class ChdbQueryStream { this._inflight = trackNative(chdbNode.StreamFetch(this._handle)); raw = await this._inflight; } catch (e) { + const cancelling = this._closed; this._closed = true; + // cancel() may already be waiting for this fetch before destroying + // the native handle. In that case only its destroy callback retires + // the stream; releasing Arrow input here would be too early. + if (!cancelling) { + activeStreams.delete(this); + flushArrowIdle(); + } throw new ChdbStreamError(asQueryError(e).message, { cause: e }); } finally { this._inflight = null; } - if (raw.done) { this._closed = true; break; } + if (raw.done) { + const cancelling = this._closed; + this._closed = true; + if (!cancelling) { + activeStreams.delete(this); + flushArrowIdle(); + } + break; + } yield new StreamChunk(raw.bytes, raw.numRows, this._format); } } finally { @@ -401,6 +438,8 @@ class ChdbQueryStream { this._closed = true; const destroy = () => { try { chdbNode.StreamCancel(this._handle); } catch (_) { /* best effort */ } + activeStreams.delete(this); + flushArrowIdle(); }; // StreamCancel destroys the stream handle synchronously, and the fetch worker // reads that handle without holding a lock. Destroying it while a fetch is @@ -416,6 +455,72 @@ class ChdbQueryStream { } } +// Typed C Data stream. Each native fetch produces at most one RecordBatch; +// bytes are copied and C callbacks released before the JS batch is exposed. +class ChdbArrowReader { + constructor(handle) { + this._handle = handle; + this._closed = false; + this._inflight = null; + this.schema = null; // available after the first fetch, even for zero rows + activeStreams.add(this); + } + get closed() { return this._closed; } + + async *[Symbol.asyncIterator]() { + try { + while (!this._closed) { + let raw; + try { + this._inflight = trackNative(chdbNode.FetchArrowStream(this._handle)); + raw = await this._inflight; + } catch (e) { + const cancelling = this._closed; + this._closed = true; + if (!cancelling) { + activeStreams.delete(this); + flushArrowIdle(); + } + throw asQueryError(e); + } finally { + this._inflight = null; + } + const table = arrowTableFromDirect(raw); + this.schema = table.schema; + if (raw.done) { + const cancelling = this._closed; + this._closed = true; + if (!cancelling) { + activeStreams.delete(this); + flushArrowIdle(); + } + break; + } + for (const batch of table.batches) yield batch; + } + } finally { + this.cancel(); + } + } + + cancel() { + if (this._closed) return; + this._closed = true; + const destroy = () => { + try { chdbNode.CancelArrowStream(this._handle); } catch (_) { /* best effort */ } + activeStreams.delete(this); + flushArrowIdle(); + }; + if (this._inflight) { + const deferred = this._inflight.then(destroy, destroy); + pendingNativeOps.add(deferred); + deferred.finally(() => pendingNativeOps.delete(deferred)); + } else { + destroy(); + } + } +} + // Standalone exported query function function query(query, format = "CSV") { if (!query) { @@ -504,10 +609,45 @@ const openSessions = new Set(); // the abort/timeout ones — close() landing mid-flight on an untracked op was the // macos-14/Node-20 120s hang (chdb-io/chdb-node#53). const pendingNativeOps = new Set(); +// Arrow sources are process-wide in core. A registration cannot be released +// while *any* native query or lazy stream may still be scanning its buffers. +// Keep teardown waits separate from native work so a deferred close does not +// wait on itself. This deliberately errs on the side of delaying release when +// an unrelated stream is open rather than guessing which SQL reads a name. +const activeNativeOps = new Set(); +const activeStreams = new Set(); +const arrowIdleTasks = new Set(); +const arrowRegistrations = new Map(); +let defaultArrowGeneration = 0; + +function arrowBusy() { + return activeNativeOps.size > 0 || activeStreams.size > 0; +} +function flushArrowIdle() { + if (arrowBusy()) return; + for (const task of [...arrowIdleTasks]) { + arrowIdleTasks.delete(task); + task(); + } +} +function whenArrowIdle(task) { + if (arrowBusy()) arrowIdleTasks.add(task); + else task(); +} +function retireArrowRegistrations(connection) { + for (const [token, entry] of arrowRegistrations) { + if (entry.connection === connection) arrowRegistrations.delete(token); + } +} function trackNative(nativePromise) { const done = nativePromise.then(() => {}, () => {}); pendingNativeOps.add(done); - done.then(() => pendingNativeOps.delete(done)); + activeNativeOps.add(done); + done.then(() => { + pendingNativeOps.delete(done); + activeNativeOps.delete(done); + flushArrowIdle(); + }); return nativePromise; } @@ -681,6 +821,10 @@ class Session { try { const key = this.path ? resolvePath(this.path) : this.path; this._key = key; // normalized directory, for the deferred-teardown bookkeeping + // Creating a Session can evict the default connection. Its Arrow names + // belong to that old connection, even if a new default is opened later. + defaultArrowGeneration++; + retireArrowRegistrations(null); this.connection = connectionArgs ? chdbNode.CreateConnection(key, connectionArgs) : chdbNode.CreateConnection(key); @@ -740,6 +884,52 @@ class Session { return withAbortTimeout(chdbNode.QueryAsyncConnection(this.connection, sql, format), opts); } + // Direct C Data result, not Arrow IPC. The native worker copies each C + // buffer before releasing its stream; JS then constructs an apache-arrow + // Table from those owned bytes. Keep the native promise tracked through + // completion so close() cannot tear down the connection mid-query. + queryArrow(query, opts = {}) { + if (!this.connection) return Promise.reject(new ChdbClosedError("No active connection available")); + if (!query) return Promise.reject(new ChdbStreamError("queryArrow requires a non-empty query")); + try { _assertArrowCore(); } catch (e) { return Promise.reject(e); } + let native; + try { native = chdbNode.QueryArrowAsync(this.connection, query, opts); } + catch (e) { return Promise.reject(e); } + trackNative(native); + return native.then((raw) => arrowTableFromDirect(raw), (e) => { throw asQueryError(e); }); + } + + queryArrowStream(query, opts = {}) { + return this.#startArrowStream(query, opts); + } + + queryArrowStreamBind(query, params = {}, opts = {}) { + return this.#startArrowStream(query, opts, formatParams(params)); + } + + async queryArrowBind(query, params = {}, opts = {}) { + const reader = this.queryArrowStreamBind(query, params, opts); + const batches = []; + for await (const batch of reader) batches.push(batch); + const arrow = require('apache-arrow'); + return new arrow.Table(reader.schema, batches); + } + + #startArrowStream(query, opts, bound) { + if (!this.connection) throw new ChdbClosedError("No active connection available"); + if (!query) throw new ChdbStreamError("queryArrowStream requires a non-empty query"); + _assertArrowCore(); + if (this._activeStream && !this._activeStream.closed) { + throw new ChdbStreamError("a stream is already active on this session; finish or cancel it first"); + } + let handle; + try { handle = chdbNode.StartArrowStream(this.connection, query, opts, bound); } + catch (e) { throw asQueryError(e); } + const stream = new ChdbArrowReader(handle); + this._activeStream = stream; + return stream; + } + queryBindAsync(query, params = {}, opts = {}) { if (!this.connection) return Promise.reject(new ChdbClosedError("No active connection available")); if (!query) return Promise.resolve(emptyResult()); @@ -826,6 +1016,7 @@ class Session { const conn = this.connection; this.connection = null; const teardown = () => { + retireArrowRegistrations(conn); if (conn) { try { chdbNode.CloseConnection(conn); } catch (_) { /* best effort */ } } if (this.isTemp) { try { this.#removeTempDir(); } catch (_) { /* best effort */ } } }; @@ -836,16 +1027,21 @@ class Session { // until they drain rather than racing them. _drainPendingOps() awaits these // deferred closes too, so a new session is never created before the prior // connection is fully released. - if (conn && pendingNativeOps.size > 0) { + const hasArrowSources = [...arrowRegistrations.values()].some((entry) => entry.connection === conn); + if (conn && (pendingNativeOps.size > 0 || (hasArrowSources && activeStreams.size > 0))) { const key = this._key; addPendingTeardown(key); // Removed in the same tick as the teardown it guards, not in a later // .finally: the entry exists to answer "is this connection still alive", // and a caller that resumes between the two would be told yes about a // connection that is already gone. - const deferred = Promise.allSettled([...pendingNativeOps]).then(() => { - try { teardown(); } finally { removePendingTeardown(key); } - }); + const deferred = Promise.allSettled([...pendingNativeOps]).then(() => new Promise((resolve) => { + const finish = () => { + try { teardown(); } finally { removePendingTeardown(key); resolve(); } + }; + if (hasArrowSources) whenArrowIdle(finish); + else finish(); + })); pendingNativeOps.add(deferred); deferred.finally(() => pendingNativeOps.delete(deferred)); } else { @@ -920,22 +1116,80 @@ function version() { }; } -// Arrow C Data Interface — low-level binding. The high-level helper lives in -// Layer 3 (`src/layer3/execute/arrow-input.ts`); this module just routes the -// call to the native addon. The connection arg is `Session.connection` (the -// External returned by CreateConnection) or null for the process-wide default. +// Arrow C Data Interface — low-level binding and registration lifetime seam. +// The high-level helper lives in Layer 3 (`src/layer3/execute/arrow-input.ts`). +// The connection arg is `Session.connection` (the External returned by +// CreateConnection) or null for the process-wide default. function _arrowRegisterColumns(connection, tableName, columns) { - return chdbNode.ArrowRegisterColumns(connection ?? null, tableName, columns); + chdbNode.ArrowRegisterColumns(connection ?? null, tableName, columns); + return recordArrowRegistration(connection, tableName); +} +function _arrowRegisterBatches(connection, tableName, batches) { + _assertArrowCore(); + chdbNode.ArrowRegisterBatches(connection ?? null, tableName, batches); + return recordArrowRegistration(connection, tableName); +} +function _arrowRegisterReader(connection, tableName, firstBatch) { + _assertArrowCore(); + const handle = chdbNode.ArrowRegisterReader(connection, tableName, firstBatch); + return { handle, token: recordArrowRegistration(connection, tableName) }; +} +function _arrowPushReaderBatch(handle, batch) { + return chdbNode.ArrowPushReaderBatch(handle, batch); +} +function _arrowFinishReader(handle, error) { + return chdbNode.ArrowFinishReader(handle, error); +} +function _arrowCancelReader(handle) { + return chdbNode.ArrowCancelReader(handle); } -function _arrowUnregister(connection, tableName) { - return chdbNode.ArrowUnregister(connection ?? null, tableName); +function recordArrowRegistration(connection, tableName) { + const token = Symbol('arrow registration'); + arrowRegistrations.set(token, { connection: connection ?? null, tableName, + generation: defaultArrowGeneration }); + return token; +} +function _arrowUnregister(connection, tableName, token, defer = true) { + // Direct callers of this internal helper retain its original immediate + // behavior; high-level handles pass their registration token so an old + // handle cannot unregister a later registration using the same name. + if (token === undefined) return chdbNode.ArrowUnregister(connection ?? null, tableName); + const entry = arrowRegistrations.get(token); + if (!entry) return true; // its connection was already closed/evicted + if (entry.connection !== (connection ?? null) || entry.tableName !== tableName) { + throw new TypeError('Arrow registration token does not match its connection and name'); + } + if (entry.connection === null && entry.generation !== defaultArrowGeneration) { + arrowRegistrations.delete(token); + return true; + } + if (arrowBusy() && !defer) { + throw new Error('Cannot refresh an Arrow source while a query or stream is active'); + } + const release = () => { + if (arrowRegistrations.get(token) !== entry) return; + try { + chdbNode.ArrowUnregister(entry.connection, entry.tableName); + } finally { + arrowRegistrations.delete(token); + } + }; + if (!arrowBusy()) { release(); return true; } + whenArrowIdle(() => { + try { release(); } catch (e) { + process.emitWarning(`chdb: deferred Arrow unregister failed: ${e.message}`); + } + }); + return false; } module.exports = { query, queryBind, queryAsync, queryBindAsync, insert, Session, version, drainPending, _closeAllSessions, _drainPendingOps, - _arrowRegisterColumns, _arrowUnregister, + _arrowRegisterColumns, _arrowRegisterBatches, _arrowRegisterReader, + _assertArrowCore, + _arrowPushReaderBatch, _arrowFinishReader, _arrowCancelReader, _arrowUnregister, }; // Layer 3: the fluent, immutable query builder. It sits on Layer 1, a sibling of diff --git a/index.mjs b/index.mjs index 30e8ad5..843697f 100644 --- a/index.mjs +++ b/index.mjs @@ -16,6 +16,11 @@ export const insert = mod.insert export const Session = mod.Session export const version = mod.version export const drainPending = mod.drainPending +export const registerArrowTable = mod.registerArrowTable +export const registerArrowSource = mod.registerArrowSource +export const registerArrowReader = mod.registerArrowReader +export const insertArrow = mod.insertArrow +export const insertArrowReader = mod.insertArrowReader // Layer 3: ChdbCompileError is the only net-new error class; export it as a // named ESM binding so it is catchable by class. The fluent builder surface diff --git a/lib/chdb_node.cpp b/lib/chdb_node.cpp index 35c49b4..2f5381b 100644 --- a/lib/chdb_node.cpp +++ b/lib/chdb_node.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -71,11 +74,9 @@ char * QueryWithConnection(ChdbConnection conn, const char * query, const char * // chdb_connect (dereferenced for chdb_query, passed as-is to chdb_close_conn), // matching the External handle layout v2 exposed to JS. // -// Thread-safety: registry mutations happen on the Node main thread (the N-API -// wrappers below), so they are already serialized by the event loop; queries run -// on libuv worker threads but only touch a connection handle, never this registry. -// g_reg_mu is a defensive guard so the invariants hold even if a future caller -// mutates the registry off the main thread. +// Thread-safety: each N-API wrapper runs on its isolate's event loop; several +// worker_threads can mutate this process-wide registry concurrently. g_reg_mu +// serializes those changes. libuv query workers only touch retained handles. struct Registry { std::string boundKey; // path the EmbeddedServer is bound to (valid while !conns.empty()) std::set conns; // every live connection (sessions + the default) @@ -83,6 +84,7 @@ struct Registry { }; static Registry g_reg; static std::mutex g_reg_mu; +static std::unordered_map g_conn_owner; // Extra argv entries are the same `--setting=value` forms a `clickhouse local` // invocation takes, and they have to be given at connect: `--path` and @@ -212,6 +214,7 @@ static void hard_close_all() { if (c) chdb_close_conn(c); } g_reg.conns.clear(); + g_conn_owner.clear(); g_reg.defaultConn = nullptr; g_reg.boundKey.clear(); } @@ -256,6 +259,21 @@ static bool g_atexit_registered = false; // its env cleanup hook counts it out. ExitTeardownWrapper reads it. static std::atomic g_live_envs{0}; +static void close_for_env(napi_env owner) { + std::lock_guard lk(g_reg_mu); + for (auto it = g_reg.conns.begin(); it != g_reg.conns.end();) { + chdb_connection * conn = *it; + auto found = g_conn_owner.find(conn); + if (found == g_conn_owner.end() || found->second != owner) { ++it; continue; } + erase_arrow_tables_for_conn(conn); + chdb_close_conn(conn); + g_conn_owner.erase(found); + if (g_reg.defaultConn == conn) g_reg.defaultConn = nullptr; + it = g_reg.conns.erase(it); + } + if (g_reg.conns.empty()) g_reg.boundKey.clear(); +} + static void ensure_atexit() { if (!g_atexit_registered) { std::atexit(shutdown_on_process_exit); @@ -265,9 +283,17 @@ static void ensure_atexit() { // Default connection for standalone query()/queryBind() (lazy, in-memory). A // single shared default per process; reused across standalone calls. -static chdb_connection *get_default_conn(char **error_message) { +static chdb_connection *get_default_conn(char **error_message, napi_env owner) { std::lock_guard lk(g_reg_mu); - if (g_reg.defaultConn) return g_reg.defaultConn; + if (g_reg.defaultConn) { + auto found = g_conn_owner.find(g_reg.defaultConn); + if (found != g_conn_owner.end() && found->second != owner) { + if (error_message && !*error_message) + *error_message = strdup("Default connection belongs to another Node isolate; use a Session"); + return nullptr; + } + return g_reg.defaultConn; + } // No default up. If a session holds a real data directory, "" cannot bind a // second one (one EmbeddedServer path per process) — reject, as v2 did. if (!g_reg.conns.empty()) { @@ -283,6 +309,7 @@ static chdb_connection *get_default_conn(char **error_message) { return nullptr; } g_reg.defaultConn = c; + g_conn_owner[c] = owner; g_reg.boundKey = ""; g_reg.conns.insert(c); // Register only after chdb_connect started the engine, so the backstop runs @@ -322,6 +349,7 @@ static chdb_connection *acquire_session_conn(const std::string &path, } erase_arrow_tables_for_conn(g_reg.defaultConn); chdb_close_conn(g_reg.defaultConn); + g_conn_owner.erase(g_reg.defaultConn); g_reg.conns.erase(g_reg.defaultConn); g_reg.defaultConn = nullptr; } @@ -356,6 +384,7 @@ static void release_session_conn(chdb_connection *conn) { // JS buffer references aren't orphaned in g_live_arrow_tables. erase_arrow_tables_for_conn(conn); chdb_close_conn(conn); + g_conn_owner.erase(conn); g_reg.conns.erase(it); if (conn == g_reg.defaultConn) g_reg.defaultConn = nullptr; // Last connection out unbinds the EmbeddedServer so a different path may bind. @@ -387,8 +416,8 @@ static char *exec_query(chdb_connection conn, const char *query, } // Query function without session (uses the registry default connection) -char *Query(const char *query, const char *format, char **error_message) { - chdb_connection *conn_ptr = get_default_conn(error_message); +char *Query(const char *query, const char *format, char **error_message, napi_env owner) { + chdb_connection *conn_ptr = get_default_conn(error_message, owner); if (!conn_ptr) return nullptr; return exec_query(*conn_ptr, query, format, error_message); } @@ -502,7 +531,7 @@ Napi::String QueryWrapper(const Napi::CallbackInfo &info) { char *error_message = nullptr; - char *result = Query(query.c_str(), format.c_str(), &error_message); + char *result = Query(query.c_str(), format.c_str(), &error_message, env); if (result == NULL) { if (error_message != NULL) { @@ -551,7 +580,7 @@ Napi::String QueryWithParamsWrapper(const Napi::CallbackInfo &info) { } char *err = nullptr; - chdb_connection *conn_ptr = get_default_conn(&err); + chdb_connection *conn_ptr = get_default_conn(&err, env); char *out = conn_ptr ? exec_query_params(*conn_ptr, sql, format, names, values, &err) : nullptr; if (!out) { if (err) { Napi::Error::New(env, err).ThrowAsJavaScriptException(); free(err); } @@ -695,7 +724,7 @@ Napi::Value QueryAsyncWrapper(const Napi::CallbackInfo &info) { return rejectedPromise(env, "param values must be pre-formatted strings"); char *err = nullptr; - chdb_connection *conn_ptr = get_default_conn(&err); // registry: main thread only + chdb_connection *conn_ptr = get_default_conn(&err, env); if (!conn_ptr) { Napi::Value p = rejectedPromise(env, err ? err : "Failed to acquire default connection"); if (err) free(err); @@ -840,7 +869,7 @@ Napi::Value InsertRawAsyncWrapper(const Napi::CallbackInfo &info) { std::string prefix = info[0].As(); char *err = nullptr; - chdb_connection *conn_ptr = get_default_conn(&err); // registry: main thread only + chdb_connection *conn_ptr = get_default_conn(&err, env); if (!conn_ptr) { Napi::Value p = rejectedPromise(env, err ? err : "Failed to acquire default connection"); if (err) free(err); @@ -1100,6 +1129,10 @@ Napi::Value CreateConnectionWrapper(const Napi::CallbackInfo & info) { return env.Null(); } + { + std::lock_guard lock(g_reg_mu); + g_conn_owner[static_cast(conn)] = env; + } return Napi::External::New(env, conn); } @@ -1183,6 +1216,429 @@ struct AbiArray { void * private_data; }; +struct AbiArrayStream { + int (*get_schema)(AbiArrayStream *, AbiSchema *); + int (*get_next)(AbiArrayStream *, AbiArray *); + const char * (*get_last_error)(AbiArrayStream *); + void (*release)(AbiArrayStream *); + void * private_data; +}; + +struct DirectArrowColumn { + std::string name; + std::string format; + bool nullable = false; + std::string dictionaryFormat; +}; +struct DirectArrowDictionary { + int64_t length = 0; + int64_t nullCount = 0; + std::vector> buffers; +}; +struct DirectArrowBatch { + int64_t length = 0; + // Each column has validity, values, and (for Utf8) offsets/data buffers. + std::vector>> columns; + std::vector nullCounts; + std::vector dictionaries; +}; + +// Copy while the C Data array is alive. Never retain its pointers in a JS +// Buffer after invoking release(); external ArrayBuffers require a separate +// ownership bridge and are not implied by this direct (non-IPC) path. +static bool copyDirectArrowBuffer(const void * ptr, size_t size, + std::vector & out, std::string & error) { + constexpr size_t maxBuffer = 512ULL * 1024 * 1024; + if (size > maxBuffer || (size && !ptr)) { + error = "Arrow buffer is missing or exceeds the 512 MiB per-buffer limit"; + return false; + } + if (size) { + const auto * bytes = static_cast(ptr); + out.assign(bytes, bytes + size); + } + return true; +} + +static void readDirectArrowStream(AbiArrayStream & stream, + std::vector & fields, + std::vector & batches, + std::string & error) { + if (!stream.release || !stream.get_schema || !stream.get_next) { + error = "Invalid Arrow result stream"; return; + } + AbiSchema schema{}; + if (stream.get_schema(&stream, &schema) != 0) error = "Arrow get_schema failed"; + else if (!schema.format || std::strcmp(schema.format, "+s") != 0 || schema.n_children < 0) + error = "Arrow result is not a struct schema"; + if (error.empty()) { + for (int64_t i = 0; i < schema.n_children; ++i) { + const AbiSchema * field = schema.children ? schema.children[i] : nullptr; + if (!field || !field->format || !field->name || field->n_children) { + error = "Unsupported Arrow result field"; break; + } + const std::string format = field->format; + const bool timestamp = format.size() >= 4 && format[0] == 't' && format[1] == 's' && + format[3] == ':' && std::strchr("smun", format[2]); + int precision = 0, scale = 0, bitWidth = 128; + const bool decimal = format.rfind("d:", 0) == 0 && + std::sscanf(format.c_str(), "d:%d,%d,%d", &precision, &scale, &bitWidth) >= 2 && + precision > 0 && (bitWidth == 128 || bitWidth == 256); + if (format != "c" && format != "C" && format != "s" && format != "S" && + format != "i" && format != "I" && format != "l" && format != "L" && + format != "f" && format != "g" && format != "b" && format != "u" && format != "z" && + !timestamp && !decimal) { + error = "Unsupported Arrow result format: " + format; break; + } + std::string dictionaryFormat; + if (field->dictionary) { + if (!field->dictionary->format || field->dictionary->n_children || field->dictionary->dictionary || + (format != "i" && format != "I") || + (std::strcmp(field->dictionary->format, "u") != 0 && + std::strcmp(field->dictionary->format, "z") != 0)) { + error = "Unsupported Arrow dictionary layout"; break; + } + dictionaryFormat = field->dictionary->format; + } + fields.push_back({field->name, format, (field->flags & 2) != 0, dictionaryFormat}); + } + } + while (error.empty()) { + AbiArray array{}; + if (stream.get_next(&stream, &array) != 0) { + error = stream.get_last_error && stream.get_last_error(&stream) + ? stream.get_last_error(&stream) : "Arrow get_next failed"; + } else if (!array.release) { + break; + } else { + DirectArrowBatch batch; + batch.length = array.length; + if (array.length < 0 || array.offset != 0 || array.n_children != static_cast(fields.size())) + error = "Unsupported Arrow result array layout"; + for (size_t i = 0; error.empty() && i < fields.size(); ++i) { + const AbiArray * child = array.children ? array.children[i] : nullptr; + if (!child || child->length != array.length || child->offset != 0 || + child->null_count < 0 || child->null_count > child->length || !child->buffers) { + error = "Unsupported Arrow result child layout"; break; + } + const auto & format = fields[i].format; + const bool variable = format == "u" || format == "z"; + const int needed = variable ? 3 : 2; + if (child->n_buffers < needed) { error = "Missing Arrow result buffers"; break; } + std::vector> buffers(needed); + const size_t validitySize = child->null_count ? static_cast((child->length + 7) / 8) : 0; + if (!copyDirectArrowBuffer(child->buffers[0], validitySize, buffers[0], error)) break; + size_t valuesSize = 0; + if (variable) valuesSize = static_cast(child->length + 1) * 4; + else if (format == "b") valuesSize = static_cast((child->length + 7) / 8); + else { + int decimalPrecision = 0, decimalScale = 0, decimalBitWidth = 128; + const bool decimal = format.rfind("d:", 0) == 0 && + std::sscanf(format.c_str(), "d:%d,%d,%d", &decimalPrecision, &decimalScale, + &decimalBitWidth) >= 2; + const size_t width = decimal ? static_cast(decimalBitWidth / 8) + : format == "c" || format == "C" ? 1 + : format == "s" || format == "S" ? 2 + : format == "i" || format == "I" || format == "f" ? 4 : 8; + valuesSize = static_cast(child->length) * width; + } + if (!copyDirectArrowBuffer(child->buffers[1], valuesSize, buffers[1], error)) break; + if (variable) { + const auto * offsets = reinterpret_cast(child->buffers[1]); + if (!offsets || offsets[child->length] < 0) { + error = "Invalid Arrow UTF-8 offsets"; break; + } + if (!copyDirectArrowBuffer(child->buffers[2], static_cast(offsets[child->length]), + buffers[2], error)) break; + } + batch.columns.push_back(std::move(buffers)); + batch.nullCounts.push_back(child->null_count); + DirectArrowDictionary dictionary; + if (!fields[i].dictionaryFormat.empty()) { + const AbiArray * source = child->dictionary; + if (!source || source->length < 0 || source->offset != 0 || source->n_buffers < 3 || + source->null_count < 0 || source->null_count > source->length || !source->buffers) { + error = "Invalid Arrow dictionary array"; break; + } + dictionary.length = source->length; + dictionary.nullCount = source->null_count; + dictionary.buffers.resize(3); + if (!copyDirectArrowBuffer(source->buffers[0], source->null_count ? + static_cast((source->length + 7) / 8) : 0, dictionary.buffers[0], error)) break; + if (!copyDirectArrowBuffer(source->buffers[1], static_cast(source->length + 1) * 4, + dictionary.buffers[1], error)) break; + const auto * offsets = reinterpret_cast(source->buffers[1]); + if (!offsets || offsets[source->length] < 0) { error = "Invalid Arrow dictionary offsets"; break; } + if (!copyDirectArrowBuffer(source->buffers[2], static_cast(offsets[source->length]), + dictionary.buffers[2], error)) break; + } + batch.dictionaries.push_back(std::move(dictionary)); + } + array.release(&array); + array.release = nullptr; + if (error.empty()) batches.push_back(std::move(batch)); + } + if (!error.empty()) { + if (array.release) array.release(&array); + break; + } + } + if (schema.release) schema.release(&schema); +} + +static Napi::Object directArrowResultToJs(Napi::Env env, + const std::vector & fieldData, + const std::vector & batchData) { + Napi::Array fields = Napi::Array::New(env, fieldData.size()); + for (size_t i = 0; i < fieldData.size(); ++i) { + Napi::Object field = Napi::Object::New(env); + field.Set("name", fieldData[i].name); + field.Set("format", fieldData[i].format); + field.Set("nullable", Napi::Boolean::New(env, fieldData[i].nullable)); + if (!fieldData[i].dictionaryFormat.empty()) field.Set("dictionaryFormat", fieldData[i].dictionaryFormat); + fields.Set(i, field); + } + Napi::Array batches = Napi::Array::New(env, batchData.size()); + for (size_t i = 0; i < batchData.size(); ++i) { + Napi::Object batch = Napi::Object::New(env); + batch.Set("length", Napi::Number::New(env, static_cast(batchData[i].length))); + Napi::Array columns = Napi::Array::New(env, batchData[i].columns.size()); + for (size_t j = 0; j < batchData[i].columns.size(); ++j) { + Napi::Object column = Napi::Object::New(env); + column.Set("nullCount", Napi::Number::New(env, static_cast(batchData[i].nullCounts[j]))); + Napi::Array buffers = Napi::Array::New(env, batchData[i].columns[j].size()); + for (size_t k = 0; k < batchData[i].columns[j].size(); ++k) { + const auto & data = batchData[i].columns[j][k]; + buffers.Set(k, Napi::Buffer::Copy(env, data.data(), data.size())); + } + column.Set("buffers", buffers); + if (!fieldData[j].dictionaryFormat.empty()) { + const auto & source = batchData[i].dictionaries[j]; + Napi::Object dictionary = Napi::Object::New(env); + dictionary.Set("length", Napi::Number::New(env, static_cast(source.length))); + dictionary.Set("nullCount", Napi::Number::New(env, static_cast(source.nullCount))); + Napi::Array dataBuffers = Napi::Array::New(env, source.buffers.size()); + for (size_t k = 0; k < source.buffers.size(); ++k) { + const auto & data = source.buffers[k]; + dataBuffers.Set(k, Napi::Buffer::Copy(env, data.data(), data.size())); + } + dictionary.Set("buffers", dataBuffers); + column.Set("dictionary", dictionary); + } + columns.Set(j, column); + } + batch.Set("columns", columns); + batches.Set(i, batch); + } + Napi::Object out = Napi::Object::New(env); + out.Set("fields", fields); + out.Set("batches", batches); + return out; +} + +static bool parseDirectArrowOptions(Napi::Env env, Napi::Value input, chdb_arrow_options & options) { + options = {0, 0, 1}; + if (input.IsUndefined() || input.IsNull()) return true; + if (!input.IsObject()) { + Napi::TypeError::New(env, "Arrow options must be an object").ThrowAsJavaScriptException(); + return false; + } + Napi::Object object = input.As(); + for (const char * key : {"unsupportedAsBinary", "stringAsString", "lowCardinalityAsDictionary"}) { + Napi::Value value = object.Get(key); + if (!value.IsUndefined() && !value.IsBoolean()) { + Napi::TypeError::New(env, std::string("Arrow option ") + key + " must be boolean").ThrowAsJavaScriptException(); + return false; + } + } + Napi::Value unsupported = object.Get("unsupportedAsBinary"); + Napi::Value stringAsString = object.Get("stringAsString"); + Napi::Value dictionary = object.Get("lowCardinalityAsDictionary"); + if (!unsupported.IsUndefined()) options.unsupported_as_binary = unsupported.As().Value(); + if (!stringAsString.IsUndefined()) options.string_as_string = stringAsString.As().Value(); + if (!dictionary.IsUndefined()) options.low_cardinality_as_dictionary = dictionary.As().Value(); + return true; +} + +class QueryArrowWorker : public Napi::AsyncWorker { +public: + QueryArrowWorker(Napi::Env env, chdb_connection conn, std::string sql, chdb_arrow_options options) + : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), + conn_(conn), sql_(std::move(sql)), options_(options), guard_(conn) {} + Napi::Promise GetPromise() { return deferred_.Promise(); } + + void Execute() override { + AbiArrayStream stream{}; + chdb_result * result = chdb_query_arrow_n( + conn_, sql_.data(), sql_.size(), reinterpret_cast(&stream), &options_); + if (!result) { SetError("chdb_query_arrow_n returned a null result"); return; } + std::string error; + if (const char * nativeError = chdb_result_error(result)) error = nativeError; + if (error.empty()) readDirectArrowStream(stream, fields_, batches_, error); + if (stream.release) stream.release(&stream); + chdb_destroy_query_result(result); + if (!error.empty()) SetError(error); + } + + void OnOK() override { + deferred_.Resolve(directArrowResultToJs(Env(), fields_, batches_)); + } + void OnError(const Napi::Error & e) override { deferred_.Reject(e.Value()); } + +private: + Napi::Promise::Deferred deferred_; + chdb_connection conn_; + std::string sql_; + chdb_arrow_options options_; + std::vector fields_; + std::vector batches_; + ConnGuard guard_; +}; + +Napi::Value QueryArrowAsyncWrapper(const Napi::CallbackInfo & info) { + Napi::Env env = info.Env(); + if (info.Length() < 2 || !info[0].IsExternal() || !info[1].IsString()) { + Napi::TypeError::New(env, "Usage: connection, SQL").ThrowAsJavaScriptException(); + return env.Undefined(); + } + auto * conn = static_cast(info[0].As>().Data()); + if (!conn || !*conn) return rejectedPromise(env, "No active connection available"); + chdb_arrow_options options; + if (!parseDirectArrowOptions(env, info.Length() >= 3 ? info[2] : env.Undefined(), options)) return env.Undefined(); + auto * worker = new QueryArrowWorker(env, *conn, info[1].As().Utf8Value(), options); + worker->Queue(); + return worker->GetPromise(); +} + +struct DirectArrowStreamState { + chdb_connection conn; + chdb_result * handle; + bool closed = false; +}; + +static void closeDirectArrowStream(DirectArrowStreamState * state) { + if (!state || state->closed) return; + state->closed = true; + if (state->handle) { + chdb_stream_cancel_query(state->conn, state->handle); + chdb_destroy_query_result(state->handle); + state->handle = nullptr; + } +} + +Napi::Value StartArrowStreamWrapper(const Napi::CallbackInfo & info) { + Napi::Env env = info.Env(); + if (info.Length() < 2 || !info[0].IsExternal() || !info[1].IsString()) { + Napi::TypeError::New(env, "Usage: connection, SQL").ThrowAsJavaScriptException(); + return env.Undefined(); + } + auto * conn = static_cast(info[0].As>().Data()); + if (!conn || !*conn) { + Napi::Error::New(env, "No active connection available").ThrowAsJavaScriptException(); + return env.Undefined(); + } + chdb_arrow_options options; + if (!parseDirectArrowOptions(env, info.Length() >= 3 ? info[2] : env.Undefined(), options)) return env.Undefined(); + const std::string sql = info[1].As().Utf8Value(); + chdb_result * handle = nullptr; + if (info.Length() >= 4 && info[3].IsObject()) { + std::vector names, values; + if (!collectParams(info[3].As(), names, values)) { + Napi::TypeError::New(env, "Arrow stream parameter values must be pre-formatted strings") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + const size_t count = names.size(); + std::vector cNames(count), cValues(count); + std::vector valueLengths(count); + for (size_t i = 0; i < count; ++i) { + cNames[i] = names[i].c_str(); + cValues[i] = values[i].data(); + valueLengths[i] = values[i].size(); + } + handle = chdb_stream_query_arrow_with_params_n(*conn, sql.data(), sql.size(), &options, + count ? cNames.data() : nullptr, nullptr, + count ? cValues.data() : nullptr, count ? valueLengths.data() : nullptr, count); + } else { + handle = chdb_stream_query_arrow_n(*conn, sql.data(), sql.size(), &options); + } + if (!handle) { + Napi::Error::New(env, "chdb_stream_query_arrow_n returned a null handle").ThrowAsJavaScriptException(); + return env.Undefined(); + } + if (const char * error = chdb_result_error(handle)) { + const std::string message = error; + chdb_destroy_query_result(handle); + Napi::Error::New(env, message).ThrowAsJavaScriptException(); + return env.Undefined(); + } + auto * state = new DirectArrowStreamState{*conn, handle, false}; + return Napi::External::New(env, state, [](Napi::Env, DirectArrowStreamState * s) { + closeDirectArrowStream(s); + delete s; + }); +} + +class FetchArrowWorker : public Napi::AsyncWorker { +public: + FetchArrowWorker(Napi::Env env, DirectArrowStreamState * state, Napi::Value handle) + : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), + state_(state), handleRef_(Napi::Persistent(handle)), guard_(state ? state->conn : nullptr) {} + Napi::Promise GetPromise() { return deferred_.Promise(); } + + void Execute() override { + if (!state_ || state_->closed || !state_->handle) { done_ = true; return; } + AbiArrayStream batch{}; + const chdb_state status = chdb_stream_fetch_arrow( + state_->conn, state_->handle, reinterpret_cast(&batch)); + std::string error; + if (status != CHDBSuccess) error = "chdb_stream_fetch_arrow failed"; + if (error.empty()) readDirectArrowStream(batch, fields_, batches_, error); + if (batch.release) batch.release(&batch); + if (!error.empty()) { + closeDirectArrowStream(state_); + SetError(error); + return; + } + if (batches_.empty()) { + done_ = true; + chdb_destroy_query_result(state_->handle); + state_->handle = nullptr; + state_->closed = true; + } + } + void OnOK() override { + Napi::Object out = directArrowResultToJs(Env(), fields_, batches_); + out.Set("done", Napi::Boolean::New(Env(), done_)); + deferred_.Resolve(out); + } + void OnError(const Napi::Error & e) override { deferred_.Reject(e.Value()); } + +private: + Napi::Promise::Deferred deferred_; + DirectArrowStreamState * state_; + Napi::Reference handleRef_; + bool done_ = false; + std::vector fields_; + std::vector batches_; + ConnGuard guard_; +}; + +Napi::Value FetchArrowStreamWrapper(const Napi::CallbackInfo & info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsExternal()) return rejectedPromise(env, "Usage: Arrow stream handle"); + auto * state = info[0].As>().Data(); + auto * worker = new FetchArrowWorker(env, state, info[0]); + worker->Queue(); + return worker->GetPromise(); +} + +Napi::Value CancelArrowStreamWrapper(const Napi::CallbackInfo & info) { + if (info.Length() >= 1 && info[0].IsExternal()) + closeDirectArrowStream(info[0].As>().Data()); + return info.Env().Undefined(); +} + // No-op release callbacks. chdb-arrow.cpp saves each child's release into a // vector before scanning, then restores it; passing NULL would segfault there. static void AbiSchemaNoopRelease(AbiSchema * s) { s->release = nullptr; } @@ -1219,7 +1675,7 @@ static bool resolveArrowConn(Napi::Env env, Napi::Value handle, chdb_connection } if (handle.IsNull() || handle.IsUndefined()) { char * err = nullptr; - chdb_connection * c = get_default_conn(&err); + chdb_connection * c = get_default_conn(&err, env); if (c == nullptr) { std::string msg = err ? err : "no default connection available"; if (err) free(err); @@ -1327,9 +1783,13 @@ struct LiveArrowTable { std::string format; // "+s" + child format strings (owned strings below) std::vector formatStorage; std::vector nameStorage; + std::vector dictionaryFormatStorage; // Children are stored contiguously and pointed at via pointer-arrays below. std::vector childSchemas; std::vector childArrays; + std::vector dictionarySchemas; + std::vector dictionaryArrays; + std::vector> dictionaryBufferPtrs; std::vector childSchemaPtrs; std::vector childArrayPtrs; AbiSchema rootSchema{}; @@ -1346,52 +1806,230 @@ struct LiveArrowKeyHash { } }; static std::unordered_map, LiveArrowKeyHash> g_live_arrow_tables; +struct LiveArrowStream { + AbiArrayStream stream{}; + std::vector> batches; + size_t next = 0; +}; +static std::unordered_map, LiveArrowKeyHash> g_live_arrow_streams; +struct LiveAsyncArrowStream; +struct AsyncArrowLease { + std::shared_ptr state; + std::unique_ptr batch; +}; +struct AsyncArrowSignal { + std::shared_ptr state; + AsyncArrowLease * released = nullptr; +}; +struct PendingAsyncArrowBatch { + std::unique_ptr batch; + Napi::Promise::Deferred deferred; +}; +struct LiveAsyncArrowStream { + explicit LiveAsyncArrowStream(Napi::Env owner) : env(owner) {} + Napi::Env env; + AbiArrayStream stream{}; + std::unique_ptr schemaBatch; + std::deque> queue; + std::deque waiting; + std::mutex mu; + std::condition_variable cv; + Napi::ThreadSafeFunction releaseTsfn; + std::weak_ptr self; + size_t outstanding = 0; + bool ended = false; + bool cancelled = false; + bool tsfnReleased = false; + std::string error; + ~LiveAsyncArrowStream() { if (releaseTsfn && !tsfnReleased) releaseTsfn.Release(); } +}; +static std::unordered_map, LiveArrowKeyHash> g_live_arrow_async; +static std::atomic g_async_arrow_delivered{0}; +static std::atomic g_async_arrow_released{0}; +static std::atomic g_arrow_input_schema_released{0}; static std::mutex g_live_arrow_mu; -// Drop every registered Arrow table belonging to `conn`, releasing the pinned -// JS buffer references and C-side struct storage. Called from the connection -// close paths so a session closed without an explicit Unregister doesn't leak. +static void countedArrowSchemaRelease(AbiSchema * schema) { + ++g_arrow_input_schema_released; + schema->release = nullptr; +} + +static void cancelAsyncArrowStream(const std::shared_ptr & state) { + if (!state) return; + std::deque waiting; + bool releaseTsfn = false; + { + std::lock_guard lock(state->mu); + state->cancelled = true; + state->ended = true; + if (state->error.empty()) state->error = "Arrow reader cancelled"; + state->queue.clear(); + waiting.swap(state->waiting); + if (state->outstanding == 0 && !state->tsfnReleased) { + state->tsfnReleased = true; + releaseTsfn = true; + } + } + state->cv.notify_all(); + if (releaseTsfn) state->releaseTsfn.Release(); + for (auto & pending : waiting) + pending.deferred.Reject(Napi::Error::New(state->env, "Arrow reader closed").Value()); +} + +static int liveAsyncArrowGetSchema(AbiArrayStream * stream, AbiSchema * out) { + auto * live = static_cast(stream->private_data); + if (!live || !live->schemaBatch) return 1; + *out = live->schemaBatch->rootSchema; + out->release = countedArrowSchemaRelease; + return 0; +} +static const char * liveAsyncArrowGetLastError(AbiArrayStream * stream) { + auto * live = static_cast(stream->private_data); + return live && !live->error.empty() ? live->error.c_str() : nullptr; +} +static void liveAsyncArrowRelease(AbiArrayStream * stream) { stream->release = nullptr; } + +static void deliverAsyncArrowSignal(Napi::Env env, Napi::Function, AsyncArrowSignal * signal) { + auto state = signal->state; + if (signal->released) delete signal->released; // Napi::Reference: main thread only + const bool released = signal->released != nullptr; + if (released) ++g_async_arrow_released; + delete signal; + std::optional ready; + bool releaseTsfn = false; + { + std::lock_guard lock(state->mu); + if (released) --state->outstanding; + if (state->cancelled && state->outstanding == 0 && !state->tsfnReleased) { + state->tsfnReleased = true; + releaseTsfn = true; + } + if (!state->cancelled && !state->waiting.empty() && state->queue.size() < 2) { + auto pending = std::move(state->waiting.front()); + state->waiting.pop_front(); + state->queue.push_back(std::move(pending.batch)); + ready = pending.deferred; + } + } + state->cv.notify_all(); + if (releaseTsfn) state->releaseTsfn.Release(); + if (ready) ready->Resolve(env.Undefined()); +} + +static void liveAsyncArrowArrayRelease(AbiArray * array) { + auto * lease = static_cast(array->private_data); + array->release = nullptr; + array->private_data = nullptr; + if (!lease) return; + // C Data release may run on a core worker. Napi::Reference destruction and + // producer promise resolution must happen on the owning JS isolate instead. + auto * signal = new AsyncArrowSignal{lease->state, lease}; + napi_status status = lease->state->releaseTsfn.NonBlockingCall(signal, deliverAsyncArrowSignal); + // During isolate shutdown there is no safe thread on which to drop pinned + // JS references. Leave the lease for process teardown rather than release it + // on a core worker. Normal operation never takes this path. + (void)status; +} + +static int liveAsyncArrowGetNext(AbiArrayStream * stream, AbiArray * out) { + auto * live = static_cast(stream->private_data); + if (!live) return 1; + std::unique_lock lock(live->mu); + live->cv.wait(lock, [live] { return !live->queue.empty() || live->ended || live->cancelled; }); + if (live->cancelled || !live->error.empty()) return 1; + if (live->queue.empty()) { *out = {}; return 0; } + auto batch = std::move(live->queue.front()); + live->queue.pop_front(); + ++live->outstanding; + lock.unlock(); + // The state is also owned by the registration map and its JS handle. The + // weak self reference avoids a permanent ownership cycle. + auto * lease = new AsyncArrowLease{live->self.lock(), std::move(batch)}; + ++g_async_arrow_delivered; + auto * capacity = new AsyncArrowSignal{lease->state, nullptr}; + (void)live->releaseTsfn.NonBlockingCall(capacity, deliverAsyncArrowSignal); + *out = lease->batch->rootArray; + out->release = liveAsyncArrowArrayRelease; + out->private_data = lease; + return 0; +} + +static int liveArrowGetSchema(AbiArrayStream * stream, AbiSchema * out) { + auto * live = static_cast(stream->private_data); + if (!live || live->batches.empty()) return 1; + *out = live->batches.front()->rootSchema; + out->release = countedArrowSchemaRelease; + return 0; +} +static int liveArrowGetNext(AbiArrayStream * stream, AbiArray * out) { + auto * live = static_cast(stream->private_data); + if (!live) return 1; + if (live->next >= live->batches.size()) { + *out = {}; + return 0; + } + *out = live->batches[live->next++]->rootArray; + out->release = AbiArrayNoopRelease; + return 0; +} +static const char * liveArrowGetLastError(AbiArrayStream *) { return nullptr; } +static void liveArrowRelease(AbiArrayStream * stream) { stream->release = nullptr; } + +// Unregister BEFORE releasing the JS buffers and C-side structs. Core's Arrow +// registry is process-wide and chdb_close_conn does not remove its entries; +// erasing only our map left dangling pointers and reserved names behind. static void erase_arrow_tables_for_conn(chdb_connection *conn) { std::lock_guard lock(g_live_arrow_mu); for (auto it = g_live_arrow_tables.begin(); it != g_live_arrow_tables.end();) { - if (it->first.first == conn) it = g_live_arrow_tables.erase(it); + if (it->first.first == conn) { + chdb_arrow_unregister_table(*conn, it->first.second.c_str()); + it = g_live_arrow_tables.erase(it); + } else ++it; } + for (auto it = g_live_arrow_streams.begin(); it != g_live_arrow_streams.end();) { + if (it->first.first == conn) { + chdb_arrow_unregister_table(*conn, it->first.second.c_str()); + it = g_live_arrow_streams.erase(it); + } else ++it; + } + for (auto it = g_live_arrow_async.begin(); it != g_live_arrow_async.end();) { + if (it->first.first == conn) { + chdb_arrow_unregister_table(*conn, it->first.second.c_str()); + cancelAsyncArrowStream(it->second); + it = g_live_arrow_async.erase(it); + } else ++it; + } } // Drop the entire registry (process teardown backstop). static void clear_all_arrow_tables() { std::lock_guard lock(g_live_arrow_mu); + // Only called by atexit. The core registry's static destructor may have + // already run; normal connection closes unregister before reaching here. g_live_arrow_tables.clear(); + g_live_arrow_streams.clear(); + g_live_arrow_async.clear(); } -// nativeArrowRegisterColumns(conn, tableName, columns) -// columns: Array<{ name, format, length, nullCount, buffers: Array }> -// - format: an Arrow C Data Interface format string ("i"/"l"/"g"/"b"/"u"/...) -// - buffers[0] = validity bitmap (null if nullCount === 0) -// - buffers[1] = data -// - buffers[2] = offsets (utf8 only) -Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { - Napi::Env env = info.Env(); - if (info.Length() < 3 || !info[1].IsString() || !info[2].IsArray()) { - Napi::TypeError::New(env, "Usage: connection, tableName, columns[]").ThrowAsJavaScriptException(); - return env.Undefined(); - } - chdb_connection * conn = nullptr; - if (!resolveArrowConn(env, info[0], &conn)) return env.Undefined(); - std::string tableName = info[1].As().Utf8Value(); - Napi::Array columns = info[2].As(); +// Build one caller-owned C Data batch from JS column buffers. Used by both +// chdb_arrow_array_scan and the multi-batch C Stream adapter below. +static std::unique_ptr makeLiveArrowTable(Napi::Env env, Napi::Array columns) { uint32_t ncols = columns.Length(); if (ncols == 0) { Napi::TypeError::New(env, "registerArrowTable: at least one column required").ThrowAsJavaScriptException(); - return env.Undefined(); + return nullptr; } auto live = std::make_unique(); live->formatStorage.reserve(ncols); live->nameStorage.reserve(ncols); + live->dictionaryFormatStorage.reserve(ncols); live->childSchemas.resize(ncols); live->childArrays.resize(ncols); + live->dictionarySchemas.resize(ncols); + live->dictionaryArrays.resize(ncols); + live->dictionaryBufferPtrs.resize(ncols); live->childSchemaPtrs.resize(ncols); live->childArrayPtrs.resize(ncols); live->buffers.bufferPtrs.resize(ncols); @@ -1403,7 +2041,7 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { Napi::Value colv = columns[i]; if (!colv.IsObject()) { Napi::TypeError::New(env, "Each column must be an object").ThrowAsJavaScriptException(); - return env.Undefined(); + return nullptr; } Napi::Object col = colv.As(); Napi::Value nameV = col.Get("name"); @@ -1413,7 +2051,7 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { Napi::Value buffersV = col.Get("buffers"); if (!nameV.IsString() || !formatV.IsString() || !lengthV.IsNumber() || !buffersV.IsArray()) { Napi::TypeError::New(env, "Column needs { name, format, length, buffers }").ThrowAsJavaScriptException(); - return env.Undefined(); + return nullptr; } live->nameStorage.emplace_back(nameV.As().Utf8Value()); live->formatStorage.emplace_back(formatV.As().Utf8Value()); @@ -1423,7 +2061,7 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { // ArrowArray.length, and a negative nullCount mismatches the validity bitmap. if (length < 0 || nullCount < 0) { Napi::TypeError::New(env, "Column length and nullCount must be non-negative").ThrowAsJavaScriptException(); - return env.Undefined(); + return nullptr; } // A separate flag (not a -1 sentinel) so a column whose length is legitimately // any value, including 0, sets the root length on the first column. @@ -1432,7 +2070,7 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { rootLengthSet = true; } else if (rootLength != length) { Napi::TypeError::New(env, "All columns must have the same length").ThrowAsJavaScriptException(); - return env.Undefined(); + return nullptr; } Napi::Array bufs = buffersV.As(); @@ -1447,7 +2085,7 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { } if (!bv.IsBuffer()) { Napi::TypeError::New(env, "Column buffers must be Node Buffers or null").ThrowAsJavaScriptException(); - return env.Undefined(); + return nullptr; } Napi::Buffer buf = bv.As>(); ptrs.push_back(buf.Data()); @@ -1484,6 +2122,56 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { ca.release = AbiArrayNoopRelease; ca.private_data = nullptr; live->childArrayPtrs[i] = &ca; + + Napi::Value dictionary = col.Get("dictionary"); + if (!dictionary.IsUndefined() && !dictionary.IsNull()) { + if (!dictionary.IsObject()) { + Napi::TypeError::New(env, "Arrow dictionary must be an object").ThrowAsJavaScriptException(); + return nullptr; + } + Napi::Object dict = dictionary.As(); + Napi::Value fmt = dict.Get("format"), count = dict.Get("length"), + nulls = dict.Get("nullCount"), values = dict.Get("buffers"); + if (!fmt.IsString() || !count.IsNumber() || !nulls.IsNumber() || !values.IsArray()) { + Napi::TypeError::New(env, "Arrow dictionary needs format, length, nullCount, buffers").ThrowAsJavaScriptException(); + return nullptr; + } + const int64_t dictLength = count.As().Int64Value(); + const int64_t dictNulls = nulls.As().Int64Value(); + if (dictLength < 0 || dictNulls < 0 || dictNulls > dictLength) { + Napi::TypeError::New(env, "Arrow dictionary length/nullCount invalid").ThrowAsJavaScriptException(); + return nullptr; + } + live->dictionaryFormatStorage.emplace_back(fmt.As().Utf8Value()); + Napi::Array dictBuffers = values.As(); + auto & pointers = live->dictionaryBufferPtrs[i]; + pointers.reserve(dictBuffers.Length()); + for (uint32_t j = 0; j < dictBuffers.Length(); ++j) { + Napi::Value value = dictBuffers[j]; + if (value.IsNull() || value.IsUndefined()) { pointers.push_back(nullptr); continue; } + if (!value.IsBuffer()) { + Napi::TypeError::New(env, "Arrow dictionary buffers must be Buffers").ThrowAsJavaScriptException(); + return nullptr; + } + pointers.push_back(value.As>().Data()); + live->buffers.jsRefs.push_back(Napi::Reference::New(value, 1)); + } + AbiSchema & ds = live->dictionarySchemas[i]; + ds = {}; + ds.format = live->dictionaryFormatStorage.back().c_str(); + ds.name = ""; + ds.flags = 2; + ds.release = AbiSchemaNoopRelease; + cs.dictionary = &ds; + AbiArray & da = live->dictionaryArrays[i]; + da = {}; + da.length = dictLength; + da.null_count = dictNulls; + da.n_buffers = static_cast(pointers.size()); + da.buffers = pointers.data(); + da.release = AbiArrayNoopRelease; + ca.dictionary = &da; + } } // Root struct schema. @@ -1514,6 +2202,22 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { live->rootArray.release = AbiArrayNoopRelease; live->rootArray.private_data = nullptr; + return live; +} + +// nativeArrowRegisterColumns(conn, tableName, columns) +Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { + Napi::Env env = info.Env(); + if (info.Length() < 3 || !info[1].IsString() || !info[2].IsArray()) { + Napi::TypeError::New(env, "Usage: connection, tableName, columns[]").ThrowAsJavaScriptException(); + return env.Undefined(); + } + chdb_connection * conn = nullptr; + if (!resolveArrowConn(env, info[0], &conn)) return env.Undefined(); + std::string tableName = info[1].As().Utf8Value(); + auto live = makeLiveArrowTable(env, info[2].As()); + if (!live) return env.Undefined(); + chdb_state st = chdb_arrow_array_scan( *conn, tableName.c_str(), @@ -1533,6 +2237,161 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { return env.Undefined(); } +// A materialized apache-arrow Table may contain several RecordBatches. The +// C Stream callbacks only read prebuilt native descriptors and pinned Buffer +// pointers; no JS callback is invoked from a core query worker thread. +Napi::Value ArrowRegisterBatchesWrapper(const Napi::CallbackInfo & info) { + Napi::Env env = info.Env(); + if (info.Length() < 3 || !info[1].IsString() || !info[2].IsArray()) { + Napi::TypeError::New(env, "Usage: connection, tableName, batches[]").ThrowAsJavaScriptException(); + return env.Undefined(); + } + chdb_connection * conn = nullptr; + if (!resolveArrowConn(env, info[0], &conn)) return env.Undefined(); + std::string tableName = info[1].As().Utf8Value(); + Napi::Array batches = info[2].As(); + if (!batches.Length()) { + Napi::TypeError::New(env, "Arrow Table needs at least one batch").ThrowAsJavaScriptException(); + return env.Undefined(); + } + auto live = std::make_unique(); + live->batches.reserve(batches.Length()); + for (uint32_t i = 0; i < batches.Length(); ++i) { + Napi::Value value = batches[i]; + if (!value.IsArray()) { + Napi::TypeError::New(env, "Each Arrow batch must contain columns[]").ThrowAsJavaScriptException(); + return env.Undefined(); + } + auto batch = makeLiveArrowTable(env, value.As()); + if (!batch) return env.Undefined(); + if (!live->batches.empty()) { + const auto & first = *live->batches.front(); + if (batch->nameStorage != first.nameStorage || batch->formatStorage != first.formatStorage) { + Napi::TypeError::New(env, "Arrow batch schemas must match").ThrowAsJavaScriptException(); + return env.Undefined(); + } + } + live->batches.push_back(std::move(batch)); + } + live->stream.get_schema = liveArrowGetSchema; + live->stream.get_next = liveArrowGetNext; + live->stream.get_last_error = liveArrowGetLastError; + live->stream.release = liveArrowRelease; + live->stream.private_data = live.get(); + if (chdb_arrow_scan(*conn, tableName.c_str(), reinterpret_cast(&live->stream)) != CHDBSuccess) { + Napi::Error::New(env, "chdb_arrow_scan failed for table " + tableName).ThrowAsJavaScriptException(); + return env.Undefined(); + } + { + std::lock_guard lock(g_live_arrow_mu); + g_live_arrow_streams[LiveArrowKey{conn, tableName}] = std::move(live); + } + return env.Undefined(); +} + +struct AsyncArrowHandle { std::shared_ptr state; }; + +Napi::Value ArrowRegisterReaderWrapper(const Napi::CallbackInfo & info) { + Napi::Env env = info.Env(); + if (info.Length() < 3 || !info[1].IsString() || !info[2].IsArray()) { + Napi::TypeError::New(env, "Usage: connection, tableName, firstBatchColumns[]").ThrowAsJavaScriptException(); + return env.Undefined(); + } + chdb_connection * conn = nullptr; + if (!resolveArrowConn(env, info[0], &conn)) return env.Undefined(); + std::string tableName = info[1].As().Utf8Value(); + auto schemaBatch = makeLiveArrowTable(env, info[2].As()); + auto firstBatch = makeLiveArrowTable(env, info[2].As()); + if (!schemaBatch || !firstBatch) return env.Undefined(); + auto state = std::make_shared(env); + state->self = state; + state->schemaBatch = std::move(schemaBatch); + state->queue.push_back(std::move(firstBatch)); + state->releaseTsfn = Napi::ThreadSafeFunction::New(env, + Napi::Function::New(env, [](const Napi::CallbackInfo &) {}), "Arrow batch release", 0, 1); + state->releaseTsfn.Unref(env); + state->stream.get_schema = liveAsyncArrowGetSchema; + state->stream.get_next = liveAsyncArrowGetNext; + state->stream.get_last_error = liveAsyncArrowGetLastError; + state->stream.release = liveAsyncArrowRelease; + state->stream.private_data = state.get(); + if (chdb_arrow_scan(*conn, tableName.c_str(), reinterpret_cast(&state->stream)) != CHDBSuccess) { + Napi::Error::New(env, "chdb_arrow_scan failed for reader " + tableName).ThrowAsJavaScriptException(); + return env.Undefined(); + } + { + std::lock_guard lock(g_live_arrow_mu); + g_live_arrow_async[LiveArrowKey{conn, tableName}] = state; + } + auto * handle = new AsyncArrowHandle{state}; + return Napi::External::New(env, handle, [](Napi::Env, AsyncArrowHandle * h) { + cancelAsyncArrowStream(h->state); + delete h; + }); +} + +Napi::Value ArrowPushReaderBatchWrapper(const Napi::CallbackInfo & info) { + Napi::Env env = info.Env(); + if (info.Length() < 2 || !info[0].IsExternal() || !info[1].IsArray()) + return rejectedPromise(env, "Usage: readerHandle, batchColumns[]"); + auto state = info[0].As>().Data()->state; + auto batch = makeLiveArrowTable(env, info[1].As()); + if (!batch) return env.Undefined(); + if (batch->nameStorage != state->schemaBatch->nameStorage || + batch->formatStorage != state->schemaBatch->formatStorage) { + Napi::TypeError::New(env, "Arrow reader batch schema changed").ThrowAsJavaScriptException(); + return env.Undefined(); + } + auto deferred = Napi::Promise::Deferred::New(env); + { + std::lock_guard lock(state->mu); + if (state->cancelled || state->ended) { + deferred.Reject(Napi::Error::New(env, "Arrow reader is closed").Value()); + } else if (state->queue.size() < 2) { + state->queue.push_back(std::move(batch)); + state->cv.notify_all(); + deferred.Resolve(env.Undefined()); + } else if (state->waiting.empty()) { + state->waiting.push_back({std::move(batch), deferred}); + } else { + deferred.Reject(Napi::Error::New(env, "Arrow reader producer must await backpressure").Value()); + } + } + return deferred.Promise(); +} + +Napi::Value ArrowFinishReaderWrapper(const Napi::CallbackInfo & info) { + if (info.Length() < 1 || !info[0].IsExternal()) return info.Env().Undefined(); + auto state = info[0].As>().Data()->state; + std::deque waiting; + { + std::lock_guard lock(state->mu); + state->ended = true; + if (info.Length() > 1 && info[1].IsString()) { + state->error = info[1].As().Utf8Value(); + waiting.swap(state->waiting); + } + } + state->cv.notify_all(); + for (auto & pending : waiting) + pending.deferred.Reject(Napi::Error::New(info.Env(), state->error).Value()); + return info.Env().Undefined(); +} + +Napi::Value ArrowCancelReaderWrapper(const Napi::CallbackInfo & info) { + if (info.Length() >= 1 && info[0].IsExternal()) + cancelAsyncArrowStream(info[0].As>().Data()->state); + return info.Env().Undefined(); +} + +Napi::Value ArrowLifetimeStatsWrapper(const Napi::CallbackInfo & info) { + Napi::Object result = Napi::Object::New(info.Env()); + result.Set("delivered", Napi::Number::New(info.Env(), static_cast(g_async_arrow_delivered.load()))); + result.Set("released", Napi::Number::New(info.Env(), static_cast(g_async_arrow_released.load()))); + result.Set("schemaReleased", Napi::Number::New(info.Env(), static_cast(g_arrow_input_schema_released.load()))); + return result; +} + // nativeArrowUnregister(conn, tableName) — drops the registry entry. Napi::Value ArrowUnregisterWrapper(const Napi::CallbackInfo & info) { Napi::Env env = info.Env(); @@ -1555,6 +2414,12 @@ Napi::Value ArrowUnregisterWrapper(const Napi::CallbackInfo & info) { { std::lock_guard lock(g_live_arrow_mu); g_live_arrow_tables.erase(LiveArrowKey{ conn, tableName }); + g_live_arrow_streams.erase(LiveArrowKey{ conn, tableName }); + auto async = g_live_arrow_async.find(LiveArrowKey{conn, tableName}); + if (async != g_live_arrow_async.end()) { + cancelAsyncArrowStream(async->second); + g_live_arrow_async.erase(async); + } } return env.Undefined(); } @@ -1828,6 +2693,7 @@ Napi::Value ExitTeardownWrapper(const Napi::CallbackInfo &info) { if (c) { erase_arrow_tables_for_conn(c); chdb_close_conn(c); + g_conn_owner.erase(c); } if (c == g_reg.defaultConn) g_reg.defaultConn = nullptr; it = g_reg.conns.erase(it); @@ -1877,20 +2743,28 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { exports.Set("ArrowRegisterArray", Napi::Function::New(env, ArrowRegisterArrayWrapper)); exports.Set("ArrowRegisterStream", Napi::Function::New(env, ArrowRegisterStreamWrapper)); exports.Set("ArrowRegisterColumns", Napi::Function::New(env, ArrowRegisterColumnsWrapper)); + exports.Set("ArrowRegisterBatches", Napi::Function::New(env, ArrowRegisterBatchesWrapper)); + exports.Set("ArrowRegisterReader", Napi::Function::New(env, ArrowRegisterReaderWrapper)); + exports.Set("ArrowPushReaderBatch", Napi::Function::New(env, ArrowPushReaderBatchWrapper)); + exports.Set("ArrowFinishReader", Napi::Function::New(env, ArrowFinishReaderWrapper)); + exports.Set("ArrowCancelReader", Napi::Function::New(env, ArrowCancelReaderWrapper)); + exports.Set("ArrowLifetimeStats", Napi::Function::New(env, ArrowLifetimeStatsWrapper)); exports.Set("ArrowUnregister", Napi::Function::New(env, ArrowUnregisterWrapper)); + exports.Set("QueryArrowAsync", Napi::Function::New(env, QueryArrowAsyncWrapper)); + exports.Set("StartArrowStream", Napi::Function::New(env, StartArrowStreamWrapper)); + exports.Set("FetchArrowStream", Napi::Function::New(env, FetchArrowStreamWrapper)); + exports.Set("CancelArrowStream", Napi::Function::New(env, CancelArrowStreamWrapper)); // For index.js's process 'exit' hook. exports.Set("ExitTeardown", Napi::Function::New(env, ExitTeardownWrapper)); - // Close every live connection on env teardown, ahead of the std::atexit - // backstop. Both call the idempotent hard_close_all, so running twice is - // harmless. Node runs no cleanup hooks on process.exit(), which is one reason - // index.js closes on the 'exit' event instead (ExitTeardownWrapper). The - // hook also counts this environment out of g_live_envs. + // Each isolate closes only its own connections. The process exit hook and + // atexit backstop handle remaining process-wide connections after workers + // have stopped, without closing another live isolate's session. g_live_envs.fetch_add(1); - env.AddCleanupHook([] { + env.AddCleanupHook([owner = static_cast(env)] { + close_for_env(owner); g_live_envs.fetch_sub(1); - hard_close_all(); }); return exports; diff --git a/lib/chdb_node.h b/lib/chdb_node.h index 24594ad..c65b312 100644 --- a/lib/chdb_node.h +++ b/lib/chdb_node.h @@ -1,4 +1,5 @@ #pragma once -char *Query(const char *query, const char *format); -char *QuerySession(const char *query, const char *format, const char *path); +#include + +char *Query(const char *query, const char *format, char **error_message, napi_env owner); diff --git a/llms-full.txt b/llms-full.txt index eebfcf2..3650a6d 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -136,7 +136,43 @@ Codegen / introspection: Drizzle / Prisma schema file. introspectDatabase / introspectTable / describeSource / emitDatabase (programmatic) parseDrizzleFile / parseDrizzleSource / parsePrismaSchema (static conversion) - registerArrowTable(...) register an in-memory Arrow/columnar dataset as arrowstream('name') + registerArrowTable(name, columns|batch|table, {session?}) register an in-memory + source as arrowstream('name'). apache-arrow RecordBatch and single-batch Table + accept unsliced primitive numeric/Bool/Utf8/Binary buffers; materialized + multi-batch Tables use C Stream. One-chunk Int32/UInt32-indexed Utf8/Binary + dictionaries and sliced primitive/bitmap buffers are supported; nested + input is not yet supported. + No IPC on input; this does not + guarantee zero ClickHouse-column copies. handle.close() defers native release + during active queries/streams; finish or cancel a lazy stream to let it drain. + registerArrowSource(columns|batch|table, {session?}) same input path, with a + generated process-unique name returned on its closable handle. + registerArrowReader(asyncIterable, {session}) one-shot Arrow + reader as an engine SQL source, with a bounded native prefetch queue and + async close(). The first batch establishes schema; empty sources need + source.schema. Reader errors propagate to SQL; no IPC or readAll(). + insertArrow(table, columns|batch|table, {session}) bulk insert one Arrow batch + using an internal scoped registration and INSERT SELECT. Destination columns + are matched by name. rowsSubmitted is input size, not a storage write ledger. + insertArrowReader(table, reader, {session}) same field-name mapping and + INSERT SELECT over the bounded reader source; rowsSubmitted is not a + storage-side write ledger. + Session.queryArrow(sql) direct C Data output as an apache-arrow Table (not IPC). + Primitive signed/unsigned integers, Float32/64, Bool, Utf8, Binary, + Timestamp and Decimal128 are + supported; buffers are copied before C stream release, so the result + outlives the Session. stringAsString:false exports String as Binary. + Session.queryArrowStream(sql, options?) pull one typed apache-arrow + RecordBatch per core C Data fetch; cancel() on partial consumption. + schema is available after the first fetch, including zero-row results. + lowCardinalityAsDictionary:true preserves Utf8/Binary dictionaries with + Int32/UInt32 indices across batches; other dictionary layouts reject. + Session.queryArrowStreamBind(sql, params, options?) same batch reader with + server-side {name:Type} binding. queryArrowBind materializes that direct + stream as a Table; neither interpolates values into SQL. + chdb/arrow optional typed subpath: queryArrow/queryArrowBind return + apache-arrow Table; queryArrowStream/queryArrowStreamBind yield RecordBatch. + Import this subpath when the optional apache-arrow peer is installed. ================================================================ LAYER 2 — @clickhouse/client integration (import 'chdb/connection') diff --git a/package-lock.json b/package-lock.json index 3edc535..1e0c567 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,10 +35,10 @@ "node": ">=18.0.0" }, "optionalDependencies": { - "@chdb/lib-darwin-arm64": "26.7.2-rc.2.1", - "@chdb/lib-darwin-x64": "26.7.2-rc.2.1", - "@chdb/lib-linux-arm64-gnu": "26.7.2-rc.2.1", - "@chdb/lib-linux-x64-gnu": "26.7.2-rc.2.1" + "@chdb/lib-darwin-arm64": "26.11.0-rc.3.1", + "@chdb/lib-darwin-x64": "26.11.0-rc.3.1", + "@chdb/lib-linux-arm64-gnu": "26.11.0-rc.3.1", + "@chdb/lib-linux-x64-gnu": "26.11.0-rc.3.1" }, "peerDependencies": { "@aws-sdk/client-s3": ">=3", diff --git a/package.json b/package.json index 125808d..a181bd2 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,11 @@ "require": "./dist/libchdb/index.js", "default": "./dist/libchdb/index.js" }, + "./arrow": { + "types": "./dist/arrow-entry.d.ts", + "import": "./arrow.mjs", + "require": "./dist/arrow-entry.js" + }, "./agents": { "types": "./integrations/agents/index.d.mts", "import": "./integrations/agents/index.mjs" @@ -92,6 +97,7 @@ "files": [ "index.js", "index.mjs", + "arrow.mjs", "index.d.ts", "dist", "example.js", @@ -121,10 +127,10 @@ "node-gyp-build": "^4.6.0" }, "optionalDependencies": { - "@chdb/lib-darwin-arm64": "26.11.0-rc.2.1", - "@chdb/lib-darwin-x64": "26.11.0-rc.2.1", - "@chdb/lib-linux-arm64-gnu": "26.11.0-rc.2.1", - "@chdb/lib-linux-x64-gnu": "26.11.0-rc.2.1" + "@chdb/lib-darwin-arm64": "26.11.0-rc.3.1", + "@chdb/lib-darwin-x64": "26.11.0-rc.3.1", + "@chdb/lib-linux-arm64-gnu": "26.11.0-rc.3.1", + "@chdb/lib-linux-x64-gnu": "26.11.0-rc.3.1" }, "peerDependencies": { "@clickhouse/client": ">=1.23.0", diff --git a/scripts/smoke-arrow-e2e.mjs b/scripts/smoke-arrow-e2e.mjs new file mode 100644 index 0000000..0662bc2 --- /dev/null +++ b/scripts/smoke-arrow-e2e.mjs @@ -0,0 +1,48 @@ +// Installed-package Arrow smoke: checks the shipped addon + libchdb pair, +// never the build tree. Run with the optional apache-arrow peer installed. +import { Session, registerArrowSource, insertArrowReader } from 'chdb' +import { queryArrow as typedQueryArrow } from 'chdb/arrow' +import { Table, Int32, vectorFromArray } from 'apache-arrow' + +const assert = (condition, message) => { if (!condition) throw new Error(message) } +const session = new Session() +try { + const input = new Table({ + x: vectorFromArray([1, 2], new Int32()), + s: vectorFromArray(['a', 'b']), // apache-arrow's default dictionary encoding + }) + const source = registerArrowSource(input, { session }) + try { + const result = await session.queryAsync(`SELECT sum(x) FROM arrowstream('${source.name}')`, { format: 'CSV' }) + assert(result.text().trim() === '3', 'A1 dictionary source') + } finally { source.close() } + + const table = await session.queryArrow('SELECT toInt32(7) AS x') + assert(table.schema.fields[0].type.toString() === 'Int32', 'A3 typed table') + assert(table.toArray()[0].x === 7, 'A3 value') + assert((await typedQueryArrow(session, 'SELECT toInt32(8) AS x')).toArray()[0].x === 8, + 'optional typed Arrow subpath') + const bound = await session.queryArrowBind('SELECT toInt32({n:Int32}) AS n', { n: 9 }) + assert(bound.toArray()[0].n === 9, 'A3/A4 server-bound parameter') + + const reader = session.queryArrowStream('SELECT toInt32(number) AS x FROM numbers(3) SETTINGS max_block_size=1') + const values = [] + for await (const batch of reader) values.push(...batch.getChild('x').toArray()) + assert(values.join(',') === '0,1,2', 'A4 incremental batches') + + await session.queryAsync('CREATE TABLE arrow_smoke (x Int32) ENGINE=Memory') + async function* batches() { + yield new Table({ x: vectorFromArray([3], new Int32()) }).batches[0] + yield new Table({ x: vectorFromArray([4], new Int32()) }).batches[0] + } + const written = await insertArrowReader('arrow_smoke', batches(), { session }) + assert(written.rowsSubmitted === 2, 'A2/A5 reader insert count') + const count = await session.queryAsync('SELECT sum(x) FROM arrow_smoke', { format: 'CSV' }) + assert(count.text().trim() === '7', 'A5 inserted values') + + const dictionary = await session.queryArrow( + "SELECT toLowCardinality('v') AS s", { lowCardinalityAsDictionary: true }, + ) + assert(dictionary.schema.fields[0].type.toString().startsWith('Dictionary<'), 'A6 dictionary option') +} finally { session.close() } +console.log('smoke-arrow-e2e OK') diff --git a/src/arrow-direct.ts b/src/arrow-direct.ts new file mode 100644 index 0000000..f5853fe --- /dev/null +++ b/src/arrow-direct.ts @@ -0,0 +1,113 @@ +/** Construct apache-arrow objects from the native C Data result (never IPC). */ +import { ChdbArrowError } from './errors' +import type { Uint32, Dictionary } from 'apache-arrow' + +interface NativeField { name: string; format: string; nullable: boolean; dictionaryFormat?: string } +interface NativeColumn { + nullCount: number + buffers: Buffer[] + dictionary?: { length: number; nullCount: number; buffers: Buffer[] } +} +interface NativeBatch { length: number; columns: NativeColumn[] } +export interface NativeArrowResult { fields: NativeField[]; batches: NativeBatch[] } + +export function arrowTableFromDirect(raw: NativeArrowResult): unknown { + let arrow: typeof import('apache-arrow') + try { arrow = require('apache-arrow') as typeof import('apache-arrow') } + catch { throw new ChdbArrowError('apache-arrow is required for queryArrow(); run npm i apache-arrow') } + + const typeFor = (format: string) => { + const decimal = /^d:(\d+),(-?\d+)(?:,(128|256))?$/.exec(format) + if (decimal) return new arrow.Decimal(Number(decimal[2]), Number(decimal[1]), Number(decimal[3] || 128)) + if (/^ts[smun]:/.test(format)) { + const timezone = format.slice(4) || undefined + switch (format[2]) { + case 's': return new arrow.TimestampSecond(timezone) + case 'm': return new arrow.TimestampMillisecond(timezone) + case 'u': return new arrow.TimestampMicrosecond(timezone) + case 'n': return new arrow.TimestampNanosecond(timezone) + } + } + switch (format) { + case 'c': return new arrow.Int8() + case 'C': return new arrow.Uint8() + case 's': return new arrow.Int16() + case 'S': return new arrow.Uint16() + case 'i': return new arrow.Int32() + case 'I': return new arrow.Uint32() + case 'l': return new arrow.Int64() + case 'L': return new arrow.Uint64() + case 'f': return new arrow.Float32() + case 'g': return new arrow.Float64() + case 'b': return new arrow.Bool() + case 'u': return new arrow.Utf8() + case 'z': return new arrow.Binary() + default: throw new ChdbArrowError(`Unsupported direct Arrow format: ${format}`) + } + } + const fields = raw.fields.map((field) => new arrow.Field(field.name, + field.dictionaryFormat + ? new arrow.Dictionary(typeFor(field.dictionaryFormat), typeFor(field.format) as Uint32) + : typeFor(field.format), field.nullable)) + const schema = new arrow.Schema(fields) + const view = (buffer: Buffer): Uint8Array => new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) + // The Buffer backing store may start at an unaligned byteOffset. Copy into + // a fresh aligned ArrayBuffer before constructing numeric typed views. + const aligned = (buffer: Buffer): ArrayBuffer => Uint8Array.from(buffer).buffer + const batches = raw.batches.map((batch) => { + if (batch.columns.length !== fields.length) throw new ChdbArrowError('Direct Arrow column count mismatch') + const children = batch.columns.map((column, i) => { + const format = raw.fields[i]!.format + const validity = column.nullCount ? view(column.buffers[0]!) : undefined + const values = column.buffers[1]! + const common = { type: fields[i]!.type, length: batch.length, nullCount: column.nullCount, nullBitmap: validity } + if (column.dictionary) { + const dictionaryFormat = raw.fields[i]!.dictionaryFormat + if (dictionaryFormat !== 'u' && dictionaryFormat !== 'z') { + throw new ChdbArrowError(`Unsupported direct Arrow dictionary: ${dictionaryFormat}`) + } + const source = column.dictionary + const dictionaryProps = { length: source.length, nullCount: source.nullCount, + nullBitmap: source.nullCount ? view(source.buffers[0]!) : undefined, + valueOffsets: new Int32Array(aligned(source.buffers[1]!)), data: view(source.buffers[2]!) } + const dictionaryData = dictionaryFormat === 'u' + ? arrow.makeData({ ...dictionaryProps, type: new arrow.Utf8() }) + : arrow.makeData({ ...dictionaryProps, type: new arrow.Binary() }) + const dictionary = new arrow.Vector([dictionaryData as any]) + const indices = format === 'i' ? new Int32Array(aligned(values)) : new Uint32Array(aligned(values)) + return arrow.makeData({ ...common, type: fields[i]!.type as Dictionary, + data: indices, dictionary }) + } + if (/^ts[smun]:/.test(format)) { + return arrow.makeData({ ...common, + type: fields[i]!.type as import('apache-arrow').TimestampMillisecond, + data: new BigInt64Array(aligned(values)) }) + } + if (/^d:\d+,-?\d+(?:,(?:128|256))?$/.test(format)) { + return arrow.makeData({ ...common, type: fields[i]!.type as import('apache-arrow').Decimal, + data: new Uint32Array(aligned(values)) }) + } + switch (format) { + case 'c': return arrow.makeData({ ...common, type: new arrow.Int8(), data: new Int8Array(aligned(values)) }) + case 'C': return arrow.makeData({ ...common, type: new arrow.Uint8(), data: new Uint8Array(aligned(values)) }) + case 's': return arrow.makeData({ ...common, type: new arrow.Int16(), data: new Int16Array(aligned(values)) }) + case 'S': return arrow.makeData({ ...common, type: new arrow.Uint16(), data: new Uint16Array(aligned(values)) }) + case 'i': return arrow.makeData({ ...common, type: new arrow.Int32(), data: new Int32Array(aligned(values)) }) + case 'I': return arrow.makeData({ ...common, type: new arrow.Uint32(), data: new Uint32Array(aligned(values)) }) + case 'l': return arrow.makeData({ ...common, type: new arrow.Int64(), data: new BigInt64Array(aligned(values)) }) + case 'L': return arrow.makeData({ ...common, type: new arrow.Uint64(), data: new BigUint64Array(aligned(values)) }) + case 'f': return arrow.makeData({ ...common, type: new arrow.Float32(), data: new Float32Array(aligned(values)) }) + case 'g': return arrow.makeData({ ...common, type: new arrow.Float64(), data: new Float64Array(aligned(values)) }) + case 'b': return arrow.makeData({ ...common, type: new arrow.Bool(), data: view(values) }) + case 'u': return arrow.makeData({ ...common, type: new arrow.Utf8(), + valueOffsets: new Int32Array(aligned(values)), data: view(column.buffers[2]!) }) + case 'z': return arrow.makeData({ ...common, type: new arrow.Binary(), + valueOffsets: new Int32Array(aligned(values)), data: view(column.buffers[2]!) }) + default: throw new ChdbArrowError(`Unsupported direct Arrow format: ${format}`) + } + }) + const data = arrow.makeData({ type: new arrow.Struct(fields), length: batch.length, children }) + return new arrow.RecordBatch(schema, data) + }) + return new arrow.Table(schema, batches) +} diff --git a/src/arrow-entry.ts b/src/arrow-entry.ts new file mode 100644 index 0000000..7837b01 --- /dev/null +++ b/src/arrow-entry.ts @@ -0,0 +1,30 @@ +/** Optional typed Arrow seam. Import this subpath when apache-arrow is installed. */ +import type { Table, RecordBatch, Schema } from 'apache-arrow' +import type { Session, DirectArrowOptions } from '../index' + +export interface TypedArrowReader extends AsyncIterable { + readonly closed: boolean + readonly schema: Schema | null + cancel(): void +} + +export function queryArrow(session: Session, sql: string, options?: DirectArrowOptions): Promise { + return session.queryArrow(sql, options) as Promise
+} + +export function queryArrowBind(session: Session, sql: string, params: object, + options?: DirectArrowOptions): Promise
{ + return session.queryArrowBind(sql, params, options) as Promise
+} + +export function queryArrowStream(session: Session, sql: string, + options?: DirectArrowOptions): TypedArrowReader { + return session.queryArrowStream(sql, options) as TypedArrowReader +} + +export function queryArrowStreamBind(session: Session, sql: string, params: object, + options?: DirectArrowOptions): TypedArrowReader { + return session.queryArrowStreamBind(sql, params, options) as TypedArrowReader +} + +export { registerArrowSource, registerArrowReader, insertArrow, insertArrowReader } from './layer3' diff --git a/src/layer3/compiler/identifier.ts b/src/layer3/compiler/identifier.ts index 942d315..b088511 100644 --- a/src/layer3/compiler/identifier.ts +++ b/src/layer3/compiler/identifier.ts @@ -18,6 +18,14 @@ function quoteSegment(segment: string): string { return '`' + segment.replace(/\\/g, '\\\\').replace(/`/g, '\\`') + '`' } +/** Quote one literal field name, where a dot is part of the name. */ +export function quoteIdentifierSegment(name: string): string { + if (typeof name !== 'string' || name.length === 0) { + throw new ChdbCompileError(`Invalid field name ${JSON.stringify(name)}: must be non-empty`) + } + return quoteSegment(name) +} + /** * Quote a possibly-dotted identifier. An empty name or an empty segment * (leading/trailing/double dot) is a build-time error, since it can only come diff --git a/src/layer3/execute/arrow-input.ts b/src/layer3/execute/arrow-input.ts index 84bc703..21af0f3 100644 --- a/src/layer3/execute/arrow-input.ts +++ b/src/layer3/execute/arrow-input.ts @@ -19,6 +19,7 @@ import { runtime, type Runtime } from '../runtime' import { ChdbCompileError } from '../../errors' +import { randomUUID } from 'node:crypto' /** * Best-effort safety net: if a handle is garbage-collected without an explicit @@ -40,15 +41,15 @@ const arrowFinalizers = new FinalizationRegistry<{ conn: unknown session?: { readonly connection: unknown } name: string - state: { live: boolean } + state: { live: boolean; token: unknown } }>(({ rt, conn, session, name, state }) => { if (!state.live) return state.live = false - // Session.close() releases the native connection and its Arrow registrations. - // The External captured at registration is no longer safe to pass to native. + // Session.close() releases the native connection and its Arrow registrations + // after outstanding scans drain. Its old External must not be used again. if (session !== undefined && session.connection !== conn) return try { - rt._arrowUnregister(conn, name) + rt._arrowUnregister(conn, name, state.token) } catch { // best effort: connection may already be closed, table already erased, etc. } @@ -70,20 +71,173 @@ interface NativeArrowColumn { nullCount: number /** [validity bitmap or null, data buffer, optional offsets buffer]. */ buffers: (Buffer | null)[] + dictionary?: { format: string; length: number; nullCount: number; buffers: (Buffer | null)[] } +} + +// Structural rather than an apache-arrow type import: it is an optional peer, +// and clients that only use SQL must still be able to typecheck without it. +export interface ArrowRecordBatchInput { + readonly schema: { readonly fields: ReadonlyArray<{ readonly name: string; readonly type: { toString(): string } }> } + readonly data: { readonly children: ReadonlyArray } + readonly numRows: number + readonly numCols: number +} +export interface ArrowTableInput { readonly batches: ReadonlyArray } + +interface ArrowDataInput { + readonly offset?: number + readonly length: number + readonly nullCount: number + readonly nullBitmap?: Uint8Array + readonly values?: ArrayBufferView + readonly valueOffsets?: Int32Array +} + +function rebaseBits(bits: Uint8Array, offset: number, length: number): Buffer { + const output = Buffer.alloc(Math.ceil(length / 8)) + for (let i = 0; i < length; i++) { + if (bits[(offset + i) >> 3]! & (1 << ((offset + i) & 7))) output[i >> 3]! |= 1 << (i & 7) + } + return output +} + +/** + * A deliberately narrow bridge from apache-arrow JS buffers to the C Data + * descriptor supported by the current addon. Materialized multi-batch tables + * use the native C Stream adapter; sliced bitmaps are rebased and one-chunk + * dictionary buffers stay columnar without flattening their rows. + */ +export function encodeArrowBatch(input: unknown): NativeArrowColumn[] { + // apache-arrow publishes separate CJS and ESM constructors. A Table made by + // one entry point need not pass instanceof against the other (notably under + // Vitest), so inspect the public batch/schema shape instead. + const source = input as ArrowTableInput & ArrowRecordBatchInput + const batch = source?.batches + ? source.batches.length === 1 ? source.batches[0]! : null + : source?.data?.children && source?.schema ? source : null + if (!batch) { + throw new ChdbCompileError('registerArrowTable: expected a RecordBatch or an apache-arrow Table with a batch') + } + if (batch.numCols === 0) throw new ChdbCompileError('registerArrowTable: at least one column required') + if (!Number.isSafeInteger(batch.numRows) || batch.numRows < 0 || + batch.schema.fields.length !== batch.numCols || batch.data.children.length !== batch.numCols) { + throw new ChdbCompileError('registerArrowTable: invalid Arrow batch shape') + } + + return batch.schema.fields.map((field, i) => { + const data = batch.data.children[i] as ArrowDataInput | undefined + const offset = data?.offset ?? 0 + if (!data || !Number.isSafeInteger(offset) || offset < 0 || data.length !== batch.numRows || + !Number.isSafeInteger(data.nullCount) || data.nullCount < 0 || data.nullCount > data.length) { + throw new ChdbCompileError(`registerArrowTable: column "${field.name}" has an invalid offset or length`) + } + let format: string + let views: ArrayBufferView[] + const kind = field.type.toString() + if (kind.startsWith('Dictionary<')) { + const dictionaryType = field.type as { indices: { toString(): string }; dictionary: { toString(): string } } + if (dictionaryType.dictionary.toString().startsWith('Dictionary<')) { + throw new ChdbCompileError('nested Arrow dictionaries are unsupported') + } + const vector = (data as ArrowDataInput & { dictionary?: { data?: ReadonlyArray } }).dictionary + if (vector?.data?.length !== 1) { + throw new ChdbCompileError(`registerArrowTable: dictionary for "${field.name}" needs one unsliced value chunk`) + } + const dictionaryData = vector.data[0] as ArrowDataInput + const index = encodeArrowBatch({ schema: { fields: [{ name: field.name, type: dictionaryType.indices }] }, + data: { children: [data] }, numCols: 1, numRows: batch.numRows })[0]! + const values = encodeArrowBatch({ schema: { fields: [{ name: '', type: dictionaryType.dictionary }] }, + data: { children: [dictionaryData] }, numCols: 1, numRows: dictionaryData.length })[0]! + if (!['i', 'I'].includes(index.format) || !['u', 'z'].includes(values.format)) { + throw new ChdbCompileError(`registerArrowTable: unsupported dictionary layout for "${field.name}"`) + } + return { ...index, dictionary: { format: values.format, length: values.length, + nullCount: values.nullCount, buffers: values.buffers } } + } + const fixedFormats: Record = { + Int8: 'c', Uint8: 'C', Int16: 's', Uint16: 'S', + Int32: 'i', Uint32: 'I', Int64: 'l', Uint64: 'L', + Float32: 'f', Float64: 'g', + } + if (fixedFormats[kind]) { + format = fixedFormats[kind]! + views = [data.values ?? new Uint8Array(0)] + } else if (kind.startsWith('Timestamp<')) { + const timestamp = field.type as { unit?: number; timezone?: string | null } + const unit = timestamp.unit == null ? undefined : 'smun'[timestamp.unit] + if (!unit) throw new ChdbCompileError(`registerArrowTable: unsupported timestamp unit for "${field.name}"`) + format = `ts${unit}:${timestamp.timezone ?? ''}` + views = [data.values ?? new BigInt64Array(0)] + } else if (kind.startsWith('Decimal[')) { + const decimal = field.type as { precision?: number; scale?: number; bitWidth?: number } + if (!decimal.precision || ![128, 256].includes(decimal.bitWidth ?? 128)) { + throw new ChdbCompileError(`registerArrowTable: unsupported decimal for "${field.name}"`) + } + format = `d:${decimal.precision},${decimal.scale ?? 0}${decimal.bitWidth === 256 ? ',256' : ''}` + views = [data.values ?? new Uint32Array(0)] + } else if (kind === 'Bool') { + format = 'b' + const bits = data.values as Uint8Array | undefined + if (offset && (!bits || bits.byteLength < Math.ceil((offset + data.length) / 8))) { + throw new ChdbCompileError(`registerArrowTable: truncated Bool values for "${field.name}"`) + } + views = [offset ? rebaseBits(bits!, offset, data.length) : bits ?? new Uint8Array(0)] + } else if (kind === 'Utf8' || kind === 'Binary') { + format = kind === 'Utf8' ? 'u' : 'z' + views = [data.valueOffsets ?? new Int32Array([0]), data.values ?? new Uint8Array(0)] + } else { + throw new ChdbCompileError(`registerArrowTable: unsupported Arrow type ${field.type} for "${field.name}"`) + } + const validity = data.nullCount > 0 ? data.nullBitmap : null + if (data.nullCount > 0 && (!validity || !ArrayBuffer.isView(validity) || + validity.byteLength < Math.ceil((offset + data.length) / 8))) { + throw new ChdbCompileError(`registerArrowTable: missing validity bitmap for "${field.name}"`) + } + const variable = format === 'u' || format === 'z' + const width = format.startsWith('d:') ? (format.endsWith(',256') ? 32 : 16) + : 'cC'.includes(format) ? 1 : 'sS'.includes(format) ? 2 + : 'iIf'.includes(format) ? 4 : 8 + const expected = variable ? (data.length + 1) * 4 + : format === 'b' ? Math.ceil(data.length / 8) : data.length * width + if (!views.every((v) => v && ArrayBuffer.isView(v) && !(v.buffer instanceof SharedArrayBuffer)) || + (validity && validity.buffer instanceof SharedArrayBuffer) || views[0]!.byteLength < expected) { + throw new ChdbCompileError(`registerArrowTable: invalid or shared buffer for "${field.name}"`) + } + if (variable) { + const offsets = views[0] as Int32Array + let valid = offsets[0]! >= 0 && offsets[data.length]! <= views[1]!.byteLength + for (let j = 1; j <= data.length; j++) valid &&= offsets[j]! >= offsets[j - 1]! + if (!valid) { + throw new ChdbCompileError(`registerArrowTable: invalid UTF-8 offsets for "${field.name}"`) + } + } + const buffer = (view: ArrayBufferView): Buffer => Buffer.from(view.buffer, view.byteOffset, view.byteLength) + const bitmapView = validity && offset ? rebaseBits(validity, offset, data.length) : validity + return { + name: field.name, format, length: data.length, nullCount: data.nullCount, + buffers: [bitmapView ? buffer(bitmapView) : null, ...views.map(buffer)], + } + }) } /** A handle returned by `registerArrowTable`. */ export interface ArrowTableHandle { - /** Drop the table from the engine and release JS-side buffer pins. */ + /** Close the handle; native unregister waits for active queries/streams. */ close(): void /** * Re-register the same data so a follow-up query can read it again. * The engine consumes the underlying array on each scan, so use this - * between repeated queries against the same table. + * between completed queries against the same table. Throws while a query + * or stream is active, since re-registering would invalidate that scan. */ refresh(): void } +export interface ArrowSourceHandle extends ArrowTableHandle { + /** Unique process-wide name to use in `arrowstream(name)` SQL. */ + readonly name: string +} + /** Validate name + same row count + at least one column. */ function validate(name: string, columns: ReadonlyArray): void { if (typeof name !== 'string' || name === '') { @@ -164,54 +318,80 @@ function encodeColumn(col: ArrowColumnInput): NativeArrowColumn { } /** - * Pin a JS columnar dataset as an `arrowstream('')` table the engine can - * read with zero IPC copy. Returns a handle owning the JS-side buffer - * references; call `.close()` to release everything. + * Pin JS columns, one apache-arrow RecordBatch, or a single-batch Table as an + * `arrowstream('')` source. Arrow objects support primitive numeric, + * Bool, Utf8, Binary, Timestamp, Decimal and one-chunk Dictionary buffers; + * sliced bitmaps are rebased. Multi-batch tables use a C Stream. This avoids Arrow IPC, but the engine may + * still copy buffers while mapping them to ClickHouse columns. + * + * Returns a handle owning JS-side buffer references. close() is idempotent; + * native unregister and buffer release are deferred while a query or lazy + * stream is active. Exhaust or cancel streams so deferred cleanup can finish. */ export function registerArrowTable( name: string, - columns: ReadonlyArray, + columns: ReadonlyArray | ArrowTableInput | ArrowRecordBatchInput, opts: { session?: { readonly connection: unknown } } = {}, ): ArrowTableHandle { - validate(name, columns) - const encoded = columns.map(encodeColumn) + if (typeof name !== 'string' || name === '') { + throw new ChdbCompileError('registerArrowTable: name must be a non-empty string') + } + let encoded: NativeArrowColumn[] + let encodedBatches: NativeArrowColumn[][] | undefined + if (Array.isArray(columns)) { + validate(name, columns) + encoded = columns.map(encodeColumn) + } else if ('batches' in columns && columns.batches.length > 1) { + encodedBatches = columns.batches.map(encodeArrowBatch) + encoded = encodedBatches[0]! + } else { + encoded = encodeArrowBatch(columns) + } const session = opts.session const conn = session !== undefined ? session.connection : null if (session !== undefined && conn == null) { throw new ChdbCompileError('registerArrowTable: session has no active connection') } const rt = runtime() + if (!Array.isArray(columns)) rt._assertArrowCore() const sessionStillOwnsConnection = (): boolean => session === undefined || session.connection === conn // Shared between close/refresh and the GC finalizer so all three agree on // whether the table is currently registered. - const state = { live: false } + const state: { live: boolean; closed: boolean; token: unknown } = { + live: false, closed: false, token: undefined, + } const register = (): void => { if (!sessionStillOwnsConnection()) { throw new ChdbCompileError('registerArrowTable: session has no active connection') } - rt._arrowRegisterColumns(conn, name, encoded) + state.token = encodedBatches + ? rt._arrowRegisterBatches(conn, name, encodedBatches) + : rt._arrowRegisterColumns(conn, name, encoded) state.live = true } - const unregister = (): void => { + const unregister = (defer = true): void => { if (!state.live) return if (!sessionStillOwnsConnection()) { // Closing the Session has already dropped its native Arrow registrations. state.live = false return } - rt._arrowUnregister(conn, name) + rt._arrowUnregister(conn, name, state.token, defer) state.live = false } register() const handle: ArrowTableHandle = { close(): void { + if (state.closed) return unregister() + state.closed = true // Explicitly closed: drop the GC safety net so the finalizer can't fire // a redundant unregister later. arrowFinalizers.unregister(handle) }, refresh(): void { - unregister() + if (state.closed) throw new ChdbCompileError('registerArrowTable: handle is closed') + unregister(false) register() }, } @@ -220,3 +400,14 @@ export function registerArrowTable( arrowFinalizers.register(handle, { rt, conn, session, name, state }, handle) return handle } + +/** Register a materialized Arrow source under a generated opaque SQL name. */ +export function registerArrowSource( + input: ReadonlyArray | ArrowTableInput | ArrowRecordBatchInput, + opts: { session?: { readonly connection: unknown } } = {}, +): ArrowSourceHandle { + runtime()._assertArrowCore() + const name = `__chdb_arrow_${randomUUID().replaceAll('-', '')}` + const handle = registerArrowTable(name, input, opts) + return Object.assign(handle, { name }) +} diff --git a/src/layer3/execute/arrow-insert.ts b/src/layer3/execute/arrow-insert.ts new file mode 100644 index 0000000..738e969 --- /dev/null +++ b/src/layer3/execute/arrow-insert.ts @@ -0,0 +1,85 @@ +/** Bulk insert through one in-memory Arrow registration and an engine-side INSERT SELECT. */ +import { ChdbCompileError } from '../../errors' +import { quoteIdentifier, quoteIdentifierSegment } from '../compiler/identifier' +import { registerArrowSource, type ArrowColumnInput, type ArrowRecordBatchInput, type ArrowTableInput } from './arrow-input' +import { registerArrowReader, type ArrowReaderSource } from './arrow-reader' + +export type ArrowInsertInput = ReadonlyArray | ArrowRecordBatchInput | ArrowTableInput + +export interface ArrowInsertSession { + readonly connection: unknown + queryAsync(query: string, opts?: { format?: string }): Promise<{ + readonly rowsRead: number; readonly bytesRead: number; readonly elapsed: number + }> +} + +/** Number submitted is known from the Arrow source; it is not a storage-side write ledger. */ +export interface ArrowInsertSummary { + rowsSubmitted: number + rowsRead: number + bytesRead: number + elapsed: number +} + +/** + * Insert a single RecordBatch/Table batch (or supported column descriptors) + * without Arrow IPC or JS row materialization. Destination names are matched + * to source fields, not positional table order. The registration is scoped to + * this awaited query and always closed, including on errors. + */ +export async function insertArrow( + table: string, + input: ArrowInsertInput, + opts: { session: ArrowInsertSession }, +): Promise { + if (!opts?.session?.connection || typeof opts.session.queryAsync !== 'function') { + throw new ChdbCompileError('insertArrow: an active Session is required') + } + const batch = !Array.isArray(input) && 'batches' in input ? input.batches[0] : input + const columns = Array.isArray(batch) + ? batch.map((column) => column.name) + : (batch as ArrowRecordBatchInput).schema?.fields.map((field) => field.name) + if (!columns?.length || new Set(columns).size !== columns.length) { + throw new ChdbCompileError('insertArrow: source needs unique column names') + } + const rowsSubmitted = !Array.isArray(input) && 'batches' in input + ? input.batches.reduce((total, current) => total + current.numRows, 0) + : Array.isArray(batch) ? batch[0]!.data.length : (batch as ArrowRecordBatchInput).numRows + const destination = quoteIdentifier(table) + const fieldList = columns.map(quoteIdentifierSegment).join(', ') + const registration = registerArrowSource(input, { session: opts.session }) + try { + // Generated name is ASCII hex only. Identifier paths/column names are + // quoted by the shared compiler, never interpolated as untrusted SQL. + const result = await opts.session.queryAsync( + `INSERT INTO ${destination} (${fieldList}) SELECT ${fieldList} FROM arrowstream('${registration.name}')`, + ) + return { rowsSubmitted, rowsRead: result.rowsRead, bytesRead: result.bytesRead, elapsed: result.elapsed } + } finally { + registration.close() + } +} + +/** Incrementally insert an async/sync RecordBatch reader with native backpressure. */ +export async function insertArrowReader( + table: string, + source: ArrowReaderSource, + opts: { session: ArrowInsertSession }, +): Promise { + const destination = quoteIdentifier(table) + const reader = await registerArrowReader(source, opts) + try { + if (new Set(reader.fields).size !== reader.fields.length) { + throw new ChdbCompileError('insertArrowReader: source needs unique column names') + } + const fieldList = reader.fields.map(quoteIdentifierSegment).join(', ') + const result = await opts.session.queryAsync( + `INSERT INTO ${destination} (${fieldList}) SELECT ${fieldList} FROM arrowstream('${reader.name}')`, + ) + await reader.producerDone + return { rowsSubmitted: reader.rowsSubmitted, rowsRead: result.rowsRead, + bytesRead: result.bytesRead, elapsed: result.elapsed } + } finally { + await reader.close() + } +} diff --git a/src/layer3/execute/arrow-reader.ts b/src/layer3/execute/arrow-reader.ts new file mode 100644 index 0000000..8f32d0f --- /dev/null +++ b/src/layer3/execute/arrow-reader.ts @@ -0,0 +1,122 @@ +/** Bounded async Arrow reader -> core C Stream source. */ +import { randomUUID } from 'node:crypto' +import { ChdbCompileError } from '../../errors' +import { runtime, type Runtime } from '../runtime' +import { encodeArrowBatch, type ArrowRecordBatchInput } from './arrow-input' + +export type ArrowReaderSource = (AsyncIterable | Iterable) & { + /** Required only when the reader yields no batches. */ + readonly schema?: ArrowRecordBatchInput['schema'] +} + +export interface ArrowReaderHandle { + readonly name: string + readonly fields: ReadonlyArray + /** Batches accepted by the bounded native queue; not a storage write ledger. */ + readonly rowsSubmitted: number + /** Settles when the JS producer has finished or failed, not when SQL finishes. */ + readonly producerDone: Promise + /** Cancel production and unregister; one-shot readers cannot be refreshed. */ + close(): Promise +} + +const readerFinalizers = new FinalizationRegistry<{ + rt: Runtime; native: unknown; conn: unknown; name: string; token: unknown +}>(({ rt, native, conn, name, token }) => { + try { + rt._arrowCancelReader(native) + rt._arrowUnregister(conn, name, token) + } catch { /* best effort on a connection already closed */ } +}) + +/** + * Register a one-shot synchronous or asynchronous RecordBatch reader. The + * native queue holds at most two batches (including a batch awaiting C Data + * release); JS produces on its own event loop and awaits queue capacity. + * Core get_next only waits on native descriptors and never invokes JS on its + * query worker. Pass the returned name to arrowstream(name) on this Session. + */ +export async function registerArrowReader( + source: ArrowReaderSource, + opts: { session: { readonly connection: unknown } }, +): Promise { + if (!opts?.session?.connection) throw new ChdbCompileError('registerArrowReader requires an active Session') + runtime()._assertArrowCore() + const conn = opts.session.connection + const asyncMethod = (source as AsyncIterable)[Symbol.asyncIterator] + const syncMethod = (source as Iterable)[Symbol.iterator] + if (typeof asyncMethod !== 'function' && typeof syncMethod !== 'function') { + throw new ChdbCompileError('registerArrowReader requires a RecordBatch iterable') + } + const iterator: AsyncIterator | Iterator = + typeof asyncMethod === 'function' ? asyncMethod.call(source) : syncMethod.call(source) + const first = await iterator.next() + if (opts.session.connection !== conn) { + try { await iterator.return?.() } catch { /* source cleanup is best effort */ } + throw new ChdbCompileError('registerArrowReader: Session closed while awaiting the first batch') + } + let firstBatch: ArrowRecordBatchInput + if (first.done) { + const schema = (source as { schema?: ArrowRecordBatchInput['schema'] }).schema + if (!schema?.fields?.length) throw new ChdbCompileError('empty Arrow reader must provide a non-empty schema') + // Do not cross apache-arrow's independent ESM/CJS constructors: build a + // zero-row structural batch from the public schema shape instead. + firstBatch = { schema, numCols: schema.fields.length, numRows: 0, + data: { children: schema.fields.map(() => ({ offset: 0, length: 0, nullCount: 0, + values: new Uint8Array(0), valueOffsets: new Int32Array([0]) })) } } + } else firstBatch = first.value + + const rt = runtime() + const name = `__chdb_arrow_reader_${randomUUID().replaceAll('-', '')}` + const fields = firstBatch.schema.fields.map((field) => field.name) + let native: unknown + let token: unknown + try { + ({ handle: native, token } = rt._arrowRegisterReader(conn, name, encodeArrowBatch(firstBatch))) + } catch (e) { + try { await iterator.return?.() } catch { /* preserve registration error */ } + throw e + } + let closing = false + let rowsSubmitted = first.done ? 0 : firstBatch.numRows + const producerDone = (async () => { + try { + if (first.done) { rt._arrowFinishReader(native); return } + while (!closing) { + const next = await iterator.next() + if (closing) return + if (next.done) { rt._arrowFinishReader(native); return } + await rt._arrowPushReaderBatch(native, encodeArrowBatch(next.value)) + rowsSubmitted += next.value.numRows + } + } catch (e) { + if (!closing) { + rt._arrowFinishReader(native, (e as Error).message || String(e) || 'Arrow reader producer failed') + throw e + } + } + })() + // SQL sees the producer error through get_next; an unobserved auxiliary + // producerDone promise must not become an unhandled rejection. + void producerDone.catch(() => {}) + + const result: ArrowReaderHandle = { + name, fields, producerDone, + get rowsSubmitted() { return rowsSubmitted }, + async close() { + if (!closing) { + closing = true + rt._arrowCancelReader(native) + rt._arrowUnregister(conn, name, token) + readerFinalizers.unregister(result) + // An arbitrary async iterator may be stuck inside next() and queue + // return() behind it forever. Native cancellation must settle without + // waiting for a non-cooperative producer; observe its eventual cleanup. + try { void Promise.resolve(iterator.return?.()).catch(() => {}) } + catch { /* cancellation is best effort */ } + } + }, + } + readerFinalizers.register(result, { rt, native, conn, name, token }, result) + return result +} diff --git a/src/layer3/index.ts b/src/layer3/index.ts index 42c19e5..9a72f3b 100644 --- a/src/layer3/index.ts +++ b/src/layer3/index.ts @@ -74,8 +74,12 @@ export type { CompiledQuery } from './compiler/compile' export type { ExecuteOptions, ExecContext } from './execute/terminal' // Arrow C Data Interface input — JS columnar data as `arrowstream('')`. -export { registerArrowTable } from './execute/arrow-input' -export type { ArrowColumnInput, ArrowTableHandle } from './execute/arrow-input' +export { registerArrowTable, registerArrowSource } from './execute/arrow-input' +export type { ArrowColumnInput, ArrowRecordBatchInput, ArrowTableInput, ArrowTableHandle, ArrowSourceHandle } from './execute/arrow-input' +export { insertArrow, insertArrowReader } from './execute/arrow-insert' +export type { ArrowInsertInput, ArrowInsertSession, ArrowInsertSummary } from './execute/arrow-insert' +export { registerArrowReader } from './execute/arrow-reader' +export type { ArrowReaderSource, ArrowReaderHandle } from './execute/arrow-reader' // AST node types. export type * from './compiler/nodes' diff --git a/src/layer3/runtime.ts b/src/layer3/runtime.ts index cf41af8..755f57c 100644 --- a/src/layer3/runtime.ts +++ b/src/layer3/runtime.ts @@ -80,9 +80,11 @@ export interface RuntimeArrowColumn { nullCount: number /** `[validity bitmap | null, data, offsets?]`. Offsets is Utf8-only. */ buffers: ReadonlyArray + dictionary?: { format: string; length: number; nullCount: number; buffers: ReadonlyArray } } export interface Runtime { + _assertArrowCore(): void Session: RuntimeSessionCtor queryAsync(query: string, opts?: RuntimeQueryOptions): Promise queryBindAsync(query: string, params: object, opts?: RuntimeQueryOptions): Promise @@ -92,9 +94,20 @@ export interface Runtime { connection: unknown, tableName: string, columns: ReadonlyArray, - ): void - /** Remove a previously registered arrow table. */ - _arrowUnregister(connection: unknown, tableName: string): void + ): unknown + _arrowRegisterBatches( + connection: unknown, + tableName: string, + batches: ReadonlyArray>, + ): unknown + _arrowRegisterReader(connection: unknown, tableName: string, firstBatch: ReadonlyArray): { + handle: unknown; token: unknown + } + _arrowPushReaderBatch(handle: unknown, batch: ReadonlyArray): Promise + _arrowFinishReader(handle: unknown, error?: string): void + _arrowCancelReader(handle: unknown): void + /** Remove a registered table, optionally deferring until native scans drain. */ + _arrowUnregister(connection: unknown, tableName: string, token: unknown, defer?: boolean): boolean } let cached: Runtime | undefined diff --git a/test/v3/arrow-direct.test.ts b/test/v3/arrow-direct.test.ts new file mode 100644 index 0000000..2ecf048 --- /dev/null +++ b/test/v3/arrow-direct.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, it } from 'vitest' +import type { RecordBatch, Schema, Table } from 'apache-arrow' +import { Session, drainPending } from '../../index.js' + +describe('Session.queryArrow — direct C Data output', () => { + it('rejects rc.2 and older native engines before calling the Arrow ABI', async () => { + const native = require('../../build/Release/chdb_node.node') as { EngineVersion(): string } + const original = native.EngineVersion + const session = new Session() + try { + for (const version of ['26.11.0-rc.1', '26.11.0-rc.2', '26.9.0']) { + native.EngineVersion = () => version + await expect(session.queryArrow('SELECT 1')).rejects.toMatchObject({ code: 'CHDB_ABI' }) + } + native.EngineVersion = () => '26.11.0-rc.3' + const table = await session.queryArrow('SELECT toInt32(1) AS x') as Table + expect(table.toArray()[0]!.x).toBe(1) + } finally { + native.EngineVersion = original + session.close() + } + }) + + it('returns typed values, nullable UTF-8, and precision-safe Int64', async () => { + const session = new Session() + try { + const table = await session.queryArrow(` + SELECT toInt32(number) AS x, + if(number=1, NULL, toString(number)) AS s, + toInt64(9223372036854775807) AS big + FROM numbers(3) + `) as Table + expect(table.schema.fields.map((field) => [field.name, field.type.toString()])).toEqual([ + ['x', 'Int32'], ['s', 'Utf8'], ['big', 'Int64'], + ]) + expect(table.toArray().map((row) => row.toJSON())).toEqual([ + { x: 0, s: '0', big: 9223372036854775807n }, + { x: 1, s: null, big: 9223372036854775807n }, + { x: 2, s: '2', big: 9223372036854775807n }, + ]) + } finally { + session.close() + } + }) + + it('retains schema for zero rows and propagates SQL errors', async () => { + const session = new Session() + try { + const table = await session.queryArrow('SELECT toInt32(1) AS x WHERE 0') as Table + expect(table.numRows).toBe(0) + expect(table.schema.fields.map((field) => [field.name, field.type.toString()])).toEqual([['x', 'Int32']]) + await expect(session.queryArrow('SELECT x FROM missing_arrow_direct_output')).rejects.toThrow(/missing_arrow_direct_output/) + await expect(session.queryArrow('SELECT [1, 2] AS items')).rejects.toThrow(/Unsupported Arrow result field/) + } finally { + session.close() + } + }) + + it('binds direct Arrow query values server-side without interpolating SQL', async () => { + const session = new Session() + try { + const value = "x' OR '1'='1" + const table = await session.queryArrowBind('SELECT {v:String} AS v', { v: value }) as Table + expect(table.toArray()[0]!.v).toBe(value) + const reader = session.queryArrowStreamBind('SELECT toInt32({n:Int32}) AS n', { n: 7 }) + const rows: number[] = [] + for await (const batch of reader) rows.push(...(batch as RecordBatch).getChild('n')!.toArray() as Int32Array) + expect(rows).toEqual([7]) + } finally { + session.close() + } + }) + + it('owns result buffers after the Session closes', async () => { + const session = new Session() + const table = await session.queryArrow('SELECT toInt32(number) AS x FROM numbers(7) SETTINGS max_block_size=2') as Table + session.close() + expect(table.toArray().map((row) => row.x)).toEqual([0, 1, 2, 3, 4, 5, 6]) + }) + + it('maps DateTime as UInt32 and honors binary String output', async () => { + const session = new Session() + try { + const time = await session.queryArrow("SELECT toDateTime('2020-01-01 00:00:00', 'UTC') AS ts") as Table + expect(time.schema.fields[0]!.type.toString()).toBe('Uint32') + expect(time.toArray()[0]!.ts).toBe(1577836800) + const precise = await session.queryArrow( + "SELECT toDateTime64('2020-01-01 00:00:00.123', 3, 'UTC') AS ts", + ) as Table + expect(precise.schema.fields[0]!.type.toString()).toBe('Timestamp') + expect(precise.toArray()[0]!.ts).toBe(1577836800123) + const decimal = await session.queryArrow('SELECT toDecimal64(1.23, 2) AS d') as Table + expect(decimal.schema.fields[0]!.type.toString()).toBe('Decimal[18e+2]') + expect(String(decimal.toArray()[0]!.d)).toBe('123') // Arrow's unscaled decimal words + const wide = await session.queryArrow('SELECT toDecimal256(1.23, 2) AS d') as Table + expect((wide.schema.fields[0]!.type as { bitWidth: number }).bitWidth).toBe(256) + expect(Array.from(wide.batches[0]!.data.children[0]!.values as Uint32Array).slice(0, 8)) + .toEqual([123, 0, 0, 0, 0, 0, 0, 0]) + const binary = await session.queryArrow("SELECT toString('é') AS s", { stringAsString: false }) as Table + expect(binary.schema.fields[0]!.type.toString()).toBe('Binary') + expect(Array.from(binary.toArray()[0]!.s as Uint8Array)).toEqual([195, 169]) + const dictionary = await session.queryArrow( + 'SELECT toLowCardinality(toString(number % 2)) AS s FROM numbers(4)', + { lowCardinalityAsDictionary: true }, + ) as Table + expect(dictionary.schema.fields[0]!.type.toString()).toBe('Dictionary') + expect(dictionary.toArray().map((row) => row.s)).toEqual(['0', '1', '0', '1']) + const unsupported = 'SELECT sumState(toUInt64(1)) AS state' + await expect(session.queryArrow(unsupported)).rejects.toThrow(/not supported for conversion/) + const fallback = await session.queryArrow(unsupported, { unsupportedAsBinary: true }) as Table + expect(fallback.schema.fields[0]!.type.toString()).toBe('Binary') + } finally { + session.close() + } + }) +}) + +describe('Session.queryArrowStream — incremental C Data batches', () => { + it('fetches typed batches incrementally and keeps a stable schema', async () => { + const session = new Session() + try { + const reader = session.queryArrowStream( + 'SELECT toInt32(number) AS x FROM numbers(7) SETTINGS max_block_size=2', + ) + const lengths: number[] = [] + const values: number[] = [] + for await (const value of reader) { + const batch = value as RecordBatch + lengths.push(batch.numRows) + values.push(...batch.getChild('x')!.toArray() as Int32Array) + } + expect(lengths).toEqual([2, 2, 2, 1]) + expect(values).toEqual([0, 1, 2, 3, 4, 5, 6]) + expect((reader.schema as Schema).fields[0]!.name).toBe('x') + expect(reader.closed).toBe(true) + } finally { + session.close() + } + }) + + it('keeps LowCardinality dictionary indices valid across batches', async () => { + const session = new Session() + try { + const reader = session.queryArrowStream( + 'SELECT toLowCardinality(toString(number % 2)) AS s FROM numbers(7) SETTINGS max_block_size=2', + { lowCardinalityAsDictionary: true }, + ) + const values: string[] = [] + for await (const value of reader) values.push(...(value as RecordBatch).toArray().map((row) => row.s)) + expect(values).toEqual(['0', '1', '0', '1', '0', '1', '0']) + expect((reader.schema as Schema).fields[0]!.type.toString()).toBe('Dictionary') + } finally { + session.close() + } + }) + + it('retains an empty result schema and cancels on early break', async () => { + const session = new Session() + try { + const empty = session.queryArrowStream('SELECT toInt32(1) AS x WHERE 0') + let count = 0 + for await (const _batch of empty) count++ + expect(count).toBe(0) + expect((empty.schema as Schema).fields[0]!.name).toBe('x') + + const reader = session.queryArrowStream('SELECT toInt32(number) AS x FROM numbers(100) SETTINGS max_block_size=2') + for await (const _batch of reader) break + expect(reader.closed).toBe(true) + const healthy = await session.queryArrow('SELECT toInt32(7) AS x') as Table + expect(healthy.toArray()[0]!.x).toBe(7) + } finally { + session.close() + } + }) + + it('defers cancel and Session close behind an in-flight native fetch', async () => { + const session = new Session() + const reader = session.queryArrowStream( + 'SELECT toInt32(number) AS x FROM numbers(4) WHERE sleepEachRow(0.03)=0 SETTINGS max_block_size=1', + ) + const iterator = reader[Symbol.asyncIterator]() + const next = iterator.next() + reader.cancel() + session.close() + try { + await next + await iterator.return?.() + await drainPending() + expect(reader.closed).toBe(true) + } finally { + reader.cancel() + session.close() + await drainPending() + } + }) +}) diff --git a/test/v3/arrow-entry.test.ts b/test/v3/arrow-entry.test.ts new file mode 100644 index 0000000..2c0eb95 --- /dev/null +++ b/test/v3/arrow-entry.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import type { Table, RecordBatch } from 'apache-arrow' +import { Session } from '../../index.js' +import { queryArrow, queryArrowStream } from 'chdb/arrow' + +describe('optional typed Arrow subpath', () => { + it('returns typed Table and RecordBatch values without a hard peer on the base entry', async () => { + const session = new Session() + try { + const table: Table = await queryArrow(session, 'SELECT toInt32(7) AS x') + expect(table.toArray()[0]!.x).toBe(7) + const stream = queryArrowStream(session, 'SELECT toInt32(number) AS x FROM numbers(2)') + const rows: number[] = [] + for await (const batch of stream) { + const typed: RecordBatch = batch + rows.push(...typed.getChild('x')!.toArray() as Int32Array) + } + expect(rows).toEqual([0, 1]) + } finally { + session.close() + } + }) +}) diff --git a/test/v3/layer3/arrow-input.test.ts b/test/v3/layer3/arrow-input.test.ts index 28dc8d1..5d793ce 100644 --- a/test/v3/layer3/arrow-input.test.ts +++ b/test/v3/layer3/arrow-input.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'vitest' -import { registerArrowTable, selectFrom, chTable, ChdbCompileError, Session } from '../../../index.js' +import { registerArrowTable, registerArrowSource, selectFrom, chTable, ChdbCompileError, Session, drainPending } from '../../../index.js' +import { Table, Field, List, Int32, Int64, Uint64, Float32, Float64, Bool, Utf8, Binary, TimestampMillisecond, Decimal, vectorFromArray } from 'apache-arrow' // Arrow C Data Interface input — register a JS columnar dataset as // `arrowstream('')` and read it through the regular builder. Single-pass @@ -7,6 +8,238 @@ import { registerArrowTable, selectFrom, chTable, ChdbCompileError, Session } fr // repeated reads against the same table. describe('registerArrowTable — round-trips JS data through the engine', () => { + it('generates distinct opaque names for independent Arrow sources', async () => { + const session = new Session() + const batch = new Table({ v: vectorFromArray([7], new Int32()) }).batches[0]! + const one = registerArrowSource(batch, { session }) + const two = registerArrowSource(batch, { session }) + try { + expect(one.name).toMatch(/^__chdb_arrow_[0-9a-f]{32}$/) + expect(two.name).not.toBe(one.name) + const result = await session.queryAsync( + `SELECT a.v + b.v AS total FROM arrowstream('${one.name}') AS a CROSS JOIN arrowstream('${two.name}') AS b`, + { format: 'CSV' }, + ) + expect(result.text().trim()).toBe('14') + } finally { + one.close() + two.close() + session.close() + } + }) + + it('accepts a one-batch apache-arrow Table with nullable primitive buffers', async () => { + const input = new Table({ + v: vectorFromArray([1, null, 3], new Int32()), + s: vectorFromArray(['é', null, '東京'], new Utf8()), + }) + expect(input.batches).toHaveLength(1) + const t = registerArrowTable('arr_js_table', input) + try { + const rows = (await selectFrom(chTable.arrowstream('arr_js_table').as('t')) + .select(['v', 's']).execute()) as { v: number | null; s: string | null }[] + expect(rows).toEqual([{ v: 1, s: 'é' }, { v: null, s: null }, { v: 3, s: '東京' }]) + } finally { + t.close() + } + }) + + it('accepts Int64, Float64, and bit-packed Bool from a RecordBatch', async () => { + const input = new Table({ + big: vectorFromArray([9_223_372_036_854_775_807n, 2n], new Int64()), + score: vectorFromArray([1.25, 2.5], new Float64()), + flag: vectorFromArray([true, false], new Bool()), + }) + const t = registerArrowTable('arr_js_primitives', input.batches[0]!) + try { + const rows = (await selectFrom(chTable.arrowstream('arr_js_primitives').as('t')) + .select(['big', 'score', 'flag']).execute()) as { big: string; score: number; flag: boolean }[] + expect(rows).toEqual([ + { big: '9223372036854775807', score: 1.25, flag: true }, + { big: '2', score: 2.5, flag: false }, + ]) + } finally { + t.close() + } + }) + + it('accepts unsigned integers, Float32, and Binary from a RecordBatch', async () => { + const input = new Table({ + u: vectorFromArray([18_446_744_073_709_551_615n, 2n], new Uint64()), + f: vectorFromArray([1.5, 2.5], new Float32()), + b: vectorFromArray([new Uint8Array([0, 255]), new Uint8Array([65])], new Binary()), + }) + const session = new Session() + const t = registerArrowTable('arr_js_more_primitives', input.batches[0]!, { session }) + try { + const result = await session.queryAsync( + "SELECT toString(u) AS u, f, hex(b) AS encoded FROM arrowstream('arr_js_more_primitives')", + { format: 'JSONEachRow' }, + ) + expect(result.text().trim().split('\n').map((line) => JSON.parse(line))).toEqual([ + { u: '18446744073709551615', f: 1.5, encoded: '00FF' }, + { u: '2', f: 2.5, encoded: '41' }, + ]) + } finally { + t.close() + session.close() + } + }) + + it('preserves a timezone-tagged Arrow timestamp as DateTime64', async () => { + const session = new Session() + const input = new Table({ ts: vectorFromArray([1577836800123], new TimestampMillisecond('UTC')) }) + const t = registerArrowSource(input, { session }) + try { + const result = await session.queryAsync( + `SELECT toUnixTimestamp64Milli(ts) FROM arrowstream('${t.name}')`, { format: 'CSV' }, + ) + expect(result.text().trim()).toBe('1577836800123') + } finally { + t.close() + session.close() + } + }) + + it('imports a Decimal128 source without a JS numeric conversion', async () => { + const session = new Session() + const input = new Table({ d: vectorFromArray([new Uint32Array([123, 0, 0, 0])], new Decimal(2, 18, 128)) }) + const t = registerArrowSource(input, { session }) + try { + const result = await session.queryAsync( + `SELECT toString(d) FROM arrowstream('${t.name}')`, { format: 'CSV' }, + ) + expect(result.text().trim()).toBe('"1.23"') + const wide = new Table({ d: vectorFromArray( + [new Uint32Array([123, 0, 0, 0, 0, 0, 0, 0])], new Decimal(2, 76, 256), + ) }) + const wideHandle = registerArrowSource(wide, { session }) + try { + const value = await session.queryAsync( + `SELECT toString(d) FROM arrowstream('${wideHandle.name}')`, { format: 'CSV' }, + ) + expect(value.text().trim()).toBe('"1.23"') + } finally { + wideHandle.close() + } + } finally { + t.close() + session.close() + } + }) + + it('reads a materialized multi-batch Table and refreshes its one-shot C Stream', async () => { + const single = new Table({ v: vectorFromArray([1, 2], new Int32()) }) + const chunks = new Table([single.batches[0]!, single.batches[0]!]) + expect(chunks.batches.length).toBeGreaterThan(1) + const t = registerArrowTable('arr_multi_batch', chunks) + try { + const read = async () => (await selectFrom(chTable.arrowstream('arr_multi_batch').as('t')) + .select('v').orderBy('v').execute()) as { v: number }[] + expect(await read()).toEqual([{ v: 1 }, { v: 1 }, { v: 2 }, { v: 2 }]) + t.refresh() + expect(await read()).toEqual([{ v: 1 }, { v: 1 }, { v: 2 }, { v: 2 }]) + } finally { + t.close() + } + }) + + it('reads the default dictionary-encoded JS string Table', async () => { + const dictionary = new Table({ s: vectorFromArray(['x', null, 'z']) }) + const t = registerArrowTable('arr_dictionary', dictionary) + try { + const rows = (await selectFrom(chTable.arrowstream('arr_dictionary').as('t')) + .select('s').execute()) as { s: string | null }[] + expect(rows).toEqual([{ s: 'x' }, { s: null }, { s: 'z' }]) + } finally { + t.close() + } + }) + + it('reads a sliced dictionary-index column with nulls', async () => { + const dictionary = new Table({ s: vectorFromArray(['a', null, 'c', 'd']) }).slice(1, 4) + const t = registerArrowTable('arr_dictionary_slice', dictionary) + try { + const rows = (await selectFrom(chTable.arrowstream('arr_dictionary_slice').as('t')) + .select('s').execute()) as { s: string | null }[] + expect(rows).toEqual([{ s: null }, { s: 'c' }, { s: 'd' }]) + } finally { + t.close() + } + }) + + it('rejects unsupported nested input explicitly instead of flattening rows', () => { + const input = new Table({ items: vectorFromArray([[1, 2]], new List(new Field('item', new Int32(), true))) }) + expect(() => registerArrowSource(input)).toThrow(/unsupported Arrow type/) + }) + + it('rebases sliced validity and bit-packed Bool without materializing rows', async () => { + const table = new Table({ + v: vectorFromArray([1, null, 3], new Int32()), + s: vectorFromArray(['a', null, 'c'], new Utf8()), + b: vectorFromArray([true, false, true], new Bool()), + }) + const t = registerArrowTable('arr_slice', table.slice(1)) + try { + const rows = (await selectFrom(chTable.arrowstream('arr_slice').as('t')) + .select(['v', 's', 'b']).execute()) as { v: number | null; s: string | null; b: boolean }[] + expect(rows).toEqual([{ v: null, s: null, b: false }, { v: 3, s: 'c', b: true }]) + } finally { + t.close() + } + }) + + it('rebases a slice starting past the first bitmap byte', async () => { + const values = Array.from({ length: 20 }, (_, i) => i === 10 ? null : i) + const table = new Table({ + v: vectorFromArray(values, new Int32()), + b: vectorFromArray(values.map((_, i) => i % 2 === 0), new Bool()), + }) + const t = registerArrowTable('arr_slice_bits', table.slice(9, 13)) + try { + const rows = (await selectFrom(chTable.arrowstream('arr_slice_bits').as('t')) + .select(['v', 'b']).execute()) as { v: number | null; b: boolean }[] + expect(rows).toEqual([ + { v: 9, b: false }, { v: null, b: true }, { v: 11, b: false }, { v: 12, b: true }, + ]) + } finally { + t.close() + } + }) + + it('keeps the original registration after a duplicate Arrow batch fails', async () => { + const input = new Table({ v: vectorFromArray([7], new Int32()) }) + const t = registerArrowTable('arr_duplicate_batch', input.batches[0]!) + try { + expect(() => registerArrowTable('arr_duplicate_batch', input.batches[0]!)).toThrow(/failed/) + const rows = (await selectFrom(chTable.arrowstream('arr_duplicate_batch').as('t')) + .select('v').execute()) as { v: number }[] + expect(rows).toEqual([{ v: 7 }]) + } finally { + t.close() + } + }) + + it('keeps the schema for an empty RecordBatch', async () => { + const empty = new Table({ v: vectorFromArray([], new Int32()) }) + const session = new Session() + try { + const t = registerArrowTable('arr_empty_batch', empty.batches[0]!, { session }) + try { + const result = await session.queryAsync( + "SELECT v FROM arrowstream('arr_empty_batch')", { format: 'arrow' }, + ) + const output = result.toArrow() as Table + expect(output.numRows).toBe(0) + expect(output.schema.fields.map((f) => [f.name, f.type.toString()])).toEqual([['v', 'Int32']]) + } finally { + t.close() + } + } finally { + session.close() + } + }) + it('registers and unregisters on the supplied Session connection', async () => { const session = new Session() try { @@ -50,6 +283,124 @@ describe('registerArrowTable — round-trips JS data through the engine', () => expect(() => t.close()).not.toThrow() }) + it('defers close during an async scan and rejects refresh until it drains', async () => { + const session = new Session() + const t = registerArrowTable('arr_close_during_scan', [ + { name: 'x', type: 'Int32', data: new Int32Array([7, 8, 9]) }, + ], { session }) + try { + const query = session.queryAsync( + "SELECT x, sleepEachRow(0.05) FROM arrowstream('arr_close_during_scan') ORDER BY x", + { format: 'JSONEachRow' }, + ) + expect(() => t.refresh()).toThrow(/active/) + t.close() + const result = await query + expect(result.text().trim().split('\n').map((line) => JSON.parse(line).x)).toEqual([7, 8, 9]) + await drainPending() + await expect(session.queryAsync( + "SELECT x FROM arrowstream('arr_close_during_scan')", { format: 'JSONEachRow' }, + )).rejects.toThrow(/arr_close_during_scan|not found/i) + } finally { + t.close() + session.close() + } + }) + + it('keeps a source registered until an idle lazy stream is cancelled', async () => { + const session = new Session() + const t = registerArrowTable('arr_lazy_close', [ + { name: 'x', type: 'Int32', data: new Int32Array([1]) }, + ], { session }) + try { + const stream = session.queryStream("SELECT x FROM arrowstream('arr_lazy_close')") + t.close() + stream.cancel() + await drainPending() + await expect(session.queryAsync( + "SELECT x FROM arrowstream('arr_lazy_close')", { format: 'JSONEachRow' }, + )).rejects.toThrow(/arr_lazy_close|not found/i) + } finally { + t.close() + session.close() + } + }) + + it('waits for an in-flight Arrow fetch and its cancellation cleanup', async () => { + const session = new Session() + const t = registerArrowTable('arr_fetch_close', [ + { name: 'x', type: 'Int32', data: new Int32Array([1, 2, 3]) }, + ], { session }) + try { + const stream = session.queryStream( + "SELECT x, sleepEachRow(0.05) FROM arrowstream('arr_fetch_close') SETTINGS max_block_size=1", + ) + const reader = stream[Symbol.asyncIterator]() + const next = reader.next() + t.close() + stream.cancel() + await next + await reader.return?.() + await drainPending() + await expect(session.queryAsync( + "SELECT x FROM arrowstream('arr_fetch_close')", { format: 'JSONEachRow' }, + )).rejects.toThrow(/arr_fetch_close|not found/i) + } finally { + t.close() + session.close() + await drainPending() + } + }) + + it('does not let a stale default-connection handle remove a later registration', async () => { + const stale = registerArrowTable('arr_default_reused', [ + { name: 'x', type: 'Int32', data: new Int32Array([1]) }, + ]) + const session = new Session() // evicts the previous default connection + session.close() + const current = registerArrowTable('arr_default_reused', [ + { name: 'x', type: 'Int32', data: new Int32Array([2]) }, + ]) + try { + stale.close() + const rows = (await selectFrom(chTable.arrowstream('arr_default_reused').as('t')) + .select('x').execute()) as { x: number }[] + expect(rows).toEqual([{ x: 2 }]) + } finally { + stale.close() + current.close() + } + }) + + it('waits for another Session\'s lazy stream before closing a source connection', async () => { + const source = new Session() + const reader = new Session(source.path) + const t = registerArrowTable('arr_cross_session', [ + { name: 'x', type: 'Int32', data: new Int32Array([1, 2]) }, + ], { session: source }) + try { + const stream = reader.queryStream("SELECT x FROM arrowstream('arr_cross_session')") + source.close() + t.close() + // A stream is an open cursor even before its first fetch. Native source + // teardown must wait for cancellation rather than dangling its buffers. + let drained = false + const draining = drainPending().then(() => { drained = true }) + await Promise.resolve() + expect(drained).toBe(false) + stream.cancel() + await draining + await expect(reader.queryAsync( + "SELECT x FROM arrowstream('arr_cross_session')", { format: 'JSONEachRow' }, + )).rejects.toThrow(/arr_cross_session|not found/i) + } finally { + t.close() + source.close() + reader.close() + await drainPending() + } + }) + it('reads Int32, Float64, and Utf8 columns in row order', async () => { const t = registerArrowTable('arr_basics', [ { name: 'id', type: 'Int32', data: new Int32Array([10, 20, 30]) }, @@ -102,25 +453,20 @@ describe('registerArrowTable — round-trips JS data through the engine', () => } }) - it('nulls show up as null after a Nullable cast', async () => { + it('preserves nullable Int32 and UTF-8 values without a cast', async () => { const t = registerArrowTable('arr_nulls', [ { name: 'v', type: 'Int32', data: new Int32Array([1, 0, 3]), nulls: [true, false, true] }, + { name: 's', type: 'String', data: ['é', '', '東京'], nulls: [true, false, true] }, ]) try { - // chdb's arrowstream column comes through as the non-nullable inferred type; - // a `nullIf`/`Nullable` projection surfaces the original NULL semantics. const rows = (await selectFrom(chTable.arrowstream('arr_nulls').as('t')) - .select('v') - .orderBy('v') - .execute()) as { v: number }[] - // The non-null rows are 1 and 3; one row's data slot is 0 (the null - // placeholder), and that's what the engine reads since we don't cast - // through a Nullable schema here. We assert "all three rows come back" - // and that the two non-null values are present. - expect(rows).toHaveLength(3) - const values = rows.map((r) => r.v).sort((a, b) => a - b) - expect(values).toContain(1) - expect(values).toContain(3) + .select(['v', 's']) + .execute()) as { v: number | null; s: string | null }[] + expect(rows).toEqual([ + { v: 1, s: 'é' }, + { v: null, s: null }, + { v: 3, s: '東京' }, + ]) } finally { t.close() } diff --git a/test/v3/layer3/arrow-insert.test.ts b/test/v3/layer3/arrow-insert.test.ts new file mode 100644 index 0000000..73beaa6 --- /dev/null +++ b/test/v3/layer3/arrow-insert.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest' +import { Int32, Table, Utf8, vectorFromArray } from 'apache-arrow' +import { insertArrow, insertArrowReader, Session } from '../../../index.js' + +describe('insertArrow — engine-side INSERT SELECT from Arrow input', () => { + it('maps source fields by name, preserves nulls, and reports submitted rows', async () => { + const session = new Session() + try { + await session.queryAsync('CREATE TABLE arrow_insert_test (id Int32, s Nullable(String)) ENGINE=Memory') + const input = new Table({ + s: vectorFromArray(['é', null, '東京'], new Utf8()), + id: vectorFromArray([3, 1, 2], new Int32()), + }) + const summary = await insertArrow('arrow_insert_test', input, { session }) + expect(summary.rowsSubmitted).toBe(3) + const result = await session.queryAsync('SELECT id, s FROM arrow_insert_test ORDER BY id', { format: 'JSONEachRow' }) + expect(result.text().trim().split('\n').map((line) => JSON.parse(line))).toEqual([ + { id: 1, s: null }, { id: 2, s: '東京' }, { id: 3, s: 'é' }, + ]) + } finally { + session.close() + } + }) + + it('releases the registration after an INSERT error and keeps the Session usable', async () => { + const session = new Session() + try { + const input = new Table({ id: vectorFromArray([1], new Int32()) }).batches[0]! + await expect(insertArrow('missing_arrow_insert_target', input, { session })).rejects.toThrow() + await session.queryAsync('CREATE TABLE arrow_insert_recovery (id Int32) ENGINE=Memory') + await expect(insertArrow('arrow_insert_recovery', input, { session })).resolves.toMatchObject({ rowsSubmitted: 1 }) + } finally { + session.close() + } + }) + + it('inserts all batches of a materialized Table', async () => { + const session = new Session() + try { + await session.queryAsync('CREATE TABLE arrow_insert_batches (id Int32) ENGINE=Memory') + const one = new Table({ id: vectorFromArray([1, 2], new Int32()) }) + const input = new Table([one.batches[0]!, one.batches[0]!]) + expect(await insertArrow('arrow_insert_batches', input, { session })).toMatchObject({ rowsSubmitted: 4 }) + const rows = await session.queryAsync('SELECT count() FROM arrow_insert_batches', { format: 'CSV' }) + expect(rows.text().trim()).toBe('4') + } finally { + session.close() + } + }) + + it('treats dots in Arrow field names as literal names, not SQL qualifiers', async () => { + const session = new Session() + try { + await session.queryAsync('CREATE TABLE arrow_insert_quoted (`a.b` Int32) ENGINE=Memory') + const input = new Table({ 'a.b': vectorFromArray([9], new Int32()) }) + await insertArrow('arrow_insert_quoted', input, { session }) + const result = await session.queryAsync('SELECT `a.b` FROM arrow_insert_quoted', { format: 'CSV' }) + expect(result.text().trim()).toBe('9') + } finally { + session.close() + } + }) + + it('inserts a RecordBatch reader incrementally with bounded native input', async () => { + const session = new Session() + try { + await session.queryAsync('CREATE TABLE arrow_insert_reader (id Int32) ENGINE=Memory') + async function* batches() { + for (let i = 0; i < 20; i++) { + yield new Table({ id: vectorFromArray([i], new Int32()) }).batches[0]! + } + } + const summary = await insertArrowReader('arrow_insert_reader', batches(), { session }) + expect(summary.rowsSubmitted).toBe(20) + const result = await session.queryAsync('SELECT count(), sum(id) FROM arrow_insert_reader', { format: 'CSV' }) + expect(result.text().trim()).toBe('20,190') + } finally { + session.close() + } + }) + + it('requires a live Session and unique source field names', async () => { + await expect(insertArrow('dest', [{ name: 'id', type: 'Int32', data: new Int32Array([1]) }], + { session: null as unknown as Session })).rejects.toThrow(/active Session/) + const session = new Session() + try { + await expect(insertArrow('dest', [ + { name: 'id', type: 'Int32', data: new Int32Array([1]) }, + { name: 'id', type: 'Int32', data: new Int32Array([2]) }, + ], { session })).rejects.toThrow(/unique column names/) + } finally { + session.close() + } + }) +}) diff --git a/test/v3/layer3/arrow-reader.test.ts b/test/v3/layer3/arrow-reader.test.ts new file mode 100644 index 0000000..e3d694d --- /dev/null +++ b/test/v3/layer3/arrow-reader.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it } from 'vitest' +import { spawnSync } from 'node:child_process' +import { Int32, Table, vectorFromArray } from 'apache-arrow' +import { registerArrowReader, Session, drainPending } from '../../../index.js' + +function batch(values: number[]) { + return new Table({ x: vectorFromArray(values, new Int32()) }).batches[0]! +} + +describe('registerArrowReader — bounded C Stream input', () => { + // chdb-core#321/#323: rc.3 must release the registration schema exactly + // once; this regression gate is no longer conditional on the engine version. + it( + 'releases the schema returned during C Stream registration', async () => { + const native = require('../../../build/Release/chdb_node.node') as { + ArrowLifetimeStats(): { schemaReleased: number } + } + const session = new Session() + const before = native.ArrowLifetimeStats().schemaReleased + const source = await registerArrowReader([batch([1])], { session }) + try { + expect(native.ArrowLifetimeStats().schemaReleased - before).toBeGreaterThanOrEqual(1) + } finally { + await source.close() + session.close() + } + }, + ) + + it('is one-shot and releases its TSFN so a child process exits naturally', () => { + const script = ` + const a = require('apache-arrow'), m = require('./index.js'); + (async () => { + const s = new m.Session(); + const batch = new a.Table({x: a.vectorFromArray([1], new a.Int32())}).batches[0]; + const h = await m.registerArrowReader([batch], {session:s}); + try { + const sql = "SELECT x FROM arrowstream('" + h.name + "')"; + const one = await s.queryAsync(sql, {format:'CSV'}); + const two = await s.queryAsync(sql, {format:'CSV'}); + if (one.text().trim() !== '1' || two.text().trim() !== '') throw Error('reader replayed'); + } finally { await h.close(); s.close(); await m.drainPending(); } + console.log('done'); + })().catch(e => { console.error(e); process.exitCode = 1; }); + ` + const child = spawnSync(process.execPath, ['-e', script], { + cwd: process.cwd(), encoding: 'utf8', timeout: 5_000, + }) + expect(child.error).toBeUndefined() + expect(child.status).toBe(0) + expect(child.stdout).toContain('done') + }) + + it('runs one C Data release callback for each delivered batch', async () => { + const native = require('../../../build/Release/chdb_node.node') as { + ArrowLifetimeStats(): { delivered: number; released: number } + } + const session = new Session() + async function* batches() { + for (let i = 0; i < 12; i++) yield batch([i]) + } + const before = native.ArrowLifetimeStats() + const source = await registerArrowReader(batches(), { session }) + try { + const result = await session.queryAsync( + `SELECT sum(x) FROM arrowstream('${source.name}')`, { format: 'CSV' }, + ) + expect(result.text().trim()).toBe('66') + await source.producerDone + } finally { + await source.close() + session.close() + await drainPending() + } + for (let i = 0; i < 50; i++) { + const state = native.ArrowLifetimeStats() + if (state.released - before.released === 12) break + await new Promise((resolve) => setImmediate(resolve)) + } + const after = native.ArrowLifetimeStats() + expect(after.delivered - before.delivered).toBe(12) + expect(after.released - before.released).toBe(12) + }) + + it('consumes async RecordBatches in a single SQL query without IPC', async () => { + const session = new Session() + async function* batches() { + yield batch([1, 2]) + yield batch([3, 4]) + yield batch([5]) + } + const source = await registerArrowReader(batches(), { session }) + try { + const result = await session.queryAsync( + `SELECT x FROM arrowstream('${source.name}') ORDER BY x`, { format: 'JSONEachRow' }, + ) + expect(result.text().trim().split('\n').map((line) => JSON.parse(line).x)).toEqual([1, 2, 3, 4, 5]) + await source.producerDone + } finally { + await source.close() + session.close() + await drainPending() + } + }) + + it('bounds prefetched batches even when the engine retains earlier arrays', async () => { + const session = new Session() + let produced = 0 + async function* batches() { + for (let i = 0; i < 40; i++) { + produced++ + yield batch([i]) + } + } + const source = await registerArrowReader(batches(), { session }) + try { + await new Promise((resolve) => setTimeout(resolve, 10)) + expect(produced).toBeLessThanOrEqual(4) // schema + two queued + one pending push + const result = await session.queryAsync( + `SELECT sum(x) FROM arrowstream('${source.name}') ORDER BY sum(x)`, { format: 'CSV' }, + ) + expect(result.text().trim()).toBe('780') + await source.producerDone + expect(produced).toBe(40) + } finally { + await source.close() + session.close() + await drainPending() + } + }) + + it('uses an empty reader schema and propagates producer failures', async () => { + const session = new Session() + const schema = batch([]).schema + const empty = await registerArrowReader({ + schema, + async *[Symbol.asyncIterator]() {}, + }, { session }) + try { + const count = await session.queryAsync( + `SELECT count() FROM arrowstream('${empty.name}')`, { format: 'CSV' }, + ) + expect(count.text().trim()).toBe('0') + } finally { + await empty.close() + } + async function* broken() { + yield batch([1]) + throw new Error('producer failed') + } + const failing = await registerArrowReader(broken(), { session }) + try { + await expect(session.queryAsync( + `SELECT x FROM arrowstream('${failing.name}')`, { format: 'CSV' }, + )).rejects.toThrow(/producer failed/) + await expect(failing.producerDone).rejects.toThrow(/producer failed/) + } finally { + await failing.close() + session.close() + await drainPending() + } + }) + + it('rejects a changed schema and cleans up after cancellation', async () => { + const session = new Session() + async function* changed() { + yield batch([1]) + yield new Table({ y: vectorFromArray([2], new Int32()) }).batches[0]! + } + const invalid = await registerArrowReader(changed(), { session }) + try { + await expect(session.queryAsync( + `SELECT x FROM arrowstream('${invalid.name}')`, { format: 'CSV' }, + )).rejects.toThrow(/schema changed/) + await expect(invalid.producerDone).rejects.toThrow(/schema changed/) + } finally { + await invalid.close() + } + + async function* slow() { + yield batch([1]) + await new Promise((resolve) => setTimeout(resolve, 50)) + yield batch([2]) + } + const cancelled = await registerArrowReader(slow(), { session }) + try { + const query = session.queryAsync( + `SELECT sum(x) FROM arrowstream('${cancelled.name}')`, { format: 'CSV' }, + ) + const rejection = expect(query).rejects.toThrow(/Arrow reader cancelled/) + await cancelled.close() + await rejection + await drainPending() + const healthy = await session.queryAsync('SELECT 7', { format: 'CSV' }) + expect(healthy.text().trim()).toBe('7') + } finally { + await cancelled.close() + session.close() + await drainPending() + } + }) + + it('does not wait for a non-cooperative async producer on close', async () => { + const session = new Session() + async function* stuck() { + yield batch([1]) + await new Promise(() => {}) + } + const source = await registerArrowReader(stuck(), { session }) + try { + const query = session.queryAsync( + `SELECT sum(x) FROM arrowstream('${source.name}')`, { format: 'CSV' }, + ) + const rejected = expect(query).rejects.toThrow(/cancelled/) + await new Promise((resolve) => setTimeout(resolve, 5)) + await Promise.race([ + source.close(), + new Promise((_, reject) => setTimeout(() => reject(new Error('close hung')), 200)), + ]) + await rejected + } finally { + await source.close() + session.close() + await drainPending() + } + }) +}) diff --git a/test/v3/layer3/arrow-worker-isolate.test.ts b/test/v3/layer3/arrow-worker-isolate.test.ts new file mode 100644 index 0000000..cc9ad4f --- /dev/null +++ b/test/v3/layer3/arrow-worker-isolate.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest' +import { spawnSync } from 'node:child_process' + +describe('Arrow source in a worker isolate', () => { + it('does not close a main-thread Session when the worker env exits', () => { + const workerCode = ` + const {parentPort,workerData}=require('node:worker_threads'); + const m=require('./index.js'),a=require('apache-arrow'); + (async()=>{const s=new m.Session(workerData.path); + const b=new a.Table({x:a.vectorFromArray([1],new a.Int32())}).batches[0]; + const h=await m.registerArrowReader([b],{session:s}); + try{await s.queryAsync("SELECT x FROM arrowstream('"+h.name+"')",{format:'CSV'})} + finally{await h.close();s.close();await m.drainPending()} + parentPort.postMessage('ok'); + })().catch(e=>{parentPort.postMessage(e.message);process.exitCode=1}); + ` + const script = ` + const {Worker}=require('node:worker_threads'),m=require('./index.js'); + (async()=>{const s=new m.Session(); + try{const worker=new Worker(${JSON.stringify(workerCode)},{eval:true,workerData:{path:s.path}}); + const value=await new Promise((resolve,reject)=>{ + worker.once('message',resolve);worker.once('error',reject) + }); + if(value!=='ok')throw Error(value); + await new Promise((resolve,reject)=>worker.once('exit',code=>code===0?resolve():reject(Error('exit '+code)))); + const result=await s.queryAsync('SELECT 7',{format:'CSV'}); + if(result.text().trim()!=='7')throw Error('main Session was closed'); + console.log('concurrent isolate OK'); + }finally{s.close();await m.drainPending()} + })().catch(e=>{console.error(e);process.exitCode=1}); + ` + const child = spawnSync(process.execPath, ['-e', script], { + cwd: process.cwd(), encoding: 'utf8', timeout: 8_000, + }) + expect(child.error).toBeUndefined() + expect(child.status).toBe(0) + expect(child.stdout).toContain('concurrent isolate OK') + }) + + it('tears down reader references before another isolate opens a Session', () => { + const script = ` + const { Worker } = require('node:worker_threads'); + const m = require('./index.js'); + const worker = new Worker(\ + "const m=require('./index.js'),a=require('apache-arrow'),{parentPort}=require('node:worker_threads');"+ + "(async()=>{const s=new m.Session();const b=new a.Table({x:a.vectorFromArray([1],new a.Int32())}).batches[0];"+ + "const h=await m.registerArrowReader([b],{session:s});try{await s.queryAsync(\\\"SELECT x FROM arrowstream('\\\"+h.name+\\\"')\\\",{format:'CSV'})}"+ + "finally{await h.close();s.close();await m.drainPending()}parentPort.postMessage('ok')})().catch(e=>{parentPort.postMessage(e.message);process.exitCode=1})", + {eval:true}); + worker.once('message', value => { if (value !== 'ok') throw Error(value); }); + worker.once('exit', async code => { + if (code !== 0) throw Error('worker exited ' + code); + const s = new m.Session(); + try { if (s.query('SELECT 7','CSV').trim() !== '7') throw Error('main query failed'); } + finally { s.close(); } + console.log('isolate OK'); + }); + ` + const child = spawnSync(process.execPath, ['-e', script], { + cwd: process.cwd(), encoding: 'utf8', timeout: 8_000, + }) + expect(child.error).toBeUndefined() + expect(child.status).toBe(0) + expect(child.stdout).toContain('isolate OK') + }) +}) diff --git a/update_libchdb.sh b/update_libchdb.sh index 8f491ec..b1ef274 100755 --- a/update_libchdb.sh +++ b/update_libchdb.sh @@ -17,15 +17,17 @@ set -e # without them. v26.7.3 is the first stable release that exports them. # # The pin is currently a release candidate, and chdb_version() on it returns the -# full "26.11.0-rc.2", suffix and all. Durable records that string in +# full "26.11.0-rc.3", suffix and all. Durable records that string in # head.engine.version and raises min_reader to it on every write. A pre-release # sorts below the release it leads to, so an object written against this RC -# stays readable by the eventual stable, but every earlier engine (26.11.0-rc.1, +# stays readable by the eventual stable, but every earlier engine (26.11.0-rc.2, # 26.9.x and before) is refused it with engine_incompatible from then on. +# The Arrow A1–A6 ABI and chdb-core#323's registration ownership fixes are +# present in rc.3. See docs/arrow-rc3-conformance.md for validation scope. # # Keep the pin on its own line and literal: the release check greps for it when # it proposes a bump. -CHDB_ENGINE_PIN=v26.11.0-rc.2 +CHDB_ENGINE_PIN=v26.11.0-rc.3 # CHDB_ENGINE_VERSION overrides the pin, which is how the release check runs the # suite against an engine this repository has not adopted yet. It is a different @@ -53,7 +55,7 @@ LATEST_RELEASE="${CHDB_ENGINE_VERSION:-$CHDB_ENGINE_PIN}" # ERR_DLOPEN_FAILED instead. # # The publish and cleanroom workflows read CHDB_LIB_VERSION from here. -LIBCHDB_NPM_VERSION=26.11.0-rc.2.1 +LIBCHDB_NPM_VERSION=26.11.0-rc.3.1 # Download the correct version based on the platform case "$(uname -s)" in From 42e57436e500f99a737f66bee70b89b0b059b827 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Fri, 13 Nov 2026 11:02:09 +1300 Subject: [PATCH 2/4] Document direct Arrow use and stabilize cleanroom runner --- .github/workflows/chdb-node-test.yml | 4 +- README.md | 9 +- docs/arrow-rc3-conformance.md | 2 + docs/arrow.md | 138 +++++++++++++++++++++++++++ docs/design/architecture.md | 1 + docs/design/native-arrow.md | 56 +++++++++++ src/layer3/execute/arrow-reader.ts | 5 +- 7 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 docs/arrow.md create mode 100644 docs/design/native-arrow.md diff --git a/.github/workflows/chdb-node-test.yml b/.github/workflows/chdb-node-test.yml index 80d0e09..96320b3 100644 --- a/.github/workflows/chdb-node-test.yml +++ b/.github/workflows/chdb-node-test.yml @@ -218,7 +218,9 @@ jobs: # installed `chdb` package — the Node analogue of chDB Python's "pip install # the built wheel, then run the suite". Proves the published shape actually # loads and runs, not just the in-repo source. - runs-on: ubuntu-latest + # Keep the portable-builder host stable; the rolling ubuntu-latest image + # (26.04) cancelled the AlmaLinux re-link step before packaging on #104. + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/README.md b/README.md index 95f8e04..7d4cb53 100644 --- a/README.md +++ b/README.md @@ -144,15 +144,20 @@ somewhere later. | Server-side parameter binding (`{name:Type}`) | ✅ | | Insert (object / positional rows) | ✅ | | Streaming results (`AsyncIterable`) | ✅ | -| Arrow **output** (`format: 'arrow'` + `toArrow()`) | ✅ | +| Arrow IPC **output** (`format: 'arrow'` + `toArrow()`) | ✅ | | AbortSignal / timeout | ✅ (single-shot is honest: rejects early; native runs to completion) | -| Arrow **scan** (`registerArrowTable`, Arrow input) | ⏳ follow-up | +| Direct Arrow **scan**, reader, typed results, and bulk insert | ✅ in PR #104 (rc.3 native packages pending) | | Arrow zero-copy (M2, `{ zeroCopy: true }`) | ⏳ follow-up | | chDB ↔ `@clickhouse/client` integration (`chdb/connection`, **experimental**) | ✅ | | Durable V1 control plane (`chdb/durable`, **experimental**) | ✅ pure TS, engine injected | | Default durable engine over the addon (`chdb/durable/node`) | ✅ | | Remote object storage for durable (`chdb/durable/s3`) | ✅ verified on AWS S3 and MinIO; R2 untested | +See the [Arrow guide](docs/arrow.md) for examples, lifecycle rules, and the +difference between direct C Data/C Stream and the existing Arrow IPC path. +The direct APIs require chdb-core v26.11.0-rc.3; they are not in the currently +published npm platform packages. + ### chDB ↔ `@clickhouse/client` integration (`chdb/connection`, experimental) > **Status**: this integration uses the experimental diff --git a/docs/arrow-rc3-conformance.md b/docs/arrow-rc3-conformance.md index 2cce54d..e30db94 100644 --- a/docs/arrow-rc3-conformance.md +++ b/docs/arrow-rc3-conformance.md @@ -15,6 +15,8 @@ platforms and registry-installed packages remain CI/release gates. The shared contract is `chdb-core/docs/NATIVE_ARROW_BINDING_CONTRACT.md`. In this binding, **direct** means C Data/C Stream without Arrow IPC. Input and output may copy buffers; no end-to-end zero-copy claim is made. +For application examples see the [Arrow guide](arrow.md); for the bounded +queue and release sequence see the [Node Arrow design](design/native-arrow.md). The optional `chdb/arrow` subpath supplies TypeScript `Table`/`RecordBatch` return types without making the base `chdb` entry require `apache-arrow`. diff --git a/docs/arrow.md b/docs/arrow.md new file mode 100644 index 0000000..d922f74 --- /dev/null +++ b/docs/arrow.md @@ -0,0 +1,138 @@ +# Arrow with chdb in Node.js + +The direct Arrow APIs let you query in-memory Arrow data, read typed results, +and insert batches without Arrow IPC. They are implemented in [PR #104](https://github.com/chdb-io/chdb-node/pull/104) +against chdb-core v26.11.0-rc.3; the matching npm platform packages are not +published yet. Once released, install both `chdb` and its optional peer: + +```sh +npm install chdb apache-arrow +``` + +## Choose a path + +| Task | API | Result | +| --- | --- | --- | +| Query an Arrow `Table` or `RecordBatch` | `registerArrowSource` + `arrowstream(name)` | SQL result in your chosen format | +| Feed batches as they arrive | `registerArrowReader` + `arrowstream(name)` | One-shot SQL source | +| Read a small typed result | `session.queryArrow(sql)` | `apache-arrow` `Table` | +| Read a large typed result | `session.queryArrowStream(sql)` | Async iterable of `RecordBatch` | +| Insert a table or a batch | `insertArrow(table, input, { session })` | Insert summary | +| Insert a batch iterable | `insertArrowReader(table, source, { session })` | Insert summary | + +The existing `queryAsync(sql, { format: 'arrow' }).toArrow()` still uses Arrow +IPC. The direct APIs use Arrow C Data/C Stream instead; **direct is not an +end-to-end zero-copy guarantee**. chdb may copy input while converting columns, +and Node copies output buffers before releasing the native result. + +## Query an in-memory Arrow table + +```js +import { Session, registerArrowSource } from 'chdb' +import { Table, Int32, vectorFromArray } from 'apache-arrow' + +const session = new Session() +const input = new Table({ id: vectorFromArray([1, 2, 3], new Int32()) }) +const source = registerArrowSource(input, { session }) // unique, opaque name +try { + const sql = `SELECT sum(id) FROM arrowstream('${source.name}')` + console.log((await session.queryAsync(sql, { format: 'CSV' })).text()) // 6 + + // A materialized source is consumed by a scan. Refresh only after the + // previous query has completed, before using it in another query. + source.refresh() + console.log((await session.queryAsync(sql, { format: 'CSV' })).text()) // 6 +} finally { + source.close() + session.close() +} +``` + +`registerArrowTable(name, input, { session })` is available when you need to +choose the name. Names are process-wide; prefer generated names unless you have +a reason to coordinate them. Close the registration after the last scan. If a +query or lazy stream is active, unregister and buffer release are deferred +until that scan finishes or is cancelled. + +## Insert Arrow data + +```js +import { Session, insertArrow, insertArrowReader } from 'chdb' +import { Table, Int32, vectorFromArray } from 'apache-arrow' + +const session = new Session() +try { + await session.queryAsync('CREATE TABLE events (id Int32) ENGINE=Memory') + const input = new Table({ id: vectorFromArray([1, 2], new Int32()) }) + await insertArrow('events', input, { session }) + + async function* batches() { + yield new Table({ id: vectorFromArray([3], new Int32()) }).batches[0] + yield new Table({ id: vectorFromArray([4], new Int32()) }).batches[0] + } + const summary = await insertArrowReader('events', batches(), { session }) + console.log(summary.rowsSubmitted) // 2; not a storage-side commit ledger +} finally { + session.close() +} +``` + +Destination fields are matched by name, not table position. Both insert +functions unregister their temporary source on success or error. For a custom +SQL transform over an incoming reader, register it with +`await registerArrowReader(batches(), { session })`, query +`arrowstream(handle.name)` once, then `await handle.close()` in `finally`. +The native queue holds at most two batches not yet handed to +core; producing another waits for capacity. Core can retain delivered batches, +so this is not a bound on the entire query's memory. A cancelled producer +should cooperate with iterator `return()` to release its own resources. + +## Read typed results + +```js +import { Session } from 'chdb' + +const session = new Session() +try { + const small = await session.queryArrow('SELECT toInt32(number) AS id FROM numbers(3)') + console.log(small.toArray().map(row => row.id)) // [0, 1, 2] + + const reader = session.queryArrowStream('SELECT toInt32(number) AS id FROM numbers(1000)') + try { + for await (const batch of reader) { + console.log(batch.numRows) + // Stop early if needed. + } + } finally { + reader.cancel() + } +} finally { + session.close() +} +``` + +Use `queryArrowBind` and `queryArrowStreamBind` for server-side +`{name:Type}` parameters, for example +`await session.queryArrowBind('SELECT {id:Int32} AS id', { id: 7 })`. +For TypeScript return types without making the base `chdb` import require +`apache-arrow`, use the helpers from `chdb/arrow`. + +## Limits and errors + +- The direct path requires a loaded chdb-core >= v26.11.0-rc.3; an older + binary raises `CHDB_ABI`. Installing `apache-arrow` alone does not upgrade + the native platform package. +- Supported input includes primitive scalars, Boolean, UTF-8/Binary, + Timestamp, Decimal128/256, slices, null bitmaps, and a single-chunk + dictionary of UTF-8/Binary values with Int32/UInt32 indices. Nested input + and multi-chunk dictionaries are rejected rather than silently converted. +- Supported output includes basic scalars, Timestamp, Decimal128/256, and + LowCardinality UTF-8/Binary dictionaries. Options include + `stringAsString`, `lowCardinalityAsDictionary`, and `unsupportedAsBinary`; + without the latter, unmappable types fail rather than silently falling back + to IPC. See [validated type scope](arrow-rc3-conformance.md). +- Keep each `Session` open until its scans and readers finish. Cancel a + partially consumed output reader, close input registrations, and await + input reader cleanup. Error paths release the native handles. + +For the thread and ownership model, see the [Node Arrow design](design/native-arrow.md). diff --git a/docs/design/architecture.md b/docs/design/architecture.md index b7ac19a..fec0982 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -77,6 +77,7 @@ Current PR #43 Layer 1 capabilities include: - CJS and ESM entry points. See [Layer 1 Native Binding Design and Implementation](./layer1-native-binding.md) for the full reviewer guide. +For the newer direct Arrow input/output path, see [Node native Arrow: threads and ownership](./native-arrow.md). ## Layer 2 - ClickHouse Client Compatibility diff --git a/docs/design/native-arrow.md b/docs/design/native-arrow.md new file mode 100644 index 0000000..2c61760 --- /dev/null +++ b/docs/design/native-arrow.md @@ -0,0 +1,56 @@ +# Node native Arrow: threads and ownership + +Direct Arrow input bridges a JS iterator and core's synchronous C Stream +consumer. The addon keeps JS buffers alive, exposes C Data descriptors, and +uses a native queue so a core worker never calls the JS iterator. Output +fetches run as libuv tasks over one stream handle. A task may reuse a worker +thread; a batch does not get a dedicated thread. + +Time runs downward; columns represent execution domains: + +```mermaid +sequenceDiagram + participant JS as JS event loop + participant Bridge as N-API bridge + bounded queue + participant Worker as libuv worker / chdb-core + + Note over JS,Worker: Input: RecordBatch reader -> arrowstream() + JS->>JS: await first batch; determine schema + JS->>Bridge: register C Stream, pin JS buffers, enqueue first batch + Bridge-->>JS: return name and close handle + JS->>Worker: queryAsync(SQL using arrowstream(name)) + par Producer on JS event loop + loop later batches + JS->>Bridge: await enqueue(batch); wait if queue is full + end + and Consumer on query worker + loop while SQL needs input + Worker->>Bridge: get_next() waits, then dequeues C Data + Bridge-->>Worker: batch descriptor + Worker->>Bridge: C release callback (may run off JS thread) + Bridge-->>JS: thread-safe notification; release N-API refs + end + end + Worker-->>JS: query settles (result or error) + JS->>Bridge: await handle.close(); cancel/unregister after active scan + + Note over JS,Worker: Output: SQL -> RecordBatch reader + JS->>Bridge: queryArrowStream(SQL) creates one stream handle + loop each awaited reader.next() + JS->>Bridge: request next batch + Bridge->>Worker: queue AsyncWorker fetch task + Worker->>Worker: chdb_stream_fetch_arrow(handle) + Worker->>Worker: copy buffers; call C release + Worker-->>JS: resolve as JS RecordBatch + end + JS->>Bridge: reader.cancel() or iteration ends + Bridge->>Worker: wait for in-flight fetch, then destroy handle +``` + +The queue admits at most two **not-yet-delivered** batches and one waiting +producer request. It applies backpressure to JS production, not to batches +that core has already dequeued and retained (for example, for sorting). +Closing during a scan defers unregister and input release until query cleanup; +the thread-safe callback releases JS-owned references on their isolate. On +output, the copy precedes C release, so a returned batch outlives the native +result. See the [user guide](../arrow.md) and [conformance scope](../arrow-rc3-conformance.md). diff --git a/src/layer3/execute/arrow-reader.ts b/src/layer3/execute/arrow-reader.ts index 8f32d0f..3c7d736 100644 --- a/src/layer3/execute/arrow-reader.ts +++ b/src/layer3/execute/arrow-reader.ts @@ -31,8 +31,9 @@ const readerFinalizers = new FinalizationRegistry<{ /** * Register a one-shot synchronous or asynchronous RecordBatch reader. The - * native queue holds at most two batches (including a batch awaiting C Data - * release); JS produces on its own event loop and awaits queue capacity. + * native queue holds at most two not-yet-delivered batches; JS produces on its + * own event loop and awaits queue capacity. Core may retain delivered batches + * beyond the queue, so this is not a total-query memory bound. * Core get_next only waits on native descriptors and never invokes JS on its * query worker. Pass the returned name to arrowstream(name) on this Session. */ From bd5fad5f234e79a53510ea7308dd20393b6ffe80 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Fri, 13 Nov 2026 11:31:04 +1300 Subject: [PATCH 3/4] Guard Arrow reader concurrency and preserve schema ownership --- docs/arrow.md | 9 +++ docs/design/native-arrow.md | 4 ++ index.js | 13 ++-- lib/chdb_node.cpp | 76 +++++++++++++++++----- src/arrow-direct.ts | 4 +- src/layer3/execute/arrow-reader.ts | 14 ++++- test/v3/arrow-direct.test.ts | 66 +++++++++++++++++++- test/v3/layer3/arrow-input.test.ts | 14 +++++ test/v3/layer3/arrow-reader.test.ts | 97 ++++++++++++++++++++++++++++- 9 files changed, 269 insertions(+), 28 deletions(-) diff --git a/docs/arrow.md b/docs/arrow.md index d922f74..880414f 100644 --- a/docs/arrow.md +++ b/docs/arrow.md @@ -86,6 +86,10 @@ The native queue holds at most two batches not yet handed to core; producing another waits for capacity. Core can retain delivered batches, so this is not a bound on the entire query's memory. A cancelled producer should cooperate with iterator `return()` to release its own resources. +Use `queryAsync` or another async query API while a JS Arrow reader is +registered: synchronous `query`/`queryBind` on its connection are rejected +because blocking the event loop would prevent the producer from refilling +the queue. ## Read typed results @@ -116,6 +120,11 @@ Use `queryArrowBind` and `queryArrowStreamBind` for server-side `await session.queryArrowBind('SELECT {id:Int32} AS id', { id: 7 })`. For TypeScript return types without making the base `chdb` import require `apache-arrow`, use the helpers from `chdb/arrow`. +Only one iterator may consume a given Arrow result reader at a time; cancel +it before closing its Session if you stop early. +Await earlier operations on the same Session before opening a direct Arrow +stream: its synchronous startup rejects a busy connection instead of blocking +the JS event loop. ## Limits and errors diff --git a/docs/design/native-arrow.md b/docs/design/native-arrow.md index 2c61760..de340ff 100644 --- a/docs/design/native-arrow.md +++ b/docs/design/native-arrow.md @@ -50,6 +50,10 @@ sequenceDiagram The queue admits at most two **not-yet-delivered** batches and one waiting producer request. It applies backpressure to JS production, not to batches that core has already dequeued and retained (for example, for sorting). +Synchronous queries cannot consume a live JS reader on the same connection: +they would block the event loop that must feed this queue. Stream creation +also refuses an in-flight operation on its connection, and fetches on one +stream handle are serialized. Closing during a scan defers unregister and input release until query cleanup; the thread-safe callback releases JS-owned references on their isolate. On output, the copy precedes C release, so a returned batch outlives the native diff --git a/index.js b/index.js index 2a57ffe..c290889 100644 --- a/index.js +++ b/index.js @@ -462,12 +462,15 @@ class ChdbArrowReader { this._handle = handle; this._closed = false; this._inflight = null; + this._iterating = false; this.schema = null; // available after the first fetch, even for zero rows activeStreams.add(this); } get closed() { return this._closed; } async *[Symbol.asyncIterator]() { + if (this._iterating) throw new ChdbStreamError("Arrow reader already has an active iterator"); + this._iterating = true; try { while (!this._closed) { let raw; @@ -485,6 +488,7 @@ class ChdbArrowReader { } finally { this._inflight = null; } + if (this._closed) break; const table = arrowTableFromDirect(raw); this.schema = table.schema; if (raw.done) { @@ -499,6 +503,7 @@ class ChdbArrowReader { for (const batch of table.batches) yield batch; } } finally { + this._iterating = false; this.cancel(); } } @@ -821,10 +826,6 @@ class Session { try { const key = this.path ? resolvePath(this.path) : this.path; this._key = key; // normalized directory, for the deferred-teardown bookkeeping - // Creating a Session can evict the default connection. Its Arrow names - // belong to that old connection, even if a new default is opened later. - defaultArrowGeneration++; - retireArrowRegistrations(null); this.connection = connectionArgs ? chdbNode.CreateConnection(key, connectionArgs) : chdbNode.CreateConnection(key); @@ -836,6 +837,10 @@ class Session { if (this.isTemp) { try { this.#removeTempDir(); } catch (_) {} } throw new ChdbConnectionError("Failed to create connection"); } + // A successful bind can evict the default connection. A rejected bind + // leaves it alive, so keep its Arrow registration tokens in that case. + defaultArrowGeneration++; + retireArrowRegistrations(null); ensureExitHook(); openSessions.add(this); diff --git a/lib/chdb_node.cpp b/lib/chdb_node.cpp index 2f5381b..fe07011 100644 --- a/lib/chdb_node.cpp +++ b/lib/chdb_node.cpp @@ -85,6 +85,13 @@ struct Registry { static Registry g_reg; static std::mutex g_reg_mu; static std::unordered_map g_conn_owner; +static bool hasLiveAsyncArrowReader(chdb_connection * conn); +static bool rejectSyncArrowReader(Napi::Env env, chdb_connection * conn) { + if (!hasLiveAsyncArrowReader(conn)) return false; + Napi::Error::New(env, "A live Arrow reader requires an async query; synchronous query would block its JS producer") + .ThrowAsJavaScriptException(); + return true; +} // Extra argv entries are the same `--setting=value` forms a `clickhouse local` // invocation takes, and they have to be given at connect: `--path` and @@ -525,6 +532,7 @@ Napi::String QueryWrapper(const Napi::CallbackInfo &info) { Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException(); return Napi::String::New(env, ""); } + if (rejectSyncArrowReader(env, nullptr)) return Napi::String::New(env, ""); std::string query = info[0].As().Utf8Value(); std::string format = info[1].As().Utf8Value(); @@ -571,6 +579,7 @@ Napi::String QueryWithParamsWrapper(const Napi::CallbackInfo &info) { Napi::TypeError::New(env, "Usage: sql, format, params").ThrowAsJavaScriptException(); return Napi::String::New(env, ""); } + if (rejectSyncArrowReader(env, nullptr)) return Napi::String::New(env, ""); std::string sql = info[0].As(); std::string format = info[1].As(); std::vector names, values; @@ -600,6 +609,7 @@ Napi::String QueryWithParamsConnectionWrapper(const Napi::CallbackInfo &info) { return Napi::String::New(env, ""); } ChdbConnection conn = info[0].As>().Data(); + if (rejectSyncArrowReader(env, static_cast(conn))) return Napi::String::New(env, ""); std::string sql = info[1].As(); std::string format = info[2].As(); std::vector names, values; @@ -1159,6 +1169,7 @@ Napi::String QueryWithConnectionWrapper(const Napi::CallbackInfo & info) { } ChdbConnection conn = info[0].As>().Data(); + if (rejectSyncArrowReader(env, static_cast(conn))) return Napi::String::New(env, ""); std::string query = info[1].As().Utf8Value(); std::string format = info[2].As().Utf8Value(); @@ -1514,10 +1525,13 @@ struct DirectArrowStreamState { chdb_connection conn; chdb_result * handle; bool closed = false; + bool fetching = false; // managed on the owning JS event loop + bool cancelRequested = false; }; static void closeDirectArrowStream(DirectArrowStreamState * state) { if (!state || state->closed) return; + if (state->fetching) { state->cancelRequested = true; return; } state->closed = true; if (state->handle) { chdb_stream_cancel_query(state->conn, state->handle); @@ -1537,6 +1551,16 @@ Napi::Value StartArrowStreamWrapper(const Napi::CallbackInfo & info) { Napi::Error::New(env, "No active connection available").ThrowAsJavaScriptException(); return env.Undefined(); } + // Starting the stream calls into the connection synchronously. A worker on + // that same connection may need this event loop to refill an Arrow reader. + { + std::lock_guard lock(g_reg_mu); + if (inflight_locked(*conn) > 0) { + Napi::Error::New(env, "Cannot start Arrow stream while this connection has an in-flight operation") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + } chdb_arrow_options options; if (!parseDirectArrowOptions(env, info.Length() >= 3 ? info[2] : env.Undefined(), options)) return env.Undefined(); const std::string sql = info[1].As().Utf8Value(); @@ -1572,7 +1596,7 @@ Napi::Value StartArrowStreamWrapper(const Napi::CallbackInfo & info) { Napi::Error::New(env, message).ThrowAsJavaScriptException(); return env.Undefined(); } - auto * state = new DirectArrowStreamState{*conn, handle, false}; + auto * state = new DirectArrowStreamState{*conn, handle}; return Napi::External::New(env, state, [](Napi::Env, DirectArrowStreamState * s) { closeDirectArrowStream(s); delete s; @@ -1587,32 +1611,36 @@ class FetchArrowWorker : public Napi::AsyncWorker { Napi::Promise GetPromise() { return deferred_.Promise(); } void Execute() override { - if (!state_ || state_->closed || !state_->handle) { done_ = true; return; } + // The event loop owns state mutation. While fetching=true it defers cancel + // and the External is pinned by handleRef_, so this handle stays live. AbiArrayStream batch{}; const chdb_state status = chdb_stream_fetch_arrow( state_->conn, state_->handle, reinterpret_cast(&batch)); std::string error; - if (status != CHDBSuccess) error = "chdb_stream_fetch_arrow failed"; + if (status != CHDBSuccess) { + if (const char * nativeError = chdb_result_error(state_->handle)) error = nativeError; + if (error.empty()) error = "chdb_stream_fetch_arrow failed"; + } if (error.empty()) readDirectArrowStream(batch, fields_, batches_, error); if (batch.release) batch.release(&batch); if (!error.empty()) { - closeDirectArrowStream(state_); SetError(error); return; } - if (batches_.empty()) { - done_ = true; - chdb_destroy_query_result(state_->handle); - state_->handle = nullptr; - state_->closed = true; - } + done_ = batches_.empty(); } void OnOK() override { Napi::Object out = directArrowResultToJs(Env(), fields_, batches_); out.Set("done", Napi::Boolean::New(Env(), done_)); + state_->fetching = false; + if (done_ || state_->cancelRequested) closeDirectArrowStream(state_); deferred_.Resolve(out); } - void OnError(const Napi::Error & e) override { deferred_.Reject(e.Value()); } + void OnError(const Napi::Error & e) override { + state_->fetching = false; + closeDirectArrowStream(state_); + deferred_.Reject(e.Value()); + } private: Napi::Promise::Deferred deferred_; @@ -1628,6 +1656,9 @@ Napi::Value FetchArrowStreamWrapper(const Napi::CallbackInfo & info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsExternal()) return rejectedPromise(env, "Usage: Arrow stream handle"); auto * state = info[0].As>().Data(); + if (state->closed || state->cancelRequested) return rejectedPromise(env, "Arrow stream is closed"); + if (state->fetching) return rejectedPromise(env, "Arrow stream fetch already in progress"); + state->fetching = true; auto * worker = new FetchArrowWorker(env, state, info[0]); worker->Queue(); return worker->GetPromise(); @@ -1844,10 +1875,16 @@ struct LiveAsyncArrowStream { ~LiveAsyncArrowStream() { if (releaseTsfn && !tsfnReleased) releaseTsfn.Release(); } }; static std::unordered_map, LiveArrowKeyHash> g_live_arrow_async; +static std::mutex g_live_arrow_mu; +static bool hasLiveAsyncArrowReader(chdb_connection * conn) { + std::lock_guard lock(g_live_arrow_mu); + for (const auto & entry : g_live_arrow_async) + if (!conn || entry.first.first == conn) return true; + return false; +} static std::atomic g_async_arrow_delivered{0}; static std::atomic g_async_arrow_released{0}; static std::atomic g_arrow_input_schema_released{0}; -static std::mutex g_live_arrow_mu; static void countedArrowSchemaRelease(AbiSchema * schema) { ++g_arrow_input_schema_released; @@ -2205,6 +2242,16 @@ static std::unique_ptr makeLiveArrowTable(Napi::Env env, Napi::A return live; } +static bool sameLiveArrowSchema(const LiveArrowTable & a, const LiveArrowTable & b) { + if (a.nameStorage != b.nameStorage || a.formatStorage != b.formatStorage) return false; + for (size_t i = 0; i < a.childSchemas.size(); ++i) { + const AbiSchema * ad = a.childSchemas[i].dictionary; + const AbiSchema * bd = b.childSchemas[i].dictionary; + if (!!ad != !!bd || (ad && std::string(ad->format) != bd->format)) return false; + } + return true; +} + // nativeArrowRegisterColumns(conn, tableName, columns) Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { Napi::Env env = info.Env(); @@ -2266,7 +2313,7 @@ Napi::Value ArrowRegisterBatchesWrapper(const Napi::CallbackInfo & info) { if (!batch) return env.Undefined(); if (!live->batches.empty()) { const auto & first = *live->batches.front(); - if (batch->nameStorage != first.nameStorage || batch->formatStorage != first.formatStorage) { + if (!sameLiveArrowSchema(*batch, first)) { Napi::TypeError::New(env, "Arrow batch schemas must match").ThrowAsJavaScriptException(); return env.Undefined(); } @@ -2337,8 +2384,7 @@ Napi::Value ArrowPushReaderBatchWrapper(const Napi::CallbackInfo & info) { auto state = info[0].As>().Data()->state; auto batch = makeLiveArrowTable(env, info[1].As()); if (!batch) return env.Undefined(); - if (batch->nameStorage != state->schemaBatch->nameStorage || - batch->formatStorage != state->schemaBatch->formatStorage) { + if (!sameLiveArrowSchema(*batch, *state->schemaBatch)) { Napi::TypeError::New(env, "Arrow reader batch schema changed").ThrowAsJavaScriptException(); return env.Undefined(); } diff --git a/src/arrow-direct.ts b/src/arrow-direct.ts index f5853fe..d40a6aa 100644 --- a/src/arrow-direct.ts +++ b/src/arrow-direct.ts @@ -45,9 +45,9 @@ export function arrowTableFromDirect(raw: NativeArrowResult): unknown { default: throw new ChdbArrowError(`Unsupported direct Arrow format: ${format}`) } } - const fields = raw.fields.map((field) => new arrow.Field(field.name, + const fields = raw.fields.map((field, i) => new arrow.Field(field.name, field.dictionaryFormat - ? new arrow.Dictionary(typeFor(field.dictionaryFormat), typeFor(field.format) as Uint32) + ? new arrow.Dictionary(typeFor(field.dictionaryFormat), typeFor(field.format) as Uint32, i) : typeFor(field.format), field.nullable)) const schema = new arrow.Schema(fields) const view = (buffer: Buffer): Uint8Array => new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) diff --git a/src/layer3/execute/arrow-reader.ts b/src/layer3/execute/arrow-reader.ts index 3c7d736..0a4e9d8 100644 --- a/src/layer3/execute/arrow-reader.ts +++ b/src/layer3/execute/arrow-reader.ts @@ -63,8 +63,16 @@ export async function registerArrowReader( // Do not cross apache-arrow's independent ESM/CJS constructors: build a // zero-row structural batch from the public schema shape instead. firstBatch = { schema, numCols: schema.fields.length, numRows: 0, - data: { children: schema.fields.map(() => ({ offset: 0, length: 0, nullCount: 0, - values: new Uint8Array(0), valueOffsets: new Int32Array([0]) })) } } + data: { children: schema.fields.map((field) => { + const type = field.type as { toString(): string; indices?: { toString(): string } } + const dictionary = type.toString().startsWith('Dictionary<') + ? { data: [{ offset: 0, length: 0, nullCount: 0, + values: new Uint8Array(0), valueOffsets: new Int32Array([0]) }] } + : undefined + return { offset: 0, length: 0, nullCount: 0, + values: type.indices?.toString() === 'Uint32' ? new Uint32Array(0) : new Int32Array(0), + valueOffsets: new Int32Array([0]), dictionary } + }) } } } else firstBatch = first.value const rt = runtime() @@ -92,7 +100,7 @@ export async function registerArrowReader( } } catch (e) { if (!closing) { - rt._arrowFinishReader(native, (e as Error).message || String(e) || 'Arrow reader producer failed') + rt._arrowFinishReader(native, (e instanceof Error ? e.message : String(e)) || 'Arrow reader producer failed') throw e } } diff --git a/test/v3/arrow-direct.test.ts b/test/v3/arrow-direct.test.ts index 2ecf048..f6d9dac 100644 --- a/test/v3/arrow-direct.test.ts +++ b/test/v3/arrow-direct.test.ts @@ -145,10 +145,17 @@ describe('Session.queryArrowStream — incremental C Data batches', () => { 'SELECT toLowCardinality(toString(number % 2)) AS s FROM numbers(7) SETTINGS max_block_size=2', { lowCardinalityAsDictionary: true }, ) - const values: string[] = [] - for await (const value of reader) values.push(...(value as RecordBatch).toArray().map((row) => row.s)) + const batches: RecordBatch[] = [] + for await (const value of reader) batches.push(value as RecordBatch) + const values = new (require('apache-arrow').Table)(reader.schema, batches) + .toArray().map((row: { s: string }) => row.s) expect(values).toEqual(['0', '1', '0', '1', '0', '1', '0']) expect((reader.schema as Schema).fields[0]!.type.toString()).toBe('Dictionary') + const bound = await session.queryArrowBind( + 'SELECT toLowCardinality(toString(number % 2)) AS s FROM numbers(5) WHERE number < {limit:UInt64} SETTINGS max_block_size=1', + { limit: 4 }, { lowCardinalityAsDictionary: true }, + ) as Table + expect(bound.toArray().map((row) => row.s)).toEqual(['0', '1', '0', '1']) } finally { session.close() } @@ -193,4 +200,59 @@ describe('Session.queryArrowStream — incremental C Data batches', () => { await drainPending() } }) + + it('rejects a synchronous start while this connection has async work', async () => { + const session = new Session() + try { + const pending = session.queryAsync('SELECT sleep(0.15)', { format: 'CSV' }) + expect(() => session.queryArrowStream('SELECT toInt32(1) AS x')).toThrow(/in-flight operation/) + await pending + const reader = session.queryArrowStream('SELECT toInt32(1) AS x') + for await (const _ of reader) { /* drains normally once idle */ } + } finally { session.close(); await drainPending() } + }) + + it('rejects overlapping native fetches and defers direct cancel until the worker finishes', async () => { + const native = require('../../build/Release/chdb_node.node') as { + StartArrowStream(conn: unknown, sql: string, opts: object): unknown + FetchArrowStream(handle: unknown): Promise + CancelArrowStream(handle: unknown): void + } + const session = new Session() + const handle = native.StartArrowStream(session.connection, + 'SELECT toInt32(number) AS x FROM numbers(4) WHERE sleepEachRow(0.03)=0', {}) + try { + const first = native.FetchArrowStream(handle) + await expect(native.FetchArrowStream(handle)).rejects.toThrow(/already in progress/) + native.CancelArrowStream(handle) + await first + await expect(native.FetchArrowStream(handle)).rejects.toThrow(/closed/) + } finally { native.CancelArrowStream(handle); session.close(); await drainPending() } + }) + + it('allows only one active JS iterator for a native stream', async () => { + const session = new Session() + const reader = session.queryArrowStream('SELECT toInt32(number) AS x FROM numbers(3)') + try { + const first = reader[Symbol.asyncIterator]() + const second = reader[Symbol.asyncIterator]() + const pending = first.next() + await expect(second.next()).rejects.toThrow(/active iterator/) + await pending + await first.return?.() + await drainPending() + } finally { reader.cancel(); session.close(); await drainPending() } + }) + + it('preserves a mid-stream engine error instead of a generic fetch failure', async () => { + const session = new Session() + try { + const reader = session.queryArrowStream( + "SELECT throwIf(number = 2, 'arrow stream failure') AS x FROM numbers(4) SETTINGS max_block_size=1", + ) + await expect((async () => { + for await (const _ of reader) { /* consume until the engine fails */ } + })()).rejects.toThrow(/arrow stream failure/) + } finally { session.close(); await drainPending() } + }) }) diff --git a/test/v3/layer3/arrow-input.test.ts b/test/v3/layer3/arrow-input.test.ts index 5d793ce..20c31a6 100644 --- a/test/v3/layer3/arrow-input.test.ts +++ b/test/v3/layer3/arrow-input.test.ts @@ -372,6 +372,20 @@ describe('registerArrowTable — round-trips JS data through the engine', () => } }) + it('keeps a default Arrow registration when Session construction fails', async () => { + const source = registerArrowSource(new Table({ x: vectorFromArray([7], new Int32()) })) + try { + expect(() => new Session('', { connectionArgs: ['--path=/invalid'] })) + .toThrow(/--path/) + const result = await selectFrom(chTable.arrowstream(source.name)).select('x').execute() + expect(result).toEqual([{ x: 7 }]) + source.close() + const replacement = registerArrowTable(source.name, + new Table({ x: vectorFromArray([8], new Int32()) })) + replacement.close() + } finally { source.close(); await drainPending() } + }) + it('waits for another Session\'s lazy stream before closing a source connection', async () => { const source = new Session() const reader = new Session(source.path) diff --git a/test/v3/layer3/arrow-reader.test.ts b/test/v3/layer3/arrow-reader.test.ts index e3d694d..c5e6ed4 100644 --- a/test/v3/layer3/arrow-reader.test.ts +++ b/test/v3/layer3/arrow-reader.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' import { spawnSync } from 'node:child_process' -import { Int32, Table, vectorFromArray } from 'apache-arrow' -import { registerArrowReader, Session, drainPending } from '../../../index.js' +import { Binary, Dictionary, Int32, Table, Utf8, vectorFromArray } from 'apache-arrow' +import { registerArrowReader, registerArrowSource, Session, drainPending } from '../../../index.js' function batch(values: number[]) { return new Table({ x: vectorFromArray(values, new Int32()) }).batches[0]! @@ -161,6 +161,99 @@ describe('registerArrowReader — bounded C Stream input', () => { } }) + it('registers an empty reader with a supported dictionary schema', async () => { + const session = new Session() + const schema = new Table({ s: vectorFromArray(['a'], new Dictionary(new Utf8(), new Int32())) }).schema + const source = await registerArrowReader({ schema, async *[Symbol.asyncIterator]() {} }, { session }) + try { + const result = await session.queryAsync(`SELECT count() FROM arrowstream('${source.name}')`, { format: 'CSV' }) + expect(result.text().trim()).toBe('0') + } finally { await source.close(); session.close(); await drainPending() } + }) + + it('unblocks SQL when a producer rejects with null', async () => { + const session = new Session() + async function* broken() { yield batch([1]); throw null } + const source = await registerArrowReader(broken(), { session }) + try { + await expect(session.queryAsync(`SELECT x FROM arrowstream('${source.name}')`, { format: 'CSV' })) + .rejects.toThrow(/null/) + await expect(source.producerDone).rejects.toBeNull() + } finally { await source.close(); session.close(); await drainPending() } + }) + + it('rejects a dictionary-to-plain schema change in a reader and materialized batches', async () => { + const session = new Session() + const dictionary = new Table({ x: vectorFromArray(['a'], new Dictionary(new Utf8(), new Int32())) }).batches[0]! + const plain = batch([1]) // same field and Int32 index format, but no dictionary + const differentValues = new Table({ + x: vectorFromArray([new Uint8Array([65])], new Dictionary(new Binary(), new Int32())), + }).batches[0]! + expect(() => registerArrowSource({ batches: [dictionary, plain] } as Table, { session })) + .toThrow(/schemas must match/) + expect(() => registerArrowSource({ batches: [dictionary, differentValues] } as Table, { session })) + .toThrow(/schemas must match/) + const source = await registerArrowReader([dictionary, plain], { session }) + try { + await expect(session.queryAsync(`SELECT x FROM arrowstream('${source.name}')`, { format: 'CSV' })) + .rejects.toThrow(/schema changed/) + await expect(source.producerDone).rejects.toThrow(/schema changed/) + } finally { await source.close(); session.close(); await drainPending() } + }) + + it('rejects synchronous SQL before a JS producer can deadlock the event loop', () => { + const script = ` + const a = require('apache-arrow'), m = require('./index.js'); + (async () => { + const s = new m.Session(); + const b = new a.Table({x: a.vectorFromArray([1], new a.Int32())}).batches[0]; + async function* batches() { for (let i = 0; i < 10; i++) yield b; } + const h = await m.registerArrowReader(batches(), {session:s}); + const sql = "SELECT count() FROM arrowstream('" + h.name + "')"; + try { + for (const run of [() => s.query(sql), () => s.queryBind(sql, {})]) { + try { run(); throw Error('sync query unexpectedly ran'); } + catch (e) { if (!/requires an async query/.test(e.message)) throw e; } + } + const result = await s.queryAsync(sql, {format:'CSV'}); + if (result.text().trim() !== '10') throw Error('async query failed'); + } finally { await h.close(); s.close(); await m.drainPending(); } + console.log('done'); + })().catch(e => { console.error(e); process.exitCode = 1; }); + ` + const child = spawnSync(process.execPath, ['-e', script], { + cwd: process.cwd(), encoding: 'utf8', timeout: 5_000, + }) + expect(child.error).toBeUndefined() + expect(child.status).toBe(0) + expect(child.stdout).toContain('done') + }) + + it('can stream direct Arrow output from a live async reader without blocking its producer', () => { + const script = ` + const a = require('apache-arrow'), m = require('./index.js'); + (async () => { + const s = new m.Session(); + const b = new a.Table({x: a.vectorFromArray([1], new a.Int32())}).batches[0]; + async function* batches() { for (let i = 0; i < 10; i++) yield b; } + const h = await m.registerArrowReader(batches(), {session:s}); + try { + const out = s.queryArrowStream("SELECT sum(x) AS total FROM arrowstream('" + h.name + "')"); + const rows = []; + for await (const part of out) rows.push(...part.toArray()); + if (rows[0].total !== 10n) throw Error('wrong result: ' + String(rows[0].total)); + } finally { await h.close(); s.close(); await m.drainPending(); } + console.log('done'); + })().catch(e => { console.error(e); process.exitCode = 1; }); + ` + const child = spawnSync(process.execPath, ['-e', script], { + cwd: process.cwd(), encoding: 'utf8', timeout: 5_000, + }) + expect(child.error).toBeUndefined() + expect(child.status).toBe(0) + expect(child.stdout).toContain('done') + }) + it('rejects a changed schema and cleans up after cancellation', async () => { const session = new Session() async function* changed() { From 495560aa55b7501145f072725be0209d07010118 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Fri, 13 Nov 2026 11:47:33 +1300 Subject: [PATCH 4/4] Prevent cross-isolate Arrow borrowing and deferred stream cancellation --- docs/arrow-rc3-conformance.md | 2 +- docs/arrow.md | 4 + docs/design/native-arrow.md | 4 + lib/chdb_node.cpp | 152 ++++++++++++++++---- test/v3/arrow-direct.test.ts | 27 +++- test/v3/layer3/arrow-worker-isolate.test.ts | 71 ++++++++- 6 files changed, 223 insertions(+), 37 deletions(-) diff --git a/docs/arrow-rc3-conformance.md b/docs/arrow-rc3-conformance.md index e30db94..c776fb1 100644 --- a/docs/arrow-rc3-conformance.md +++ b/docs/arrow-rc3-conformance.md @@ -22,7 +22,7 @@ return types without making the base `chdb` entry require `apache-arrow`. | Capability | Implemented now | Further validation / documented scope | | --- | --- | --- | -| A1 SQL source | `registerArrowTable` accepts supported column descriptors, `RecordBatch`, and materialized multi-batch `Table`; C Stream callbacks use pinned JS buffers. `registerArrowSource` generates opaque process-unique names. Primitive, Timestamp, Decimal128/256, sliced buffers/bitmaps, and one-chunk Dictionary input are tested. | Nested types and multi-chunk dictionary input explicitly reject; more platform/isolate stress remains. | +| A1 SQL source | `registerArrowTable` accepts supported column descriptors, `RecordBatch`, and materialized multi-batch `Table`; C Stream callbacks use pinned JS buffers. `registerArrowSource` generates opaque process-unique names. Primitive, Timestamp, Decimal128/256, sliced buffers/bitmaps, and one-chunk Dictionary input are tested. Borrowed sources cannot coexist with a Session in another Node isolate. | Nested types and multi-chunk dictionary input explicitly reject; more platform/isolate stress remains. | | A2 reader source | `registerArrowReader` accepts sync/async RecordBatch iterables. The JS producer awaits capacity in a two-batch native queue; worker-thread `get_next` never calls JS. Empty schema, producer error, schema mismatch, cancellation, one-release-per-delivered-batch, one-shot behavior and worker-isolate teardown are tested. | ASan/RSS stress and platform matrix. Core may retain batches after pulling them (e.g. sorting); queue capacity counts only batches not yet handed to core. `close()` cancels native promptly even if a user producer never resolves `next()`; that producer must cooperate to release its own resources. | | A3 typed result | `Session.queryArrow` calls `chdb_query_arrow_n`, copies C buffers before release, and constructs an `apache-arrow.Table`; `queryArrowBind` materializes the bound A4 stream. Empty schema and SQL errors tested. | Nested types and large-result memory stress. | | A4 typed batches | `Session.queryArrowStream` pulls C Data per batch; `queryArrowStreamBind` uses `chdb_stream_query_arrow_with_params_n` with server-side binding. Early-break, in-flight cancel and bound values are tested. | Nested types, repeated cancellation and memory stress. | diff --git a/docs/arrow.md b/docs/arrow.md index 880414f..8dc3cb5 100644 --- a/docs/arrow.md +++ b/docs/arrow.md @@ -143,5 +143,9 @@ the JS event loop. - Keep each `Session` open until its scans and readers finish. Cancel a partially consumed output reader, close input registrations, and await input reader cleanup. Error paths release the native handles. +- Arrow source registrations are isolate-local. While one isolate owns a + registration, another Node worker cannot open a Session; close the source + before transferring work between isolates. The engine's registry names are + process-wide, but a borrowed JS buffer cannot safely outlive its isolate. For the thread and ownership model, see the [Node Arrow design](design/native-arrow.md). diff --git a/docs/design/native-arrow.md b/docs/design/native-arrow.md index de340ff..3e20f45 100644 --- a/docs/design/native-arrow.md +++ b/docs/design/native-arrow.md @@ -54,6 +54,10 @@ Synchronous queries cannot consume a live JS reader on the same connection: they would block the event loop that must feed this queue. Stream creation also refuses an in-flight operation on its connection, and fetches on one stream handle are serialized. +Because core's Arrow registry shares raw borrowed pointers process-wide, +source registration excludes live Sessions in other Node isolates, and an +isolate cannot open a Session while another owns a source. This prevents a +foreign scan from retaining a source after its owning isolate tears down. Closing during a scan defers unregister and input release until query cleanup; the thread-safe callback releases JS-owned references on their isolate. On output, the copy precedes C release, so a returned batch outlives the native diff --git a/lib/chdb_node.cpp b/lib/chdb_node.cpp index fe07011..ff19f7a 100644 --- a/lib/chdb_node.cpp +++ b/lib/chdb_node.cpp @@ -17,10 +17,10 @@ #include typedef void * ChdbConnection; -ChdbConnection CreateConnection(const char * path, char ** error_message); +ChdbConnection CreateConnection(const char * path, char ** error_message, napi_env owner); ChdbConnection CreateConnectionWithArgs(const char * path, const std::vector & extraArgs, - char ** error_message); + char ** error_message, napi_env owner); void CloseConnection(ChdbConnection conn); char * QueryWithConnection(ChdbConnection conn, const char * query, const char * format, char ** error_message); @@ -85,6 +85,44 @@ struct Registry { static Registry g_reg; static std::mutex g_reg_mu; static std::unordered_map g_conn_owner; +// A C Stream borrows JS buffers from its owning isolate. Core exposes names +// process-wide, so do not allow another isolate to open a connection while +// registrations exist, or register a source while another isolate has one. +static napi_env g_arrow_registration_owner = nullptr; +static size_t g_arrow_registration_pending = 0; +static std::set> g_arrow_registration_names; +static void resetArrowOwnerIfIdleLocked() { + if (!g_arrow_registration_pending && g_arrow_registration_names.empty()) + g_arrow_registration_owner = nullptr; +} +static bool beginArrowRegistration(Napi::Env env) { + std::lock_guard lock(g_reg_mu); + const napi_env owner = env; + if (g_arrow_registration_owner && g_arrow_registration_owner != owner) return false; + for (auto * conn : g_reg.conns) { + auto found = g_conn_owner.find(conn); + if (found != g_conn_owner.end() && found->second != owner) return false; + } + g_arrow_registration_owner = owner; + ++g_arrow_registration_pending; + return true; +} +static bool reserveArrowRegistration(Napi::Env env) { + if (beginArrowRegistration(env)) return true; + Napi::Error::New(env, "Arrow sources cannot be shared with a live Session in another Node isolate") + .ThrowAsJavaScriptException(); + return false; +} +static void finishArrowRegistration(chdb_connection * conn, const std::string & name, bool success) { + std::lock_guard lock(g_reg_mu); + --g_arrow_registration_pending; + if (success) g_arrow_registration_names.emplace(conn, name); + resetArrowOwnerIfIdleLocked(); +} +static void endArrowRegistrationLocked(chdb_connection * conn, const std::string & name) { + g_arrow_registration_names.erase({conn, name}); + resetArrowOwnerIfIdleLocked(); +} static bool hasLiveAsyncArrowReader(chdb_connection * conn); static bool rejectSyncArrowReader(Napi::Env env, chdb_connection * conn) { if (!hasLiveAsyncArrowReader(conn)) return false; @@ -169,6 +207,9 @@ static chdb_connection *open_raw(const std::string &path, // appears. //===--------------------------------------------------------------------===// static std::unordered_map g_inflight; +struct DirectArrowStreamState; +static void finishDirectArrowStream(DirectArrowStreamState * state); +static std::vector>> g_deferred_arrow_streams; static void retain_conn(chdb_connection conn) { if (!conn) return; @@ -181,7 +222,15 @@ static void release_conn(chdb_connection conn) { std::lock_guard lk(g_reg_mu); auto it = g_inflight.find(conn); if (it == g_inflight.end()) return; - if (--it->second <= 0) g_inflight.erase(it); + if (--it->second > 0) return; + g_inflight.erase(it); + // No worker can still hold this connection. Finish any cancellation that + // would have blocked its JS producer if performed on the event loop. + for (auto pending = g_deferred_arrow_streams.begin(); pending != g_deferred_arrow_streams.end();) { + if (pending->first != conn) { ++pending; continue; } + finishDirectArrowStream(pending->second.get()); + pending = g_deferred_arrow_streams.erase(pending); + } } // Caller must already hold g_reg_mu: the count is read as part of an eviction @@ -224,6 +273,9 @@ static void hard_close_all() { g_conn_owner.clear(); g_reg.defaultConn = nullptr; g_reg.boundKey.clear(); + g_arrow_registration_names.clear(); + g_arrow_registration_pending = 0; + g_arrow_registration_owner = nullptr; } // The atexit backstop: close whatever is still open when the process reaches C @@ -292,6 +344,11 @@ static void ensure_atexit() { // single shared default per process; reused across standalone calls. static chdb_connection *get_default_conn(char **error_message, napi_env owner) { std::lock_guard lk(g_reg_mu); + if (g_arrow_registration_owner && g_arrow_registration_owner != owner) { + if (error_message && !*error_message) + *error_message = strdup("Arrow sources belong to another Node isolate; close them first"); + return nullptr; + } if (g_reg.defaultConn) { auto found = g_conn_owner.find(g_reg.defaultConn); if (found != g_conn_owner.end() && found->second != owner) { @@ -330,8 +387,13 @@ static chdb_connection *get_default_conn(char **error_message, napi_env owner) { // live in-memory default yields; a different data directory is rejected. static chdb_connection *acquire_session_conn(const std::string &path, const std::vector &extraArgs, - char **error_message) { + char **error_message, napi_env owner) { std::lock_guard lk(g_reg_mu); + if (g_arrow_registration_owner && g_arrow_registration_owner != owner) { + if (error_message && !*error_message) + *error_message = strdup("Arrow sources belong to another Node isolate; close them first"); + return nullptr; + } if (!g_reg.conns.empty()) { if (g_reg.boundKey.empty()) { // Only the in-memory default is up (boundKey ""). It is transient and must @@ -378,6 +440,7 @@ static chdb_connection *acquire_session_conn(const std::string &path, } g_reg.boundKey = path; g_reg.conns.insert(c); + g_conn_owner[c] = owner; ensure_atexit(); return c; } @@ -478,17 +541,17 @@ static char *exec_query_params(chdb_connection conn, return output; } -ChdbConnection CreateConnection(const char * path, char ** error_message) { - return CreateConnectionWithArgs(path, {}, error_message); +ChdbConnection CreateConnection(const char * path, char ** error_message, napi_env owner) { + return CreateConnectionWithArgs(path, {}, error_message, owner); } ChdbConnection CreateConnectionWithArgs(const char * path, const std::vector & extraArgs, - char ** error_message) { + char ** error_message, napi_env owner) { // Sessions always pass a real path (a temp dir for in-memory sessions), so // an empty key here never collides with the default connection's "" key. std::string p = (path && path[0]) ? std::string(path) : std::string(); - return static_cast(acquire_session_conn(p, extraArgs, error_message)); + return static_cast(acquire_session_conn(p, extraArgs, error_message, owner)); } void CloseConnection(ChdbConnection conn) { @@ -1130,7 +1193,7 @@ Napi::Value CreateConnectionWrapper(const Napi::CallbackInfo & info) { } char *error_message = nullptr; - ChdbConnection conn = CreateConnectionWithArgs(path.c_str(), settings, &error_message); + ChdbConnection conn = CreateConnectionWithArgs(path.c_str(), settings, &error_message, env); if (!conn) { std::string msg = error_message ? error_message : "Failed to create connection"; @@ -1139,10 +1202,6 @@ Napi::Value CreateConnectionWrapper(const Napi::CallbackInfo & info) { return env.Null(); } - { - std::lock_guard lock(g_reg_mu); - g_conn_owner[static_cast(conn)] = env; - } return Napi::External::New(env, conn); } @@ -1529,14 +1588,24 @@ struct DirectArrowStreamState { bool cancelRequested = false; }; -static void closeDirectArrowStream(DirectArrowStreamState * state) { +static void finishDirectArrowStream(DirectArrowStreamState * state) { + if (!state || !state->handle) return; + chdb_stream_cancel_query(state->conn, state->handle); + chdb_destroy_query_result(state->handle); + state->handle = nullptr; +} + +static void closeDirectArrowStream(const std::shared_ptr & state) { if (!state || state->closed) return; if (state->fetching) { state->cancelRequested = true; return; } + std::lock_guard lock(g_reg_mu); + if (state->closed) return; state->closed = true; - if (state->handle) { - chdb_stream_cancel_query(state->conn, state->handle); - chdb_destroy_query_result(state->handle); - state->handle = nullptr; + if (inflight_locked(state->conn) > 0) { + state->cancelRequested = true; + g_deferred_arrow_streams.emplace_back(state->conn, state); + } else { + finishDirectArrowStream(state.get()); } } @@ -1596,16 +1665,18 @@ Napi::Value StartArrowStreamWrapper(const Napi::CallbackInfo & info) { Napi::Error::New(env, message).ThrowAsJavaScriptException(); return env.Undefined(); } - auto * state = new DirectArrowStreamState{*conn, handle}; - return Napi::External::New(env, state, [](Napi::Env, DirectArrowStreamState * s) { - closeDirectArrowStream(s); + auto * state = new std::shared_ptr( + std::make_shared(DirectArrowStreamState{*conn, handle})); + return Napi::External>::New(env, state, + [](Napi::Env, std::shared_ptr * s) { + closeDirectArrowStream(*s); delete s; }); } class FetchArrowWorker : public Napi::AsyncWorker { public: - FetchArrowWorker(Napi::Env env, DirectArrowStreamState * state, Napi::Value handle) + FetchArrowWorker(Napi::Env env, std::shared_ptr state, Napi::Value handle) : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), state_(state), handleRef_(Napi::Persistent(handle)), guard_(state ? state->conn : nullptr) {} Napi::Promise GetPromise() { return deferred_.Promise(); } @@ -1644,7 +1715,7 @@ class FetchArrowWorker : public Napi::AsyncWorker { private: Napi::Promise::Deferred deferred_; - DirectArrowStreamState * state_; + std::shared_ptr state_; Napi::Reference handleRef_; bool done_ = false; std::vector fields_; @@ -1655,7 +1726,7 @@ class FetchArrowWorker : public Napi::AsyncWorker { Napi::Value FetchArrowStreamWrapper(const Napi::CallbackInfo & info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsExternal()) return rejectedPromise(env, "Usage: Arrow stream handle"); - auto * state = info[0].As>().Data(); + auto state = *info[0].As>>().Data(); if (state->closed || state->cancelRequested) return rejectedPromise(env, "Arrow stream is closed"); if (state->fetching) return rejectedPromise(env, "Arrow stream fetch already in progress"); state->fetching = true; @@ -1666,7 +1737,7 @@ Napi::Value FetchArrowStreamWrapper(const Napi::CallbackInfo & info) { Napi::Value CancelArrowStreamWrapper(const Napi::CallbackInfo & info) { if (info.Length() >= 1 && info[0].IsExternal()) - closeDirectArrowStream(info[0].As>().Data()); + closeDirectArrowStream(*info[0].As>>().Data()); return info.Env().Undefined(); } @@ -1756,11 +1827,13 @@ Napi::Value ArrowRegisterArrayWrapper(const Napi::CallbackInfo & info) { if (!pointerArg(env, info[2], "schemaPtr", &schemaPtr)) return env.Undefined(); if (!pointerArg(env, info[3], "arrayPtr", &arrayPtr)) return env.Undefined(); + if (!reserveArrowRegistration(env)) return env.Undefined(); chdb_state st = chdb_arrow_array_scan( *conn, tableName.c_str(), reinterpret_cast(static_cast(schemaPtr)), reinterpret_cast(static_cast(arrayPtr))); + finishArrowRegistration(conn, tableName, st == CHDBSuccess); if (st != CHDBSuccess) { Napi::Error::New(env, "chdb_arrow_array_scan failed for table " + tableName).ThrowAsJavaScriptException(); return env.Undefined(); @@ -1781,10 +1854,12 @@ Napi::Value ArrowRegisterStreamWrapper(const Napi::CallbackInfo & info) { uint64_t streamPtr = 0; if (!pointerArg(env, info[2], "streamPtr", &streamPtr)) return env.Undefined(); + if (!reserveArrowRegistration(env)) return env.Undefined(); chdb_state st = chdb_arrow_scan( *conn, tableName.c_str(), reinterpret_cast(static_cast(streamPtr))); + finishArrowRegistration(conn, tableName, st == CHDBSuccess); if (st != CHDBSuccess) { Napi::Error::New(env, "chdb_arrow_scan failed for table " + tableName).ThrowAsJavaScriptException(); return env.Undefined(); @@ -2017,22 +2092,27 @@ static void liveArrowRelease(AbiArrayStream * stream) { stream->release = nullpt // erasing only our map left dangling pointers and reserved names behind. static void erase_arrow_tables_for_conn(chdb_connection *conn) { std::lock_guard lock(g_live_arrow_mu); + // Caller holds g_reg_mu. Include low-level caller-built registrations that + // have no JS-buffer entry in the maps below. + for (auto it = g_arrow_registration_names.begin(); it != g_arrow_registration_names.end();) { + if (it->first != conn) { ++it; continue; } + chdb_arrow_unregister_table(*conn, it->second.c_str()); + it = g_arrow_registration_names.erase(it); + } + resetArrowOwnerIfIdleLocked(); for (auto it = g_live_arrow_tables.begin(); it != g_live_arrow_tables.end();) { if (it->first.first == conn) { - chdb_arrow_unregister_table(*conn, it->first.second.c_str()); it = g_live_arrow_tables.erase(it); } else ++it; } for (auto it = g_live_arrow_streams.begin(); it != g_live_arrow_streams.end();) { if (it->first.first == conn) { - chdb_arrow_unregister_table(*conn, it->first.second.c_str()); it = g_live_arrow_streams.erase(it); } else ++it; } for (auto it = g_live_arrow_async.begin(); it != g_live_arrow_async.end();) { if (it->first.first == conn) { - chdb_arrow_unregister_table(*conn, it->first.second.c_str()); cancelAsyncArrowStream(it->second); it = g_live_arrow_async.erase(it); } else ++it; @@ -2265,11 +2345,13 @@ Napi::Value ArrowRegisterColumnsWrapper(const Napi::CallbackInfo & info) { auto live = makeLiveArrowTable(env, info[2].As()); if (!live) return env.Undefined(); + if (!reserveArrowRegistration(env)) return env.Undefined(); chdb_state st = chdb_arrow_array_scan( *conn, tableName.c_str(), reinterpret_cast(&live->rootSchema), reinterpret_cast(&live->rootArray)); + finishArrowRegistration(conn, tableName, st == CHDBSuccess); if (st != CHDBSuccess) { Napi::Error::New(env, "chdb_arrow_array_scan failed for table " + tableName).ThrowAsJavaScriptException(); return env.Undefined(); @@ -2325,7 +2407,10 @@ Napi::Value ArrowRegisterBatchesWrapper(const Napi::CallbackInfo & info) { live->stream.get_last_error = liveArrowGetLastError; live->stream.release = liveArrowRelease; live->stream.private_data = live.get(); - if (chdb_arrow_scan(*conn, tableName.c_str(), reinterpret_cast(&live->stream)) != CHDBSuccess) { + if (!reserveArrowRegistration(env)) return env.Undefined(); + const chdb_state st = chdb_arrow_scan(*conn, tableName.c_str(), reinterpret_cast(&live->stream)); + finishArrowRegistration(conn, tableName, st == CHDBSuccess); + if (st != CHDBSuccess) { Napi::Error::New(env, "chdb_arrow_scan failed for table " + tableName).ThrowAsJavaScriptException(); return env.Undefined(); } @@ -2362,7 +2447,10 @@ Napi::Value ArrowRegisterReaderWrapper(const Napi::CallbackInfo & info) { state->stream.get_last_error = liveAsyncArrowGetLastError; state->stream.release = liveAsyncArrowRelease; state->stream.private_data = state.get(); - if (chdb_arrow_scan(*conn, tableName.c_str(), reinterpret_cast(&state->stream)) != CHDBSuccess) { + if (!reserveArrowRegistration(env)) return env.Undefined(); + const chdb_state st = chdb_arrow_scan(*conn, tableName.c_str(), reinterpret_cast(&state->stream)); + finishArrowRegistration(conn, tableName, st == CHDBSuccess); + if (st != CHDBSuccess) { Napi::Error::New(env, "chdb_arrow_scan failed for reader " + tableName).ThrowAsJavaScriptException(); return env.Undefined(); } @@ -2467,6 +2555,10 @@ Napi::Value ArrowUnregisterWrapper(const Napi::CallbackInfo & info) { g_live_arrow_async.erase(async); } } + { + std::lock_guard lock(g_reg_mu); + endArrowRegistrationLocked(conn, tableName); + } return env.Undefined(); } diff --git a/test/v3/arrow-direct.test.ts b/test/v3/arrow-direct.test.ts index f6d9dac..68bef3b 100644 --- a/test/v3/arrow-direct.test.ts +++ b/test/v3/arrow-direct.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { RecordBatch, Schema, Table } from 'apache-arrow' -import { Session, drainPending } from '../../index.js' +import { Session, registerArrowReader, drainPending } from '../../index.js' describe('Session.queryArrow — direct C Data output', () => { it('rejects rc.2 and older native engines before calling the Arrow ABI', async () => { @@ -255,4 +255,29 @@ describe('Session.queryArrowStream — incremental C Data batches', () => { })()).rejects.toThrow(/arrow stream failure/) } finally { session.close(); await drainPending() } }) + + it('does not block the JS producer when cancelling an idle stream on a busy connection', async () => { + const session = new Session() + const idle = session.queryArrowStream('SELECT toInt32(1) AS x') + const arrow = require('apache-arrow') as typeof import('apache-arrow') + const batch = (x: number) => new arrow.Table({ + x: arrow.vectorFromArray([x], new arrow.Int32()), + }).batches[0]! + async function* batches() { + yield batch(1) + await new Promise((resolve) => setTimeout(resolve, 60)) + yield batch(2) + } + const source = await registerArrowReader(batches(), { session }) + try { + const pending = session.queryAsync(`SELECT sum(x) FROM arrowstream('${source.name}')`, { format: 'CSV' }) + const started = Date.now() + idle.cancel() + expect(Date.now() - started).toBeLessThan(50) + expect((await pending).text().trim()).toBe('3') + await source.producerDone + await drainPending() + await expect(session.queryArrow('SELECT toInt32(4) AS x')).resolves.toBeDefined() + } finally { idle.cancel(); await source.close(); session.close(); await drainPending() } + }) }) diff --git a/test/v3/layer3/arrow-worker-isolate.test.ts b/test/v3/layer3/arrow-worker-isolate.test.ts index cc9ad4f..f7c683f 100644 --- a/test/v3/layer3/arrow-worker-isolate.test.ts +++ b/test/v3/layer3/arrow-worker-isolate.test.ts @@ -5,12 +5,10 @@ describe('Arrow source in a worker isolate', () => { it('does not close a main-thread Session when the worker env exits', () => { const workerCode = ` const {parentPort,workerData}=require('node:worker_threads'); - const m=require('./index.js'),a=require('apache-arrow'); + const m=require('./index.js'); (async()=>{const s=new m.Session(workerData.path); - const b=new a.Table({x:a.vectorFromArray([1],new a.Int32())}).batches[0]; - const h=await m.registerArrowReader([b],{session:s}); - try{await s.queryAsync("SELECT x FROM arrowstream('"+h.name+"')",{format:'CSV'})} - finally{await h.close();s.close();await m.drainPending()} + try{await s.queryAsync('SELECT 1',{format:'CSV'})} + finally{s.close();await m.drainPending()} parentPort.postMessage('ok'); })().catch(e=>{parentPort.postMessage(e.message);process.exitCode=1}); ` @@ -37,6 +35,69 @@ describe('Arrow source in a worker isolate', () => { expect(child.stdout).toContain('concurrent isolate OK') }) + it('refuses a second isolate while a borrowed Arrow registration is live', () => { + const script = ` + const {Worker}=require('node:worker_threads'),m=require('./index.js'),a=require('apache-arrow'); + (async()=>{const s=new m.Session(); + const b=new a.Table({x:a.vectorFromArray([1],new a.Int32())}); + const h=m.registerArrowSource(b,{session:s}); + try { + const worker=new Worker("const m=require('./index.js'),{parentPort,workerData}=require('node:worker_threads');"+ + "try{new m.Session(workerData.path);parentPort.postMessage('unexpected success')}"+ + "catch(e){parentPort.postMessage(e.message)}",{eval:true,workerData:{path:s.path}}); + const message=await new Promise((resolve,reject)=>{worker.once('message',resolve);worker.once('error',reject)}); + if(!/another Node isolate/.test(message))throw Error(message); + await new Promise((resolve,reject)=>worker.once('exit',code=>code===0?resolve():reject(Error('exit '+code)))); + h.close(); + const reopened=new Worker("const m=require('./index.js'),{parentPort,workerData}=require('node:worker_threads');"+ + "try{const s=new m.Session(workerData.path);s.close();parentPort.postMessage('ok')}"+ + "catch(e){parentPort.postMessage(e.message)}",{eval:true,workerData:{path:s.path}}); + const ok=await new Promise((resolve,reject)=>{reopened.once('message',resolve);reopened.once('error',reject)}); + if(ok!=='ok')throw Error(ok); + await new Promise((resolve,reject)=>reopened.once('exit',code=>code===0?resolve():reject(Error('exit '+code)))); + } finally {h.close();s.close();await m.drainPending()} + console.log('isolated'); + })().catch(e=>{console.error(e);process.exitCode=1}); + ` + const child = spawnSync(process.execPath, ['-e', script], { + cwd: process.cwd(), encoding: 'utf8', timeout: 8_000, + }) + expect(child.error).toBeUndefined() + expect(child.status).toBe(0) + expect(child.stdout).toContain('isolated') + }) + + it('refuses a borrowed source while another isolate holds a Session', () => { + const script = ` + const {Worker}=require('node:worker_threads'),m=require('./index.js'),a=require('apache-arrow'); + (async()=>{const s=new m.Session(); + const worker=new Worker("const m=require('./index.js'),{parentPort,workerData}=require('node:worker_threads');"+ + "const s=new m.Session(workerData.path);parentPort.postMessage('open');"+ + "parentPort.once('message',()=>{s.close();parentPort.postMessage('closed')})", + {eval:true,workerData:{path:s.path}}); + try { + const opened=await new Promise((resolve,reject)=>{worker.once('message',resolve);worker.once('error',reject)}); + if(opened!=='open')throw Error(opened); + const b=new a.Table({x:a.vectorFromArray([1],new a.Int32())}); + try {m.registerArrowSource(b,{session:s});throw Error('unexpected source registration')} + catch(e){if(!/another Node isolate/.test(e.message))throw e} + worker.postMessage('close'); + const closed=await new Promise((resolve,reject)=>{worker.once('message',resolve);worker.once('error',reject)}); + if(closed!=='closed')throw Error(closed); + await new Promise((resolve,reject)=>worker.once('exit',code=>code===0?resolve():reject(Error('exit '+code)))); + const source=m.registerArrowSource(b,{session:s});source.close(); + } finally {s.close();await m.drainPending()} + console.log('ownership enforced'); + })().catch(e=>{console.error(e);process.exitCode=1}); + ` + const child = spawnSync(process.execPath, ['-e', script], { + cwd: process.cwd(), encoding: 'utf8', timeout: 8_000, + }) + expect(child.error).toBeUndefined() + expect(child.status).toBe(0) + expect(child.stdout).toContain('ownership enforced') + }) + it('tears down reader references before another isolate opens a Session', () => { const script = ` const { Worker } = require('node:worker_threads');