From 6004b0a0f9d3805d1dcf5c2174f3d842c6ca94b7 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Fri, 13 Nov 2026 13:34:17 +1300 Subject: [PATCH 1/2] Improve Arrow empty sources and cancellation in Node --- docs/arrow.md | 20 +++++++++++- index.d.ts | 2 ++ index.js | 43 ++++++++++++++++++++++--- src/layer3/execute/arrow-input.ts | 24 +++++++++++++- src/layer3/execute/arrow-insert.ts | 21 ++++++++---- src/layer3/execute/arrow-reader.ts | 50 ++++++++++++++++++----------- test/v3/arrow-direct.test.ts | 44 +++++++++++++++++++++++++ test/v3/layer3/arrow-input.test.ts | 15 ++++++++- test/v3/layer3/arrow-insert.test.ts | 36 ++++++++++++++++++++- test/v3/layer3/arrow-reader.test.ts | 22 +++++++++++++ 10 files changed, 245 insertions(+), 32 deletions(-) diff --git a/docs/arrow.md b/docs/arrow.md index 8dc3cb5..3493151 100644 --- a/docs/arrow.md +++ b/docs/arrow.md @@ -53,6 +53,8 @@ choose the name. Names are process-wide; prefer generated names unless you have a reason to coordinate them. Close the registration after the last scan. If a query or lazy stream is active, unregister and buffer release are deferred until that scan finishes or is cancelled. +An Arrow `Table` with a non-empty schema and no batches is a valid zero-row +source; `insertArrow` likewise accepts it as a zero-row insert. ## Insert Arrow data @@ -86,6 +88,11 @@ The native queue holds at most two batches not yet handed to core; producing another waits for capacity. Core can retain delivered batches, so this is not a bound on the entire query's memory. A cancelled producer should cooperate with iterator `return()` to release its own resources. +Pass `{ session, signal }` to `registerArrowReader` or `insertArrowReader` to +cancel the native reader, including while waiting for its first batch. An +upstream iterator blocked inside `next()` cannot be forcibly interrupted; +the caller must arrange for it to cooperate with `return()` or its own signal. +An insert cancelled after it starts is not promised to roll back committed rows. Use `queryAsync` or another async query API while a JS Arrow reader is registered: synchronous `query`/`queryBind` on its connection are rejected because blocking the event loop would prevent the producer from refilling @@ -101,7 +108,9 @@ try { const small = await session.queryArrow('SELECT toInt32(number) AS id FROM numbers(3)') console.log(small.toArray().map(row => row.id)) // [0, 1, 2] - const reader = session.queryArrowStream('SELECT toInt32(number) AS id FROM numbers(1000)') + const controller = new AbortController() + const reader = session.queryArrowStream('SELECT toInt32(number) AS id FROM numbers(1000)', + { signal: controller.signal }) try { for await (const batch of reader) { console.log(batch.numRows) @@ -118,6 +127,11 @@ try { Use `queryArrowBind` and `queryArrowStreamBind` for server-side `{name:Type}` parameters, for example `await session.queryArrowBind('SELECT {id:Int32} AS id', { id: 7 })`. +All four direct output methods accept `signal`. A buffered `queryArrow` settles +early when aborted, but its native computation may still run to completion; +the Session defers teardown until it finishes. A streamed reader cancels its +native handle after any in-flight fetch completes. Neither behavior promises +transaction rollback. For TypeScript return types without making the base `chdb` import require `apache-arrow`, use the helpers from `chdb/arrow`. Only one iterator may consume a given Arrow result reader at a time; cancel @@ -135,6 +149,10 @@ the JS event loop. Timestamp, Decimal128/256, slices, null bitmaps, and a single-chunk dictionary of UTF-8/Binary values with Int32/UInt32 indices. Nested input and multi-chunk dictionaries are rejected rather than silently converted. +- Direct input accepts Arrow objects and batch iterables, not an Arrow IPC + byte stream from a file, `fetch()` response, or Node `Readable`. Decode IPC + to batches with `apache-arrow` before using `registerArrowReader`; the + existing Arrow IPC output path is separate. - Supported output includes basic scalars, Timestamp, Decimal128/256, and LowCardinality UTF-8/Binary dictionaries. Options include `stringAsString`, `lowCardinalityAsDictionary`, and `unsupportedAsBinary`; diff --git a/index.d.ts b/index.d.ts index d5e49b6..efd6e6e 100644 --- a/index.d.ts +++ b/index.d.ts @@ -249,6 +249,8 @@ export interface ChdbArrowReader extends AsyncIterable { } export interface DirectArrowOptions { + /** Abort the caller's wait; an already-running buffered query finishes natively before teardown. */ + signal?: AbortSignal; /** Degrade otherwise unmappable engine types to Arrow Binary. Default false. */ unsupportedAsBinary?: boolean; /** Export ClickHouse String as Arrow Utf8 (default) or Binary. */ diff --git a/index.js b/index.js index c290889..74b126a 100644 --- a/index.js +++ b/index.js @@ -246,6 +246,28 @@ function withAbortTimeout(nativePromise, opts) { }); } +// Direct Arrow returns an apache-arrow Table rather than a ChdbResult. Match +// queryAsync's early-settle abort without wrapping the typed value or releasing +// the connection while its native worker is still running. +function withArrowAbort(promise, signal) { + trackNative(promise); + if (!signal) return promise; + return new Promise((resolve, reject) => { + let settled = false; + const finish = (fn, value) => { + if (settled) return; + settled = true; + signal.removeEventListener('abort', onAbort); + fn(value); + }; + const onAbort = () => finish(reject, new ChdbAbortError( + 'Arrow query aborted (the underlying computation may still run to completion in the background)')); + if (signal.aborted) onAbort(); + else signal.addEventListener('abort', onAbort, { once: true }); + promise.then((value) => finish(resolve, value), (error) => finish(reject, error)); + }); +} + // Parameterized queries set+reset parameter state around each call // (chdb-core's CApiQueryParameterGuard). With the bundled libchdb this set/reset // is NOT isolated per connection under real concurrency: two parameterized @@ -458,20 +480,27 @@ class ChdbQueryStream { // Typed C Data stream. Each native fetch produces at most one RecordBatch; // bytes are copied and C callbacks released before the JS batch is exposed. class ChdbArrowReader { - constructor(handle) { + constructor(handle, signal) { this._handle = handle; this._closed = false; this._inflight = null; this._iterating = false; + this._signal = signal; + this._onAbort = signal ? () => this.cancel() : null; + if (signal) signal.addEventListener('abort', this._onAbort, { once: true }); this.schema = null; // available after the first fetch, even for zero rows activeStreams.add(this); } get closed() { return this._closed; } + _removeAbortListener() { + if (this._signal) this._signal.removeEventListener('abort', this._onAbort); + } async *[Symbol.asyncIterator]() { if (this._iterating) throw new ChdbStreamError("Arrow reader already has an active iterator"); this._iterating = true; try { + if (this._signal && this._signal.aborted) throw new ChdbAbortError('Arrow stream aborted'); while (!this._closed) { let raw; try { @@ -480,20 +509,24 @@ class ChdbArrowReader { } catch (e) { const cancelling = this._closed; this._closed = true; + this._removeAbortListener(); if (!cancelling) { activeStreams.delete(this); flushArrowIdle(); } + if (this._signal && this._signal.aborted) throw new ChdbAbortError('Arrow stream aborted', { cause: e }); throw asQueryError(e); } finally { this._inflight = null; } + if (this._signal && this._signal.aborted) throw new ChdbAbortError('Arrow stream aborted'); if (this._closed) break; const table = arrowTableFromDirect(raw); this.schema = table.schema; if (raw.done) { const cancelling = this._closed; this._closed = true; + this._removeAbortListener(); if (!cancelling) { activeStreams.delete(this); flushArrowIdle(); @@ -511,6 +544,7 @@ class ChdbArrowReader { cancel() { if (this._closed) return; this._closed = true; + this._removeAbortListener(); const destroy = () => { try { chdbNode.CancelArrowStream(this._handle); } catch (_) { /* best effort */ } activeStreams.delete(this); @@ -896,12 +930,12 @@ class Session { queryArrow(query, opts = {}) { if (!this.connection) return Promise.reject(new ChdbClosedError("No active connection available")); if (!query) return Promise.reject(new ChdbStreamError("queryArrow requires a non-empty query")); + if (opts.signal && opts.signal.aborted) return Promise.reject(new ChdbAbortError('Arrow query aborted')); try { _assertArrowCore(); } catch (e) { return Promise.reject(e); } let native; try { native = chdbNode.QueryArrowAsync(this.connection, query, opts); } catch (e) { return Promise.reject(e); } - trackNative(native); - return native.then((raw) => arrowTableFromDirect(raw), (e) => { throw asQueryError(e); }); + return withArrowAbort(native.then((raw) => arrowTableFromDirect(raw), (e) => { throw asQueryError(e); }), opts.signal); } queryArrowStream(query, opts = {}) { @@ -923,6 +957,7 @@ class Session { #startArrowStream(query, opts, bound) { if (!this.connection) throw new ChdbClosedError("No active connection available"); if (!query) throw new ChdbStreamError("queryArrowStream requires a non-empty query"); + if (opts.signal && opts.signal.aborted) throw new ChdbAbortError('Arrow stream aborted'); _assertArrowCore(); if (this._activeStream && !this._activeStream.closed) { throw new ChdbStreamError("a stream is already active on this session; finish or cancel it first"); @@ -930,7 +965,7 @@ class Session { let handle; try { handle = chdbNode.StartArrowStream(this.connection, query, opts, bound); } catch (e) { throw asQueryError(e); } - const stream = new ChdbArrowReader(handle); + const stream = new ChdbArrowReader(handle, opts.signal); this._activeStream = stream; return stream; } diff --git a/src/layer3/execute/arrow-input.ts b/src/layer3/execute/arrow-input.ts index 21af0f3..be3310f 100644 --- a/src/layer3/execute/arrow-input.ts +++ b/src/layer3/execute/arrow-input.ts @@ -82,7 +82,27 @@ export interface ArrowRecordBatchInput { readonly numRows: number readonly numCols: number } -export interface ArrowTableInput { readonly batches: ReadonlyArray } +export interface ArrowTableInput { + /** Required for a zero-batch Table; existing nonempty structural inputs need not supply it. */ + readonly schema?: ArrowRecordBatchInput['schema'] + readonly batches: ReadonlyArray +} + +/** A schema-bearing empty batch for Tables/readers with no data batches. */ +export function emptyArrowBatch(schema: ArrowRecordBatchInput['schema'] | undefined): ArrowRecordBatchInput { + if (!schema?.fields?.length) throw new ChdbCompileError('empty Arrow source must provide a non-empty schema') + return { schema, numCols: schema.fields.length, numRows: 0, + data: { children: schema.fields.map((field) => { + const type = field.type as { toString(): string; indices?: { toString(): string } } + const dictionary = type.toString().startsWith('Dictionary<') + ? { data: [{ offset: 0, length: 0, nullCount: 0, + values: new Uint8Array(0), valueOffsets: new Int32Array([0]) }] } + : undefined + return { offset: 0, length: 0, nullCount: 0, + values: type.indices?.toString() === 'Uint32' ? new Uint32Array(0) : new Int32Array(0), + valueOffsets: new Int32Array([0]), dictionary } + }) } } +} interface ArrowDataInput { readonly offset?: number @@ -341,6 +361,8 @@ export function registerArrowTable( if (Array.isArray(columns)) { validate(name, columns) encoded = columns.map(encodeColumn) + } else if ('batches' in columns && columns.batches.length === 0) { + encoded = encodeArrowBatch(emptyArrowBatch(columns.schema)) } else if ('batches' in columns && columns.batches.length > 1) { encodedBatches = columns.batches.map(encodeArrowBatch) encoded = encodedBatches[0]! diff --git a/src/layer3/execute/arrow-insert.ts b/src/layer3/execute/arrow-insert.ts index 738e969..c1cdef0 100644 --- a/src/layer3/execute/arrow-insert.ts +++ b/src/layer3/execute/arrow-insert.ts @@ -1,5 +1,5 @@ /** Bulk insert through one in-memory Arrow registration and an engine-side INSERT SELECT. */ -import { ChdbCompileError } from '../../errors' +import { ChdbAbortError, ChdbCompileError } from '../../errors' import { quoteIdentifier, quoteIdentifierSegment } from '../compiler/identifier' import { registerArrowSource, type ArrowColumnInput, type ArrowRecordBatchInput, type ArrowTableInput } from './arrow-input' import { registerArrowReader, type ArrowReaderSource } from './arrow-reader' @@ -38,7 +38,8 @@ export async function insertArrow( const batch = !Array.isArray(input) && 'batches' in input ? input.batches[0] : input const columns = Array.isArray(batch) ? batch.map((column) => column.name) - : (batch as ArrowRecordBatchInput).schema?.fields.map((field) => field.name) + : (batch as ArrowRecordBatchInput | undefined)?.schema?.fields.map((field) => field.name) + ?? (!Array.isArray(input) && 'batches' in input ? input.schema?.fields.map((field) => field.name) : undefined) if (!columns?.length || new Set(columns).size !== columns.length) { throw new ChdbCompileError('insertArrow: source needs unique column names') } @@ -64,7 +65,7 @@ export async function insertArrow( export async function insertArrowReader( table: string, source: ArrowReaderSource, - opts: { session: ArrowInsertSession }, + opts: { session: ArrowInsertSession; signal?: AbortSignal }, ): Promise { const destination = quoteIdentifier(table) const reader = await registerArrowReader(source, opts) @@ -72,10 +73,18 @@ export async function insertArrowReader( if (new Set(reader.fields).size !== reader.fields.length) { throw new ChdbCompileError('insertArrowReader: source needs unique column names') } + if (opts.signal?.aborted) throw new ChdbAbortError('Arrow insert aborted') const fieldList = reader.fields.map(quoteIdentifierSegment).join(', ') - const result = await opts.session.queryAsync( - `INSERT INTO ${destination} (${fieldList}) SELECT ${fieldList} FROM arrowstream('${reader.name}')`, - ) + let result: Awaited> + try { + result = await opts.session.queryAsync( + `INSERT INTO ${destination} (${fieldList}) SELECT ${fieldList} FROM arrowstream('${reader.name}')`, + ) + } catch (error) { + if (opts.signal?.aborted) throw new ChdbAbortError('Arrow insert aborted', { cause: error }) + throw error + } + if (opts.signal?.aborted) throw new ChdbAbortError('Arrow insert aborted') await reader.producerDone return { rowsSubmitted: reader.rowsSubmitted, rowsRead: result.rowsRead, bytesRead: result.bytesRead, elapsed: result.elapsed } diff --git a/src/layer3/execute/arrow-reader.ts b/src/layer3/execute/arrow-reader.ts index 0a4e9d8..ab166ff 100644 --- a/src/layer3/execute/arrow-reader.ts +++ b/src/layer3/execute/arrow-reader.ts @@ -1,8 +1,8 @@ /** Bounded async Arrow reader -> core C Stream source. */ import { randomUUID } from 'node:crypto' -import { ChdbCompileError } from '../../errors' +import { ChdbAbortError, ChdbCompileError } from '../../errors' import { runtime, type Runtime } from '../runtime' -import { encodeArrowBatch, type ArrowRecordBatchInput } from './arrow-input' +import { encodeArrowBatch, emptyArrowBatch, type ArrowRecordBatchInput } from './arrow-input' export type ArrowReaderSource = (AsyncIterable | Iterable) & { /** Required only when the reader yields no batches. */ @@ -39,9 +39,10 @@ const readerFinalizers = new FinalizationRegistry<{ */ export async function registerArrowReader( source: ArrowReaderSource, - opts: { session: { readonly connection: unknown } }, + opts: { session: { readonly connection: unknown }; signal?: AbortSignal }, ): Promise { if (!opts?.session?.connection) throw new ChdbCompileError('registerArrowReader requires an active Session') + if (opts.signal?.aborted) throw new ChdbAbortError('Arrow reader registration aborted') runtime()._assertArrowCore() const conn = opts.session.connection const asyncMethod = (source as AsyncIterable)[Symbol.asyncIterator] @@ -51,7 +52,25 @@ export async function registerArrowReader( } const iterator: AsyncIterator | Iterator = typeof asyncMethod === 'function' ? asyncMethod.call(source) : syncMethod.call(source) - const first = await iterator.next() + // Registration must not hold the caller hostage when a network-backed + // iterator has not produced its first batch. An uncooperative next() cannot + // be forcibly stopped; return() is requested without awaiting it. + const first = opts.signal ? await new Promise>((resolve, reject) => { + const onAbort = () => { + try { void Promise.resolve(iterator.return?.()).catch(() => {}) } catch { /* best effort */ } + reject(new ChdbAbortError('Arrow reader registration aborted')) + } + if (opts.signal!.aborted) { onAbort(); return } + opts.signal!.addEventListener('abort', onAbort, { once: true }) + Promise.resolve().then(() => iterator.next()).then( + (value) => { opts.signal!.removeEventListener('abort', onAbort); resolve(value) }, + (error) => { opts.signal!.removeEventListener('abort', onAbort); reject(error) }, + ) + }) : await iterator.next() + if (opts.signal?.aborted) { + try { void Promise.resolve(iterator.return?.()).catch(() => {}) } catch { /* best effort */ } + throw new ChdbAbortError('Arrow reader registration aborted') + } if (opts.session.connection !== conn) { try { await iterator.return?.() } catch { /* source cleanup is best effort */ } throw new ChdbCompileError('registerArrowReader: Session closed while awaiting the first batch') @@ -59,20 +78,8 @@ export async function registerArrowReader( let firstBatch: ArrowRecordBatchInput if (first.done) { const schema = (source as { schema?: ArrowRecordBatchInput['schema'] }).schema - if (!schema?.fields?.length) throw new ChdbCompileError('empty Arrow reader must provide a non-empty schema') - // Do not cross apache-arrow's independent ESM/CJS constructors: build a - // zero-row structural batch from the public schema shape instead. - firstBatch = { schema, numCols: schema.fields.length, numRows: 0, - data: { children: schema.fields.map((field) => { - const type = field.type as { toString(): string; indices?: { toString(): string } } - const dictionary = type.toString().startsWith('Dictionary<') - ? { data: [{ offset: 0, length: 0, nullCount: 0, - values: new Uint8Array(0), valueOffsets: new Int32Array([0]) }] } - : undefined - return { offset: 0, length: 0, nullCount: 0, - values: type.indices?.toString() === 'Uint32' ? new Uint32Array(0) : new Int32Array(0), - valueOffsets: new Int32Array([0]), dictionary } - }) } } + // Apache Arrow has separate ESM/CJS constructors; use its public schema shape. + firstBatch = emptyArrowBatch(schema) } else firstBatch = first.value const rt = runtime() @@ -87,6 +94,7 @@ export async function registerArrowReader( throw e } let closing = false + let onAbort: (() => void) | undefined let rowsSubmitted = first.done ? 0 : firstBatch.numRows const producerDone = (async () => { try { @@ -115,6 +123,7 @@ export async function registerArrowReader( async close() { if (!closing) { closing = true + if (onAbort) opts.signal!.removeEventListener('abort', onAbort) rt._arrowCancelReader(native) rt._arrowUnregister(conn, name, token) readerFinalizers.unregister(result) @@ -127,5 +136,10 @@ export async function registerArrowReader( }, } readerFinalizers.register(result, { rt, native, conn, name, token }, result) + if (opts.signal) { + onAbort = () => { void result.close() } + if (opts.signal.aborted) onAbort() + else opts.signal.addEventListener('abort', onAbort, { once: true }) + } return result } diff --git a/test/v3/arrow-direct.test.ts b/test/v3/arrow-direct.test.ts index 68bef3b..adcfbc1 100644 --- a/test/v3/arrow-direct.test.ts +++ b/test/v3/arrow-direct.test.ts @@ -3,6 +3,32 @@ import type { RecordBatch, Schema, Table } from 'apache-arrow' import { Session, registerArrowReader, drainPending } from '../../index.js' describe('Session.queryArrow — direct C Data output', () => { + it('honors an already-aborted signal without starting a native query', async () => { + const session = new Session() + const controller = new AbortController() + controller.abort() + try { + await expect(session.queryArrow('SELECT 1', { signal: controller.signal })) + .rejects.toMatchObject({ name: 'AbortError', code: 'CHDB_ABORT' }) + expect(() => session.queryArrowStream('SELECT 1', { signal: controller.signal })) + .toThrow(/aborted/) + expect((await session.queryArrow('SELECT toInt32(1) AS x') as Table).numRows).toBe(1) + } finally { session.close() } + }) + + it('settles a buffered Arrow query early on abort but drains native work before close', async () => { + const session = new Session() + const controller = new AbortController() + try { + const pending = session.queryArrow('SELECT toInt32(1) AS x WHERE sleep(0.1)=0', + { signal: controller.signal }) + controller.abort() + await expect(pending).rejects.toMatchObject({ name: 'AbortError', code: 'CHDB_ABORT' }) + await drainPending() + expect((await session.queryArrow('SELECT toInt32(7) AS x') as Table).toArray()[0]!.x).toBe(7) + } finally { session.close(); await drainPending() } + }) + it('rejects rc.2 and older native engines before calling the Arrow ABI', async () => { const native = require('../../build/Release/chdb_node.node') as { EngineVersion(): string } const original = native.EngineVersion @@ -116,6 +142,24 @@ describe('Session.queryArrow — direct C Data output', () => { }) describe('Session.queryArrowStream — incremental C Data batches', () => { + it('aborts an in-flight fetch, then leaves the Session usable', async () => { + const session = new Session() + const controller = new AbortController() + const reader = session.queryArrowStream( + 'SELECT toInt32(number) AS x FROM numbers(4) WHERE sleepEachRow(0.03)=0 SETTINGS max_block_size=1', + { signal: controller.signal }, + ) + try { + const iterator = reader[Symbol.asyncIterator]() + const next = iterator.next() + controller.abort() + await expect(next).rejects.toMatchObject({ name: 'AbortError', code: 'CHDB_ABORT' }) + await drainPending() + expect(reader.closed).toBe(true) + expect((await session.queryArrow('SELECT toInt32(7) AS x') as Table).toArray()[0]!.x).toBe(7) + } finally { reader.cancel(); session.close(); await drainPending() } + }) + it('fetches typed batches incrementally and keeps a stable schema', async () => { const session = new Session() try { diff --git a/test/v3/layer3/arrow-input.test.ts b/test/v3/layer3/arrow-input.test.ts index 20c31a6..3d19caa 100644 --- a/test/v3/layer3/arrow-input.test.ts +++ b/test/v3/layer3/arrow-input.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest' import { registerArrowTable, registerArrowSource, selectFrom, chTable, ChdbCompileError, Session, drainPending } from '../../../index.js' -import { Table, Field, List, Int32, Int64, Uint64, Float32, Float64, Bool, Utf8, Binary, TimestampMillisecond, Decimal, vectorFromArray } from 'apache-arrow' +import { Table, Schema, Field, List, Int32, Int64, Uint64, Float32, Float64, Bool, Utf8, Binary, TimestampMillisecond, Decimal, vectorFromArray } from 'apache-arrow' // Arrow C Data Interface input — register a JS columnar dataset as // `arrowstream('')` and read it through the regular builder. Single-pass @@ -8,6 +8,19 @@ import { Table, Field, List, Int32, Int64, Uint64, Float32, Float64, Bool, Utf8, // repeated reads against the same table. describe('registerArrowTable — round-trips JS data through the engine', () => { + it('registers a schema-bearing Table with no batches as a zero-row source', async () => { + const session = new Session() + const input = new Table(new Schema([new Field('x', new Int32())])) + expect(input.batches).toHaveLength(0) + const source = registerArrowSource(input, { session }) + try { + const sql = `SELECT count() FROM arrowstream('${source.name}')` + expect((await session.queryAsync(sql, { format: 'CSV' })).text().trim()).toBe('0') + source.refresh() + expect((await session.queryAsync(sql, { format: 'CSV' })).text().trim()).toBe('0') + } finally { source.close(); session.close() } + }) + it('generates distinct opaque names for independent Arrow sources', async () => { const session = new Session() const batch = new Table({ v: vectorFromArray([7], new Int32()) }).batches[0]! diff --git a/test/v3/layer3/arrow-insert.test.ts b/test/v3/layer3/arrow-insert.test.ts index 73beaa6..3f68f52 100644 --- a/test/v3/layer3/arrow-insert.test.ts +++ b/test/v3/layer3/arrow-insert.test.ts @@ -1,8 +1,19 @@ import { describe, expect, it } from 'vitest' -import { Int32, Table, Utf8, vectorFromArray } from 'apache-arrow' +import { Field, Int32, Schema, Table, Utf8, vectorFromArray } from 'apache-arrow' import { insertArrow, insertArrowReader, Session } from '../../../index.js' describe('insertArrow — engine-side INSERT SELECT from Arrow input', () => { + it('accepts an empty schema-bearing Table as a zero-row insert', async () => { + const session = new Session() + try { + await session.queryAsync('CREATE TABLE arrow_insert_empty (id Int32) ENGINE=Memory') + const input = new Table(new Schema([new Field('id', new Int32())])) + expect(await insertArrow('arrow_insert_empty', input, { session })).toMatchObject({ rowsSubmitted: 0 }) + const result = await session.queryAsync('SELECT count() FROM arrow_insert_empty', { format: 'CSV' }) + expect(result.text().trim()).toBe('0') + } finally { session.close() } + }) + it('maps source fields by name, preserves nulls, and reports submitted rows', async () => { const session = new Session() try { @@ -79,6 +90,29 @@ describe('insertArrow — engine-side INSERT SELECT from Arrow input', () => { } }) + it('aborts a live reader insert and keeps the Session usable', async () => { + const session = new Session() + const controller = new AbortController() + let reachedSecondBatch!: () => void + const waitingForSecondBatch = new Promise((resolve) => { reachedSecondBatch = resolve }) + try { + await session.queryAsync('CREATE TABLE arrow_insert_aborted (id Int32) ENGINE=Memory') + async function* batches() { + yield new Table({ id: vectorFromArray([1], new Int32()) }).batches[0]! + reachedSecondBatch() + await new Promise(() => {}) // an uncooperative upstream source + } + const pending = insertArrowReader('arrow_insert_aborted', batches(), + { session, signal: controller.signal }) + await waitingForSecondBatch + await new Promise((resolve) => setImmediate(resolve)) // let INSERT start consuming the first batch + controller.abort() + await expect(pending).rejects.toMatchObject({ name: 'AbortError', code: 'CHDB_ABORT' }) + const result = await session.queryAsync('SELECT 7', { format: 'CSV' }) + expect(result.text().trim()).toBe('7') + } finally { session.close() } + }) + it('requires a live Session and unique source field names', async () => { await expect(insertArrow('dest', [{ name: 'id', type: 'Int32', data: new Int32Array([1]) }], { session: null as unknown as Session })).rejects.toThrow(/active Session/) diff --git a/test/v3/layer3/arrow-reader.test.ts b/test/v3/layer3/arrow-reader.test.ts index c5e6ed4..4d85fa9 100644 --- a/test/v3/layer3/arrow-reader.test.ts +++ b/test/v3/layer3/arrow-reader.test.ts @@ -8,6 +8,28 @@ function batch(values: number[]) { } describe('registerArrowReader — bounded C Stream input', () => { + it('can abort registration before a network-backed first batch arrives', async () => { + const session = new Session() + const controller = new AbortController() + let returned = false + const source = { + [Symbol.asyncIterator]() { + return { + next: () => new Promise>>(() => {}), + return: async () => { returned = true; return { done: true as const, value: undefined } }, + } + }, + } + try { + const registration = registerArrowReader(source, { session, signal: controller.signal }) + controller.abort() + await expect(registration).rejects.toMatchObject({ name: 'AbortError', code: 'CHDB_ABORT' }) + expect(returned).toBe(true) + const healthy = await session.queryAsync('SELECT 1', { format: 'CSV' }) + expect(healthy.text().trim()).toBe('1') + } finally { session.close() } + }) + // chdb-core#321/#323: rc.3 must release the registration schema exactly // once; this regression gate is no longer conditional on the engine version. it( From 14ca7fa195cf12401a677cfde60173603d92dc04 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Fri, 13 Nov 2026 17:06:13 +1300 Subject: [PATCH 2/2] Report Arrow stream abort after yielded batch --- index.js | 7 ++++++- test/v3/arrow-direct.test.ts | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 74b126a..68e840c 100644 --- a/index.js +++ b/index.js @@ -533,7 +533,12 @@ class ChdbArrowReader { } break; } - for (const batch of table.batches) yield batch; + for (const batch of table.batches) { + yield batch; + // An abort while the caller holds a batch closes this reader. When + // iteration resumes, report truncation instead of silently ending. + if (this._signal && this._signal.aborted) throw new ChdbAbortError('Arrow stream aborted'); + } } } finally { this._iterating = false; diff --git a/test/v3/arrow-direct.test.ts b/test/v3/arrow-direct.test.ts index adcfbc1..718cde6 100644 --- a/test/v3/arrow-direct.test.ts +++ b/test/v3/arrow-direct.test.ts @@ -142,6 +142,25 @@ describe('Session.queryArrow — direct C Data output', () => { }) describe('Session.queryArrowStream — incremental C Data batches', () => { + it('reports abort after a yielded batch instead of silently ending the iterator', async () => { + const session = new Session() + const controller = new AbortController() + const reader = session.queryArrowStream( + 'SELECT toInt32(number) AS x FROM numbers(4) SETTINGS max_block_size=1', + { signal: controller.signal }, + ) + try { + const iterator = reader[Symbol.asyncIterator]() + const first = await iterator.next() + expect(first.done).toBe(false) + controller.abort() + await expect(iterator.next()).rejects.toMatchObject({ name: 'AbortError', code: 'CHDB_ABORT' }) + await drainPending() + expect(reader.closed).toBe(true) + expect((await session.queryArrow('SELECT toInt32(7) AS x') as Table).toArray()[0]!.x).toBe(7) + } finally { reader.cancel(); session.close(); await drainPending() } + }) + it('aborts an in-flight fetch, then leaves the Session usable', async () => { const session = new Session() const controller = new AbortController()