From 226e9c6f9927eacabc6a57b0232e286a5a8b2179 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Mon, 16 Nov 2026 13:35:22 +1300 Subject: [PATCH 1/2] Verify Arrow from prerelease npm installs --- .github/workflows/prebuild-publish.yml | 8 ++++++-- test/v3/layer3/arrow-worker-isolate.test.ts | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/prebuild-publish.yml b/.github/workflows/prebuild-publish.yml index d2a5106..372f192 100644 --- a/.github/workflows/prebuild-publish.yml +++ b/.github/workflows/prebuild-publish.yml @@ -206,17 +206,19 @@ jobs: VER="${GITHUB_REF_NAME#v}" work="$(mktemp -d)" cp -R "$GITHUB_WORKSPACE/test/pack" "$work/test" + cp "$GITHUB_WORKSPACE/scripts/smoke-arrow-e2e.mjs" "$work/smoke-arrow-e2e.mjs" cd "$work" npm init -y >/dev/null # The version may lag in the registry right after publish; retry. ok= for i in 1 2 3 4 5 6; do - if npm install "chdb@$VER" vitest@2 >/tmp/i.log 2>&1; then ok=1; break; fi + if npm install "chdb@$VER" vitest@2 apache-arrow@21.1.0 >/tmp/i.log 2>&1; then ok=1; break; fi echo "install attempt $i failed (registry propagation?); retry in 15s"; sleep 15 done [ -n "$ok" ] || { echo "could not install chdb@$VER"; tail -20 /tmp/i.log; exit 1; } node -e "console.log('verifying installed chdb', require('chdb/package.json').version, 'on', process.version, process.platform, process.arch)" npx vitest run test + node smoke-arrow-e2e.mjs verify-old-distro: # Every leg of `verify` runs on a GitHub runner image, and all four of them @@ -243,15 +245,17 @@ jobs: echo "distro: $(. /etc/os-release; echo "$PRETTY_NAME"), glibc $(ldd --version | head -1 | grep -oE '[0-9]+\.[0-9]+$')" work="$(mktemp -d)" cp -R "$GITHUB_WORKSPACE/test/pack" "$work/test" + cp "$GITHUB_WORKSPACE/scripts/smoke-arrow-e2e.mjs" "$work/smoke-arrow-e2e.mjs" cd "$work" npm init -y >/dev/null ok= for i in 1 2 3 4 5 6; do - if npm install "chdb@$VER" vitest@2 >/tmp/i.log 2>&1; then ok=1; break; fi + if npm install "chdb@$VER" vitest@2 apache-arrow@21.1.0 >/tmp/i.log 2>&1; then ok=1; break; fi echo "install attempt $i failed (registry propagation?); retry in 15s"; sleep 15 done [ -n "$ok" ] || { echo "could not install chdb@$VER"; tail -20 /tmp/i.log; exit 1; } npx vitest run test + node smoke-arrow-e2e.mjs promote: needs: [verify, verify-old-distro] diff --git a/test/v3/layer3/arrow-worker-isolate.test.ts b/test/v3/layer3/arrow-worker-isolate.test.ts index f7c683f..7febcef 100644 --- a/test/v3/layer3/arrow-worker-isolate.test.ts +++ b/test/v3/layer3/arrow-worker-isolate.test.ts @@ -16,11 +16,12 @@ describe('Arrow source in a worker isolate', () => { const {Worker}=require('node:worker_threads'),m=require('./index.js'); (async()=>{const s=new m.Session(); try{const worker=new Worker(${JSON.stringify(workerCode)},{eval:true,workerData:{path:s.path}}); + const exited=new Promise((resolve,reject)=>worker.once('exit',code=>code===0?resolve():reject(Error('exit '+code)))); const value=await new Promise((resolve,reject)=>{ worker.once('message',resolve);worker.once('error',reject) }); if(value!=='ok')throw Error(value); - await new Promise((resolve,reject)=>worker.once('exit',code=>code===0?resolve():reject(Error('exit '+code)))); + await exited; const result=await s.queryAsync('SELECT 7',{format:'CSV'}); if(result.text().trim()!=='7')throw Error('main Session was closed'); console.log('concurrent isolate OK'); From 5f935d16acd83e3cfee733fe94aef28ec91c5ad4 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Tue, 17 Nov 2026 09:32:00 +1300 Subject: [PATCH 2/2] Prepare Node Arrow release on chdb-core 26.11.1 --- .github/workflows/chdb-node-test.yml | 2 +- .github/workflows/prebuild-publish.yml | 14 ++++++++------ README.md | 6 +++--- docs/arrow.md | 4 ++-- package-lock.json | 8 ++++---- update_libchdb.sh | 12 ++++-------- 6 files changed, 22 insertions(+), 24 deletions(-) diff --git a/.github/workflows/chdb-node-test.yml b/.github/workflows/chdb-node-test.yml index 96320b3..84e90d8 100644 --- a/.github/workflows/chdb-node-test.yml +++ b/.github/workflows/chdb-node-test.yml @@ -10,7 +10,7 @@ on: # update_libchdb.sh pins a published chdb-core libchdb across all four target # platforms, so the full v2 + v3 suite is expected green here. The pin is -# currently the v26.11.0-rc.2 release candidate. v26.7.3 was the first stable +# currently the v26.11.1 stable release. v26.7.3 was the first stable # release to export the Durable V1 ABI (chdb_backup_database_n / # chdb_restore_database_n / chdb_classify_query_n); v26.7.2-rc.2 was the # candidate that introduced them. diff --git a/.github/workflows/prebuild-publish.yml b/.github/workflows/prebuild-publish.yml index 372f192..05a12b0 100644 --- a/.github/workflows/prebuild-publish.yml +++ b/.github/workflows/prebuild-publish.yml @@ -211,9 +211,10 @@ jobs: npm init -y >/dev/null # The version may lag in the registry right after publish; retry. ok= - for i in 1 2 3 4 5 6; do - if npm install "chdb@$VER" vitest@2 apache-arrow@21.1.0 >/tmp/i.log 2>&1; then ok=1; break; fi - echo "install attempt $i failed (registry propagation?); retry in 15s"; sleep 15 + for i in $(seq 1 20); do + if npm install --prefer-online "chdb@$VER" vitest@2 apache-arrow@21.1.0 >/tmp/i.log 2>&1 && + node -e "const p=require('chdb/package.json');const n='@chdb/lib-'+process.platform+'-'+process.arch+(process.platform==='linux'?'-gnu':'');if(require(n+'/package.json').version!==p.optionalDependencies[n])process.exit(1)" >>/tmp/i.log 2>&1; then ok=1; break; fi + echo "install attempt $i failed (registry propagation?); retry in 20s"; sleep 20 done [ -n "$ok" ] || { echo "could not install chdb@$VER"; tail -20 /tmp/i.log; exit 1; } node -e "console.log('verifying installed chdb', require('chdb/package.json').version, 'on', process.version, process.platform, process.arch)" @@ -249,9 +250,10 @@ jobs: cd "$work" npm init -y >/dev/null ok= - for i in 1 2 3 4 5 6; do - if npm install "chdb@$VER" vitest@2 apache-arrow@21.1.0 >/tmp/i.log 2>&1; then ok=1; break; fi - echo "install attempt $i failed (registry propagation?); retry in 15s"; sleep 15 + for i in $(seq 1 20); do + if npm install --prefer-online "chdb@$VER" vitest@2 apache-arrow@21.1.0 >/tmp/i.log 2>&1 && + node -e "const p=require('chdb/package.json');const n='@chdb/lib-'+process.platform+'-'+process.arch+(process.platform==='linux'?'-gnu':'');if(require(n+'/package.json').version!==p.optionalDependencies[n])process.exit(1)" >>/tmp/i.log 2>&1; then ok=1; break; fi + echo "install attempt $i failed (registry propagation?); retry in 20s"; sleep 20 done [ -n "$ok" ] || { echo "could not install chdb@$VER"; tail -20 /tmp/i.log; exit 1; } npx vitest run test diff --git a/README.md b/README.md index a8bf2cf..273de40 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ somewhere later. | Streaming results (`AsyncIterable`) | ✅ | | Arrow IPC **output** (`format: 'arrow'` + `toArrow()`) | ✅ | | AbortSignal / timeout | ✅ (single-shot is honest: rejects early; native runs to completion) | -| Direct Arrow **scan**, reader, typed results, and bulk insert | ✅ in source (matching rc.3 native packages pending) | +| Direct Arrow **scan**, reader, typed results, and bulk insert | ✅ with the matching native package | | Arrow zero-copy (M2, `{ zeroCopy: true }`) | ⏳ follow-up | | chDB ↔ `@clickhouse/client` integration (`chdb/connection`, **experimental**) | ✅ | | Durable V1 control plane (`chdb/durable`, **experimental**) | ✅ pure TS, engine injected | @@ -155,8 +155,8 @@ somewhere later. See the [Arrow guide](docs/arrow.md) for examples, lifecycle rules, and the difference between direct C Data/C Stream and the existing Arrow IPC path. -The direct APIs require chdb-core v26.11.0-rc.3; they are not in the currently -published npm platform packages. +The direct APIs require chdb-core v26.11.0-rc.3 or newer. This release pins +v26.11.1 and packages its native library for each supported platform. ### chDB ↔ `@clickhouse/client` integration (`chdb/connection`, experimental) diff --git a/docs/arrow.md b/docs/arrow.md index eb94389..fd57f2a 100644 --- a/docs/arrow.md +++ b/docs/arrow.md @@ -2,11 +2,11 @@ The direct Arrow APIs let you query in-memory Arrow data, read typed results, and insert batches without Arrow IPC. They require chdb-core v26.11.0-rc.3 -or newer. The matching npm platform packages are not published yet. +or newer; this release pins v26.11.1 in its matching npm platform packages. Arrow Inter-Process Communication (IPC) encodes data as a byte stream or file. The direct APIs here exchange in-memory Arrow buffers and batches with the native engine instead; they may still copy data while converting it. A `RecordBatch` is a typed group of columnar rows, and a reader supplies successive batches. -Once the matching packages are released, install `chdb` and its optional peer: +Install `chdb` and its optional peer: ```sh npm install chdb apache-arrow diff --git a/package-lock.json b/package-lock.json index fd3ceb2..9cdf91e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,10 +35,10 @@ "node": ">=18.0.0" }, "optionalDependencies": { - "@chdb/lib-darwin-arm64": "26.11.0-rc.3.1", - "@chdb/lib-darwin-x64": "26.11.0-rc.3.1", - "@chdb/lib-linux-arm64-gnu": "26.11.0-rc.3.1", - "@chdb/lib-linux-x64-gnu": "26.11.0-rc.3.1" + "@chdb/lib-darwin-arm64": "26.11.1-stable.1", + "@chdb/lib-darwin-x64": "26.11.1-stable.1", + "@chdb/lib-linux-arm64-gnu": "26.11.1-stable.1", + "@chdb/lib-linux-x64-gnu": "26.11.1-stable.1" }, "peerDependencies": { "@aws-sdk/client-s3": ">=3", diff --git a/update_libchdb.sh b/update_libchdb.sh index 26dda2e..28e3d87 100755 --- a/update_libchdb.sh +++ b/update_libchdb.sh @@ -16,14 +16,10 @@ set -e # count and target-database proof. chdb/durable's engine adapter cannot be built # without them. v26.7.3 is the first stable release that exports them. # -# The pin is currently a release candidate, and chdb_version() on it returns the -# full "26.11.0-rc.3", suffix and all. Durable records that string in -# head.engine.version and raises min_reader to it on every write. A pre-release -# sorts below the release it leads to, so an object written against this RC -# stays readable by the eventual stable, but every earlier engine (26.11.0-rc.2, -# 26.9.x and before) is refused it with engine_incompatible from then on. -# The Arrow A1–A6 ABI and chdb-core#323's registration ownership fixes are -# present in rc.3. See docs/arrow-rc3-conformance.md for validation scope. +# The pin is a stable release containing the direct Arrow ABI and registration +# ownership fixes first validated against rc.3. Durable records the engine +# version in head.engine.version and refuses earlier incompatible engines. +# See docs/arrow-rc3-conformance.md for the earlier candidate's validation scope. # # Keep the pin on its own line and literal: the release check greps for it when # it proposes a bump.