From 597647e1f34dc245b3f5308a61279be059c3d39e Mon Sep 17 00:00:00 2001 From: Changshuo Chen Date: Wed, 24 Jun 2026 06:55:55 +0000 Subject: [PATCH 1/7] Add the Layer 3 fluent query builder core Scaffold the Layer 3 package and ChdbCompileError, and implement the SQL compiler with server-side value binding ({pN:Type} placeholders + raw values via Layer 1 queryBindAsync; identifiers backtick-quoted). Add the SELECT/INSERT/UPDATE(ALTER..UPDATE)/DELETE(ALTER..DELETE)/INSERT..SELECT builders, expression helpers, terminals, and root factory; ClickHouse dialect sugar (final/sample/prewhere/settings/format/limitBy); chTable and chFn helpers; and golden-SQL, injection, and execution tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- index.d.ts | 6 + index.js | 10 + index.mjs | 6 + src/errors.ts | 11 + src/layer3/builder/delete.ts | 43 ++++ src/layer3/builder/expression.ts | 122 +++++++++++ src/layer3/builder/functions.ts | 83 +++++++ src/layer3/builder/insert.ts | 87 ++++++++ src/layer3/builder/select.ts | 275 ++++++++++++++++++++++++ src/layer3/builder/table-functions.ts | 180 ++++++++++++++++ src/layer3/builder/update.ts | 58 +++++ src/layer3/compiler/compile.ts | 297 ++++++++++++++++++++++++++ src/layer3/compiler/identifier.ts | 52 +++++ src/layer3/compiler/nodes.ts | 168 +++++++++++++++ src/layer3/compiler/param.ts | 82 +++++++ src/layer3/database.ts | 103 +++++++++ src/layer3/execute/format.ts | 72 +++++++ src/layer3/execute/terminal.ts | 76 +++++++ src/layer3/index.ts | 60 ++++++ src/layer3/runtime.ts | 77 +++++++ test/v3/layer3/compile.test.ts | 151 +++++++++++++ test/v3/layer3/dialect.test.ts | 72 +++++++ test/v3/layer3/execute.test.ts | 93 ++++++++ test/v3/layer3/functions.test.ts | 104 +++++++++ test/v3/layer3/injection.test.ts | Bin 0 -> 4500 bytes 25 files changed, 2288 insertions(+) create mode 100644 src/layer3/builder/delete.ts create mode 100644 src/layer3/builder/expression.ts create mode 100644 src/layer3/builder/functions.ts create mode 100644 src/layer3/builder/insert.ts create mode 100644 src/layer3/builder/select.ts create mode 100644 src/layer3/builder/table-functions.ts create mode 100644 src/layer3/builder/update.ts create mode 100644 src/layer3/compiler/compile.ts create mode 100644 src/layer3/compiler/identifier.ts create mode 100644 src/layer3/compiler/nodes.ts create mode 100644 src/layer3/compiler/param.ts create mode 100644 src/layer3/database.ts create mode 100644 src/layer3/execute/format.ts create mode 100644 src/layer3/execute/terminal.ts create mode 100644 src/layer3/index.ts create mode 100644 src/layer3/runtime.ts create mode 100644 test/v3/layer3/compile.test.ts create mode 100644 test/v3/layer3/dialect.test.ts create mode 100644 test/v3/layer3/execute.test.ts create mode 100644 test/v3/layer3/functions.test.ts create mode 100644 test/v3/layer3/injection.test.ts diff --git a/index.d.ts b/index.d.ts index 4a8d793..5dadda5 100644 --- a/index.d.ts +++ b/index.d.ts @@ -356,3 +356,9 @@ export function version(): { arch: string; napi?: number; }; + +// Layer 3: the fluent, immutable query builder, a sibling of the pluggable +// Connection surface (`chdb/connection`), both sitting on Layer 1. `selectFrom`, +// `database`, `insertInto`, `chFn`, `chTable`, the builders, the compile output +// types, and ChdbCompileError. Generated declarations live in dist/layer3. +export * from './dist/layer3'; diff --git a/index.js b/index.js index f2aacfc..dad6085 100644 --- a/index.js +++ b/index.js @@ -729,3 +729,13 @@ function version() { } module.exports = { query, queryBind, queryAsync, queryBindAsync, insert, Session, version, _closeAllSessions, _drainPendingOps }; + +// Layer 3: the fluent, immutable query builder. It sits on Layer 1, a sibling of +// the pluggable Connection surface (`chdb/connection`). Required at the BOTTOM, +// after module.exports is populated, so the lazy Layer 1 accessor in dist/layer3 +// sees a fully-formed export object. ChdbCompileError (the only net-new error) +// rides along via dist/layer3's own re-export. +const layer3 = require('./dist/layer3/index.js'); +for (const name of Object.keys(layer3)) { + if (module.exports[name] === undefined) module.exports[name] = layer3[name]; +} diff --git a/index.mjs b/index.mjs index 65055e2..b67b235 100644 --- a/index.mjs +++ b/index.mjs @@ -16,4 +16,10 @@ export const insert = mod.insert export const Session = mod.Session export const version = mod.version +// Layer 3: ChdbCompileError is the only net-new error class; export it as a +// named ESM binding so it is catchable by class. The fluent builder surface +// (selectFrom / database / insertInto / connect / chFn / chTable / sql / eb) is +// reached through the default export (`import chdb from 'chdb'`). +export const ChdbCompileError = mod.ChdbCompileError + export default mod diff --git a/src/errors.ts b/src/errors.ts index ee8e7e4..82c5981 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -77,6 +77,17 @@ export class ChdbBindError extends ChdbError { readonly code = 'CHDB_BIND' } +/** + * The fluent query builder could not turn a chain into SQL — e.g. a SELECT with + * no source, an unsupported expression, or an empty IN list. This is the only + * net-new error the builder layer introduces: it is raised while building SQL, + * before any query runs, so it never carries a ClickHouse code. Runtime failures + * (syntax, connection, timeout, …) still surface as the existing classes above. + */ +export class ChdbCompileError extends ChdbError { + readonly code = 'CHDB_COMPILE' +} + /** Progress snapshot carried by streaming-insert callbacks and errors (payload-side ledger). */ export interface InsertProgress { /** Rows flushed to the engine so far (non-empty payload lines; exact for line-delimited formats). */ diff --git a/src/layer3/builder/delete.ts b/src/layer3/builder/delete.ts new file mode 100644 index 0000000..7957fc6 --- /dev/null +++ b/src/layer3/builder/delete.ts @@ -0,0 +1,43 @@ +/** + * DELETE chain: `deleteFrom(t).where(...).execute()`. Like UPDATE, this compiles + * to a ClickHouse mutation (`ALTER TABLE … DELETE WHERE …`), not an OLTP delete. + * A WHERE is required. + */ + +import type { ChdbResult } from '../../result' +import { compileQuery, type CompiledQuery } from '../compiler/compile' +import type { DeleteQueryNode } from '../compiler/nodes' +import { + buildPredicate, + combinePredicate, + type ExprInput, + type PredicateArgs, +} from './expression' +import { executeStatement, type ExecContext, type ExecuteOptions } from '../execute/terminal' + +export class DeleteQueryBuilder { + constructor( + private readonly ctx: ExecContext, + private readonly node: DeleteQueryNode, + ) { + Object.freeze(node) + } + + private derive(patch: Partial): DeleteQueryBuilder { + return new DeleteQueryBuilder(this.ctx, { ...this.node, ...patch }) + } + + where(lhs: ExprInput): DeleteQueryBuilder + where(lhs: ExprInput, op: string, rhs: unknown): DeleteQueryBuilder + where(...args: PredicateArgs): DeleteQueryBuilder { + return this.derive({ where: combinePredicate(this.node.where, buildPredicate(args), 'And') }) + } + + compile(): CompiledQuery { + return compileQuery(this.node) + } + + execute(opts?: ExecuteOptions): Promise { + return executeStatement(this.ctx, this.node, opts) + } +} diff --git a/src/layer3/builder/expression.ts b/src/layer3/builder/expression.ts new file mode 100644 index 0000000..fb0673c --- /dev/null +++ b/src/layer3/builder/expression.ts @@ -0,0 +1,122 @@ +/** + * Expression builders: the small set of helpers that turn user input into the + * node tree the compiler walks. + * + * - `sql` — a tagged template for any SQL the verbs don't cover. Interpolated + * values are BOUND (`{pN:Type}`), never spliced; use `sql.ref()` for an + * identifier and `sql.raw()` for trusted, author-controlled SQL text. + * - `ref(name)` — a column/table reference (quote-escaped). + * - `val(value, type?)` — an explicitly bound value (optionally typed). + * - `fn(name, args)` — a function call; string args are column references, + * `ChExpression` args are nested expressions, anything else is a bound value. + * + * `eb` bundles the same helpers for an ergonomic single import. + */ + +import { ChdbCompileError } from '../../errors' +import type { Expr } from '../compiler/nodes' + +/** A wrapped expression node, carrying `.as()` for aliasing. */ +export class ChExpression { + constructor(readonly node: Expr) {} + + /** `expr AS alias`. */ + as(alias: string): ChExpression { + return new ChExpression({ kind: 'Alias', node: this.node, alias }) + } +} + +/** Anything the builder verbs accept where an expression is expected. */ +export type ExprInput = string | ChExpression + +/** A column / table reference. */ +export function ref(name: string): ChExpression { + return new ChExpression({ kind: 'Reference', name }) +} + +/** An explicitly bound value, with an optional ClickHouse placeholder type. */ +export function val(value: unknown, chType?: string): ChExpression { + return new ChExpression({ kind: 'Value', value, chType }) +} + +/** A function call. String args are references; other primitives are bound. */ +export function fn(name: string, args: ReadonlyArray = []): ChExpression { + return new ChExpression({ + kind: 'Function', + name, + args: args.map((a) => { + if (a instanceof ChExpression) return a.node + if (typeof a === 'string') return { kind: 'Reference', name: a } + return { kind: 'Value', value: a } + }), + }) +} + +interface SqlTag { + (strings: TemplateStringsArray, ...values: unknown[]): ChExpression + /** A raw, author-controlled SQL fragment (NOT bound — never pass user input). */ + raw(text: string): ChExpression + /** A quote-escaped identifier. */ + ref(name: string): ChExpression + /** A bound value (optionally typed). */ + val(value: unknown, chType?: string): ChExpression +} + +const sqlTag = ((strings: TemplateStringsArray, ...values: unknown[]): ChExpression => { + const exprs: Expr[] = values.map((v) => + v instanceof ChExpression ? v.node : { kind: 'Value', value: v }, + ) + return new ChExpression({ kind: 'Raw', fragments: [...strings], values: exprs }) +}) as SqlTag + +sqlTag.raw = (text: string) => new ChExpression({ kind: 'Raw', fragments: [text], values: [] }) +sqlTag.ref = ref +sqlTag.val = val + +/** Tagged-template SQL escape hatch (see module docs). */ +export const sql: SqlTag = sqlTag + +/** Bundled expression helpers for a single import. */ +export const eb = { ref, val, fn, sql } + +/** Convert an expression-position input (string ⇒ reference, `*` ⇒ star). */ +export function toExpr(input: ExprInput): Expr { + if (input instanceof ChExpression) return input.node + if (typeof input !== 'string') { + throw new ChdbCompileError(`Expected a column name or expression, got ${typeof input}`) + } + if (input === '*') return { kind: 'Star' } + if (input.endsWith('.*')) return { kind: 'Star', table: input.slice(0, -2) } + const aliasMatch = /^(.+?)\s+as\s+(.+)$/i.exec(input) + if (aliasMatch) { + return { kind: 'Alias', node: { kind: 'Reference', name: aliasMatch[1]!.trim() }, alias: aliasMatch[2]!.trim() } + } + return { kind: 'Reference', name: input } +} + +/** Convert a value-position input: an expression wrapper stays an expression, + * anything else becomes a bound value. */ +export function toValue(input: unknown): Expr { + if (input instanceof ChExpression) return input.node + return { kind: 'Value', value: input } +} + +/** A where/having argument list: `(lhs, op, rhs)` or a single expression. */ +export type PredicateArgs = [ExprInput] | [ExprInput, string, unknown] + +/** Turn a predicate argument list into one expression node. */ +export function buildPredicate(args: PredicateArgs): Expr { + if (args.length === 1) return toExpr(args[0]) + return { kind: 'Binary', left: toExpr(args[0]), op: args[1], right: toValue(args[2]) } +} + +/** Combine an existing predicate with a new one under AND / OR, flattening. */ +export function combinePredicate( + existing: Expr | undefined, + next: Expr, + kind: 'And' | 'Or', +): Expr { + if (existing === undefined) return next + if (existing.kind === kind) return { kind, items: [...existing.items, next] } + return { kind, items: [existing, next] } +} diff --git a/src/layer3/builder/functions.ts b/src/layer3/builder/functions.ts new file mode 100644 index 0000000..a1920c9 --- /dev/null +++ b/src/layer3/builder/functions.ts @@ -0,0 +1,83 @@ +/** + * `chFn` — typed wrappers for the ClickHouse-specific functions an analyst + * reaches for most. The 190+ other functions stay reachable through the generic + * `fn(name, args)` (you supply the return type) or `sql\`...\``; these 8 are the + * common ones worth a named helper. + * + * String args are column references, `ChExpression` args are nested expressions, + * and any other value is bound — the same rule as `fn()`. Parametric aggregates + * (`name(params)(args)`) are assembled so their parameters bind too. + */ + +import { ChExpression, fn, val, type ExprInput } from './expression' +import { validateFunctionName } from '../compiler/identifier' +import type { Expr } from '../compiler/nodes' + +function toArg(input: ExprInput | unknown): Expr { + if (input instanceof ChExpression) return input.node + if (typeof input === 'string') return { kind: 'Reference', name: input } + return { kind: 'Value', value: input } +} + +/** + * Build a parametric aggregate `name(params)(args)` as a raw node whose params + * and args are still compiled (so values among them bind). Function name is + * validated eagerly. + */ +function parametric( + name: string, + params: ReadonlyArray, + args: ReadonlyArray, +): ChExpression { + validateFunctionName(name) + const values = [...params.map(toArg), ...args.map(toArg)] + const fragments: string[] = [`${name}(`] + for (let i = 1; i < params.length; i++) fragments.push(', ') + fragments.push(')(') + for (let i = 1; i < args.length; i++) fragments.push(', ') + fragments.push(')') + return new ChExpression({ kind: 'Raw', fragments, values }) +} + +export const chFn = { + /** `argMax(arg, val)` — the `arg` value at the row maximizing `val`. */ + argMax(arg: ExprInput, val: ExprInput): ChExpression { + return fn('argMax', [arg, val]) + }, + + /** `argMin(arg, val)` — the `arg` value at the row minimizing `val`. */ + argMin(arg: ExprInput, val: ExprInput): ChExpression { + return fn('argMin', [arg, val]) + }, + + /** `uniqExact(cols…)` — exact distinct count. */ + uniqExact(...columns: ExprInput[]): ChExpression { + return fn('uniqExact', columns) + }, + + /** `retention(conds…)` — retention flags across the given conditions. */ + retention(...conditions: ExprInput[]): ChExpression { + return fn('retention', conditions) + }, + + /** `topK(k)(column)` — the k most frequent values. */ + topK(k: number, column: ExprInput): ChExpression { + return parametric('topK', [k], [column]) + }, + + /** `quantileTDigest(level)(column)` — approximate quantile via t-digest. */ + quantileTDigest(level: number, column: ExprInput): ChExpression { + return parametric('quantileTDigest', [level], [column]) + }, + + /** `windowFunnel(window)(timestamp, conds…)` — funnel analysis. */ + windowFunnel(window: number, timestamp: ExprInput, ...conditions: ExprInput[]): ChExpression { + return parametric('windowFunnel', [window], [timestamp, ...conditions]) + }, + + /** `sequenceMatch(pattern)(timestamp, conds…)` — event-sequence matching. */ + sequenceMatch(pattern: string, timestamp: ExprInput, ...conditions: ExprInput[]): ChExpression { + // The pattern is a bound value, not a column reference, so wrap it in val(). + return parametric('sequenceMatch', [val(pattern)], [timestamp, ...conditions]) + }, +} diff --git a/src/layer3/builder/insert.ts b/src/layer3/builder/insert.ts new file mode 100644 index 0000000..033615a --- /dev/null +++ b/src/layer3/builder/insert.ts @@ -0,0 +1,87 @@ +/** + * INSERT chain: `insertInto(table).values(source).execute()`. The `source` is + * polymorphic (a single verb extended by its argument type, not several verbs): + * + * - an object/array-of-values row array → forwarded to Layer 1's `insert` + * (the multithreaded engine parser does the work; no SQL is built here); + * - a SelectQueryBuilder → compiled to `INSERT INTO … SELECT` and run. + * + * Arrow / TypedArray / stream sources are handled in a later step (the Arrow IPC + * input path); this is the row-array and subquery foundation. + */ + +import { ChdbCompileError } from '../../errors' +import type { ChdbResult } from '../../result' +import type { CompiledQuery } from '../compiler/compile' +import { compileQuery } from '../compiler/compile' +import type { InsertSelectNode } from '../compiler/nodes' +import { runtime, type RuntimeInsertParams, type RuntimeInsertSummary } from '../runtime' +import { executeStatement, type ExecContext, type ExecuteOptions } from '../execute/terminal' +import { SelectQueryBuilder } from './select' + +export type InsertRow = Record | ReadonlyArray +export type InsertColumns = ReadonlyArray | { except: ReadonlyArray } + +/** Terminal for a row-array insert (forwards to Layer 1 `insert`). */ +export class InsertValuesExecutable { + constructor( + private readonly ctx: ExecContext, + private readonly table: string, + private readonly rows: ReadonlyArray, + private readonly cols?: InsertColumns, + ) {} + + execute(): Promise { + const params: RuntimeInsertParams = { table: this.table, values: this.rows, columns: this.cols } + return this.ctx.session ? this.ctx.session.insert(params) : runtime().insert(params) + } +} + +/** Terminal for `INSERT INTO … SELECT` (compiles to SQL like the SELECT path). */ +export class InsertSelectExecutable { + constructor( + private readonly ctx: ExecContext, + private readonly node: InsertSelectNode, + ) {} + + /** Emit `{ sql, parameters }` without running anything. */ + compile(): CompiledQuery { + return compileQuery(this.node) + } + + execute(opts?: ExecuteOptions): Promise { + return executeStatement(this.ctx, this.node, opts) + } +} + +export class InsertQueryBuilder { + constructor( + private readonly ctx: ExecContext, + private readonly table: string, + private readonly cols?: ReadonlyArray, + ) {} + + /** Restrict the target columns: `INSERT INTO t (a, b) …`. */ + columns(columns: ReadonlyArray): InsertQueryBuilder { + return new InsertQueryBuilder(this.ctx, this.table, columns) + } + + /** Insert object/array rows (forwarded to Layer 1's engine-parsed insert). */ + values(rows: ReadonlyArray): InsertValuesExecutable + /** Insert the result of a SELECT (`INSERT INTO … SELECT`). */ + values(query: SelectQueryBuilder): InsertSelectExecutable + values(source: ReadonlyArray | SelectQueryBuilder): InsertValuesExecutable | InsertSelectExecutable { + if (source instanceof SelectQueryBuilder) { + return new InsertSelectExecutable(this.ctx, { + kind: 'InsertSelect', + table: this.table, + columns: this.cols, + select: source.toNode(), + }) + } + if (!Array.isArray(source)) { + throw new ChdbCompileError('values() expects a row array or a SELECT query builder') + } + return new InsertValuesExecutable(this.ctx, this.table, source, this.cols) + } +} diff --git a/src/layer3/builder/select.ts b/src/layer3/builder/select.ts new file mode 100644 index 0000000..2091a4d --- /dev/null +++ b/src/layer3/builder/select.ts @@ -0,0 +1,275 @@ +/** + * SelectQueryBuilder — the immutable SELECT chain. Every method returns a NEW + * builder over a NEW frozen node, so `const q2 = q1.where(...)` derives from `q1` + * without mutating it and an unexecuted query can be passed around and composed. + * + * Verb names and clause order copy the Kysely v0.29.x shape (subject-first: + * `selectFrom().where().groupBy().orderBy().limit()`); the runtime is + * chDB's own, not Kysely's. + */ + +import { ChdbCompileError } from '../../errors' +import { compileQuery, type CompiledQuery } from '../compiler/compile' +import type { + Expr, + JoinKind, + OrderByItem, + SelectQueryNode, + SetOperator, +} from '../compiler/nodes' +import { ChExpression, toExpr, toValue, type ExprInput } from './expression' +import { + executeSelect, + type ExecContext, + type ExecuteOptions, +} from '../execute/terminal' + +/** Combine an existing predicate with a new one under AND / OR, flattening. */ +function combine(existing: Expr | undefined, next: Expr, kind: 'And' | 'Or'): Expr { + if (existing === undefined) return next + if (existing.kind === kind) return { kind, items: [...existing.items, next] } + return { kind, items: [existing, next] } +} + +/** One element of a where/having call: either `(lhs, op, rhs)` or a single expression. */ +type ColumnInput = ExprInput + +export class SelectQueryBuilder> { + constructor( + private readonly ctx: ExecContext, + private readonly node: SelectQueryNode, + ) { + Object.freeze(node) + } + + private derive(patch: Partial): SelectQueryBuilder { + return new SelectQueryBuilder(this.ctx, { ...this.node, ...patch }) + } + + // ---- projection ----------------------------------------------------------- + + /** Add selections (accumulates across calls). Each is a column / expression. */ + select(selection: ColumnInput | ReadonlyArray): SelectQueryBuilder { + const incoming = (Array.isArray(selection) ? selection : [selection]).map(toExpr) + return this.derive({ selections: [...(this.node.selections ?? []), ...incoming] }) + } + + /** Select everything (`SELECT *`). */ + selectAll(): SelectQueryBuilder { + return this.derive({ selections: [{ kind: 'Star' }] }) + } + + distinct(): SelectQueryBuilder { + return this.derive({ distinct: true }) + } + + // ---- filtering ------------------------------------------------------------ + + private predicate(args: [ColumnInput] | [ColumnInput, string, unknown]): Expr { + if (args.length === 1) return toExpr(args[0]) + const [lhs, op, rhs] = args + return { kind: 'Binary', left: toExpr(lhs), op, right: toValue(rhs) } + } + + /** AND a predicate. `(col, op, value)` or a single expression. */ + where(lhs: ColumnInput): SelectQueryBuilder + where(lhs: ColumnInput, op: string, rhs: unknown): SelectQueryBuilder + where(...args: [ColumnInput] | [ColumnInput, string, unknown]): SelectQueryBuilder { + return this.derive({ where: combine(this.node.where, this.predicate(args), 'And') }) + } + + /** Alias for {@link where} (explicit AND). */ + andWhere(lhs: ColumnInput): SelectQueryBuilder + andWhere(lhs: ColumnInput, op: string, rhs: unknown): SelectQueryBuilder + andWhere(...args: [ColumnInput] | [ColumnInput, string, unknown]): SelectQueryBuilder { + return this.derive({ where: combine(this.node.where, this.predicate(args), 'And') }) + } + + /** OR a predicate into the WHERE clause. */ + orWhere(lhs: ColumnInput): SelectQueryBuilder + orWhere(lhs: ColumnInput, op: string, rhs: unknown): SelectQueryBuilder + orWhere(...args: [ColumnInput] | [ColumnInput, string, unknown]): SelectQueryBuilder { + return this.derive({ where: combine(this.node.where, this.predicate(args), 'Or') }) + } + + having(lhs: ColumnInput): SelectQueryBuilder + having(lhs: ColumnInput, op: string, rhs: unknown): SelectQueryBuilder + having(...args: [ColumnInput] | [ColumnInput, string, unknown]): SelectQueryBuilder { + return this.derive({ having: combine(this.node.having, this.predicate(args), 'And') }) + } + + // ---- grouping / ordering / paging ---------------------------------------- + + groupBy(columns: ColumnInput | ReadonlyArray): SelectQueryBuilder { + const incoming = (Array.isArray(columns) ? columns : [columns]).map(toExpr) + return this.derive({ groupBy: [...(this.node.groupBy ?? []), ...incoming] }) + } + + orderBy(column: ColumnInput, direction: 'asc' | 'desc' = 'asc'): SelectQueryBuilder { + const item: OrderByItem = { expr: toExpr(column), direction } + return this.derive({ orderBy: [...(this.node.orderBy ?? []), item] }) + } + + limit(count: number): SelectQueryBuilder { + return this.derive({ limit: count }) + } + + offset(count: number): SelectQueryBuilder { + return this.derive({ offset: count }) + } + + // ---- joins ---------------------------------------------------------------- + + private join( + joinType: JoinKind, + source: ExprInput, + leftKey?: string, + rightKey?: string, + ): SelectQueryBuilder { + const on = + leftKey !== undefined && rightKey !== undefined + ? ({ + kind: 'Binary', + left: { kind: 'Reference', name: leftKey }, + op: '=', + right: { kind: 'Reference', name: rightKey }, + } as Expr) + : undefined + return this.derive({ + joins: [...(this.node.joins ?? []), { kind: 'Join', joinType, source: toExpr(source), on }], + }) + } + + innerJoin(source: ExprInput, leftKey: string, rightKey: string): SelectQueryBuilder { + return this.join('Inner', source, leftKey, rightKey) + } + + leftJoin(source: ExprInput, leftKey: string, rightKey: string): SelectQueryBuilder { + return this.join('Left', source, leftKey, rightKey) + } + + fullJoin(source: ExprInput, leftKey: string, rightKey: string): SelectQueryBuilder { + return this.join('Full', source, leftKey, rightKey) + } + + crossJoin(source: ExprInput): SelectQueryBuilder { + return this.join('Cross', source) + } + + // ---- ClickHouse dialect sugar -------------------------------------------- + + /** `FROM t FINAL` — merge parts on read (ReplacingMergeTree, etc.). */ + final(): SelectQueryBuilder { + return this.derive({ final: true }) + } + + /** `SAMPLE rate` — approximate query over a fraction (or absolute count) of rows. */ + sample(rate: number): SelectQueryBuilder { + return this.derive({ sample: rate }) + } + + /** + * `PREWHERE` — ClickHouse reads these columns first and filters before the + * rest of the row is read. Same argument shapes as {@link where}; multiple + * calls are AND-combined. + */ + prewhere(lhs: ColumnInput): SelectQueryBuilder + prewhere(lhs: ColumnInput, op: string, rhs: unknown): SelectQueryBuilder + prewhere(...args: [ColumnInput] | [ColumnInput, string, unknown]): SelectQueryBuilder { + return this.derive({ prewhere: combine(this.node.prewhere, this.predicate(args), 'And') }) + } + + /** Append a trailing `SETTINGS k = v, …` clause (merges across calls). */ + settings(settings: Record): SelectQueryBuilder { + return this.derive({ settings: { ...this.node.settings, ...settings } }) + } + + /** + * `FORMAT x` — the SQL-level output format (changes how ClickHouse serializes + * the result). This is distinct from `.execute({ format })`, which picks the + * view over the bytes (`Row[]` / Arrow `Table` / raw). Use this only when you + * read the raw result yourself. + */ + format(name: string): SelectQueryBuilder { + return this.derive({ format: name }) + } + + /** + * `LIMIT n BY (cols)` — keep the first n rows per distinct value of the + * columns. Independent of (and combinable with) the trailing `LIMIT`. + */ + limitBy(count: number, columns: ColumnInput | ReadonlyArray): SelectQueryBuilder { + const cols = (Array.isArray(columns) ? columns : [columns]).map(toExpr) + return this.derive({ limitBy: { count, columns: cols } }) + } + + // ---- set operations ------------------------------------------------------- + + private setOp(operator: SetOperator, other: SelectQueryBuilder): SelectQueryBuilder { + return this.derive({ + setOps: [...(this.node.setOps ?? []), { operator, query: other.toNode() }], + }) + } + + union(other: SelectQueryBuilder): SelectQueryBuilder { + return this.setOp('union', other) + } + + unionAll(other: SelectQueryBuilder): SelectQueryBuilder { + return this.setOp('unionAll', other) + } + + intersect(other: SelectQueryBuilder): SelectQueryBuilder { + return this.setOp('intersect', other) + } + + except(other: SelectQueryBuilder): SelectQueryBuilder { + return this.setOp('except', other) + } + + // ---- composition ---------------------------------------------------------- + + /** The underlying node (for use as a subquery / set-operation operand). */ + toNode(): SelectQueryNode { + return this.node + } + + /** Wrap this query as an aliased subquery usable as a source or in a join. */ + as(alias: string): ChExpression { + return new ChExpression({ + kind: 'Alias', + node: { kind: 'Subquery', query: this.node }, + alias, + }) + } + + // ---- terminals ------------------------------------------------------------ + + /** Emit `{ sql, parameters }` without running anything (debug / audit / tools). */ + compile(): CompiledQuery { + return compileQuery(this.node) + } + + /** Run and return rows (default), an Arrow Table (`{format:'arrow'}`), or a raw result. */ + execute(opts?: ExecuteOptions & { format?: undefined | 'json' }): Promise + execute(opts: ExecuteOptions & { format: 'arrow' }): Promise + execute(opts: ExecuteOptions & { format: string }): Promise + execute(opts?: ExecuteOptions): Promise { + return executeSelect(this.ctx, this.node, opts) + } + + /** Run and return the first row, or `undefined` when there are none. */ + async executeTakeFirst(opts?: ExecuteOptions): Promise { + const rows = (await executeSelect(this.ctx, this.node, opts)) as O[] + return rows[0] + } + + /** Run and return the first row, throwing if there are none. */ + async executeTakeFirstOrThrow(opts?: ExecuteOptions): Promise { + const first = await this.executeTakeFirst(opts) + if (first === undefined) { + throw new ChdbCompileError('executeTakeFirstOrThrow: the query returned no rows') + } + return first + } +} diff --git a/src/layer3/builder/table-functions.ts b/src/layer3/builder/table-functions.ts new file mode 100644 index 0000000..ef3ff26 --- /dev/null +++ b/src/layer3/builder/table-functions.ts @@ -0,0 +1,180 @@ +/** + * `chTable` — typed helpers for the ClickHouse table functions used most often + * as federation / file sources. Each returns an expression you alias and pass to + * `selectFrom`: + * + * selectFrom(chTable.s3({ url, format: 'Parquet' }).as('t')).selectAll() + * + * Every argument is bound server-side (`{pN:Type}`) exactly like a value in a + * WHERE clause — a URL or path is data, never spliced into the SQL — and chDB + * accepts bound parameters in table-function argument position, so this keeps + * the zero-interpolation rule intact for sources too. The 8 most-used functions + * are here; the long tail stays reachable via `sql.raw(...)`. + */ + +import { ChExpression } from './expression' +import type { Expr } from '../compiler/nodes' + +/** Build a table-function expression with bound arguments. */ +function tableFn(name: string, args: ReadonlyArray<{ value: unknown; type?: string }>): ChExpression { + const argNodes: Expr[] = args.map((a) => ({ kind: 'Value', value: a.value, chType: a.type })) + return new ChExpression({ kind: 'Function', name, args: argNodes }) +} + +/** Drop trailing undefined optional args (ClickHouse positional signatures). */ +function compact( + args: ReadonlyArray<{ value: unknown; type?: string } | undefined>, +): { value: unknown; type?: string }[] { + const out: { value: unknown; type?: string }[] = [] + for (const a of args) { + if (a === undefined) break + out.push(a) + } + return out +} + +const str = (value: string) => ({ value, type: 'String' }) + +export interface S3Options { + url: string + format?: string + structure?: string + accessKeyId?: string + secretAccessKey?: string +} + +export interface FileOptions { + path: string + format?: string + structure?: string +} + +export interface UrlOptions { + url: string + format?: string + structure?: string +} + +export interface PgOptions { + /** `host:port`. */ + host: string + database: string + table: string + user?: string + password?: string + schema?: string +} + +export interface MySqlOptions { + host: string + database: string + table: string + user?: string + password?: string +} + +export interface LakeOptions { + url: string + accessKeyId?: string + secretAccessKey?: string +} + +export const chTable = { + /** `numbers(count)` or `numbers(start, count)` — generated integer source. */ + numbers(countOrStart: number | bigint, count?: number | bigint): ChExpression { + const args = + count === undefined + ? [{ value: countOrStart, type: 'UInt64' }] + : [ + { value: countOrStart, type: 'UInt64' }, + { value: count, type: 'UInt64' }, + ] + return tableFn('numbers', args) + }, + + /** `s3(url[, access_key, secret], [format], [structure])`. */ + s3(opts: S3Options): ChExpression { + const creds = + opts.accessKeyId !== undefined && opts.secretAccessKey !== undefined + ? [str(opts.accessKeyId), str(opts.secretAccessKey)] + : [] + return tableFn( + 's3', + compact([ + str(opts.url), + ...creds.map((c) => c as { value: unknown; type?: string }), + opts.format !== undefined ? str(opts.format) : undefined, + opts.structure !== undefined ? str(opts.structure) : undefined, + ]), + ) + }, + + /** `file(path[, format][, structure])`. */ + file(opts: FileOptions): ChExpression { + return tableFn( + 'file', + compact([ + str(opts.path), + opts.format !== undefined ? str(opts.format) : undefined, + opts.structure !== undefined ? str(opts.structure) : undefined, + ]), + ) + }, + + /** `url(url[, format][, structure])`. */ + url(opts: UrlOptions): ChExpression { + return tableFn( + 'url', + compact([ + str(opts.url), + opts.format !== undefined ? str(opts.format) : undefined, + opts.structure !== undefined ? str(opts.structure) : undefined, + ]), + ) + }, + + /** `postgresql(host:port, database, table[, user, password][, schema])`. */ + postgresql(opts: PgOptions): ChExpression { + const auth = + opts.user !== undefined && opts.password !== undefined + ? [str(opts.user), str(opts.password)] + : [] + return tableFn( + 'postgresql', + compact([ + str(opts.host), + str(opts.database), + str(opts.table), + ...auth, + opts.schema !== undefined ? str(opts.schema) : undefined, + ]), + ) + }, + + /** `mysql(host:port, database, table, user, password)`. */ + mysql(opts: MySqlOptions): ChExpression { + const auth = + opts.user !== undefined && opts.password !== undefined + ? [str(opts.user), str(opts.password)] + : [] + return tableFn('mysql', compact([str(opts.host), str(opts.database), str(opts.table), ...auth])) + }, + + /** `iceberg(url[, access_key, secret])`. */ + iceberg(opts: LakeOptions): ChExpression { + const creds = + opts.accessKeyId !== undefined && opts.secretAccessKey !== undefined + ? [str(opts.accessKeyId), str(opts.secretAccessKey)] + : [] + return tableFn('iceberg', compact([str(opts.url), ...creds])) + }, + + /** `deltaLake(url[, access_key, secret])`. */ + deltaLake(opts: LakeOptions): ChExpression { + const creds = + opts.accessKeyId !== undefined && opts.secretAccessKey !== undefined + ? [str(opts.accessKeyId), str(opts.secretAccessKey)] + : [] + return tableFn('deltaLake', compact([str(opts.url), ...creds])) + }, +} diff --git a/src/layer3/builder/update.ts b/src/layer3/builder/update.ts new file mode 100644 index 0000000..af83f0f --- /dev/null +++ b/src/layer3/builder/update.ts @@ -0,0 +1,58 @@ +/** + * UPDATE chain: `updateTable(t).set({...}).where(...).execute()`. ClickHouse has + * no row-level UPDATE statement, so this compiles to an `ALTER TABLE … UPDATE …` + * mutation — documented as mutation semantics (asynchronous, heavyweight), not + * an OLTP point update. A WHERE is required (the compiler refuses an unguarded + * whole-table mutation). + */ + +import type { ChdbResult } from '../../result' +import { compileQuery, type CompiledQuery } from '../compiler/compile' +import type { Expr, UpdateQueryNode } from '../compiler/nodes' +import { + buildPredicate, + combinePredicate, + toValue, + type ExprInput, + type PredicateArgs, +} from './expression' +import { executeStatement, type ExecContext, type ExecuteOptions } from '../execute/terminal' + +export class UpdateQueryBuilder { + constructor( + private readonly ctx: ExecContext, + private readonly node: UpdateQueryNode, + ) { + Object.freeze(node) + } + + private derive(patch: Partial): UpdateQueryBuilder { + return new UpdateQueryBuilder(this.ctx, { ...this.node, ...patch }) + } + + /** + * Column assignments. Values are bound (`{pN:Type}`) unless wrapped as an + * expression (e.g. `sql\`col + 1\``). Accumulates across calls. + */ + set(assignments: Record): UpdateQueryBuilder { + const incoming = Object.keys(assignments).map((column) => ({ + column, + value: toValue(assignments[column]) as Expr, + })) + return this.derive({ assignments: [...this.node.assignments, ...incoming] }) + } + + where(lhs: ExprInput): UpdateQueryBuilder + where(lhs: ExprInput, op: string, rhs: unknown): UpdateQueryBuilder + where(...args: PredicateArgs): UpdateQueryBuilder { + return this.derive({ where: combinePredicate(this.node.where, buildPredicate(args), 'And') }) + } + + compile(): CompiledQuery { + return compileQuery(this.node) + } + + execute(opts?: ExecuteOptions): Promise { + return executeStatement(this.ctx, this.node, opts) + } +} diff --git a/src/layer3/compiler/compile.ts b/src/layer3/compiler/compile.ts new file mode 100644 index 0000000..a48ad95 --- /dev/null +++ b/src/layer3/compiler/compile.ts @@ -0,0 +1,297 @@ +/** + * The compiler: an immutable node tree → `{ sql, parameters }`. The SQL string + * contains only keywords, quoted identifiers, and `{pN:Type}` placeholders; + * every user value lives in `parameters` and is bound server-side by Layer 1. + * + * One {@link ParamCollector} is shared across the whole statement (subqueries + * and CTEs included) so placeholder names never collide. + */ + +import { ChdbCompileError } from '../../errors' +import { ParamCollector } from './param' +import { quoteIdentifier, validateFunctionName } from './identifier' +import type { + DeleteQueryNode, + Expr, + InsertSelectNode, + QueryNode, + SelectQueryNode, + UpdateQueryNode, +} from './nodes' + +/** The compiled output handed to Layer 1 `queryBindAsync`. */ +export interface CompiledQuery { + readonly sql: string + readonly parameters: Record +} + +// Comparison / arithmetic / logical operators the builder may emit, mapped to +// their canonical SQL token. The builder controls which token reaches here, but +// validating again means a stray operator can never become injected SQL. +const OPERATORS: Readonly> = { + '=': '=', + '!=': '!=', + '<>': '<>', + '<': '<', + '<=': '<=', + '>': '>', + '>=': '>=', + '+': '+', + '-': '-', + '*': '*', + '/': '/', + '%': '%', + like: 'LIKE', + 'not like': 'NOT LIKE', + ilike: 'ILIKE', + 'not ilike': 'NOT ILIKE', + in: 'IN', + 'not in': 'NOT IN', + is: 'IS', + 'is not': 'IS NOT', +} + +export function canonicalOperator(op: string): string { + const canon = OPERATORS[op.toLowerCase()] + if (canon === undefined) { + throw new ChdbCompileError(`Unsupported operator ${JSON.stringify(op)}`) + } + return canon +} + +class Compiler { + readonly params = new ParamCollector() + + expr(node: Expr): string { + switch (node.kind) { + case 'Value': + return this.params.bind(node.value, node.chType) + case 'Reference': + return quoteIdentifier(node.name) + case 'Star': + return node.table ? `${quoteIdentifier(node.table)}.*` : '*' + case 'Raw': { + // Interleave the literal fragments with their bound values: fragment[i] + // is followed by values[i]. The fragments are author-controlled SQL; the + // values are bound, never spliced. + let out = node.fragments[0] ?? '' + for (let i = 0; i < node.values.length; i++) { + out += this.expr(node.values[i] as Expr) + (node.fragments[i + 1] ?? '') + } + return out + } + case 'Function': { + const name = validateFunctionName(node.name) + return `${name}(${node.args.map((a) => this.expr(a)).join(', ')})` + } + case 'Binary': { + const op = canonicalOperator(node.op) + return `${this.expr(node.left)} ${op} ${this.expr(node.right)}` + } + case 'And': + case 'Or': { + if (node.items.length === 0) { + throw new ChdbCompileError(`Empty ${node.kind === 'And' ? 'AND' : 'OR'} group`) + } + const sep = node.kind === 'And' ? ' AND ' : ' OR ' + return `(${node.items.map((i) => this.expr(i)).join(sep)})` + } + case 'Alias': + return `${this.expr(node.node)} AS ${quoteIdentifier(node.alias)}` + case 'Subquery': + return `(${this.select(node.query)})` + } + } + + /** Compile a SELECT without trailing SETTINGS / FORMAT (so set operations can wrap it). */ + private selectBody(node: SelectQueryNode): string { + if (node.from === undefined && (node.selections === undefined || node.selections.length === 0)) { + throw new ChdbCompileError('SELECT requires a source (selectFrom) or explicit selections') + } + const parts: string[] = ['SELECT'] + if (node.distinct) parts.push('DISTINCT') + parts.push( + node.selections && node.selections.length > 0 + ? node.selections.map((s) => this.expr(s)).join(', ') + : '*', + ) + + if (node.from !== undefined) { + let from = `FROM ${this.expr(node.from)}` + if (node.final) from += ' FINAL' + if (node.sample !== undefined) from += ` SAMPLE ${formatNumber(node.sample)}` + parts.push(from) + } + + for (const join of node.joins ?? []) { + const kw = + join.joinType === 'Inner' + ? 'INNER JOIN' + : join.joinType === 'Left' + ? 'LEFT JOIN' + : join.joinType === 'Full' + ? 'FULL JOIN' + : 'CROSS JOIN' + let clause = `${kw} ${this.expr(join.source)}` + if (join.on !== undefined) clause += ` ON ${this.expr(join.on)}` + parts.push(clause) + } + + if (node.prewhere !== undefined) parts.push(`PREWHERE ${this.expr(node.prewhere)}`) + if (node.where !== undefined) parts.push(`WHERE ${this.expr(node.where)}`) + if (node.groupBy && node.groupBy.length > 0) { + parts.push(`GROUP BY ${node.groupBy.map((g) => this.expr(g)).join(', ')}`) + } + if (node.having !== undefined) parts.push(`HAVING ${this.expr(node.having)}`) + if (node.orderBy && node.orderBy.length > 0) { + parts.push( + `ORDER BY ${node.orderBy + .map((o) => `${this.expr(o.expr)} ${o.direction === 'desc' ? 'DESC' : 'ASC'}`) + .join(', ')}`, + ) + } + if (node.limitBy) { + parts.push( + `LIMIT ${formatInteger(node.limitBy.count)} BY ${node.limitBy.columns + .map((c) => this.expr(c)) + .join(', ')}`, + ) + } + if (node.limit !== undefined) parts.push(`LIMIT ${formatInteger(node.limit)}`) + if (node.offset !== undefined) parts.push(`OFFSET ${formatInteger(node.offset)}`) + return parts.join(' ') + } + + select(node: SelectQueryNode): string { + let sql = '' + if (node.ctes && node.ctes.length > 0) { + const recursive = node.ctes.some((c) => c.recursive) + sql += + 'WITH ' + + (recursive ? 'RECURSIVE ' : '') + + node.ctes + .map((c) => `${quoteIdentifier(c.name)} AS (${this.select(c.query)})`) + .join(', ') + + ' ' + } + sql += this.selectBody(node) + for (const op of node.setOps ?? []) { + const kw = + op.operator === 'union' + ? 'UNION DISTINCT' + : op.operator === 'unionAll' + ? 'UNION ALL' + : op.operator === 'intersect' + ? 'INTERSECT' + : 'EXCEPT' + sql += ` ${kw} ${this.selectBody(op.query)}` + } + if (node.settings && Object.keys(node.settings).length > 0) { + sql += ' SETTINGS ' + formatSettings(node.settings) + } + if (node.format !== undefined) sql += ` FORMAT ${validateFormatName(node.format)}` + return sql + } + + update(node: UpdateQueryNode): string { + if (node.assignments.length === 0) { + throw new ChdbCompileError('updateTable requires at least one .set() assignment') + } + const sets = node.assignments + .map((a) => `${quoteIdentifier(a.column)} = ${this.expr(a.value)}`) + .join(', ') + // ClickHouse mutations require a WHERE; a guard prevents an accidental + // whole-table mutation from a forgotten predicate. + if (node.where === undefined) { + throw new ChdbCompileError( + 'updateTable requires a .where(); use .where(chdb.sql`1`) to update every row on purpose', + ) + } + return `ALTER TABLE ${quoteIdentifier(node.table)} UPDATE ${sets} WHERE ${this.expr(node.where)}` + } + + delete(node: DeleteQueryNode): string { + if (node.where === undefined) { + throw new ChdbCompileError( + 'deleteFrom requires a .where(); use .where(chdb.sql`1`) to delete every row on purpose', + ) + } + return `ALTER TABLE ${quoteIdentifier(node.table)} DELETE WHERE ${this.expr(node.where)}` + } + + insertSelect(node: InsertSelectNode): string { + const cols = + node.columns && node.columns.length > 0 + ? ` (${node.columns.map((c) => quoteIdentifier(c)).join(', ')})` + : '' + return `INSERT INTO ${quoteIdentifier(node.table)}${cols} ${this.select(node.select)}` + } +} + +/** Compile any top-level query node into `{ sql, parameters }`. */ +export function compileQuery(node: QueryNode): CompiledQuery { + const c = new Compiler() + let sql: string + switch (node.kind) { + case 'SelectQuery': + sql = c.select(node) + break + case 'UpdateQuery': + sql = c.update(node) + break + case 'DeleteQuery': + sql = c.delete(node) + break + case 'InsertSelect': + sql = c.insertSelect(node) + break + } + return { sql, parameters: c.params.parameters } +} + +function formatNumber(n: number): string { + if (!Number.isFinite(n)) throw new ChdbCompileError(`Expected a finite number, got ${n}`) + return String(n) +} + +function formatInteger(n: number): string { + if (!Number.isInteger(n) || n < 0) { + throw new ChdbCompileError(`Expected a non-negative integer, got ${n}`) + } + return String(n) +} + +// A settings name is an identifier; the value is bound shape-checked here (not +// param-bound, since SETTINGS does not accept placeholders) and rendered as a +// safe literal: numbers/booleans verbatim, strings single-quoted+escaped. +const SETTING_NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/ + +function formatSettings(settings: Readonly>): string { + return Object.keys(settings) + .map((name) => { + if (!SETTING_NAME_RE.test(name)) { + throw new ChdbCompileError(`Invalid setting name ${JSON.stringify(name)}`) + } + const v = settings[name] + let rendered: string + if (typeof v === 'number') { + if (!Number.isFinite(v)) throw new ChdbCompileError(`Invalid setting value for ${name}`) + rendered = String(v) + } else if (typeof v === 'boolean') { + rendered = v ? '1' : '0' + } else { + rendered = `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` + } + return `${name} = ${rendered}` + }) + .join(', ') +} + +const FORMAT_NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/ + +function validateFormatName(name: string): string { + if (!FORMAT_NAME_RE.test(name)) { + throw new ChdbCompileError(`Invalid FORMAT name ${JSON.stringify(name)}`) + } + return name +} diff --git a/src/layer3/compiler/identifier.ts b/src/layer3/compiler/identifier.ts new file mode 100644 index 0000000..942d315 --- /dev/null +++ b/src/layer3/compiler/identifier.ts @@ -0,0 +1,52 @@ +/** + * Identifier quoting for table / column / alias names. Identifiers are the one + * thing that legitimately enters the SQL string (a value never does), so they + * are backtick-quoted and the two characters that could break out of a + * backtick-quoted ClickHouse identifier — the backslash and the backtick — are + * escaped. This makes an arbitrary identifier safe to emit without restricting + * it to an alphanumeric whitelist. + * + * A dotted name is treated as a qualifier path (`db.table`, `t.col`): each + * segment is quoted independently so `events.country` becomes + * `` `events`.`country` ``, not a single quoted name containing a dot. + */ + +import { ChdbCompileError } from '../../errors' + +function quoteSegment(segment: string): string { + // Escape the backslash first (it is the escape introducer), then the backtick. + return '`' + segment.replace(/\\/g, '\\\\').replace(/`/g, '\\`') + '`' +} + +/** + * Quote a possibly-dotted identifier. An empty name or an empty segment + * (leading/trailing/double dot) is a build-time error, since it can only come + * from a malformed builder call. + */ +export function quoteIdentifier(name: string): string { + if (typeof name !== 'string' || name.length === 0) { + throw new ChdbCompileError(`Invalid identifier ${JSON.stringify(name)}: must be a non-empty string`) + } + const segments = name.split('.') + if (segments.some((s) => s.length === 0)) { + throw new ChdbCompileError( + `Invalid identifier ${JSON.stringify(name)}: empty segment (a leading, trailing, or doubled '.')`, + ) + } + return segments.map(quoteSegment).join('.') +} + +// Function and aggregate-combinator names are emitted into the SQL text, so they +// are validated against a strict pattern rather than quoted (ClickHouse function +// names are unquoted). This blocks any attempt to smuggle SQL through a function +// name passed to the generic `fn()` helper. +const FUNCTION_NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/ + +export function validateFunctionName(name: string): string { + if (typeof name !== 'string' || !FUNCTION_NAME_RE.test(name)) { + throw new ChdbCompileError( + `Invalid function name ${JSON.stringify(name)}: expected [A-Za-z_][A-Za-z0-9_]*`, + ) + } + return name +} diff --git a/src/layer3/compiler/nodes.ts b/src/layer3/compiler/nodes.ts new file mode 100644 index 0000000..3dc615a --- /dev/null +++ b/src/layer3/compiler/nodes.ts @@ -0,0 +1,168 @@ +/** + * The immutable node tree the fluent builder accumulates and the compiler walks. + * + * Every chained call returns a new builder wrapping a new, frozen node — values + * the user supplies live in `ValueNode`s and never touch the SQL string; the + * compiler turns each into a `{pN:Type}` placeholder bound server-side. Anything + * that names a column/table is a `ReferenceNode` (quote-escaped, never bound). + */ + +/** A user value to bind server-side. `chType` overrides the inferred type. */ +export interface ValueNode { + readonly kind: 'Value' + readonly value: unknown + /** Explicit ClickHouse type for the `{pN:Type}` placeholder, when known. */ + readonly chType?: string +} + +/** A column / table / alias reference (quote-escaped, dotted parts split). */ +export interface ReferenceNode { + readonly kind: 'Reference' + readonly name: string +} + +/** Pre-built SQL with its own ordered bound values (chdb.sql tagged template). */ +export interface RawNode { + readonly kind: 'Raw' + /** SQL fragments; between fragment i and i+1 sits values[i]. */ + readonly fragments: readonly string[] + readonly values: readonly Expr[] +} + +/** `name(arg, arg, …)` — a function call. */ +export interface FunctionNode { + readonly kind: 'Function' + readonly name: string + readonly args: readonly Expr[] +} + +/** `left right` (op is a fixed operator token, never user text). */ +export interface BinaryNode { + readonly kind: 'Binary' + readonly left: Expr + readonly op: string + readonly right: Expr +} + +/** A list of expressions joined by AND / OR. */ +export interface JunctionNode { + readonly kind: 'And' | 'Or' + readonly items: readonly Expr[] +} + +/** `expr AS alias`. */ +export interface AliasNode { + readonly kind: 'Alias' + readonly node: Expr + readonly alias: string +} + +/** `*` (optionally table-qualified). */ +export interface StarNode { + readonly kind: 'Star' + readonly table?: string +} + +/** A parenthesized sub-select used as a source or scalar. */ +export interface SubqueryNode { + readonly kind: 'Subquery' + readonly query: SelectQueryNode +} + +export type Expr = + | ValueNode + | ReferenceNode + | RawNode + | FunctionNode + | BinaryNode + | JunctionNode + | AliasNode + | StarNode + | SubqueryNode + +export type JoinKind = 'Inner' | 'Left' | 'Full' | 'Cross' + +export interface JoinNode { + readonly kind: 'Join' + readonly joinType: JoinKind + readonly source: Expr + /** `ON` predicate (absent for CROSS JOIN). */ + readonly on?: Expr +} + +export interface OrderByItem { + readonly expr: Expr + readonly direction: 'asc' | 'desc' +} + +/** `LIMIT n BY (cols)` — independent of the trailing LIMIT clause. */ +export interface LimitByNode { + readonly count: number + readonly columns: readonly Expr[] +} + +export type SetOperator = 'union' | 'unionAll' | 'intersect' | 'except' + +export interface SetOperationNode { + readonly operator: SetOperator + readonly query: SelectQueryNode +} + +export interface CteNode { + readonly name: string + readonly recursive: boolean + readonly query: SelectQueryNode +} + +export interface SelectQueryNode { + readonly kind: 'SelectQuery' + readonly from?: Expr + readonly selections?: readonly Expr[] + readonly distinct?: boolean + readonly joins?: readonly JoinNode[] + readonly where?: Expr + readonly prewhere?: Expr + readonly groupBy?: readonly Expr[] + readonly having?: Expr + readonly orderBy?: readonly OrderByItem[] + readonly limit?: number + readonly offset?: number + readonly limitBy?: LimitByNode + readonly ctes?: readonly CteNode[] + readonly setOps?: readonly SetOperationNode[] + // ClickHouse dialect modifiers (Layer 3 extension surface). + readonly final?: boolean + readonly sample?: number + readonly settings?: Readonly> + /** SQL-level `FORMAT x` (distinct from the execute() output view). */ + readonly format?: string +} + +/** `ALTER TABLE t UPDATE col = expr, … WHERE pred` (ClickHouse mutation). */ +export interface UpdateQueryNode { + readonly kind: 'UpdateQuery' + readonly table: string + readonly assignments: readonly { readonly column: string; readonly value: Expr }[] + readonly where?: Expr +} + +/** `ALTER TABLE t DELETE WHERE pred` (ClickHouse mutation). */ +export interface DeleteQueryNode { + readonly kind: 'DeleteQuery' + readonly table: string + readonly where?: Expr +} + +/** `INSERT INTO t (cols) SELECT …`. Row-array inserts go through Layer 1 insert. */ +export interface InsertSelectNode { + readonly kind: 'InsertSelect' + readonly table: string + readonly columns?: readonly string[] + readonly select: SelectQueryNode +} + +export type QueryNode = + | SelectQueryNode + | UpdateQueryNode + | DeleteQueryNode + | InsertSelectNode diff --git a/src/layer3/compiler/param.ts b/src/layer3/compiler/param.ts new file mode 100644 index 0000000..7df8fd6 --- /dev/null +++ b/src/layer3/compiler/param.ts @@ -0,0 +1,82 @@ +/** + * Server-side parameter binding: every user value the builder carries becomes a + * ClickHouse named placeholder `{pN:Type}` in the SQL string, while the value + * itself goes into a separate `parameters` map handed to Layer 1's + * `queryBindAsync`. The engine binds each value by its declared type; no value + * is ever spliced into the SQL text. This is the hard security boundary — + * injection is impossible by construction, not by runtime filtering. + * + * Layer 1 owns the value→literal serialization (it applies the same TSV/Escaped + * encoding its own queryBind path uses), so this module only has to (a) hand out + * unique placeholder names and (b) pick the ClickHouse type for each placeholder. + * The raw JS value is what ends up in the `parameters` map, which keeps + * `.compile()` output readable and auditable. + */ + +/** + * Infer the ClickHouse type for a `{pN:Type}` placeholder from a JS value. + * + * Precision-safe mapping (widenings are deliberate and documented): + * - string → String + * - boolean → Bool + * - integer number → Int64 (a non-integer number → Float64) + * - bigint → Int64 (pass an explicit type for Int128/256) + * - Date → DateTime (sub-second is dropped; use an explicit + * DateTime64 type to keep it) + * - Array / TypedArray → Array(elem), elem inferred from the first + * non-null item; Nullable(elem) if any item is null + * - Map / plain object → Map(String, String) (override for typed maps) + * - null / undefined → Nullable(String) + * + * Callers can always override with an explicit type (see {@link ParamCollector.bind}). + */ +export function inferChType(value: unknown): string { + if (value === null || value === undefined) return 'Nullable(String)' + switch (typeof value) { + case 'string': + return 'String' + case 'boolean': + return 'Bool' + case 'bigint': + return 'Int64' + case 'number': + return Number.isInteger(value) ? 'Int64' : 'Float64' + } + if (value instanceof Date) return 'DateTime' + if (Array.isArray(value) || (ArrayBuffer.isView(value) && !(value instanceof DataView))) { + const arr = Array.isArray(value) + ? value + : Array.from(value as unknown as ArrayLike) + const sample = arr.find((x) => x !== null && x !== undefined) + const elem = sample === undefined ? 'String' : inferChType(sample) + const hasNull = arr.some((x) => x === null || x === undefined) + // Only scalar element types take a Nullable() wrapper — ClickHouse rejects a + // Nullable over a composite (Array/Map) type. + const wrap = hasNull && !elem.startsWith('Array(') && !elem.startsWith('Map(') + return `Array(${wrap ? `Nullable(${elem})` : elem})` + } + // Map and plain objects map to Map(String, String) by default; a typed map + // needs an explicit placeholder type. + return 'Map(String, String)' +} + +/** + * Hands out unique `{pN:Type}` placeholders and accumulates the raw values into + * a single `parameters` map for the whole statement (one collector per compile, + * so names stay unique across subqueries and CTEs). + */ +export class ParamCollector { + readonly parameters: Record = {} + #next = 0 + + /** + * Bind a value and return its placeholder text. `chType` overrides the + * inferred ClickHouse type (e.g. a schema-known `UInt64` or `DateTime64(3)`). + */ + bind(value: unknown, chType?: string): string { + const name = `p${this.#next++}` + this.parameters[name] = value + const type = chType ?? inferChType(value) + return `{${name}:${type}}` + } +} diff --git a/src/layer3/database.ts b/src/layer3/database.ts new file mode 100644 index 0000000..1226bbb --- /dev/null +++ b/src/layer3/database.ts @@ -0,0 +1,103 @@ +/** + * Root factory for the fluent builder. + * + * - `selectFrom(table)` — the top-level shortcut on the default connection. + * - `database(opts?)` — a typed root; pass a Session to pin the connection. + * - `session(path?)` — open a Layer 1 Session and bind the builder to it (for + * persistent / multi-step pipelines); call `.close()` when done. + * + * A `Database` is just a thin holder of the execution context (which connection + * to run on); the builders carry all the query state. + */ + +import { runtime, type RuntimeSession } from './runtime' +import type { Expr, SelectQueryNode } from './compiler/nodes' +import { toExpr, type ExprInput } from './builder/expression' +import { SelectQueryBuilder } from './builder/select' +import { InsertQueryBuilder } from './builder/insert' +import { UpdateQueryBuilder } from './builder/update' +import { DeleteQueryBuilder } from './builder/delete' +import type { ExecContext } from './execute/terminal' + +/** Anything accepted as a SELECT source: a table name, an expression, or a subquery builder. */ +export type FromInput = ExprInput | SelectQueryBuilder + +function toFrom(source: FromInput): Expr { + if (source instanceof SelectQueryBuilder) { + return { kind: 'Subquery', query: source.toNode() } + } + return toExpr(source) +} + +/** A builder root bound to a specific execution context (connection / session). */ +export class Database> { + constructor(private readonly ctx: ExecContext) {} + + /** Start a SELECT from a table, expression, table function, or subquery. */ + selectFrom>(source: FromInput): SelectQueryBuilder { + const node: SelectQueryNode = { kind: 'SelectQuery', from: toFrom(source) } + return new SelectQueryBuilder(this.ctx, node) + } + + /** Start an INSERT into a table (row arrays or `INSERT … SELECT`). */ + insertInto(table: string): InsertQueryBuilder { + return new InsertQueryBuilder(this.ctx, table) + } + + /** Start an UPDATE (compiles to a ClickHouse `ALTER TABLE … UPDATE` mutation). */ + updateTable(table: string): UpdateQueryBuilder { + return new UpdateQueryBuilder(this.ctx, { kind: 'UpdateQuery', table, assignments: [] }) + } + + /** Start a DELETE (compiles to a ClickHouse `ALTER TABLE … DELETE` mutation). */ + deleteFrom(table: string): DeleteQueryBuilder { + return new DeleteQueryBuilder(this.ctx, { kind: 'DeleteQuery', table }) + } + + /** The bound Session, if this root was created with one. */ + get session(): RuntimeSession | undefined { + return this.ctx.session + } + + /** Close the bound Session (no-op on the default connection). */ + close(): void { + this.ctx.session?.close() + } +} + +const DEFAULT = new Database({}) + +/** Start a SELECT on the default connection (the README hero shortcut). */ +export function selectFrom>(source: FromInput): SelectQueryBuilder { + return DEFAULT.selectFrom(source) +} + +/** Start an INSERT on the default connection. */ +export function insertInto(table: string): InsertQueryBuilder { + return DEFAULT.insertInto(table) +} + +/** Start an UPDATE (ClickHouse mutation) on the default connection. */ +export function updateTable(table: string): UpdateQueryBuilder { + return DEFAULT.updateTable(table) +} + +/** Start a DELETE (ClickHouse mutation) on the default connection. */ +export function deleteFrom(table: string): DeleteQueryBuilder { + return DEFAULT.deleteFrom(table) +} + +/** Create a typed builder root. Pass `{ session }` to pin the connection. */ +export function database>(opts?: { session?: RuntimeSession }): Database { + return new Database({ session: opts?.session }) +} + +/** + * Open a Layer 1 Session and bind the builder to it. The returned root owns the + * session — call `.close()` to release it (and remove the temp dir for an + * unnamed session). + */ +export function session>(path?: string): Database { + const s = new (runtime().Session)(path) as unknown as RuntimeSession + return new Database({ session: s }) +} diff --git a/src/layer3/execute/format.ts b/src/layer3/execute/format.ts new file mode 100644 index 0000000..940e40a --- /dev/null +++ b/src/layer3/execute/format.ts @@ -0,0 +1,72 @@ +/** + * Maps the `.execute({ format })` OUTPUT-VIEW selector to the ClickHouse engine + * format plus how to read the result back. This is deliberately distinct from + * the `.format()` SQL clause: `.format()` changes what ClickHouse serializes, + * while `execute({format})` only chooses which view of the bytes you get back + * (`Row[]` / Arrow `Table` / raw `ChdbResult`). + */ + +import { ChdbCompileError } from '../../errors' + +export type OutputView = 'rows' | 'arrow' | 'raw' + +export interface FormatPlan { + /** The format string handed to Layer 1's queryBindAsync. */ + readonly chFormat: string + /** How the terminal reads the ChdbResult back. */ + readonly view: OutputView + /** Settings the view needs injected (e.g. quote 64-bit ints for precision). */ + readonly settings?: Record +} + +// The default row view: JSONEachRow with 64-bit integers quoted so UInt64/Int64 +// survive as strings instead of losing precision through JS numbers (the +// precision-safe type policy). The bytes are parsed line-by-line, not as one +// JSON document. +const ROWS_PLAN: FormatPlan = { + chFormat: 'JSONEachRow', + view: 'rows', + settings: { output_format_json_quote_64bit_integers: 1 }, +} + +// Common shorthands → canonical ClickHouse format names. An unknown name is +// passed through verbatim (the engine validates it) so any ClickHouse format +// stays reachable without a new mapping. +const RAW_FORMATS: Readonly> = { + csv: 'CSV', + csvwithnames: 'CSVWithNames', + tsv: 'TSV', + tabseparated: 'TabSeparated', + parquet: 'Parquet', + jsoneachrow: 'JSONEachRow', + json: 'JSON', + pretty: 'Pretty', +} + +/** + * Resolve a Layer 3 output format. `'json'` (or omitted) → parsed `Row[]`; + * `'arrow'` → an apache-arrow `Table`; any other name → a raw `ChdbResult` the + * caller reads via `.text()` / `.bytes()`. + */ +export function planFormat(format?: string): FormatPlan { + if (format === undefined || format.toLowerCase() === 'json') return ROWS_PLAN + // Pass the lowercase 'arrow' alias through verbatim: Layer 1 maps it to + // ClickHouse's ArrowStream and disables Arrow output compression (which + // apache-arrow JS cannot decode), so toArrow() works. + if (format.toLowerCase() === 'arrow') return { chFormat: 'arrow', view: 'arrow' } + const key = format.toLowerCase() + const chFormat = RAW_FORMATS[key] ?? format + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(chFormat)) { + throw new ChdbCompileError(`Invalid output format ${JSON.stringify(format)}`) + } + return { chFormat, view: 'raw' } +} + +/** Parse a JSONEachRow payload (newline-delimited JSON objects) into rows. */ +export function parseRows(text: string): T[] { + const out: T[] = [] + for (const line of text.split('\n')) { + if (line.length > 0) out.push(JSON.parse(line) as T) + } + return out +} diff --git a/src/layer3/execute/terminal.ts b/src/layer3/execute/terminal.ts new file mode 100644 index 0000000..7df1d38 --- /dev/null +++ b/src/layer3/execute/terminal.ts @@ -0,0 +1,76 @@ +/** + * The execution boundary. The build phase is pure (it only assembles nodes and + * throws ChdbCompileError); IO happens only here, where a compiled + * `{ sql, parameters }` is forwarded to Layer 1 `queryBindAsync`. Layer 1 owns + * value serialization and the typed runtime-error hierarchy, so this layer adds + * no new runtime error types. + */ + +import type { ChdbResult } from '../../result' +import { runtime, type RuntimeSession } from '../runtime' +import { compileQuery, type CompiledQuery } from '../compiler/compile' +import type { QueryNode, SelectQueryNode } from '../compiler/nodes' +import { planFormat, parseRows } from './format' + +/** The connection a builder runs on: a bound Session, or the default connection. */ +export interface ExecContext { + readonly session?: RuntimeSession +} + +/** Pass-through runtime options for a terminal call. */ +export interface ExecuteOptions { + /** Output view: 'json' (default) ⇒ Row[], 'arrow' ⇒ Table, else raw ChdbResult. */ + format?: string + signal?: AbortSignal + timeout?: number + /** Extra ClickHouse settings merged into the statement's SETTINGS clause. */ + settings?: Record +} + +function mergeSettings( + node: SelectQueryNode, + extra?: Record, +): SelectQueryNode { + if (extra === undefined || Object.keys(extra).length === 0) return node + return { ...node, settings: { ...node.settings, ...extra } } +} + +async function run( + ctx: ExecContext, + compiled: CompiledQuery, + chFormat: string, + opts: ExecuteOptions, +): Promise { + const runOpts = { format: chFormat, signal: opts.signal, timeout: opts.timeout } + return ctx.session + ? ctx.session.queryBindAsync(compiled.sql, compiled.parameters, runOpts) + : runtime().queryBindAsync(compiled.sql, compiled.parameters, runOpts) +} + +/** Compile a SELECT and run it, returning rows / an Arrow Table / a raw result. */ +export async function executeSelect( + ctx: ExecContext, + node: SelectQueryNode, + opts: ExecuteOptions = {}, +): Promise { + const plan = planFormat(opts.format) + // The row view's precision setting and any caller-supplied settings ride in + // the statement's SETTINGS clause (server-side), not the engine format flag. + const effective = mergeSettings(node, { ...plan.settings, ...opts.settings }) + const compiled = compileQuery(effective) + const result = await run(ctx, compiled, plan.chFormat, opts) + if (plan.view === 'rows') return parseRows(result.text()) + if (plan.view === 'arrow') return result.toArrow() + return result +} + +/** Compile and run any query node that returns no row view (UPDATE/DELETE/INSERT…SELECT). */ +export async function executeStatement( + ctx: ExecContext, + node: QueryNode, + opts: ExecuteOptions = {}, +): Promise { + const compiled = compileQuery(node) + // Mutations and INSERT…SELECT have no row payload; CSV keeps the response tiny. + return run(ctx, compiled, 'CSV', opts) +} diff --git a/src/layer3/index.ts b/src/layer3/index.ts new file mode 100644 index 0000000..7270c1d --- /dev/null +++ b/src/layer3/index.ts @@ -0,0 +1,60 @@ +/** + * Layer 3 public surface — the fluent, immutable query builder. It borrows the + * verb names and clause order of the Kysely v0.29.x shape but vendors none of + * its runtime: a chain accumulates a node tree, `.compile()` emits + * `{ sql, parameters }`, and execution forwards to Layer 1. Every user value is + * bound server-side via `{pN:Type}` placeholders — no value is ever spliced into + * the SQL string. + * + * This is a sibling of the pluggable Connection surface (`chdb/connection`), not + * built on top of it; both sit directly on Layer 1. + */ + +export { ChdbCompileError } from '../errors' + +// Root factory. +export { + selectFrom, + insertInto, + updateTable, + deleteFrom, + database, + session, + Database, +} from './database' +export type { FromInput } from './database' + +// Builder + expression helpers. +export { SelectQueryBuilder } from './builder/select' +export { + InsertQueryBuilder, + InsertValuesExecutable, + InsertSelectExecutable, +} from './builder/insert' +export type { InsertRow, InsertColumns } from './builder/insert' +export { UpdateQueryBuilder } from './builder/update' +export { DeleteQueryBuilder } from './builder/delete' +export { sql, eb, ref, val, fn, ChExpression } from './builder/expression' +export type { ExprInput } from './builder/expression' + +// ClickHouse table-function and function helpers. +export { chTable } from './builder/table-functions' +export type { + S3Options, + FileOptions, + UrlOptions, + PgOptions, + MySqlOptions, + LakeOptions, +} from './builder/table-functions' +export { chFn } from './builder/functions' + +// Compiler output (for `.compile()` consumers and tooling). +export { compileQuery } from './compiler/compile' +export type { CompiledQuery } from './compiler/compile' + +// Execution options. +export type { ExecuteOptions, ExecContext } from './execute/terminal' + +// AST node types. +export type * from './compiler/nodes' diff --git a/src/layer3/runtime.ts b/src/layer3/runtime.ts new file mode 100644 index 0000000..5fe0654 --- /dev/null +++ b/src/layer3/runtime.ts @@ -0,0 +1,77 @@ +/** + * Lazy accessor for the Layer 1 runtime (the package's CommonJS entry, + * `index.js`). The fluent builder is a sibling of the pluggable Connection + * surface (`chdb/connection`): it + * compiles a chain to `{ sql, parameters }` and forwards execution to Layer 1's + * `queryBindAsync` / `Session`. It never touches the native addon directly. + * + * The require is lazy (resolved on first call, not at module load) so the + * load-time cycle is broken: `index.js` re-exports this surface at its bottom, + * while this module pulls Layer 1 back in. By the time any builder terminal + * actually runs, `index.js`'s exports are fully populated. + */ + +import type { ChdbResult } from '../result' + +export interface RuntimeQueryOptions { + format?: string + signal?: AbortSignal + timeout?: number + /** + * When set, the `parameters` passed to `queryBindAsync` are already the + * engine's `{name: literal}` bound map and are bound verbatim. The fluent + * compiler always pre-serializes its values (so it owns the JS→ClickHouse + * type mapping), so it always sets this. + */ + preformatted?: boolean +} + +export interface RuntimeStreamOptions { + format?: string + signal?: AbortSignal +} + +export interface RuntimeInsertParams { + table: string + values: ReadonlyArray | ReadonlyArray> + columns?: ReadonlyArray | { except: ReadonlyArray } +} + +export interface RuntimeInsertSummary { + rowsWritten: number + bytesRead: number + elapsed: number +} + +/** The subset of Layer 1's Session surface the fluent builder uses. */ +export interface RuntimeSession { + readonly path: string + readonly isTemp: boolean + readonly open: boolean + queryAsync(query: string, opts?: RuntimeQueryOptions): Promise + queryBindAsync(query: string, params: object, opts?: RuntimeQueryOptions): Promise + insert(params: RuntimeInsertParams): Promise + queryStream(query: string, opts?: RuntimeStreamOptions): unknown + close(): void +} + +export interface RuntimeSessionCtor { + new (path?: string, opts?: { installSignalHandlers?: boolean }): RuntimeSession +} + +export interface Runtime { + Session: RuntimeSessionCtor + queryAsync(query: string, opts?: RuntimeQueryOptions): Promise + queryBindAsync(query: string, params: object, opts?: RuntimeQueryOptions): Promise + insert(params: RuntimeInsertParams): Promise +} + +let cached: Runtime | undefined + +export function runtime(): Runtime { + if (cached === undefined) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + cached = require('../../index.js') as Runtime + } + return cached +} diff --git a/test/v3/layer3/compile.test.ts b/test/v3/layer3/compile.test.ts new file mode 100644 index 0000000..d20f634 --- /dev/null +++ b/test/v3/layer3/compile.test.ts @@ -0,0 +1,151 @@ +import { describe, it, expect } from 'vitest' +import { + selectFrom, + insertInto, + updateTable, + deleteFrom, + sql, + fn, + ref, + val, +} from '../../../index.js' + +// Golden-SQL tests: assert .compile()'s { sql, parameters } exactly. Every user +// value must show up as a {pN:Type} placeholder in the SQL and as a separate +// entry in parameters — so the zero-interpolation rule is auditable at a glance. + +describe('SELECT compilation', () => { + it('compiles the canonical filtered, ordered, limited query', () => { + const q = selectFrom('events').where('country', '=', 'US').limit(10) + expect(q.compile()).toEqual({ + sql: 'SELECT * FROM `events` WHERE `country` = {p0:String} LIMIT 10', + parameters: { p0: 'US' }, + }) + }) + + it('quotes dotted identifiers segment by segment', () => { + const q = selectFrom('db.events').select('events.country') + expect(q.compile().sql).toBe('SELECT `events`.`country` FROM `db`.`events`') + }) + + it('accumulates projections and supports aliases via "as"', () => { + const q = selectFrom('t').select(['a', 'b as bee']).select('c') + expect(q.compile().sql).toBe('SELECT `a`, `b` AS `bee`, `c` FROM `t`') + }) + + it('chains AND / OR predicates with flattening and grouping', () => { + const q = selectFrom('t') + .where('a', '=', 1) + .andWhere('b', '>', 2) + .orWhere('c', '<', 3) + expect(q.compile()).toEqual({ + sql: 'SELECT * FROM `t` WHERE ((`a` = {p0:Int64} AND `b` > {p1:Int64}) OR `c` < {p2:Int64})', + parameters: { p0: 1, p1: 2, p2: 3 }, + }) + }) + + it('binds an IN list as a single Array parameter', () => { + const q = selectFrom('t').where('id', 'in', [1, 2, 3]) + expect(q.compile()).toEqual({ + sql: 'SELECT * FROM `t` WHERE `id` IN {p0:Array(Int64)}', + parameters: { p0: [1, 2, 3] }, + }) + }) + + it('compiles joins with ON key equality', () => { + const q = selectFrom('users as u') + .innerJoin('orders as o', 'u.id', 'o.user_id') + .leftJoin('refunds as r', 'o.id', 'r.order_id') + .select(['u.name', 'o.total']) + expect(q.compile().sql).toBe( + 'SELECT `u`.`name`, `o`.`total` FROM `users` AS `u` ' + + 'INNER JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` ' + + 'LEFT JOIN `refunds` AS `r` ON `o`.`id` = `r`.`order_id`', + ) + }) + + it('compiles group by / having / order by / offset', () => { + const q = selectFrom('t') + .select(['country', sql`count()`.as('c')]) + .groupBy('country') + .having('c', '>', 100) + .orderBy('c', 'desc') + .limit(5) + .offset(10) + expect(q.compile()).toEqual({ + sql: + 'SELECT `country`, count() AS `c` FROM `t` GROUP BY `country` ' + + 'HAVING `c` > {p0:Int64} ORDER BY `c` DESC LIMIT 5 OFFSET 10', + parameters: { p0: 100 }, + }) + }) + + it('supports DISTINCT and set operations with unique placeholders', () => { + const a = selectFrom('t').select('x').where('x', '>', 1).distinct() + const b = selectFrom('u').select('y').where('y', '<', 9) + expect(a.unionAll(b).compile()).toEqual({ + sql: + 'SELECT DISTINCT `x` FROM `t` WHERE `x` > {p0:Int64} ' + + 'UNION ALL SELECT `y` FROM `u` WHERE `y` < {p1:Int64}', + parameters: { p0: 1, p1: 9 }, + }) + }) + + it('compiles a subquery source via .as()', () => { + const sub = selectFrom('t').select('id').where('id', '>', 0) + const q = selectFrom(sub.as('s')).select('s.id') + expect(q.compile()).toEqual({ + sql: 'SELECT `s`.`id` FROM (SELECT `id` FROM `t` WHERE `id` > {p0:Int64}) AS `s`', + parameters: { p0: 0 }, + }) + }) + + it('binds explicit types via val() and references via ref()', () => { + const q = selectFrom('t').where('id', '=', val(7, 'UInt64')).where('a', '=', ref('b')) + expect(q.compile()).toEqual({ + sql: 'SELECT * FROM `t` WHERE (`id` = {p0:UInt64} AND `a` = `b`)', + parameters: { p0: 7 }, + }) + }) + + it('binds sql`` interpolations but not fn() column refs', () => { + const q = selectFrom('t').select(fn('toStartOfDay', ['ts']).as('d')).where(sql`age > ${18}`) + expect(q.compile()).toEqual({ + sql: 'SELECT toStartOfDay(`ts`) AS `d` FROM `t` WHERE age > {p0:Int64}', + parameters: { p0: 18 }, + }) + }) +}) + +describe('mutation compilation', () => { + it('compiles UPDATE to an ALTER TABLE … UPDATE mutation', () => { + const q = updateTable('t').set({ n: 99, name: 'x' }).where('id', '=', 1) + expect(q.compile()).toEqual({ + sql: 'ALTER TABLE `t` UPDATE `n` = {p0:Int64}, `name` = {p1:String} WHERE `id` = {p2:Int64}', + parameters: { p0: 99, p1: 'x', p2: 1 }, + }) + }) + + it('compiles DELETE to an ALTER TABLE … DELETE mutation', () => { + const q = deleteFrom('t').where('id', 'in', [2, 3]) + expect(q.compile()).toEqual({ + sql: 'ALTER TABLE `t` DELETE WHERE `id` IN {p0:Array(Int64)}', + parameters: { p0: [2, 3] }, + }) + }) + + it('compiles INSERT … SELECT', () => { + const q = insertInto('dst') + .columns(['id', 'v']) + .values(selectFrom('src').select(['id', 'v']).where('v', '>', 0)) + expect(q.compile()).toEqual({ + sql: 'INSERT INTO `dst` (`id`, `v`) SELECT `id`, `v` FROM `src` WHERE `v` > {p0:Int64}', + parameters: { p0: 0 }, + }) + }) + + it('refuses an unguarded UPDATE / DELETE at compile time', () => { + expect(() => updateTable('t').set({ n: 1 }).compile()).toThrow(/requires a \.where/) + expect(() => deleteFrom('t').compile()).toThrow(/requires a \.where/) + }) +}) diff --git a/test/v3/layer3/dialect.test.ts b/test/v3/layer3/dialect.test.ts new file mode 100644 index 0000000..93dfdae --- /dev/null +++ b/test/v3/layer3/dialect.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { selectFrom, session } from '../../../index.js' + +// ClickHouse dialect sugar: FINAL / SAMPLE / PREWHERE / SETTINGS / FORMAT / +// LIMIT BY. Golden-SQL for clause placement, plus execution where the behaviour +// is observable (FINAL dedup, LIMIT BY per-group). + +describe('dialect sugar — compilation', () => { + it('places FINAL and SAMPLE on the FROM clause', () => { + const q = selectFrom('t').final().sample(0.1).selectAll() + expect(q.compile().sql).toBe('SELECT * FROM `t` FINAL SAMPLE 0.1') + }) + + it('emits PREWHERE before WHERE, both bound', () => { + const q = selectFrom('t').prewhere('a', '=', 1).where('b', '=', 2) + expect(q.compile()).toEqual({ + sql: 'SELECT * FROM `t` PREWHERE `a` = {p0:Int64} WHERE `b` = {p1:Int64}', + parameters: { p0: 1, p1: 2 }, + }) + }) + + it('appends a SETTINGS clause, merging across calls', () => { + const q = selectFrom('t').settings({ max_threads: 8 }).settings({ max_block_size: 1000 }) + expect(q.compile().sql).toBe('SELECT * FROM `t` SETTINGS max_threads = 8, max_block_size = 1000') + }) + + it('emits a SQL-level FORMAT clause distinct from execute({format})', () => { + const q = selectFrom('t').select('a').format('JSONEachRow') + expect(q.compile().sql).toBe('SELECT `a` FROM `t` FORMAT JSONEachRow') + }) + + it('emits LIMIT BY independently of the trailing LIMIT', () => { + const q = selectFrom('t').limitBy(1, ['country', 'city']).limit(100) + expect(q.compile().sql).toBe('SELECT * FROM `t` LIMIT 1 BY `country`, `city` LIMIT 100') + }) + + it('rejects an invalid SETTINGS name at compile time', () => { + expect(() => selectFrom('t').settings({ 'bad name': 1 } as never).compile()).toThrow( + /Invalid setting name/, + ) + }) +}) + +describe('dialect sugar — execution', () => { + let db: ReturnType + beforeEach(async () => { + db = session() + await db.session!.queryAsync( + 'CREATE TABLE r (id UInt64, v UInt64) ENGINE = ReplacingMergeTree(v) ORDER BY id', + { format: 'CSV' }, + ) + await db.insertInto('r').values([{ id: 1, v: 1 }]).execute() + await db.insertInto('r').values([{ id: 1, v: 2 }]).execute() + }) + + it('FINAL collapses replaced rows', async () => { + const withFinal = (await db.selectFrom('r').select('v').final().execute()) as { v: string }[] + expect(withFinal).toEqual([{ v: '2' }]) + }) + + it('LIMIT BY keeps one row per group', async () => { + await db.insertInto('r').values([{ id: 2, v: 9 }]).execute() + const rows = (await db + .selectFrom('r') + .select(['id', 'v']) + .final() + .limitBy(1, 'id') + .orderBy('id') + .execute()) as { id: string; v: string }[] + expect(rows.map((r) => r.id)).toEqual(['1', '2']) + }) +}) diff --git a/test/v3/layer3/execute.test.ts b/test/v3/layer3/execute.test.ts new file mode 100644 index 0000000..77cfa43 --- /dev/null +++ b/test/v3/layer3/execute.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { session, sql } from '../../../index.js' + +// Execution-correctness: run compiled SQL on real chDB through a bound Session +// and assert rows / types / order. Focus on ClickHouse-unique types so the +// precision-safe JS↔CH mapping is verified end to end. +// +// The global afterEach (setup.ts) force-closes every session after each test +// (single-connection constraint), so the fixture is rebuilt per test — chDB is +// fast enough that re-seeding a tiny table is negligible. + +let db: ReturnType + +beforeEach(async () => { + db = session() + await db.session!.queryAsync( + `CREATE TABLE t ( + id UInt64, + name String, + score Float64, + tags Array(String), + maybe Nullable(Int32), + ts DateTime + ) ENGINE = MergeTree ORDER BY id`, + { format: 'CSV' }, + ) + await db + .insertInto('t') + .values([ + { id: 1, name: "O'Brien", score: 1.5, tags: ['a', 'b'], maybe: 7, ts: '2024-01-02 03:04:05' }, + { id: 2, name: 'Bob', score: 2.5, tags: [], maybe: null, ts: '2024-06-07 08:09:10' }, + { id: 18446744073709551615n as unknown as number, name: 'Max', score: 0, tags: ['x'], maybe: 0, ts: '2020-01-01 00:00:00' }, + ]) + .execute() +}) + +describe('execute — round-trips', () => { + it('filters with a bound value and returns matching rows', async () => { + const rows = await db.selectFrom('t').select(['id', 'name']).where('name', '=', "O'Brien").execute() + expect(rows).toEqual([{ id: '1', name: "O'Brien" }]) + }) + + it('keeps 64-bit ids as strings (precision-safe), big value intact', async () => { + const rows = (await db.selectFrom('t').select('id').orderBy('id').execute()) as { id: string }[] + expect(rows.map((r) => r.id)).toEqual(['1', '2', '18446744073709551615']) + }) + + it('maps Array, Nullable, and DateTime correctly', async () => { + const row = (await db + .selectFrom('t') + .select(['tags', 'maybe', 'ts']) + .where('id', '=', 2) + .executeTakeFirst()) as { tags: string[]; maybe: number | null; ts: string } | undefined + expect(row).toEqual({ tags: [], maybe: null, ts: '2024-06-07 08:09:10' }) + }) + + it('groups and aggregates with HAVING', async () => { + const rows = (await db + .selectFrom('t') + .select([sql`count()`.as('c')]) + .where('score', '>', 1) + .execute()) as { c: string }[] + expect(rows[0]!.c).toBe('2') + }) + + it('binds an IN list as one Array parameter', async () => { + const rows = await db.selectFrom('t').select('id').where('id', 'in', [1, 2]).orderBy('id').execute() + expect(rows).toEqual([{ id: '1' }, { id: '2' }]) + }) + + it('executeTakeFirstOrThrow throws on an empty result', async () => { + await expect( + db.selectFrom('t').select('id').where('id', '=', 999).executeTakeFirstOrThrow(), + ).rejects.toThrow(/no rows/) + }) + + it('returns an Arrow Table for { format: "arrow" }', async () => { + const table = (await db.selectFrom('t').select('id').execute({ format: 'arrow' })) as { + numRows: number + } + expect(table.numRows).toBe(3) + }) + + it('runs an UPDATE mutation and reflects it', async () => { + await db.updateTable('t').set({ name: 'Robert' }).where('id', '=', 2).execute() + const row = (await db + .selectFrom('t') + .select('name') + .where('id', '=', 2) + .executeTakeFirst()) as { name: string } | undefined + expect(row).toEqual({ name: 'Robert' }) + }) +}) diff --git a/test/v3/layer3/functions.test.ts b/test/v3/layer3/functions.test.ts new file mode 100644 index 0000000..c71b7c6 --- /dev/null +++ b/test/v3/layer3/functions.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect, beforeEach, afterAll } from 'vitest' +import { mkdtempSync, writeFileSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { selectFrom, chTable, chFn, session } from '../../../index.js' + +// Table-function and CH-function helpers. Their arguments bind server-side like +// any value (table-function args included), so the golden SQL shows only +// placeholders; one execution test reads a real local file through file(). + +describe('chTable — compilation (arguments are bound)', () => { + it('numbers(count) and numbers(start, count)', () => { + expect(selectFrom(chTable.numbers(5).as('t')).select('number').compile()).toEqual({ + sql: 'SELECT `number` FROM numbers({p0:UInt64}) AS `t`', + parameters: { p0: 5 }, + }) + expect(selectFrom(chTable.numbers(10, 5).as('t')).selectAll().compile().parameters).toEqual({ + p0: 10, + p1: 5, + }) + }) + + it('s3 with and without credentials keeps the URL out of the SQL', () => { + const noCreds = selectFrom(chTable.s3({ url: 's3://b/*.parquet', format: 'Parquet' }).as('e')) + .selectAll() + .compile() + expect(noCreds).toEqual({ + sql: 'SELECT * FROM s3({p0:String}, {p1:String}) AS `e`', + parameters: { p0: 's3://b/*.parquet', p1: 'Parquet' }, + }) + const withCreds = selectFrom( + chTable.s3({ url: 's3://b/x', accessKeyId: 'AK', secretAccessKey: 'SK', format: 'CSV' }).as('e'), + ) + .selectAll() + .compile() + expect(withCreds.parameters).toEqual({ p0: 's3://b/x', p1: 'AK', p2: 'SK', p3: 'CSV' }) + expect(withCreds.sql).not.toContain('SK') + }) + + it('postgresql binds host/db/table/auth', () => { + const q = selectFrom( + chTable.postgresql({ host: 'h:5432', database: 'd', table: 'u', user: 'usr', password: 'pw' }).as('p'), + ).selectAll() + expect(q.compile()).toEqual({ + sql: 'SELECT * FROM postgresql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}) AS `p`', + parameters: { p0: 'h:5432', p1: 'd', p2: 'u', p3: 'usr', p4: 'pw' }, + }) + }) +}) + +describe('chFn — compilation', () => { + it('argMax / argMin / uniqExact take column references', () => { + expect(selectFrom('t').select(chFn.argMax('name', 'score').as('m')).compile().sql).toBe( + 'SELECT argMax(`name`, `score`) AS `m` FROM `t`', + ) + expect(selectFrom('t').select(chFn.uniqExact('a', 'b').as('u')).compile().sql).toBe( + 'SELECT uniqExact(`a`, `b`) AS `u` FROM `t`', + ) + }) + + it('parametric aggregates render name(params)(args) with bound params', () => { + expect(selectFrom('t').select(chFn.topK(3, 'country').as('k')).compile()).toEqual({ + sql: 'SELECT topK({p0:Int64})(`country`) AS `k` FROM `t`', + parameters: { p0: 3 }, + }) + expect(selectFrom('t').select(chFn.quantileTDigest(0.9, 'latency').as('q')).compile()).toEqual({ + sql: 'SELECT quantileTDigest({p0:Float64})(`latency`) AS `q` FROM `t`', + parameters: { p0: 0.9 }, + }) + }) + + it('sequenceMatch binds the pattern string (not a column ref)', () => { + const q = selectFrom('t').select(chFn.sequenceMatch('(?1)(?2)', 'ts', 'a', 'b').as('s')) + expect(q.compile()).toEqual({ + sql: 'SELECT sequenceMatch({p0:String})(`ts`, `a`, `b`) AS `s` FROM `t`', + parameters: { p0: '(?1)(?2)' }, + }) + }) +}) + +describe('chTable — execution against a real local file', () => { + const dir = mkdtempSync(join(tmpdir(), 'l3-tf-')) + const csv = join(dir, 'data.csv') + afterAll(() => rmSync(dir, { recursive: true, force: true })) + + let db: ReturnType + beforeEach(() => { + writeFileSync(csv, '1,US\n2,FR\n3,US\n') + db = session() + }) + + it('reads rows through file() with a bound path', async () => { + const rows = (await db + .selectFrom(chTable.file({ path: csv, format: 'CSV', structure: 'id UInt64, country String' }).as('f')) + .select(['id', 'country']) + .where('country', '=', 'US') + .orderBy('id') + .execute()) as { id: string; country: string }[] + expect(rows).toEqual([ + { id: '1', country: 'US' }, + { id: '3', country: 'US' }, + ]) + }) +}) diff --git a/test/v3/layer3/injection.test.ts b/test/v3/layer3/injection.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..b683c651b19c5b76b2739a630ce1d954bf28b8ad GIT binary patch literal 4500 zcmc&&&2AgX5zd)UQ4kiGq0Al0i(n7b%7AQhu>!P`SE8~9(+Qh1O|qB6?(uYwB!(cc z50GP$LoPW6d4rsjHwgmdnn%c2-9vInNk{-^5x}saQFV3weD!_RR`bG?j+T^3o0e)M zJEUBPu4m|St`m>$&8eyVIfK}M>M4!lF%7l_I}a3CQ%eplq4((8ZB@mOx;vc zYUTyJWpwl1MM7unY%Ek>m6E0sf6b-Pe8vYsr4;ED#(WY7$huPL37x{*52%bXnru~$ zMJn-aRH-~`I83NNRhH(eEKO;>j|;ItMqiM~keL;w5;+{8>`AMY8jbwExw`3}U-Y_k zEk$N@zBnsOvG{698cBBR08T=9>mO%l&#`qPouPhjudm*1jSr#| z|AqQzuP=JO>GlNY_ixEe1#s*q5m~XIgQ+^$zpTj7hpEXklsGqLCUP}vBp)q{>q#m{Xc*A zn}7ZB4?m`2{I@^v2jPb{H+s@=eHJC9#PGTJ1xEIocMuX`)DU)m4i~^_ztJ;2xD9aar%lFEM|CbB zh3`?jwF)$cfC`^qE+@*VIkVkxSzLDeIJO!#L1w{bND-vOnK7|&!nqVPBq5kdYVNDq zC>0vC4N@L^e1pD4>hc9K&OuoX_);vBNkRt=$RD&j?9rM!I57*?23loW z!n;>)%6Bpo=FJ-{J1gbf++lG3{^M`yp1X$+e4gtylZ$(F!2)_J16QcA0_AmJ$26-= zozon{9Jo5!c9m~6m3y}xXd65E;MNV?bh6n;HelrOq4;R9EC$(&?)zV^KD-#LjwT(7 z{`#lTr44w3{9T$0RD8$(hA=~O83IOS{<3f;elEgxJAtw#&b%?&p%k%)l?TUKdgEv- z9$dJfe7l{_A!MkcrU2;I=(slueBNw$3-@G_V5dvhPE>kFevOZ)xa;6pA)Xhe*0HGyn|Is@=lhj|+{m4ToqFc%m@5;BBBwKS{y)rWBEV zw)AL83!LK8?DR zUUQdyW|BPbdcI1EW5z=7VGyo@g;$eO-0w93 zEPF=0gYvq{^1}`6w;5vK3)aM)=Eh3T$_-!qQp4*2 zk5a6t{xRmwLp_3Dhi2gkc z?X}iTOH)mz?Y+j-iG+c9MwA84M7NjcSCf-Q!ndxZD3Y^M`wvbxXnS1O PbM33MTeRK60xkar%5bsI literal 0 HcmV?d00001 From 083e4ea2070f61d3c5abaf417a4359cf9022cc3b Mon Sep 17 00:00:00 2001 From: Changshuo Chen Date: Wed, 24 Jun 2026 06:55:55 +0000 Subject: [PATCH 2/7] Add connect() federation for external data sources Map all 14 URL schemes to table functions with every arg bound, and expose Connection.table()/describe()/databases()/tables()/snapshot(). Wire metadata for ClickHouse, Postgres, and MySQL; forward clickhouseSettings to chDB and accept-and-ignore remote-only options. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/layer3/compiler/compile.ts | 12 + src/layer3/connect/connect.ts | 155 +++++++++++ src/layer3/connect/snapshot.ts | 27 ++ src/layer3/connect/url-scheme.ts | 315 +++++++++++++++++++++++ src/layer3/database.ts | 12 + src/layer3/index.ts | 6 + test/v3/layer3/connect.test.ts | 424 +++++++++++++++++++++++++++++++ 7 files changed, 951 insertions(+) create mode 100644 src/layer3/connect/connect.ts create mode 100644 src/layer3/connect/snapshot.ts create mode 100644 src/layer3/connect/url-scheme.ts create mode 100644 test/v3/layer3/connect.test.ts diff --git a/src/layer3/compiler/compile.ts b/src/layer3/compiler/compile.ts index a48ad95..9f997c4 100644 --- a/src/layer3/compiler/compile.ts +++ b/src/layer3/compiler/compile.ts @@ -249,6 +249,18 @@ export function compileQuery(node: QueryNode): CompiledQuery { return { sql, parameters: c.params.parameters } } +/** Render a `SETTINGS` clause body (`k = v, …`) with names validated and values escaped. */ +export function renderSettings(settings: Readonly>): string { + return formatSettings(settings) +} + +/** Compile a single expression to `{ sql, parameters }` (e.g. a table-function read). */ +export function compileExpr(node: Expr): CompiledQuery { + const c = new Compiler() + const sql = c.expr(node) + return { sql, parameters: c.params.parameters } +} + function formatNumber(n: number): string { if (!Number.isFinite(n)) throw new ChdbCompileError(`Expected a finite number, got ${n}`) return String(n) diff --git a/src/layer3/connect/connect.ts b/src/layer3/connect/connect.ts new file mode 100644 index 0000000..0a64547 --- /dev/null +++ b/src/layer3/connect/connect.ts @@ -0,0 +1,155 @@ +/** + * `connect(config)` — the federation entry point. It opens a logical connection + * to an external data source (a ClickHouse server, Postgres, MySQL, MongoDB, + * object storage, a URL, or a local file) and lets you read from it through the + * local chDB engine. The engine runs in-process; the data can live anywhere a + * ClickHouse table function can reach. + * + * A `Connection` is a thin handle over a resolved source plan: + * + * const pg = chdb.connect({ url: 'postgres://h/db', username, password }) + * await chdb.selectFrom(pg.table('users').as('u')) + * .innerJoin(chTable.s3({ url, format: 'Parquet' }).as('e'), 'u.id', 'e.user_id') + * .selectAll().execute() + * + * Every host, credential, path, and option is bound server-side, so a + * connection string never reaches the SQL text. The build phase is pure; only + * the async methods touch the engine, where the typed Layer 1 errors surface. + */ + +import type { ChdbResult } from '../../result' +import { runtime } from '../runtime' +import type { ExecContext } from '../execute/terminal' +import { executeStatement } from '../execute/terminal' +import { parseRows } from '../execute/format' +import { compileExpr, compileQuery, renderSettings, type CompiledQuery } from '../compiler/compile' +import type { Expr, SelectQueryNode } from '../compiler/nodes' +import { ChExpression } from '../builder/expression' +import { buildSource, type CatalogQuery, type ConnectConfig, type SourcePlan } from './url-scheme' +import { buildSnapshotNode } from './snapshot' + +/** A column of `DESCRIBE TABLE`. */ +export interface ColumnInfo { + name: string + type: string + [key: string]: unknown +} + +/** Engine settings applied to the queries a connection issues itself. */ +export type EngineSettings = Readonly> + +function hasSettings(settings?: EngineSettings): settings is EngineSettings { + return settings !== undefined && Object.keys(settings).length > 0 +} + +/** Build the `DESCRIBE TABLE ` statement for a table / location. */ +export function compileDescribe(plan: SourcePlan, table?: string, settings?: EngineSettings): CompiledQuery { + const source = compileExpr(plan.table(table)) + const tail = hasSettings(settings) ? ` SETTINGS ${renderSettings(settings)}` : '' + return { sql: `DESCRIBE TABLE ${source.sql}${tail}`, parameters: source.parameters } +} + +/** + * Build a catalog SELECT that returns one column, always aliased `name`, so the + * caller reads the same shape regardless of the source's native column naming. + */ +function catalogSelect(cat: CatalogQuery, database?: string, settings?: EngineSettings): SelectQueryNode { + const where: Expr | undefined = + database !== undefined && cat.databaseColumn !== undefined + ? { + kind: 'Binary', + left: { kind: 'Reference', name: cat.databaseColumn }, + op: '=', + right: { kind: 'Value', value: database, chType: 'String' }, + } + : undefined + return { + kind: 'SelectQuery', + from: cat.source, + selections: [{ kind: 'Alias', node: { kind: 'Reference', name: cat.nameColumn }, alias: 'name' }], + where, + orderBy: [{ expr: { kind: 'Reference', name: 'name' }, direction: 'asc' }], + settings: hasSettings(settings) ? settings : undefined, + } +} + +/** Build the database-listing query (ClickHouse / Postgres / MySQL sources). */ +export function compileDatabases(plan: SourcePlan, settings?: EngineSettings): CompiledQuery { + return compileQuery(catalogSelect(plan.catalog('databases'), undefined, settings)) +} + +/** Build the table-listing query, optionally scoped to one database / schema. */ +export function compileTables(plan: SourcePlan, database?: string, settings?: EngineSettings): CompiledQuery { + return compileQuery(catalogSelect(plan.catalog('tables'), database, settings)) +} + +/** A logical connection to one external data source. */ +export class Connection { + private readonly plan: SourcePlan + // ClickHouse engine settings applied to the queries this connection issues + // (describe / databases / tables / snapshot). Data queries built with + // selectFrom(conn.table()) carry settings through the builder's .settings(). + private readonly settings?: EngineSettings + + constructor( + private readonly ctx: ExecContext, + config: ConnectConfig, + ) { + this.plan = buildSource(config) + this.settings = config.clickhouseSettings + } + + /** The source family, e.g. `'postgres'`, `'s3'`, `'clickhouse'`. */ + get sourceType(): string { + return this.plan.sourceType + } + + /** + * An aliasable source expression for a table/collection on this connection + * (server sources), or for the configured location (object-storage / URL / + * file sources, where the name is unused). Pass it to `selectFrom`/joins: + * + * chdb.selectFrom(conn.table('schema.users').as('u')) + */ + table(name?: string): ChExpression { + return new ChExpression(this.plan.table(name)) + } + + /** Column names and types of a table/location (`DESCRIBE TABLE`). */ + async describe(table?: string): Promise { + return (await this.runRows(compileDescribe(this.plan, table, this.settings))) as ColumnInfo[] + } + + /** + * List the source's databases (ClickHouse / Postgres / MySQL). Postgres + * groups tables by schema, so this returns its schema names — the same values + * `tables()` filters on. Throws for sources without a SQL catalog (MongoDB, + * object storage, URL, file); use `describe()` there instead. + */ + async databases(): Promise { + const rows = await this.runRows(compileDatabases(this.plan, this.settings)) + return rows.map((r) => String(r.name)) + } + + /** List table names, optionally scoped to one database / schema. */ + async tables(database?: string): Promise { + const rows = await this.runRows(compileTables(this.plan, database, this.settings)) + return rows.map((r) => String(r.name)) + } + + /** + * Copy a remote table into a local table once (`INSERT … SELECT`). There is + * no ongoing sync — it reads the source a single time. + */ + async snapshot(table: string, opts: { destination: string }): Promise { + return executeStatement(this.ctx, buildSnapshotNode(this.plan, table, opts.destination, this.settings)) + } + + private async runRows(compiled: CompiledQuery): Promise[]> { + const runOpts = { format: 'JSONEachRow' } + const result = this.ctx.session + ? await this.ctx.session.queryBindAsync(compiled.sql, compiled.parameters, runOpts) + : await runtime().queryBindAsync(compiled.sql, compiled.parameters, runOpts) + return parseRows(result.text()) + } +} diff --git a/src/layer3/connect/snapshot.ts b/src/layer3/connect/snapshot.ts new file mode 100644 index 0000000..261419f --- /dev/null +++ b/src/layer3/connect/snapshot.ts @@ -0,0 +1,27 @@ +/** + * One-shot materialization: copy a remote table into a local table once. This + * compiles to `INSERT INTO SELECT * FROM ` + * — there is no change-data-capture or ongoing sync, just a single read. + * + * The destination is a local table name (quote-escaped by the compiler); the + * source is the bound table-function read produced by the connection's plan. + */ + +import type { InsertSelectNode, SelectQueryNode } from '../compiler/nodes' +import type { SourcePlan } from './url-scheme' + +/** Build the `INSERT … SELECT` node that materializes `table` into `destination`. */ +export function buildSnapshotNode( + plan: SourcePlan, + table: string, + destination: string, + settings?: Readonly>, +): InsertSelectNode { + const select: SelectQueryNode = { + kind: 'SelectQuery', + from: plan.table(table), + selections: [{ kind: 'Star' }], + settings: settings && Object.keys(settings).length > 0 ? settings : undefined, + } + return { kind: 'InsertSelect', table: destination, select } +} diff --git a/src/layer3/connect/url-scheme.ts b/src/layer3/connect/url-scheme.ts new file mode 100644 index 0000000..cd9e661 --- /dev/null +++ b/src/layer3/connect/url-scheme.ts @@ -0,0 +1,315 @@ +/** + * Maps a connection URL to a ClickHouse table function. `connect()` takes a + * single `url` whose scheme selects the data source (Postgres, MySQL, S3, a + * ClickHouse server, a local file, …) plus flat auth/config fields; this module + * turns that into a table-function expression the builder can read from. + * + * Every credential, host, path, and option is carried as a bound value + * (`{pN:Type}`), exactly like a value in a WHERE clause — chDB accepts bound + * parameters in table-function argument position, so a URL, password, or bucket + * name is always data and never spliced into the SQL string. + * + * Ignored remote-only fields: a ClickHouse table function takes a fixed list of + * positional arguments, and some `@clickhouse/client`-style config fields have + * no slot in it — `region`, `sessionToken`, `headers`, `catalogConfig`. These + * describe how a remote service is reached, not how the local chDB engine runs + * the query, so connect() accepts them for parity and ignores them rather than + * failing (the same maximally-compatible stance Layer 2 takes for remote/auth + * fields). Put a region in the endpoint host, or use chTable/sql for the rest. + * `clickhouseSettings`, by contrast, is an engine setting and IS forwarded. + */ + +import { ChdbCompileError } from '../../errors' +import type { Expr } from '../compiler/nodes' + +/** A server source (host + table) versus a self-contained location (a URL/path). */ +export type SourceKind = 'server' | 'location' + +/** + * Configuration accepted by `connect()`. The shape mirrors + * `@clickhouse/client.createClient`: a `url` field whose scheme drives the + * source type, plus flat auth/config fields named to match that client. Fields + * not applicable to the chosen scheme are ignored. + */ +export interface ConnectConfig { + /** Data-source address; the scheme selects the source type. Required. */ + url: string + username?: string + password?: string + /** Default database (ClickHouse / Postgres / MySQL). */ + database?: string + /** TLS toggle for a ClickHouse server source (cloud is always secure). */ + secure?: boolean + /** + * ClickHouse engine settings (e.g. `max_threads`, `max_memory_usage`). These + * are applied to the local chDB engine as a `SETTINGS` clause on the queries + * this connection issues (describe / databases / tables / snapshot). For data + * queries you build with `selectFrom(conn.table())`, pass them through the + * builder's own `.settings()` / `execute({ settings })`. + */ + clickhouseSettings?: Record + /** Supabase service-role key (used as the password when set). */ + serviceRoleKey?: string + /** Supabase anon key (used as the password when no service-role key is set). */ + anonKey?: string + /** Default schema for Postgres / Supabase (a dotted table name overrides it). */ + schema?: string + accessKeyId?: string + secretAccessKey?: string + /** Data format for object-storage / URL / file sources (sniffed if omitted). */ + format?: string + // The fields below describe how a REMOTE service is reached. The ClickHouse + // table functions chDB drives have no argument slot for them, so connect() + // accepts them for @clickhouse/client parity but does not apply them — they do + // not change what the local engine executes. See "Ignored remote-only fields" + // in the module notes. Express them through the url/endpoint or chTable/sql. + /** Cloud region for object storage (put it in the endpoint host instead). */ + region?: string + /** Session token for temporary object-storage credentials. */ + sessionToken?: string + /** Extra HTTP headers for a URL source. */ + headers?: Record + /** Catalog config for an Iceberg source. */ + catalogConfig?: Record +} + +interface SchemeSpec { + /** Stable label for the source family (used by metadata discovery). */ + sourceType: string + /** The ClickHouse table function this scheme reads through. */ + fn: string + kind: SourceKind +} + +// URL scheme (URL.protocol, with its trailing colon) → table function. +const SCHEMES: Readonly> = { + 'clickhouse:': { sourceType: 'clickhouse', fn: 'remote', kind: 'server' }, + 'clickhouse-cloud:': { sourceType: 'clickhouse', fn: 'remoteSecure', kind: 'server' }, + 'supabase:': { sourceType: 'postgres', fn: 'postgresql', kind: 'server' }, + 'postgres:': { sourceType: 'postgres', fn: 'postgresql', kind: 'server' }, + 'postgresql:': { sourceType: 'postgres', fn: 'postgresql', kind: 'server' }, + 'mysql:': { sourceType: 'mysql', fn: 'mysql', kind: 'server' }, + 'mongodb:': { sourceType: 'mongodb', fn: 'mongodb', kind: 'server' }, + 'mongodb+srv:': { sourceType: 'mongodb', fn: 'mongodb', kind: 'server' }, + 's3:': { sourceType: 's3', fn: 's3', kind: 'location' }, + 'gcs:': { sourceType: 'gcs', fn: 'gcs', kind: 'location' }, + 'gs:': { sourceType: 'gcs', fn: 'gcs', kind: 'location' }, + 'azureblob:': { sourceType: 'azure', fn: 'azureBlobStorage', kind: 'location' }, + 'iceberg:': { sourceType: 'iceberg', fn: 'iceberg', kind: 'location' }, + 'delta:': { sourceType: 'delta', fn: 'deltaLake', kind: 'location' }, + 'hudi:': { sourceType: 'hudi', fn: 'hudi', kind: 'location' }, + 'https:': { sourceType: 'http', fn: 'url', kind: 'location' }, + 'http:': { sourceType: 'http', fn: 'url', kind: 'location' }, + 'file:': { sourceType: 'file', fn: 'file', kind: 'location' }, +} + +/** How to list a source's databases/tables: which catalog to read and which columns hold the names. */ +export interface CatalogQuery { + /** Table-function read of the source's catalog. */ + readonly source: Expr + /** Column holding the database/table name. */ + readonly nameColumn: string + /** Column to filter `tables()` by database (present only on the tables catalog). */ + readonly databaseColumn?: string +} + +/** A resolved source: enough to build a table-function read for any table/location. */ +export interface SourcePlan { + readonly scheme: string + readonly sourceType: string + readonly kind: SourceKind + readonly fn: string + /** + * Table-function expression for a named table/collection (server sources) or + * the configured location (object-storage / URL / file sources, where the + * name is unused). Throws if a server source is given no name. + */ + table(name?: string): Expr + /** + * The catalog read for listing databases or tables. Supported for ClickHouse, + * Postgres, and MySQL (via their system / information_schema tables); throws + * for sources without a SQL catalog (MongoDB, object storage, URL, file). + */ + catalog(kind: 'databases' | 'tables'): CatalogQuery +} + +const bound = (value: unknown, chType = 'String'): Expr => ({ kind: 'Value', value, chType }) +const call = (name: string, args: Expr[]): Expr => ({ kind: 'Function', name, args }) + +const decode = (raw: string): string | undefined => + raw === '' ? undefined : decodeURIComponent(raw) + +/** Split a `qualifier.object` name into its parts (a bare name has no qualifier). */ +function splitName(name: string): [string | undefined, string] { + const dot = name.indexOf('.') + return dot === -1 ? [undefined, name] : [name.slice(0, dot), name.slice(dot + 1)] +} + +/** Resolve a connection config into a reusable source plan. */ +export function buildSource(config: ConnectConfig): SourcePlan { + if (config === null || typeof config !== 'object' || typeof config.url !== 'string' || config.url === '') { + throw new ChdbCompileError('connect(config) requires a non-empty url') + } + + let url: URL + try { + url = new URL(config.url) + } catch { + throw new ChdbCompileError(`Invalid connect url ${JSON.stringify(config.url)}`) + } + + const resolved = SCHEMES[url.protocol] + if (resolved === undefined) { + throw new ChdbCompileError(`Unsupported connect url scheme ${JSON.stringify(url.protocol.replace(/:$/, ''))}`) + } + const spec: SchemeSpec = resolved + + // A ClickHouse server reached with `secure: true` uses the TLS table function, + // matching the always-secure `clickhouse-cloud://` scheme. + const fn = spec.fn === 'remote' && config.secure === true ? 'remoteSecure' : spec.fn + + const addr = url.host + const username = config.username ?? decode(url.username) + const password = + config.password ?? config.serviceRoleKey ?? config.anonKey ?? decode(url.password) + const database = config.database ?? decode(url.pathname.replace(/^\//, '')) + + /** Require both credentials (or neither) and append them in order. */ + function requireAuth(): [Expr, Expr] { + if (username === undefined || password === undefined) { + throw new ChdbCompileError(`A ${spec.sourceType} source requires a username and password`) + } + return [bound(username), bound(password)] + } + + function remoteArgs(database_: string, table: string): Expr[] { + const args = [bound(addr), bound(database_), bound(table)] + if (password !== undefined && username === undefined) { + throw new ChdbCompileError('A ClickHouse source password requires a username') + } + if (username !== undefined) { + args.push(bound(username)) + if (password !== undefined) args.push(bound(password)) + } + return args + } + + function locationArgs(): Expr[] { + const location = spec.fn === 'file' ? (decode(url.pathname) ?? '') : config.url + const args: Expr[] = [bound(location)] + + const hasKey = config.accessKeyId !== undefined + const hasSecret = config.secretAccessKey !== undefined + if (hasKey !== hasSecret) { + throw new ChdbCompileError(`A ${spec.sourceType} source needs both accessKeyId and secretAccessKey, or neither`) + } + const credentialed = spec.fn === 's3' || spec.fn === 'gcs' || spec.fn === 'iceberg' || spec.fn === 'deltaLake' || spec.fn === 'hudi' + if (credentialed && hasKey && hasSecret) { + args.push(bound(config.accessKeyId)) + args.push(bound(config.secretAccessKey)) + } + + // file/url/s3/gcs/azure accept a trailing format; the lakehouse functions + // detect it from the table metadata, so it is not passed there. + const takesFormat = spec.fn === 'file' || spec.fn === 'url' || spec.fn === 's3' || spec.fn === 'gcs' || spec.fn === 'azureBlobStorage' + if (takesFormat && config.format !== undefined) args.push(bound(config.format)) + return args + } + + // Catalog reads for metadata discovery. Each source exposes its catalog + // differently: ClickHouse through `system`, Postgres through a database's + // `information_schema` schema, MySQL through the `information_schema` database. + // Postgres groups tables by schema, so its "databases" are schemas — the names + // returned by databases() are exactly the values tables() accepts. + function catalog(kind: 'databases' | 'tables'): CatalogQuery { + switch (spec.sourceType) { + case 'clickhouse': { + const source = call(fn, remoteArgs('system', kind === 'databases' ? 'databases' : 'tables')) + return kind === 'databases' + ? { source, nameColumn: 'name' } + : { source, nameColumn: 'name', databaseColumn: 'database' } + } + case 'postgres': { + if (database === undefined) { + throw new ChdbCompileError('A Postgres source needs a database for metadata discovery: pass connect({ database }) or set it in the url') + } + const [user, pass] = requireAuth() + const relation = kind === 'databases' ? 'schemata' : 'tables' + const source = call('postgresql', [bound(addr), bound(database), bound(relation), user, pass, bound('information_schema')]) + return kind === 'databases' + ? { source, nameColumn: 'schema_name' } + : { source, nameColumn: 'table_name', databaseColumn: 'table_schema' } + } + case 'mysql': { + const [user, pass] = requireAuth() + // MySQL information_schema relation and column names are upper-case. + const relation = kind === 'databases' ? 'SCHEMATA' : 'TABLES' + const source = call('mysql', [bound(addr), bound('information_schema'), bound(relation), user, pass]) + return kind === 'databases' + ? { source, nameColumn: 'SCHEMA_NAME' } + : { source, nameColumn: 'TABLE_NAME', databaseColumn: 'TABLE_SCHEMA' } + } + default: + throw new ChdbCompileError( + `Listing databases/tables is not available for a ${spec.sourceType} source; use describe() to inspect a known table`, + ) + } + } + + function table(name?: string): Expr { + if (spec.kind === 'location') return call(spec.fn, locationArgs()) + + if (name === undefined) { + throw new ChdbCompileError(`A ${spec.sourceType} source needs a table name: connection.table('database.table')`) + } + const [qualifier, object] = splitName(name) + + switch (spec.fn) { + case 'remote': + case 'remoteSecure': { + const db = qualifier ?? database + if (db === undefined) { + throw new ChdbCompileError("A ClickHouse source needs a database: pass connect({ database }) or table('database.table')") + } + return call(fn, remoteArgs(db, object)) + } + case 'postgresql': { + if (database === undefined) { + throw new ChdbCompileError('A Postgres source needs a database: pass connect({ database }) or set it in the url') + } + const [user, pass] = requireAuth() + const args = [bound(addr), bound(database), bound(object), user, pass] + const schema = qualifier ?? config.schema + if (schema !== undefined) args.push(bound(schema)) + return call('postgresql', args) + } + case 'mysql': { + const db = qualifier ?? database + if (db === undefined) { + throw new ChdbCompileError('A MySQL source needs a database: pass connect({ database }) or table(\'database.table\')') + } + const [user, pass] = requireAuth() + return call('mysql', [bound(addr), bound(db), bound(object), user, pass]) + } + case 'mongodb': { + const db = qualifier ?? database + if (db === undefined) { + throw new ChdbCompileError('A MongoDB source needs a database: pass connect({ database }) or table(\'database.collection\')') + } + const [user, pass] = requireAuth() + return call('mongodb', [bound(addr), bound(db), bound(object), user, pass]) + } + default: + throw new ChdbCompileError(`Unsupported server source ${spec.sourceType}`) + } + } + + return { + scheme: url.protocol.replace(/:$/, ''), + sourceType: spec.sourceType, + kind: spec.kind, + fn, + table, + catalog, + } +} diff --git a/src/layer3/database.ts b/src/layer3/database.ts index 1226bbb..c536903 100644 --- a/src/layer3/database.ts +++ b/src/layer3/database.ts @@ -17,6 +17,8 @@ import { SelectQueryBuilder } from './builder/select' import { InsertQueryBuilder } from './builder/insert' import { UpdateQueryBuilder } from './builder/update' import { DeleteQueryBuilder } from './builder/delete' +import { Connection } from './connect/connect' +import type { ConnectConfig } from './connect/url-scheme' import type { ExecContext } from './execute/terminal' /** Anything accepted as a SELECT source: a table name, an expression, or a subquery builder. */ @@ -54,6 +56,11 @@ export class Database> { return new DeleteQueryBuilder(this.ctx, { kind: 'DeleteQuery', table }) } + /** Open a federated connection to an external data source (read through chDB). */ + connect(config: ConnectConfig): Connection { + return new Connection(this.ctx, config) + } + /** The bound Session, if this root was created with one. */ get session(): RuntimeSession | undefined { return this.ctx.session @@ -87,6 +94,11 @@ export function deleteFrom(table: string): DeleteQueryBuilder { return DEFAULT.deleteFrom(table) } +/** Open a federated connection to an external data source on the default connection. */ +export function connect(config: ConnectConfig): Connection { + return DEFAULT.connect(config) +} + /** Create a typed builder root. Pass `{ session }` to pin the connection. */ export function database>(opts?: { session?: RuntimeSession }): Database { return new Database({ session: opts?.session }) diff --git a/src/layer3/index.ts b/src/layer3/index.ts index 7270c1d..e46b734 100644 --- a/src/layer3/index.ts +++ b/src/layer3/index.ts @@ -18,12 +18,18 @@ export { insertInto, updateTable, deleteFrom, + connect, database, session, Database, } from './database' export type { FromInput } from './database' +// Federated connections (connect). +export { Connection } from './connect/connect' +export type { ColumnInfo } from './connect/connect' +export type { ConnectConfig, SourceKind, SourcePlan } from './connect/url-scheme' + // Builder + expression helpers. export { SelectQueryBuilder } from './builder/select' export { diff --git a/test/v3/layer3/connect.test.ts b/test/v3/layer3/connect.test.ts new file mode 100644 index 0000000..6965070 --- /dev/null +++ b/test/v3/layer3/connect.test.ts @@ -0,0 +1,424 @@ +import { describe, it, expect, beforeEach, afterAll } from 'vitest' +import { mkdtempSync, writeFileSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { selectFrom, connect, session, query, sql } from '../../../index.js' +import { buildSource } from '../../../src/layer3/connect/url-scheme' +import { buildSnapshotNode } from '../../../src/layer3/connect/snapshot' +import { compileDatabases, compileTables, compileDescribe } from '../../../src/layer3/connect/connect' +import { compileExpr, compileQuery } from '../../../src/layer3/compiler/compile' + +// connect() maps a connection URL to a ClickHouse table function. The scheme +// picks the function; every host, credential, path, and option is BOUND +// ({pN:Type}) — a connection string never appears as a byte in the SQL. The +// golden tests pin each scheme's mapping; the injection tests prove hostile +// values stay in parameters; the execution tests read real local files. + +const table = (cfg: Parameters[0], name?: string) => + compileExpr(buildSource(cfg).table(name)) + +describe('connect — server source mapping (table args are bound)', () => { + it('clickhouse:// → remote(addr, db, table, user, pw)', () => { + expect(table({ url: 'clickhouse://u:p@h:9000/prod' }, 'events')).toEqual({ + sql: 'remote({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String})', + parameters: { p0: 'h:9000', p1: 'prod', p2: 'events', p3: 'u', p4: 'p' }, + }) + }) + + it('clickhouse:// with secure → remoteSecure', () => { + const c = table({ url: 'clickhouse://h:9440', database: 'prod', secure: true }, 'events') + expect(c.sql).toBe('remoteSecure({p0:String}, {p1:String}, {p2:String})') + expect(c.parameters).toEqual({ p0: 'h:9440', p1: 'prod', p2: 'events' }) + }) + + it('clickhouse-cloud:// → remoteSecure', () => { + const c = table( + { url: 'clickhouse-cloud://x.clickhouse.cloud:9440', username: 'default', password: 'pw', database: 'prod' }, + 'prod.users', + ) + expect(c.sql).toBe('remoteSecure({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String})') + // A dotted table name overrides the default database. + expect(c.parameters).toEqual({ p0: 'x.clickhouse.cloud:9440', p1: 'prod', p2: 'users', p3: 'default', p4: 'pw' }) + }) + + it('postgres:// → postgresql(addr, db, table, user, pw)', () => { + expect(table({ url: 'postgres://u:p@h:5432/app' }, 'users')).toEqual({ + sql: 'postgresql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String})', + parameters: { p0: 'h:5432', p1: 'app', p2: 'users', p3: 'u', p4: 'p' }, + }) + }) + + it('postgres:// with a dotted name adds the schema arg', () => { + const c = table({ url: 'postgres://u:p@h:5432/app' }, 'reporting.users') + expect(c.sql).toBe('postgresql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}, {p5:String})') + expect(c.parameters).toEqual({ p0: 'h:5432', p1: 'app', p2: 'users', p3: 'u', p4: 'p', p5: 'reporting' }) + }) + + it('supabase:// uses the service-role key as the password', () => { + const c = table( + { url: 'supabase://abc.supabase.co', database: 'postgres', username: 'postgres', serviceRoleKey: 'srk', schema: 'public' }, + 'billing', + ) + expect(c.sql).toBe('postgresql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}, {p5:String})') + expect(c.parameters).toEqual({ p0: 'abc.supabase.co', p1: 'postgres', p2: 'billing', p3: 'postgres', p4: 'srk', p5: 'public' }) + }) + + it('mysql:// → mysql(addr, db, table, user, pw)', () => { + expect(table({ url: 'mysql://u:p@h:3306/shop' }, 'orders')).toEqual({ + sql: 'mysql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String})', + parameters: { p0: 'h:3306', p1: 'shop', p2: 'orders', p3: 'u', p4: 'p' }, + }) + }) + + it('mongodb:// and mongodb+srv:// → mongodb(addr, db, collection, user, pw)', () => { + expect(table({ url: 'mongodb://u:p@h:27017/app' }, 'events').sql).toBe( + 'mongodb({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String})', + ) + expect(table({ url: 'mongodb+srv://u:p@cluster.net/app' }, 'events').parameters).toEqual({ + p0: 'cluster.net', + p1: 'app', + p2: 'events', + p3: 'u', + p4: 'p', + }) + }) +}) + +describe('connect — location source mapping', () => { + it('s3:// → s3(url[, key, secret][, format]); name is ignored', () => { + expect(table({ url: 's3://b/*.parquet', format: 'Parquet' })).toEqual({ + sql: 's3({p0:String}, {p1:String})', + parameters: { p0: 's3://b/*.parquet', p1: 'Parquet' }, + }) + const withCreds = table({ url: 's3://b/x', accessKeyId: 'AK', secretAccessKey: 'SK', format: 'CSV' }) + expect(withCreds.sql).toBe('s3({p0:String}, {p1:String}, {p2:String}, {p3:String})') + expect(withCreds.parameters).toEqual({ p0: 's3://b/x', p1: 'AK', p2: 'SK', p3: 'CSV' }) + }) + + it('gcs:// and gs:// → gcs(url, ...)', () => { + expect(table({ url: 'gcs://b/a.csv', format: 'CSV' }).sql).toBe('gcs({p0:String}, {p1:String})') + expect(table({ url: 'gs://b/a.csv' }).sql).toBe('gcs({p0:String})') + }) + + it('azureblob:// → azureBlobStorage(url[, format])', () => { + expect(table({ url: 'azureblob://acct/container/blob', format: 'Parquet' })).toEqual({ + sql: 'azureBlobStorage({p0:String}, {p1:String})', + parameters: { p0: 'azureblob://acct/container/blob', p1: 'Parquet' }, + }) + }) + + it('iceberg://, delta://, hudi:// → lakehouse functions (no format arg)', () => { + expect(table({ url: 'iceberg://b/t' }).sql).toBe('iceberg({p0:String})') + expect(table({ url: 'delta://b/t', accessKeyId: 'AK', secretAccessKey: 'SK' }).sql).toBe( + 'deltaLake({p0:String}, {p1:String}, {p2:String})', + ) + expect(table({ url: 'hudi://b/t' }).sql).toBe('hudi({p0:String})') + }) + + it('http(s):// → url(url[, format])', () => { + expect(table({ url: 'https://e.com/d.csv', format: 'CSV' })).toEqual({ + sql: 'url({p0:String}, {p1:String})', + parameters: { p0: 'https://e.com/d.csv', p1: 'CSV' }, + }) + }) + + it('file:// → file(path[, format]); the path is the decoded url path', () => { + expect(table({ url: 'file:///data/x.csv', format: 'CSV' })).toEqual({ + sql: 'file({p0:String}, {p1:String})', + parameters: { p0: '/data/x.csv', p1: 'CSV' }, + }) + }) +}) + +describe('connect — input validation', () => { + it('rejects a missing or unsupported url', () => { + expect(() => buildSource({ url: '' } as any)).toThrow(/requires a non-empty url/) + expect(() => buildSource({ url: 'redis://h/0' })).toThrow(/Unsupported connect url scheme/) + }) + + it('requires a username and password for postgres/mysql/mongodb', () => { + expect(() => buildSource({ url: 'postgres://h:5432/app' }).table('users')).toThrow(/username and password/) + expect(() => buildSource({ url: 'mysql://h:3306/d' }).table('orders')).toThrow(/username and password/) + }) + + it('requires a table name for a server source', () => { + expect(() => buildSource({ url: 'clickhouse://h:9000/db' }).table()).toThrow(/needs a table name/) + }) + + it('requires both object-storage credentials or neither', () => { + expect(() => buildSource({ url: 's3://b/x', accessKeyId: 'AK' }).table()).toThrow(/both accessKeyId and secretAccessKey/) + }) +}) + +describe('connect — injection (urls, credentials, names are bound)', () => { + const HOSTILE = "'; DROP TABLE users; --" + + it('keeps a hostile password out of the SQL', () => { + const c = table({ url: 'postgres://h:5432/app', username: 'u', password: HOSTILE }, 'users') + expect(c.sql).not.toContain('DROP') + expect(Object.values(c.parameters)).toContain(HOSTILE) + }) + + it('keeps a hostile table name out of the SQL (it is a bound table-function arg)', () => { + const c = table({ url: 'clickhouse://u:p@h:9000/db' }, HOSTILE) + expect(c.sql).toBe('remote({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String})') + expect(Object.values(c.parameters)).toContain(HOSTILE) + }) + + it('keeps a hostile url out of the SQL', () => { + const c = table({ url: `s3://b/x`, accessKeyId: HOSTILE, secretAccessKey: HOSTILE }) + expect(c.sql).not.toContain('DROP') + expect(Object.values(c.parameters)).toContain(HOSTILE) + }) +}) + +describe('connect — snapshot (INSERT … SELECT)', () => { + it('materializes a source table into a local destination', () => { + const plan = buildSource({ url: 'postgres://u:p@h:5432/app' }) + const compiled = compileQuery(buildSnapshotNode(plan, 'users', 'users_local')) + expect(compiled).toEqual({ + sql: 'INSERT INTO `users_local` SELECT * FROM postgresql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String})', + parameters: { p0: 'h:5432', p1: 'app', p2: 'users', p3: 'u', p4: 'p' }, + }) + }) + + it('quote-escapes a dotted / hostile destination identifier', () => { + const plan = buildSource({ url: 's3://b/x', format: 'Parquet' }) + const compiled = compileQuery(buildSnapshotNode(plan, '', 'db.events`local')) + expect(compiled.sql).toBe('INSERT INTO `db`.`events\\`local` SELECT * FROM s3({p0:String}, {p1:String})') + }) +}) + +describe('connect — metadata discovery SQL (catalog reads, output column aliased `name`)', () => { + it('ClickHouse reads system.databases / system.tables', () => { + expect(compileDatabases(buildSource({ url: 'clickhouse://u:p@h:9000' }))).toEqual({ + sql: 'SELECT `name` AS `name` FROM remote({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}) ORDER BY `name` ASC', + parameters: { p0: 'h:9000', p1: 'system', p2: 'databases', p3: 'u', p4: 'p' }, + }) + const tbl = compileTables(buildSource({ url: 'clickhouse://h:9000' }), 'prod') + expect(tbl.sql).toBe( + 'SELECT `name` AS `name` FROM remote({p0:String}, {p1:String}, {p2:String}) WHERE `database` = {p3:String} ORDER BY `name` ASC', + ) + expect(tbl.parameters).toEqual({ p0: 'h:9000', p1: 'system', p2: 'tables', p3: 'prod' }) + }) + + it('Postgres reads information_schema.schemata / .tables (schemas are its databases)', () => { + expect(compileDatabases(buildSource({ url: 'postgres://u:p@h:5432/app' })).sql).toBe( + 'SELECT `schema_name` AS `name` FROM postgresql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}, {p5:String}) ORDER BY `name` ASC', + ) + const tbl = compileTables(buildSource({ url: 'postgres://u:p@h:5432/app' }), 'public') + expect(tbl.sql).toBe( + 'SELECT `table_name` AS `name` FROM postgresql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}, {p5:String}) WHERE `table_schema` = {p6:String} ORDER BY `name` ASC', + ) + expect(tbl.parameters).toEqual({ p0: 'h:5432', p1: 'app', p2: 'tables', p3: 'u', p4: 'p', p5: 'information_schema', p6: 'public' }) + }) + + it('MySQL reads information_schema.SCHEMATA / .TABLES (upper-case relations/columns)', () => { + expect(compileDatabases(buildSource({ url: 'mysql://u:p@h:3306/shop' })).sql).toBe( + 'SELECT `SCHEMA_NAME` AS `name` FROM mysql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}) ORDER BY `name` ASC', + ) + const tbl = compileTables(buildSource({ url: 'mysql://u:p@h:3306/shop' }), 'shop') + expect(tbl.sql).toBe( + 'SELECT `TABLE_NAME` AS `name` FROM mysql({p0:String}, {p1:String}, {p2:String}, {p3:String}, {p4:String}) WHERE `TABLE_SCHEMA` = {p5:String} ORDER BY `name` ASC', + ) + expect(tbl.parameters).toEqual({ p0: 'h:3306', p1: 'information_schema', p2: 'TABLES', p3: 'u', p4: 'p', p5: 'shop' }) + }) + + it('describe() compiles DESCRIBE TABLE over any source', () => { + const c = compileDescribe(buildSource({ url: 'file:///d/x.csv', format: 'CSV' })) + expect(c).toEqual({ + sql: 'DESCRIBE TABLE file({p0:String}, {p1:String})', + parameters: { p0: '/d/x.csv', p1: 'CSV' }, + }) + }) + + it('databases()/tables() reject sources without a SQL catalog', () => { + expect(() => compileDatabases(buildSource({ url: 'mongodb://u:p@h/app' }))).toThrow(/not available for a mongodb source/) + expect(() => compileTables(buildSource({ url: 's3://b/x' }))).toThrow(/not available for a s3 source/) + }) +}) + +describe('connect — config field arbitration (mirrors Layer 2)', () => { + it('forwards clickhouseSettings to the chDB engine as a SETTINGS clause', () => { + const plan = buildSource({ url: 'clickhouse://u:p@h:9000' }) + expect(compileDatabases(plan, { max_threads: 4 }).sql).toMatch(/ SETTINGS max_threads = 4$/) + expect(compileTables(plan, 'prod', { max_threads: 4 }).sql).toMatch(/ SETTINGS max_threads = 4$/) + expect(compileDescribe(buildSource({ url: 'file:///d/x.csv' }), undefined, { max_memory_usage: 1000 }).sql).toBe( + 'DESCRIBE TABLE file({p0:String}) SETTINGS max_memory_usage = 1000', + ) + const snap = compileQuery( + buildSnapshotNode(buildSource({ url: 'postgres://u:p@h:5432/app' }), 'users', 'dst', { max_threads: 2 }), + ) + expect(snap.sql).toMatch(/ SETTINGS max_threads = 2$/) + }) + + it('accepts remote-only fields without error and never emits them in SQL', () => { + // region/sessionToken/headers/catalogConfig have no table-function slot and + // do not change what the local engine executes, so they are ignored. + const c = table({ + url: 's3://b/x', + region: 'us-east-1', + sessionToken: 'tok', + headers: { Authorization: 'secret' }, + catalogConfig: { uri: 'https://catalog' }, + }) + expect(c).toEqual({ sql: 's3({p0:String})', parameters: { p0: 's3://b/x' } }) + expect(JSON.stringify(c)).not.toMatch(/us-east-1|tok|secret|catalog/) + }) +}) + +describe('connect — execution against real local files', () => { + const dir = mkdtempSync(join(tmpdir(), 'l3-connect-')) + const users = join(dir, 'users.csv') + const events = join(dir, 'events.csv') + afterAll(() => rmSync(dir, { recursive: true, force: true })) + + beforeEach(() => { + writeFileSync(users, 'id,country\n1,US\n2,FR\n3,US\n') + writeFileSync(events, 'uid,action\n1,login\n1,buy\n3,login\n') + }) + + it('reads rows through a file:// connection', async () => { + const conn = connect({ url: `file://${users}`, format: 'CSVWithNames' }) + const rows = (await selectFrom(conn.table().as('u')) + .select('country') + .where('country', '=', 'US') + .orderBy('country') + .execute()) as { country: string }[] + expect(rows).toEqual([{ country: 'US' }, { country: 'US' }]) + }) + + it('describe() returns the inferred columns', async () => { + const conn = connect({ url: `file://${users}`, format: 'CSVWithNames' }) + const cols = await conn.describe() + expect(cols.map((c) => c.name)).toContain('country') + }) + + it('joins two file:// connections (cross-source JOIN)', async () => { + const u = connect({ url: `file://${users}`, format: 'CSVWithNames' }) + const e = connect({ url: `file://${events}`, format: 'CSVWithNames' }) + const rows = (await selectFrom(u.table().as('u')) + .innerJoin(e.table().as('e'), 'u.id', 'e.uid') + .select(['u.country', 'e.action']) + .orderBy('e.action') + .execute()) as { country: string; action: string }[] + expect(rows).toEqual([ + { country: 'US', action: 'buy' }, + { country: 'US', action: 'login' }, + { country: 'US', action: 'login' }, + ]) + }) + + it('snapshot() materializes a file source into a local table', async () => { + const db = session() + try { + await db.session!.queryAsync('CREATE TABLE snap (id Int64, country String) ENGINE = Memory') + const conn = db.connect({ url: `file://${users}`, format: 'CSVWithNames' }) + await conn.snapshot('', { destination: 'snap' }) + const rows = (await db.selectFrom('snap').select('country').orderBy('id').execute()) as { + country: string + }[] + expect(rows).toEqual([{ country: 'US' }, { country: 'FR' }, { country: 'US' }]) + } finally { + db.close() + } + }) +}) + +// Real federation against live Postgres / MySQL servers. These prove the table +// functions, metadata catalogs, snapshots, and cross-source joins work against +// actual databases — not just that the SQL looks right. They skip cleanly when +// no server is reachable (e.g. a CI job without the services), and the URLs are +// overridable so CI can point at its own instances. +const PG_URL = process.env.CHDB_TEST_PG_URL ?? 'postgres://postgres:pw@127.0.0.1:55432/app' +const MY_URL = process.env.CHDB_TEST_MYSQL_URL ?? 'mysql://app:pw@127.0.0.1:33060/shop' + +function reachable(probe: string): boolean { + try { + query(probe, 'CSV') + return true + } catch { + return false + } +} + +const PG_OK = reachable(`SELECT 1 FROM ${tfn(PG_URL, 'users')} LIMIT 1`) +const MY_OK = reachable(`SELECT 1 FROM ${tfn(MY_URL, 'orders')} LIMIT 1`) + +// Build the raw probe table-function call from a connection url (so the probe +// and the public connect() path read the same servers). +function tfn(url: string, name: string): string { + const c = compileExpr(buildSource({ url }).table(name)) + let sql = c.sql + // Inline the bound probe params (probe only — the library never does this). + for (const [k, v] of Object.entries(c.parameters)) { + sql = sql.replace(`{${k}:String}`, `'${String(v).replace(/'/g, "''")}'`) + } + return sql +} + +describe.skipIf(!PG_OK)('connect — live Postgres federation', () => { + it('reads rows through a postgres:// connection', async () => { + const pg = connect({ url: PG_URL }) + const rows = (await selectFrom(pg.table('users').as('u')) + .select('country') + .where('country', '=', 'US') + .orderBy('country') + .execute()) as { country: string }[] + expect(rows).toEqual([{ country: 'US' }, { country: 'US' }]) + }) + + it('databases() lists schemas and tables(schema) lists tables', async () => { + const pg = connect({ url: PG_URL }) + expect(await pg.databases()).toContain('public') + expect(await pg.tables('public')).toContain('users') + }) + + it('describe() returns the inferred columns', async () => { + const cols = await connect({ url: PG_URL }).describe('users') + expect(cols.map((c) => c.name)).toEqual(['id', 'country']) + }) + + it('snapshot() materializes a remote table into a local one', async () => { + const db = session() + try { + await db.session!.queryAsync('CREATE TABLE snap (id Int64, country String) ENGINE = Memory') + await db.connect({ url: PG_URL }).snapshot('users', { destination: 'snap' }) + const n = (await db.selectFrom('snap').select('country').where('country', '=', 'US').execute()) as unknown[] + expect(n).toHaveLength(2) + } finally { + db.close() + } + }) +}) + +describe.skipIf(!MY_OK)('connect — live MySQL federation', () => { + it('reads rows and lists the catalog', async () => { + const my = connect({ url: MY_URL }) + const rows = (await selectFrom(my.table('orders').as('o')) + .select('country') + .where('country', '=', 'US') + .orderBy('country') + .execute()) as { country: string }[] + expect(rows).toEqual([{ country: 'US' }, { country: 'US' }]) + expect(await my.databases()).toContain('shop') + expect(await my.tables('shop')).toContain('orders') + }) +}) + +describe.skipIf(!(PG_OK && MY_OK))('connect — cross-source JOIN (Postgres × MySQL)', () => { + it('joins a Postgres table to a MySQL table through the local engine', async () => { + const pg = connect({ url: PG_URL }) + const my = connect({ url: MY_URL }) + const rows = (await selectFrom(pg.table('users').as('u')) + .innerJoin(my.table('orders').as('o'), 'u.id', 'o.id') + .select(['u.id', sql.raw('`u`.`country`').as('pg_country'), sql.raw('`o`.`country`').as('my_country')]) + .orderBy('u.id') + .execute()) as { id: string; pg_country: string; my_country: string }[] + expect(rows).toEqual([ + { id: '1', pg_country: 'US', my_country: 'US' }, + { id: '2', pg_country: 'FR', my_country: 'FR' }, + { id: '3', pg_country: 'US', my_country: 'US' }, + ]) + }) +}) From 4cf30db5ac07648a35254d35c5f680a713f0f28f Mon Sep 17 00:00:00 2001 From: Changshuo Chen Date: Wed, 24 Jun 2026 06:55:55 +0000 Subject: [PATCH 3/7] Add typed row inference and the gen-types CLI Infer row types from a typed Database through to .execute(). Add the chdb-gen-types CLI with runtime introspection helpers, plus static Drizzle and Prisma schema conversion. Co-Authored-By: Claude Opus 4.8 (1M context) --- package.json | 3 + src/layer3/builder/select.ts | 5 + src/layer3/codegen/cli.ts | 212 +++++++++++++++++ src/layer3/codegen/emit.ts | 58 +++++ src/layer3/codegen/from-drizzle.ts | 231 +++++++++++++++++++ src/layer3/codegen/from-prisma.ts | 200 ++++++++++++++++ src/layer3/codegen/introspect.ts | 108 +++++++++ src/layer3/database.ts | 22 +- src/layer3/index.ts | 21 ++ src/layer3/types/ch-types.ts | 84 +++++++ src/layer3/types/infer.ts | 51 ++++ test/v3/layer3/codegen.test.ts | 231 +++++++++++++++++++ test/v3/layer3/fixtures/drizzle-schema.ts | 40 ++++ test/v3/layer3/fixtures/prisma-schema.prisma | 44 ++++ test/v3/layer3/static-codegen.test.ts | 156 +++++++++++++ test/v3/layer3/types.test.ts | 130 +++++++++++ 16 files changed, 1589 insertions(+), 7 deletions(-) create mode 100644 src/layer3/codegen/cli.ts create mode 100644 src/layer3/codegen/emit.ts create mode 100644 src/layer3/codegen/from-drizzle.ts create mode 100644 src/layer3/codegen/from-prisma.ts create mode 100644 src/layer3/codegen/introspect.ts create mode 100644 src/layer3/types/ch-types.ts create mode 100644 src/layer3/types/infer.ts create mode 100644 test/v3/layer3/codegen.test.ts create mode 100644 test/v3/layer3/fixtures/drizzle-schema.ts create mode 100644 test/v3/layer3/fixtures/prisma-schema.prisma create mode 100644 test/v3/layer3/static-codegen.test.ts create mode 100644 test/v3/layer3/types.test.ts diff --git a/package.json b/package.json index e863145..39aef0b 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "description": "chDB bindings for nodejs", "main": "index.js", "types": "index.d.ts", + "bin": { + "chdb-gen-types": "dist/layer3/codegen/cli.js" + }, "exports": { ".": { "types": "./index.d.ts", diff --git a/src/layer3/builder/select.ts b/src/layer3/builder/select.ts index 2091a4d..2b3c329 100644 --- a/src/layer3/builder/select.ts +++ b/src/layer3/builder/select.ts @@ -35,6 +35,11 @@ function combine(existing: Expr | undefined, next: Expr, kind: 'And' | 'Or'): Ex type ColumnInput = ExprInput export class SelectQueryBuilder> { + // A phantom type marker so callers can recover the row type with + // `RowOf` — TypeScript's ReturnType picks .execute's last overload + // (the non-JSON fallback returning `unknown`), which loses the typed row. + declare readonly _row: O + constructor( private readonly ctx: ExecContext, private readonly node: SelectQueryNode, diff --git a/src/layer3/codegen/cli.ts b/src/layer3/codegen/cli.ts new file mode 100644 index 0000000..97b1a15 --- /dev/null +++ b/src/layer3/codegen/cli.ts @@ -0,0 +1,212 @@ +#!/usr/bin/env node +/** + * `chdb gen-types` — introspect a data source and write a TypeScript Database + * interface (column types as ClickHouse string literals) ready to feed into + * `chdb.database()`. The CLI is a thin wrapper over `introspectDatabase` + * and `emitDatabase`; everything programmatic lives in those two modules. + * + * Source modes: + * --from-file DESCRIBE a local file via the file() table function + * --from-table DESCRIBE a local table (the default connection) + * --from-url Open via connect() and DESCRIBE + * + * Common options: + * --table Repeatable. Server sources (--from-url postgres://..., + * mysql://, clickhouse://) need at least one. Each + * becomes a property on the emitted interface. + * --format File format (auto-detected if omitted) + * --structure File structure (auto-detected if omitted) + * --name Emitted interface name (default: Database) + * --out Write to file (default: stdout) + * --label