diff --git a/crates/krishiv-exec/src/context.rs b/crates/krishiv-exec/src/context.rs index e9b58aa0f..220c9c84e 100644 --- a/crates/krishiv-exec/src/context.rs +++ b/crates/krishiv-exec/src/context.rs @@ -398,6 +398,11 @@ pub struct TaskContext { name: &'static str, memory: OnceLock, dependency: Option, + /// The driver's scheduler-assigned globally unique partition id within + /// its pipeline group (Spark's partition index for the partition the + /// driver computes): stateful functions own one value stream per + /// partition. The first driver of a group reads partition 0. + partition: u32, } impl fmt::Debug for TaskContext { @@ -426,9 +431,24 @@ impl TaskContext { name, memory: OnceLock::new(), dependency: None, + partition: 0, } } + /// The same context for the driver `partition` (its scheduler-assigned + /// globally unique id within its pipeline group). + #[must_use] + pub(crate) fn with_partition(mut self, partition: u32) -> Self { + self.partition = partition; + self + } + + /// The driver's scheduler-assigned partition id (set when the driver is + /// spawned; the first driver of a group reads 0). + pub fn partition(&self) -> u32 { + self.partition + } + pub(crate) fn with_dependency(mut self, state: GlobalState) -> Self { self.dependency = Some(state); self diff --git a/crates/krishiv-exec/src/driver.rs b/crates/krishiv-exec/src/driver.rs index 8bdab1c43..7f7c7d600 100644 --- a/crates/krishiv-exec/src/driver.rs +++ b/crates/krishiv-exec/src/driver.rs @@ -119,14 +119,16 @@ fn since(executor: &dyn Executor, start: Instant) -> Duration { executor.now().saturating_duration_since(start) } -/// Spawns one driver of `run`. -pub(crate) fn spawn_driver(run: Arc) -> TaskHandle { +/// Spawns one driver of `run`: `partition` is the driver's +/// scheduler-assigned globally unique partition id within its pipeline +/// group (see [`TaskContext::with_partition`]). +pub(crate) fn spawn_driver(run: Arc, partition: u32) -> TaskHandle { let pipeline = run.pipeline.index; let query = Arc::clone(&run.query); let executor = Arc::clone(&run.executor); let q = Arc::clone(&query); let body = async move { - if let Err(e) = drive(run).await { + if let Err(e) = drive(run, partition).await { q.fail(e); } }; @@ -145,8 +147,8 @@ pub(crate) fn spawn_driver(run: Arc) -> TaskHandle { executor.spawn(query.class(), Box::pin(timed)) } -async fn drive(run: Arc) -> Result<()> { - Driver::new(run)?.run().await +async fn drive(run: Arc, partition: u32) -> Result<()> { + Driver::new(run, partition)?.run().await } /// Times every poll of a driver: the stall detector. A poll is a @@ -319,7 +321,7 @@ struct Driver { } impl Driver { - fn new(run: Arc) -> Result { + fn new(run: Arc, partition: u32) -> Result { let owner = SpillOwner::new(); let ctx = |id, name| { TaskContext::new( @@ -329,6 +331,7 @@ impl Driver { id, name, ) + .with_partition(partition) }; let p = &run.pipeline; let (source_id, source) = &p.source; diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index 7606aeab1..d24d49514 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -516,8 +516,9 @@ async fn reverse( let hashing = hashing.clone(); let handle = Arc::clone(ctx.consumer().consumer()); let query = Arc::clone(ctx.query()); + let partition = ctx.partition(); let out = ctx - .run_blocking(move || reverse_blocking(&shared, &hashing, task, &handle, &query)) + .run_blocking(move || reverse_blocking(&shared, &hashing, task, &handle, &query, partition)) .await; ctx.consumer().consumer().release_credit(); out @@ -536,6 +537,7 @@ fn reverse_blocking( task: Task, handle: &Arc, query: &QueryContext, + partition: u32, ) -> Result { // Everything the join holds is reserved before it is allocated, and the // reversal declines while it still can (before the build stream starts): @@ -648,10 +650,10 @@ fn reverse_blocking( } } if pairs.len() >= PAIR_BATCH { - mark_pairs(shared, &probe, &mut pairs, &mut matched)?; + mark_pairs(shared, &probe, &mut pairs, &mut matched, partition)?; } } - mark_pairs(shared, &probe, &mut pairs, &mut matched) + mark_pairs(shared, &probe, &mut pairs, &mut matched, partition) }; for_each( build, @@ -684,6 +686,7 @@ fn mark_pairs( probe: &Batch, pairs: &mut Vec<(u32, &[u8])>, matched: &mut [bool], + partition: u32, ) -> Result<()> { let Some(filter) = &shared.filter else { pairs.clear(); @@ -701,7 +704,7 @@ fn mark_pairs( let mut columns: Vec = taken.data().columns().to_vec(); columns.extend(build_cols); let candidates = super::record(&shared.filter_schema, columns, m)?; - let mask = filter.evaluate(&candidates)?; + let mask = filter.evaluate_in(&candidates, partition)?; if mask.len() != m { return Err(internal( "a join filter returned the wrong row count".to_owned(), @@ -770,10 +773,11 @@ pub(crate) async fn step( .get() .ok_or_else(|| internal("hash join probed before its build".to_owned()))?; let limit = ctx.query().config().batch_size.max(1); + let partition = ctx.partition(); if let Some(a) = &mut st.active { if let Some(c) = &mut a.cursor { let tables = Tables::Single(&a.table); - let (out, done) = c.next(shared, &tables, limit)?; + let (out, done) = c.next(shared, &tables, limit, partition)?; if done { a.cursor = None; } @@ -1676,9 +1680,14 @@ mod tests { let mut filler = krishiv_memory::Reservation::new(Arc::clone(&other), id); filler.try_grow(LIMIT - root.reserved()).unwrap(); let handle = Arc::clone(ctx.consumer().consumer()); - let Ok(Reversed::Declined(task)) = - reverse_blocking(shared, &hashing, task, &handle, ctx.query()) - else { + let Ok(Reversed::Declined(task)) = reverse_blocking( + shared, + &hashing, + task, + &handle, + ctx.query(), + ctx.partition(), + ) else { panic!("the probe rows cannot fit"); }; assert_eq!(task.probe.files.len(), files, "nothing was consumed"); @@ -1686,9 +1695,14 @@ mod tests { // With room, the same task joins: a semi join with no build rows // emits nothing, and the probe files are gone with the task. drop(filler); - let Ok(Reversed::Joined(out)) = - reverse_blocking(shared, &hashing, task, &handle, ctx.query()) - else { + let Ok(Reversed::Joined(out)) = reverse_blocking( + shared, + &hashing, + task, + &handle, + ctx.query(), + ctx.partition(), + ) else { panic!("the probe rows fit now"); }; assert!(out.is_none()); diff --git a/crates/krishiv-exec/src/join/nested.rs b/crates/krishiv-exec/src/join/nested.rs index 441d6a587..511d2dbcc 100644 --- a/crates/krishiv-exec/src/join/nested.rs +++ b/crates/krishiv-exec/src/join/nested.rs @@ -221,6 +221,8 @@ struct ProbeLocal { offset: usize, /// Per probe row of the chunk: whether it matched. matched: Vec, + /// The driver's partition: the residual filter evaluates in it. + partition: u32, } impl Operator for ProbeOp { @@ -250,6 +252,7 @@ impl Operator for ProbeOp { batch: 0, offset: 0, matched: Vec::new(), + partition: ctx.partition(), })) } @@ -280,8 +283,10 @@ impl Operator for ProbeOp { let n = (b.num_rows() - l.offset).min((TARGET_PAIRS / (l.end - l.start)).max(1)); let slice = b.slice(l.offset, n); l.offset += n; + let (partition, start, end) = (l.partition, l.start, l.end); + let matched = &mut l.matched; if let Some(pairs) = - self.pairs(input.data(), l.start, l.end, &slice, &mut l.matched)? + self.pairs(input.data(), start, end, &slice, matched, partition)? { out.push(pairs); } @@ -325,6 +330,7 @@ impl ProbeOp { end: usize, build: &RecordBatch, matched: &mut [bool], + partition: u32, ) -> Result> { let b = build.num_rows(); if b == 0 { @@ -356,7 +362,7 @@ impl ProbeOp { let mask = match &self.0.spec.filter { None => BooleanArray::from(vec![true; n]), Some(f) => { - let m = f.evaluate(&paired)?; + let m = f.evaluate_in(&paired, partition)?; // NULL is not a match. match m.nulls() { Some(v) => BooleanArray::new(m.values() & v.inner(), None), diff --git a/crates/krishiv-exec/src/join/probe.rs b/crates/krishiv-exec/src/join/probe.rs index 7678d1cbc..226d5816f 100644 --- a/crates/krishiv-exec/src/join/probe.rs +++ b/crates/krishiv-exec/src/join/probe.rs @@ -154,6 +154,7 @@ impl Cursor { shared: &Shared, tables: &Tables<'_>, limit: usize, + partition: u32, ) -> Result<(Option, bool)> { let n = self.batch.num_rows(); let quick = !shared.join_type.emits_build() && shared.filter.is_none(); @@ -190,7 +191,10 @@ impl Cursor { } } if !probe.is_empty() { - return Ok((self.pairs(shared, tables, &probe, &build)?, false)); + return Ok(( + self.pairs(shared, tables, &probe, &build, partition)?, + false, + )); } Ok((self.tail(shared)?, true)) } @@ -203,6 +207,7 @@ impl Cursor { tables: &Tables<'_>, probe: &[u32], build: &[(usize, u32)], + partition: u32, ) -> Result> { let arena = |p: usize| tables.get(p).map(JoinTable::arena); let build_cols = shared.build_columns(self.rows, build, arena)?; @@ -213,7 +218,7 @@ impl Cursor { let mask = match &shared.filter { Some(f) => { let candidates = record(&shared.filter_schema, columns.clone(), m)?; - let mask = f.evaluate(&candidates)?; + let mask = f.evaluate_in(&candidates, partition)?; if mask.len() != m { return Err(internal( "a join filter returned the wrong row count".to_owned(), @@ -368,6 +373,8 @@ struct ProbeLocal { signal: Arc, flush: Flush, batch_size: usize, + /// The driver's partition: the residual filter evaluates in it. + partition: u32, } fn local_mut(local: &mut LocalState) -> Result<&mut ProbeLocal> { @@ -428,12 +435,13 @@ impl ProbeOp { l.cursor = Some(cursor); } let limit = l.batch_size; + let partition = l.partition; let cursor = l .cursor .as_mut() .ok_or_else(|| internal("no probe cursor".to_owned()))?; loop { - let (out, done) = cursor.next(s, &tables, limit)?; + let (out, done) = cursor.next(s, &tables, limit, partition)?; if done { l.cursor = None; return Ok(OperatorOutput::NeedMoreInput(out)); @@ -473,6 +481,7 @@ impl Operator for ProbeOp { signal: Arc::clone(memory.signal()), flush: Flush::default(), batch_size: ctx.query().config().batch_size.max(1), + partition: ctx.partition(), })) } diff --git a/crates/krishiv-exec/src/operators/expand.rs b/crates/krishiv-exec/src/operators/expand.rs index aedb7a718..fca4cae65 100644 --- a/crates/krishiv-exec/src/operators/expand.rs +++ b/crates/krishiv-exec/src/operators/expand.rs @@ -8,7 +8,7 @@ use arrow_array::{ArrayRef, RecordBatch, new_null_array}; use arrow_schema::{DataType, SchemaRef}; use krishiv_types::batch::Batch; -use super::{Project, ScalarExpr, local_mut, no_state}; +use super::{PartitionState, Project, ScalarExpr, local_mut}; use crate::context::TaskContext; use crate::error::{Result, internal}; use crate::operator::{LocalState, Operator, OperatorOutput}; @@ -56,8 +56,14 @@ impl Expand { } } -/// The next projection to emit for the current input batch. -struct Cursor(usize); +/// The next projection to emit for the current input batch, and the +/// driver's partition: the projections evaluate row expressions, which draw +/// the partition's stream. +#[derive(Debug, Clone, Copy)] +struct State { + next: usize, + partition: u32, +} impl Operator for Expand { fn name(&self) -> &'static str { @@ -68,29 +74,35 @@ impl Operator for Expand { Arc::clone(&self.schema) } - fn local_state(&self, _ctx: &TaskContext) -> Result { - Ok(Box::new(Cursor(0))) + fn local_state(&self, ctx: &TaskContext) -> Result { + Ok(Box::new(State { + next: 0, + partition: ctx.partition(), + })) } fn execute(&self, input: &Batch, local: &mut LocalState) -> Result { - let cursor = local_mut::(local, "Expand")?; + let state = local_mut::(local, "Expand")?; if input.num_rows() == 0 { - cursor.0 = 0; + state.next = 0; return Ok(OperatorOutput::NeedMoreInput(None)); } let project = self .projects - .get(cursor.0) + .get(state.next) .ok_or_else(|| internal("an expansion past its last projection".to_owned()))?; - let OperatorOutput::NeedMoreInput(Some(out)) = project.execute(input, &mut no_state())? + let mut partition: LocalState = Box::new(PartitionState { + partition: state.partition, + }); + let OperatorOutput::NeedMoreInput(Some(out)) = project.execute(input, &mut partition)? else { return Err(internal("a projection did not output a batch".to_owned())); }; - cursor.0 += 1; - if cursor.0 < self.projects.len() { + state.next += 1; + if state.next < self.projects.len() { return Ok(OperatorOutput::HaveMoreOutput(out)); } - cursor.0 = 0; + state.next = 0; Ok(OperatorOutput::NeedMoreInput(Some(out))) } } diff --git a/crates/krishiv-exec/src/operators/expr.rs b/crates/krishiv-exec/src/operators/expr.rs index a9aea6646..fbd1666ab 100644 --- a/crates/krishiv-exec/src/operators/expr.rs +++ b/crates/krishiv-exec/src/operators/expr.rs @@ -20,6 +20,16 @@ pub trait Predicate: Send + Sync + fmt::Debug { /// Any error fails the query (for example a division by zero under /// ANSI mode). fn evaluate(&self, batch: &RecordBatch) -> Result; + /// As [`Self::evaluate`], in the executing partition `partition` + /// (Spark's partition index). The default runs [`Self::evaluate`], for + /// the stateless predicates. + /// + /// # Errors + /// As [`Self::evaluate`]. + fn evaluate_in(&self, batch: &RecordBatch, partition: u32) -> Result { + let _ = partition; + self.evaluate(batch) + } } /// A scalar expression evaluated over a batch, for [`Project`](super::Project). @@ -33,6 +43,16 @@ pub trait ScalarExpr: Send + Sync + fmt::Debug { /// # Errors /// Any error fails the query. fn evaluate(&self, batch: &RecordBatch) -> Result; + /// As [`Self::evaluate`], in the executing partition `partition` + /// (Spark's partition index). The default runs [`Self::evaluate`], for + /// the stateless expressions. + /// + /// # Errors + /// As [`Self::evaluate`]. + fn evaluate_in(&self, batch: &RecordBatch, partition: u32) -> Result { + let _ = partition; + self.evaluate(batch) + } } /// A [`Predicate`] from a closure. diff --git a/crates/krishiv-exec/src/operators/filter.rs b/crates/krishiv-exec/src/operators/filter.rs index 5c850bda1..17a039cbd 100644 --- a/crates/krishiv-exec/src/operators/filter.rs +++ b/crates/krishiv-exec/src/operators/filter.rs @@ -7,7 +7,7 @@ use arrow_schema::SchemaRef; use krishiv_types::batch::Batch; use super::expr::Predicate; -use super::no_state; +use super::{PartitionState, local_mut}; use crate::context::TaskContext; use crate::error::Result; use crate::operator::{LocalState, Operator, OperatorOutput}; @@ -38,15 +38,18 @@ impl Operator for Filter { Arc::clone(&self.schema) } - fn local_state(&self, _ctx: &TaskContext) -> Result { - Ok(no_state()) + fn local_state(&self, ctx: &TaskContext) -> Result { + Ok(Box::new(PartitionState { + partition: ctx.partition(), + })) } - fn execute(&self, input: &Batch, _local: &mut LocalState) -> Result { + fn execute(&self, input: &Batch, local: &mut LocalState) -> Result { if input.num_rows() == 0 { return Ok(OperatorOutput::NeedMoreInput(None)); } - let mask = self.predicate.evaluate(input.data())?; + let partition = local_mut::(local, "Filter")?.partition; + let mask = self.predicate.evaluate_in(input.data(), partition)?; let out = input.filter(&mask)?; Ok(OperatorOutput::NeedMoreInput( (out.num_rows() > 0).then_some(out), diff --git a/crates/krishiv-exec/src/operators/mod.rs b/crates/krishiv-exec/src/operators/mod.rs index 4ebc36e92..ddb7c1f6b 100644 --- a/crates/krishiv-exec/src/operators/mod.rs +++ b/crates/krishiv-exec/src/operators/mod.rs @@ -84,5 +84,13 @@ pub(crate) fn no_state() -> LocalState { Box::new(()) } +/// Per-driver local state of an operator evaluating row expressions: the +/// driver's partition index (see [`TaskContext::partition`](crate::context::TaskContext::partition)). +#[derive(Debug, Clone, Copy)] +pub(crate) struct PartitionState { + /// The driver's partition index. + pub(crate) partition: u32, +} + #[cfg(all(test, not(loom)))] mod tests; diff --git a/crates/krishiv-exec/src/operators/project.rs b/crates/krishiv-exec/src/operators/project.rs index a6dbdd3ba..27ce49145 100644 --- a/crates/krishiv-exec/src/operators/project.rs +++ b/crates/krishiv-exec/src/operators/project.rs @@ -9,7 +9,7 @@ use arrow_schema::{Field, Schema, SchemaRef}; use krishiv_types::batch::Batch; use super::expr::ScalarExpr; -use super::no_state; +use super::{PartitionState, local_mut}; use crate::context::TaskContext; use crate::error::{Result, internal}; use crate::operator::{LocalState, Operator, OperatorOutput}; @@ -138,15 +138,18 @@ impl Operator for Project { Arc::clone(&self.schema) } - fn local_state(&self, _ctx: &TaskContext) -> Result { - Ok(no_state()) + fn local_state(&self, ctx: &TaskContext) -> Result { + Ok(Box::new(PartitionState { + partition: ctx.partition(), + })) } - fn execute(&self, input: &Batch, _local: &mut LocalState) -> Result { + fn execute(&self, input: &Batch, local: &mut LocalState) -> Result { let rows = input.num_rows(); if rows == 0 { return Ok(OperatorOutput::NeedMoreInput(None)); } + let partition = local_mut::(local, "Project")?.partition; let data = input.data(); let columns = self .items @@ -158,7 +161,7 @@ impl Operator for Project { .get(*index) .map(Arc::clone) .ok_or_else(|| internal(format!("project index {index} out of range"))), - ProjectItem::Expr { expr, .. } => expr.evaluate(data), + ProjectItem::Expr { expr, .. } => expr.evaluate_in(data, partition), } }) .collect::>>()?; diff --git a/crates/krishiv-exec/src/operators/replicate.rs b/crates/krishiv-exec/src/operators/replicate.rs index 3fd4395b2..e5aac7f24 100644 --- a/crates/krishiv-exec/src/operators/replicate.rs +++ b/crates/krishiv-exec/src/operators/replicate.rs @@ -38,6 +38,8 @@ impl ReplicateRows { /// Where the operator is in the current input batch. struct Cursor { batch_size: usize, + /// The driver's partition: `count` evaluates stateful functions in it. + partition: u32, /// The input's counts, while some of its copies are still to emit. counts: Option, /// The row being copied. @@ -58,6 +60,7 @@ impl Operator for ReplicateRows { fn local_state(&self, ctx: &TaskContext) -> Result { Ok(Box::new(Cursor { batch_size: ctx.query().config().batch_size.max(1), + partition: ctx.partition(), counts: None, row: 0, emitted: 0, @@ -72,7 +75,7 @@ impl Operator for ReplicateRows { if input.num_rows() == 0 { return Ok(OperatorOutput::NeedMoreInput(None)); } - let v = self.count.evaluate(input.data())?; + let v = self.count.evaluate_in(input.data(), cur.partition)?; let c = v .as_primitive_opt::() .ok_or_else(|| internal("a replication count is not BIGINT".to_owned()))? diff --git a/crates/krishiv-exec/src/operators/scan.rs b/crates/krishiv-exec/src/operators/scan.rs index 770f9b637..3663e332c 100644 --- a/crates/krishiv-exec/src/operators/scan.rs +++ b/crates/krishiv-exec/src/operators/scan.rs @@ -195,6 +195,9 @@ pub(super) struct ScanLocal { pub(super) claimed: u64, /// The runtime filters' row stage. pub(super) rows: super::RowStage, + /// The driver's partition: pushed filters evaluate stateful functions + /// in it (see [`Predicate::evaluate_in`]). + pub(super) partition: u32, } impl MemoryScan { @@ -278,7 +281,12 @@ impl MemoryScan { /// Reads the next piece of the claimed (non-empty) morsel, filtered and /// projected. - pub(super) fn next_piece(&self, local: &mut ScanLocal, batch_rows: usize) -> Result { + pub(super) fn next_piece( + &self, + local: &mut ScanLocal, + batch_rows: usize, + partition: u32, + ) -> Result { let batch = self .batches .get(local.batches.start) @@ -301,7 +309,7 @@ impl MemoryScan { if piece.num_rows() == 0 { break; } - let mask = filter.evaluate(piece.data())?; + let mask = filter.evaluate_in(piece.data(), partition)?; piece = piece.filter(&mask)?; } // After the pushed filters, as Spark evaluates a runtime filter @@ -352,7 +360,8 @@ impl MemoryScan { // One piece per call, even an empty input batch or one a pushed // filter emptied: the driver must reach a batch boundary (cancel, // abort and spill checks) after every piece read. - let piece = self.next_piece(local, g.batch_rows)?; + let partition = local.partition; + let piece = self.next_piece(local, g.batch_rows, partition)?; g.returned .fetch_add(piece.num_rows() as u64, Ordering::Relaxed); Ok(Some(piece)) @@ -421,6 +430,7 @@ impl Source for MemoryScan { offset: 0, claimed: 0, rows: super::RowStage::new(&g.runtime, &self.table, self.runtime.settings()), + partition: _ctx.partition(), })) } diff --git a/crates/krishiv-exec/src/operators/tests.rs b/crates/krishiv-exec/src/operators/tests.rs index d628fb75a..38342e8ce 100644 --- a/crates/krishiv-exec/src/operators/tests.rs +++ b/crates/krishiv-exec/src/operators/tests.rs @@ -126,8 +126,9 @@ fn filter_keeps_selected_rows_with_their_weights() { #[test] fn filter_edge_cases_produce_no_output_and_null_does_not_select() { + let (_root, ctx) = ctx(); let f = Filter::new(schema(), gt1()); - let mut local = unlocal(); + let mut local = f.local_state(&ctx).unwrap_or_else(|e| unreachable!("{e}")); // Empty batch. let empty = ints(&[]); assert!( @@ -182,7 +183,11 @@ fn filter_errors_propagate() { Ok(BooleanArray::from(vec![true])) })), ); - let e = bad_len.execute(&ints(&[1, 2]), &mut unlocal()).unwrap_err(); + let (_root, ctx) = ctx(); + let mut local = bad_len + .local_state(&ctx) + .unwrap_or_else(|e| unreachable!("{e}")); + let e = bad_len.execute(&ints(&[1, 2]), &mut local).unwrap_err(); assert_eq!(e.error_class(), ErrorClass::InternalError); let failing = Filter::new( schema(), @@ -192,7 +197,7 @@ fn filter_errors_propagate() { })) })), ); - assert!(failing.execute(&ints(&[1]), &mut unlocal()).is_err()); + assert!(failing.execute(&ints(&[1]), &mut local).is_err()); assert!( format!( "{:?}", @@ -254,9 +259,11 @@ fn project_reorders_renames_and_computes() { .map(|f| f.name().clone()) .collect(); assert_eq!(names, ["s", "x", "y", "s"]); + let (_root, ctx) = ctx(); + let mut local = p.local_state(&ctx).unwrap_or_else(|e| unreachable!("{e}")); let input = weighted(&[1, 2], &[3, -4]); let out = out_batch( - p.execute(&input, &mut unlocal()) + p.execute(&input, &mut local) .unwrap_or_else(|e| unreachable!("{e}")), ) .unwrap_or_else(|| unreachable!("rows")); @@ -287,14 +294,16 @@ fn project_reorders_renames_and_computes() { #[test] fn project_edge_cases() { + let (_root, ctx) = ctx(); let all_null = batch(&[None, None]); let p = Project::try_new( &schema(), vec![ProjectItem::column(0), ProjectItem::expr("y", plus_one())], ) .unwrap_or_else(|e| unreachable!("{e}")); + let mut local = p.local_state(&ctx).unwrap_or_else(|e| unreachable!("{e}")); let out = out_batch( - p.execute(&all_null, &mut unlocal()) + p.execute(&all_null, &mut local) .unwrap_or_else(|e| unreachable!("{e}")), ) .unwrap_or_else(|| unreachable!("rows")); @@ -303,15 +312,18 @@ fn project_edge_cases() { // Empty input: no output. assert!( out_batch( - p.execute(&ints(&[]), &mut unlocal()) + p.execute(&ints(&[]), &mut local) .unwrap_or_else(|e| unreachable!("{e}")) ) .is_none() ); // No columns: the row count survives. let none = Project::try_new(&schema(), vec![]).unwrap_or_else(|e| unreachable!("{e}")); + let mut none_local = none + .local_state(&ctx) + .unwrap_or_else(|e| unreachable!("{e}")); let out = out_batch( - none.execute(&ints(&[1, 2, 3]), &mut unlocal()) + none.execute(&ints(&[1, 2, 3]), &mut none_local) .unwrap_or_else(|e| unreachable!("{e}")), ); assert_eq!( @@ -322,6 +334,7 @@ fn project_edge_cases() { #[test] fn project_rejects_bad_plans_and_bad_evaluators() { + let (_root, ctx) = ctx(); let e = Project::try_new(&schema(), vec![ProjectItem::column(2)]).unwrap_err(); assert_eq!(e.error_class(), ErrorClass::InternalError); // An evaluator whose column is shorter than the batch, or of another type. @@ -331,7 +344,8 @@ fn project_rejects_bad_plans_and_bad_evaluators() { vec![ProjectItem::expr("y", Arc::new(PlusOne { ty, short }))], ) .unwrap_or_else(|e| unreachable!("{e}")); - let e = p.execute(&ints(&[1, 2]), &mut unlocal()).unwrap_err(); + let mut local = p.local_state(&ctx).unwrap_or_else(|e| unreachable!("{e}")); + let e = p.execute(&ints(&[1, 2]), &mut local).unwrap_err(); assert_eq!(e.error_class(), ErrorClass::InternalError); } // An index that is valid for the plan but not for the batch. @@ -345,7 +359,22 @@ fn project_rejects_bad_plans_and_bad_evaluators() { ) .unwrap_or_else(|e| unreachable!("{e}")), ); - assert!(p.execute(&narrow, &mut unlocal()).is_err()); + let mut local = p.local_state(&ctx).unwrap_or_else(|e| unreachable!("{e}")); + assert!(p.execute(&narrow, &mut local).is_err()); +} + +#[test] +fn filter_and_project_reject_foreign_local_state() { + // A stateful operator evaluated with another operator's state fails + // closed instead of silently reading partition 0's stream. + let e = Filter::new(schema(), gt1()) + .execute(&ints(&[2]), &mut unlocal()) + .unwrap_err(); + assert_eq!(e.error_class(), ErrorClass::InternalError); + let p = Project::try_new(&schema(), vec![ProjectItem::column(0)]) + .unwrap_or_else(|e| unreachable!("{e}")); + let e = p.execute(&ints(&[2]), &mut unlocal()).unwrap_err(); + assert_eq!(e.error_class(), ErrorClass::InternalError); } fn run_limit(offset: u64, limit: u64, sizes: &[usize]) -> (Vec, usize) { @@ -711,8 +740,9 @@ fn scan_reports_inconsistent_state_as_internal_errors() { offset: 0, claimed: 0, rows: super::RowStage::new(&[], &schema(), super::RowStageSettings::default()), + partition: 0, }; - let e = scan.next_piece(&mut local, 4).unwrap_err(); + let e = scan.next_piece(&mut local, 4, 0).unwrap_err(); assert_eq!(e.error_class(), ErrorClass::InternalError); // A cursor that hands out a morsel the global state does not have. let global: crate::GlobalState = Arc::new(ScanGlobal { @@ -965,3 +995,89 @@ fn expand_rejects_bad_plans() { .unwrap_or_else(|| unreachable!("an error")); assert_eq!(err.error_class(), ErrorClass::InternalError); } + +/// A stateful function sees the driver's partition: `rand(0)` draws +/// partition 0's stream on one driver and partition 1's on another, even +/// when partition 1 binds first, and the same partition draws the same +/// stream on a fresh evaluation. +#[test] +fn project_threads_the_drivers_partition_to_stateful_functions() { + use arrow_array::NullArray; + use arrow_array::types::Float64Type; + use krishiv_common::config::Config; + use krishiv_plan::{ + Expr, ExprNode, FunctionId, FunctionKind, Literal, ResolvedExpr, ResolvedFunction, + }; + + use crate::planner::eval::{Compiled, EvalEnv, Input, compile}; + + // `rand(0)` analyzed: a non-nullable double of a folded seed. + let call = |seed: i32| { + let f = ResolvedFunction { + id: FunctionId::Registry { + name: "rand".to_owned(), + id: 0, + }, + kind: FunctionKind::Scalar, + args: vec![Arc::new(Expr::new(ExprNode::Literal(Literal::Integer( + seed, + ))))], + distinct: false, + filter: None, + data_type: krishiv_types::DataType::Double, + nullable: false, + deterministic: false, + }; + Expr::new(ExprNode::Resolved(ResolvedExpr::Function(f))) + }; + let one_row = || { + let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Null, true)])); + let data = RecordBatch::try_new(schema, vec![Arc::new(NullArray::new(1))]) + .unwrap_or_else(|e| unreachable!("{e}")); + Batch::new(data) + }; + let project = |seed: i32| { + let config = Config::new(); + let env = EvalEnv::new(&config, Arc::from("test")); + let schema = Arc::new(Schema::empty()); + let compiled = compile( + &call(seed), + Input { + layout: &[], + schema: &schema, + env: &env, + subqueries: &[], + }, + ) + .unwrap_or_else(|e| unreachable!("{e}")); + let input = one_row(); + Project::try_new( + &input.schema(), + vec![ProjectItem::expr( + "r", + Arc::new(Compiled(compiled)) as Arc, + )], + ) + .unwrap_or_else(|e| unreachable!("{e}")) + }; + let run = |p: &Arc, partition: u32| { + let (_root, ctx) = ctx(); + let ctx = ctx.with_partition(partition); + let mut local = p.local_state(&ctx).unwrap_or_else(|e| unreachable!("{e}")); + let out = out_batch( + p.execute(&one_row(), &mut local) + .unwrap_or_else(|e| unreachable!("{e}")), + ) + .unwrap_or_else(|| unreachable!("a row")); + out.data().column(0).as_primitive::().value(0) + }; + // Partition 1 binds first on the shared binding: partition 0 still + // draws its own stream (each slot draws `base + slot`), and both are + // Spark's first draws for their seeds. + let p = project(0); + assert_eq!(run(&p, 1), 0.6363787615254752); + assert_eq!(run(&p, 0), 0.7604953758285915); + // A fresh evaluation draws the same stream again: the stream is a pure + // function of (seed, partition, row). + assert_eq!(run(&project(0), 0), 0.7604953758285915); +} diff --git a/crates/krishiv-exec/src/planner/eval/arith.rs b/crates/krishiv-exec/src/planner/eval/arith.rs index 07d1f37ee..0864dd26f 100644 --- a/crates/krishiv-exec/src/planner/eval/arith.rs +++ b/crates/krishiv-exec/src/planner/eval/arith.rs @@ -151,8 +151,11 @@ impl PhysExpr for Binary { self.nullable } - fn eval(&self, batch: &RecordBatch) -> Result { - let (mut l, r) = (self.l.eval(batch)?, self.r.eval(batch)?); + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let (mut l, r) = ( + self.l.eval(batch, partition)?, + self.r.eval(batch, partition)?, + ); let scalar = l.is_scalar() && r.is_scalar(); if scalar { l = Value::Array(Arc::clone(l.array())); @@ -213,8 +216,8 @@ impl PhysExpr for Negative { self.e.nullable() } - fn eval(&self, batch: &RecordBatch) -> Result { - let v = self.e.eval(batch)?; + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let v = self.e.eval(batch, partition)?; let a = v.array(); let ctx = &self.kernels; macro_rules! int { @@ -294,8 +297,11 @@ impl PhysExpr for IntegralDivide { true } - fn eval(&self, batch: &RecordBatch) -> Result { - let (mut l, r) = (self.l.eval(batch)?, self.r.eval(batch)?); + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let (mut l, r) = ( + self.l.eval(batch, partition)?, + self.r.eval(batch, partition)?, + ); let scalar = l.is_scalar() && r.is_scalar(); if scalar { l = Value::Array(Arc::clone(l.array())); diff --git a/crates/krishiv-exec/src/planner/eval/cast.rs b/crates/krishiv-exec/src/planner/eval/cast.rs index 3cae59546..96fa0a7a2 100644 --- a/crates/krishiv-exec/src/planner/eval/cast.rs +++ b/crates/krishiv-exec/src/planner/eval/cast.rs @@ -149,8 +149,8 @@ impl PhysExpr for Cast { self.nullable } - fn eval(&self, batch: &RecordBatch) -> Result { - let v = self.e.eval(batch)?; + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let v = self.e.eval(batch, partition)?; let out = apply(&self.parts, v.array())?; Ok(match v { Value::Scalar(_) => Value::Scalar(out), diff --git a/crates/krishiv-exec/src/planner/eval/cond.rs b/crates/krishiv-exec/src/planner/eval/cond.rs index 943c5f6be..b4d750804 100644 --- a/crates/krishiv-exec/src/planner/eval/cond.rs +++ b/crates/krishiv-exec/src/planner/eval/cond.rs @@ -93,7 +93,7 @@ impl PhysExpr for When { || self.branches.iter().any(|(_, v)| v.nullable()) } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { let rows = batch.num_rows(); // `rest`: the rows no branch took yet, as a batch and their row // numbers in `batch`. @@ -106,7 +106,7 @@ impl PhysExpr for When { if rest_rows.is_empty() { break; } - let c = cond.eval(&rest)?.into_array(rest.num_rows())?; + let c = cond.eval(&rest, partition)?.into_array(rest.num_rows())?; let c = c .as_boolean_opt() .ok_or_else(|| internal("CASE condition"))?; @@ -121,7 +121,9 @@ impl PhysExpr for When { let mask = BooleanArray::new(taken.clone(), None); let chosen = filter_record_batch(&rest, &mask).map_err(|e| internal(&e.to_string()))?; let v = cast_null( - value.eval(&chosen)?.into_array(chosen.num_rows())?, + value + .eval(&chosen, partition)? + .into_array(chosen.num_rows())?, &self.out, ); let part = parts.len(); @@ -143,7 +145,10 @@ impl PhysExpr for When { } if !rest_rows.is_empty() { let v = match &self.otherwise { - Some(e) => cast_null(e.eval(&rest)?.into_array(rest.num_rows())?, &self.out), + Some(e) => cast_null( + e.eval(&rest, partition)?.into_array(rest.num_rows())?, + &self.out, + ), None => new_null_array(&self.out, rest_rows.len()), }; let part = parts.len(); @@ -262,12 +267,12 @@ impl PhysExpr for NullIf { true } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { let rows = batch.num_rows(); // `value` is evaluated once: the comparison casts its array, the // result keeps its values. - let a = self.value.eval(batch)?; - let b = self.other.eval(batch)?; + let a = self.value.eval(batch, partition)?; + let b = self.other.eval(batch, partition)?; let scalar = a.is_scalar() && b.is_scalar(); let n = if scalar { 1 } else { rows }; let a_full = a.into_array(n)?; @@ -348,10 +353,10 @@ impl PhysExpr for Nanvl { self.a.nullable() || self.b.nullable() } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { let rows = batch.num_rows(); // `a` is evaluated once; NULL is not NaN. - let a = self.a.eval(batch)?.into_array(rows)?; + let a = self.a.eval(batch, partition)?.into_array(rows)?; let nan = match a.data_type() { ArrowType::Float32 => BooleanBuffer::from_iter( a.as_primitive::() @@ -369,7 +374,7 @@ impl PhysExpr for Nanvl { return Ok(Value::Array(a)); } // `b` sees only the NaN rows, so `nanvl(1, 1 / 0)` never raises. - let b = super::eval_where(self.b.as_ref(), batch, &nan)?; + let b = super::eval_where(self.b.as_ref(), batch, &nan, partition)?; let out = zip(&BooleanArray::new(nan, None), &b, &a).map_err(|e| internal(&e.to_string()))?; Ok(Value::Array(out)) @@ -457,7 +462,10 @@ mod tests { /// Evaluates `e` over `batch` as a column of `batch.num_rows()` rows. fn values(e: &Arc, batch: &RecordBatch) -> ArrayRef { - e.eval(batch).unwrap().into_array(batch.num_rows()).unwrap() + e.eval(batch, 0) + .unwrap() + .into_array(batch.num_rows()) + .unwrap() } /// `1 / 0`: `NULL` without ANSI, `DIVIDE_BY_ZERO` with it. @@ -529,7 +537,7 @@ mod tests { ); let e = coalesce(&f, vec![one_over_zero(&env), lit(&Literal::Double(1.0))]).unwrap(); if ansi { - let err = e.eval(&batch).unwrap_err(); + let err = e.eval(&batch, 0).unwrap_err(); assert_eq!(err.error_class(), ErrorClass::DivideByZero); } else { assert_eq!( @@ -666,7 +674,7 @@ mod tests { .unwrap(); match want { None => { - let err = e.eval(&batch).unwrap_err(); + let err = e.eval(&batch, 0).unwrap_err(); assert_eq!(err.error_class(), ErrorClass::CastInvalidInput); assert_eq!(err.sql_state(), Some("22018")); } @@ -736,9 +744,9 @@ mod tests { self.inner.nullable() } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { self.n.fetch_add(1, Ordering::SeqCst); - self.inner.eval(batch) + self.inner.eval(batch, partition) } } @@ -868,7 +876,7 @@ mod tests { ) .unwrap(); if ansi { - let err = e.eval(&batch).unwrap_err(); + let err = e.eval(&batch, 0).unwrap_err(); assert_eq!(err.error_class(), ErrorClass::DivideByZero); } else { assert!(values(&e, &batch).as_primitive::().is_null(0)); diff --git a/crates/krishiv-exec/src/planner/eval/field.rs b/crates/krishiv-exec/src/planner/eval/field.rs index f73e94a4e..8a8c34300 100644 --- a/crates/krishiv-exec/src/planner/eval/field.rs +++ b/crates/krishiv-exec/src/planner/eval/field.rs @@ -29,8 +29,8 @@ impl PhysExpr for GetField { self.nullable } - fn eval(&self, batch: &RecordBatch) -> Result { - Ok(match self.base.eval(batch)? { + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + Ok(match self.base.eval(batch, partition)? { Value::Array(a) => Value::Array(get(&a, self.ordinal)?), Value::Scalar(s) => Value::Scalar(get(&s, self.ordinal)?), }) diff --git a/crates/krishiv-exec/src/planner/eval/func.rs b/crates/krishiv-exec/src/planner/eval/func.rs index 3f35f4e37..fae6ace1a 100644 --- a/crates/krishiv-exec/src/planner/eval/func.rs +++ b/crates/krishiv-exec/src/planner/eval/func.rs @@ -54,7 +54,8 @@ pub(super) fn compile( .with_kernel(input.env.kernels) .with_time_zone(Arc::new(zone)) .with_query_start(input.env.query_start_micros) - .with_user(Arc::clone(&input.env.user)); + .with_user(Arc::clone(&input.env.user)) + .with_seed_salt(Arc::clone(&input.env.seed_salt)); let function = bind_scalar(name, &bind_args, &opts).map_err(|err| { if err.error_class() == krishiv_common::error::ErrorClass::KrishivUnsupportedFeature { unsupported_expr(e) @@ -101,18 +102,20 @@ impl PhysExpr for Call { self.nullable } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { let args = self .args .iter() .map(|a| { - Ok(match a.eval(batch)? { + Ok(match a.eval(batch, partition)? { Value::Array(v) => Arg::Array(v), Value::Scalar(v) => Arg::Scalar(v), }) }) .collect::>>()?; - let out = self.function.invoke(&args, batch.num_rows())?; + let out = self + .function + .invoke_in(&args, batch.num_rows(), partition)?; Ok(Value::Array(if self.view { to_view(&out)? } else { out })) } } diff --git a/crates/krishiv-exec/src/planner/eval/logic.rs b/crates/krishiv-exec/src/planner/eval/logic.rs index c77a05772..f08d64884 100644 --- a/crates/krishiv-exec/src/planner/eval/logic.rs +++ b/crates/krishiv-exec/src/planner/eval/logic.rs @@ -62,8 +62,8 @@ impl PhysExpr for AndOr { self.l.nullable() || self.r.nullable() } - fn eval(&self, batch: &RecordBatch) -> Result { - let lv = self.l.eval(batch)?; + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let lv = self.l.eval(batch, partition)?; if lv.is_scalar() && batch.num_rows() > 0 { // A literal left side decides for every row or for none. let l = lv @@ -87,9 +87,9 @@ impl PhysExpr for AndOr { let decided = if self.and { &l_ok & &!&lb } else { &l_ok & &lb }; let open = !&decided; let rv = if open.count_set_bits() == rows { - self.r.eval(batch)? + self.r.eval(batch, partition)? } else { - Value::Array(super::eval_where(self.r.as_ref(), batch, &open)?) + Value::Array(super::eval_where(self.r.as_ref(), batch, &open, partition)?) }; let scalar = lv.is_scalar() && rv.is_scalar(); let rows = if scalar { 1 } else { rows }; @@ -133,8 +133,8 @@ impl PhysExpr for Not { self.0.nullable() } - fn eval(&self, batch: &RecordBatch) -> Result { - let v = self.0.eval(batch)?; + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let v = self.0.eval(batch, partition)?; let a = v .array() .as_boolean_opt() @@ -167,8 +167,8 @@ impl PhysExpr for IsNull { false } - fn eval(&self, batch: &RecordBatch) -> Result { - let v = self.e.eval(batch)?; + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let v = self.e.eval(batch, partition)?; let a = v.array(); let valid = match (a.data_type(), a.logical_nulls()) { (ArrowType::Null, _) => BooleanBuffer::new_unset(a.len()), @@ -246,8 +246,11 @@ impl PhysExpr for Compare { self.kind.is_some() && (maybe_null(&self.l) || maybe_null(&self.r)) } - fn eval(&self, batch: &RecordBatch) -> Result { - let (mut l, r) = (self.l.eval(batch)?, self.r.eval(batch)?); + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let (mut l, r) = ( + self.l.eval(batch, partition)?, + self.r.eval(batch, partition)?, + ); let scalar = l.is_scalar() && r.is_scalar(); if scalar { // Kernels take at most one literal side. diff --git a/crates/krishiv-exec/src/planner/eval/mod.rs b/crates/krishiv-exec/src/planner/eval/mod.rs index a7721863d..f2f578889 100644 --- a/crates/krishiv-exec/src/planner/eval/mod.rs +++ b/crates/krishiv-exec/src/planner/eval/mod.rs @@ -33,6 +33,7 @@ mod strings; use std::fmt; use std::sync::Arc; +use std::sync::atomic::AtomicU64; use arrow_array::cast::AsArray; use arrow_array::{Array, ArrayRef, BooleanArray, RecordBatch, UInt32Array, new_null_array}; @@ -94,11 +95,13 @@ pub(crate) fn broadcast(s: &ArrayRef, rows: usize) -> Result { /// `e` over the rows of `batch` selected by `mask`, as a column of every /// row (NULL where not selected): `e` sees only the selected rows, so it -/// raises only for them. +/// raises only for them. `partition` is the executing partition's index +/// (see [`PhysExpr::eval`]). pub(crate) fn eval_where( e: &dyn PhysExpr, batch: &RecordBatch, mask: &arrow_buffer::BooleanBuffer, + partition: u32, ) -> Result { let rows = batch.num_rows(); let selected = mask.count_set_bits(); @@ -108,7 +111,7 @@ pub(crate) fn eval_where( let chosen = arrow_select::filter::filter_record_batch(batch, &BooleanArray::new(mask.clone(), None)) .map_err(|err| internal(&err.to_string()))?; - let values = e.eval(&chosen)?.into_array(selected)?; + let values = e.eval(&chosen, partition)?.into_array(selected)?; scatter(values.as_ref(), mask) } @@ -148,8 +151,13 @@ pub(crate) trait PhysExpr: Send + Sync + fmt::Debug { fn data_type(&self) -> &ArrowType; /// Whether a value may be NULL. fn nullable(&self) -> bool; - /// Its values over `batch`. - fn eval(&self, batch: &RecordBatch) -> Result; + /// Its values over `batch` in the executing partition `partition` + /// (Spark's partition index): a stateful function (`rand`, `uuid`, + /// `monotonically_increasing_id`, ...) owns one value stream per + /// partition, and this evaluates the next values of that stream. The + /// same query runs each partition's stream from its start, so one + /// partition's values never depend on another's rows. + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result; } /// What every expression of one query is evaluated with. @@ -169,6 +177,9 @@ pub(crate) struct EvalEnv { pub(crate) query_start_micros: i64, /// The session user (`current_user()`). pub(crate) user: Arc, + /// Counts the unseeded random binds of one plan instantiation (see + /// [`BindOptions::seed_salt`](krishiv_functions::scalar::BindOptions::seed_salt)). + pub(crate) seed_salt: Arc, } impl EvalEnv { @@ -197,6 +208,7 @@ impl EvalEnv { kernels: KernelCtx::new(), query_start_micros, user, + seed_salt: Arc::new(AtomicU64::new(0)), } } @@ -209,6 +221,7 @@ impl EvalEnv { kernels: KernelCtx::new(), query_start_micros: env.query_start_micros, user: Arc::clone(&env.user), + seed_salt: Arc::new(AtomicU64::new(0)), } } @@ -406,7 +419,7 @@ impl PhysExpr for Column { self.nullable } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, _partition: u32) -> Result { batch .columns() .get(self.index) @@ -439,7 +452,7 @@ impl PhysExpr for Lit { self.value.logical_null_count() > 0 } - fn eval(&self, _batch: &RecordBatch) -> Result { + fn eval(&self, _batch: &RecordBatch, _partition: u32) -> Result { Ok(Value::Scalar(Arc::clone(&self.value))) } } @@ -493,18 +506,22 @@ pub(crate) fn literal_array(l: &Literal) -> Result { pub(crate) struct Compiled(pub(crate) Arc); impl Compiled { - /// The values over `batch`. When evaluation fails, the error is the + /// The values over `batch` in `partition`. When evaluation fails, the error is the /// first failing row's, as Spark's row-at-a-time evaluation raises it: /// a batch evaluates sub-expressions one after another (and each lazy /// `CASE` branch over its own rows), so the first error found need not /// be the first row's. Only the failing path pays for the search. - fn values(&self, batch: &RecordBatch) -> Result { + fn values(&self, batch: &RecordBatch, partition: u32) -> Result { let rows = batch.num_rows(); - match self.0.eval(batch).and_then(|v| v.into_array(rows)) { + match self + .0 + .eval(batch, partition) + .and_then(|v| v.into_array(rows)) + { Ok(v) => Ok(v), Err(e) if rows > 1 => { for i in 0..rows { - self.0.eval(&batch.slice(i, 1))?; + self.0.eval(&batch.slice(i, 1), partition)?; } Err(e) } @@ -515,7 +532,11 @@ impl Compiled { impl Predicate for Compiled { fn evaluate(&self, batch: &RecordBatch) -> Result { - let v = self.values(batch)?; + ::evaluate_in(self, batch, 0) + } + + fn evaluate_in(&self, batch: &RecordBatch, partition: u32) -> Result { + let v = self.values(batch, partition)?; v.as_boolean_opt() .cloned() .ok_or_else(|| internal("a predicate is not boolean")) @@ -532,7 +553,11 @@ impl ScalarExpr for Compiled { } fn evaluate(&self, batch: &RecordBatch) -> Result { - self.values(batch) + ::evaluate_in(self, batch, 0) + } + + fn evaluate_in(&self, batch: &RecordBatch, partition: u32) -> Result { + self.values(batch, partition) } } diff --git a/crates/krishiv-exec/src/planner/eval/strings.rs b/crates/krishiv-exec/src/planner/eval/strings.rs index 49e2b2a82..fe91a1f7c 100644 --- a/crates/krishiv-exec/src/planner/eval/strings.rs +++ b/crates/krishiv-exec/src/planner/eval/strings.rs @@ -80,9 +80,10 @@ struct Like { pub(super) fn like(mut args: Vec>) -> Result> { let escape = if args.len() == 3 { let e = args.pop().ok_or_else(|| internal("LIKE arity"))?; - let v = e.eval(&RecordBatch::new_empty(Arc::new( - arrow_schema::Schema::empty(), - ))); + let v = e.eval( + &RecordBatch::new_empty(Arc::new(arrow_schema::Schema::empty())), + 0, + ); match v { Ok(Value::Scalar(s)) if is_string(s.data_type()) && s.is_valid(0) => { let text = str_value(s.as_ref(), 0); @@ -114,8 +115,11 @@ impl PhysExpr for Like { self.s.nullable() || self.pattern.nullable() } - fn eval(&self, batch: &RecordBatch) -> Result { - let (s, p) = (self.s.eval(batch)?, self.pattern.eval(batch)?); + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { + let (s, p) = ( + self.s.eval(batch, partition)?, + self.pattern.eval(batch, partition)?, + ); let scalar = s.is_scalar() && p.is_scalar(); let rows = if scalar { 1 } else { batch.num_rows() }; let (sa, pa) = (s.array(), p.array()); @@ -176,11 +180,11 @@ impl PhysExpr for Concat { self.nullable } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, partition: u32) -> Result { let values = self .args .iter() - .map(|a| a.eval(batch)) + .map(|a| a.eval(batch, partition)) .collect::>>()?; let scalar = values.iter().all(Value::is_scalar); let arrays = values @@ -230,7 +234,7 @@ mod tests { true } - fn eval(&self, batch: &RecordBatch) -> Result { + fn eval(&self, batch: &RecordBatch, _partition: u32) -> Result { Ok(Value::Array(Arc::clone(batch.column(self.0)))) } } @@ -282,7 +286,7 @@ mod tests { true, ) .unwrap(); - let Value::Array(out) = e.eval(&batch).unwrap() else { + let Value::Array(out) = e.eval(&batch, 0).unwrap() else { panic!("a column in, a column out"); }; let out: Vec<_> = out.as_string_view().iter().collect(); diff --git a/crates/krishiv-exec/src/planner/subquery.rs b/crates/krishiv-exec/src/planner/subquery.rs index 77323d579..bbc65bb52 100644 --- a/crates/krishiv-exec/src/planner/subquery.rs +++ b/crates/krishiv-exec/src/planner/subquery.rs @@ -141,7 +141,7 @@ impl PhysExpr for SubqueryValue { self.kind == Kind::Scalar } - fn eval(&self, _batch: &RecordBatch) -> Result { + fn eval(&self, _batch: &RecordBatch, _partition: u32) -> Result { let v = self .slot .0 diff --git a/crates/krishiv-exec/src/runner.rs b/crates/krishiv-exec/src/runner.rs index db691c680..c3c982779 100644 --- a/crates/krishiv-exec/src/runner.rs +++ b/crates/krishiv-exec/src/runner.rs @@ -401,6 +401,12 @@ fn start_group( let (sink_id, sink) = &first.sink; let sink_global = sink.global_state(&ctx((*sink_id, sink.name())))?; let mut started = Vec::with_capacity(members.len()); + // Scheduler-assigned globally unique partition ids: the drivers of + // every member share one id space, in pipeline order, so two drivers + // never compute the same partition (stateful functions own one value + // stream per partition). An index past `u32` is rejected rather than + // wrapped to `u32::MAX`, which would collide. + let mut next_partition: u64 = 0; for pipeline in members { let (source_id, source) = &pipeline.source; let mut source_ctx = ctx((*source_id, source.name())); @@ -421,11 +427,18 @@ fn start_group( .min(source.max_parallelism(&source_global)) .max(1) }; + let base = u32::try_from(next_partition) + .map_err(|_| internal("too many drivers for u32 partition ids".to_owned()))?; + // `drivers` fits `usize`; the end index must fit `u32` as well. + u32::try_from(next_partition + drivers as u64) + .map_err(|_| internal("too many drivers for u32 partition ids".to_owned()))?; + next_partition += drivers as u64; started.push(MemberStart { pipeline, operator_deps, source_global, drivers, + base_partition: base, }); } // One barrier per operator, over the drivers of every pipeline it is in. @@ -455,6 +468,8 @@ struct MemberStart { operator_deps: Vec>, source_global: GlobalState, drivers: usize, + /// The first scheduler-assigned partition id of its drivers. + base_partition: u32, } /// A pipeline group planned but not yet running: the state its members @@ -495,6 +510,21 @@ fn member_barriers( .ok_or_else(|| internal("an operator without a barrier".to_owned())) } +/// One driver per index of `member`: the scheduler-assigned partition ids +/// `base..base + count`. Rejects an id past `u32` instead of wrapping it. +fn drivers(run: &Arc, base: u32, count: usize) -> Result> { + let mut out = Vec::with_capacity(count); + for i in 0..count { + let offset = u32::try_from(i) + .map_err(|_| internal("too many drivers for u32 partition ids".to_owned()))?; + let partition = base + .checked_add(offset) + .ok_or_else(|| internal("too many drivers for u32 partition ids".to_owned()))?; + out.push(spawn_driver(Arc::clone(run), partition)); + } + Ok(out) +} + /// Starts every member of `group` together, sharing the sink's state. fn spawn_group( executor: &Arc, @@ -522,7 +552,7 @@ fn spawn_group( query: Arc::clone(query), executor: Arc::clone(executor), }); - handles.extend((0..m.drivers).map(|_| spawn_driver(Arc::clone(&run)))); + handles.extend(drivers(&run, m.base_partition, m.drivers)?); runs.push(run); } Ok((runs, handles)) @@ -559,7 +589,7 @@ async fn run_group_sequential( let mut handles = Drivers(FuturesUnordered::new()); handles .0 - .extend((0..m.drivers).map(|_| spawn_driver(Arc::clone(&run)))); + .extend(drivers(&run, m.base_partition, m.drivers)?); await_drivers(query, &mut handles).await; record_extra_metrics(query, std::slice::from_ref(&run)); query.check_cancelled()?; diff --git a/crates/krishiv-functions/src/lib.rs b/crates/krishiv-functions/src/lib.rs index 6fc14331c..50e70a4b0 100644 --- a/crates/krishiv-functions/src/lib.rs +++ b/crates/krishiv-functions/src/lib.rs @@ -9,6 +9,9 @@ //! [`scalar::ScalarFunction`] over the `krishiv-kernels` kernels, which //! the physical planner calls once per batch. //! +//! - `uniform`: the `uniform` result-type rule both layers share (not a +//! layer: the analyzer and the executor map their own types to its kinds). +//! //! Aggregates are executed by `krishiv-exec`; the canonical operators and //! `CAST` by the expression evaluator. @@ -19,3 +22,4 @@ mod python_wrappers; pub mod registry; pub mod scalar; pub mod signature; +pub(crate) mod uniform; diff --git a/crates/krishiv-functions/src/python_wrappers/table.rs b/crates/krishiv-functions/src/python_wrappers/table.rs index e5fe8a809..675b2218f 100644 --- a/crates/krishiv-functions/src/python_wrappers/table.rs +++ b/crates/krishiv-functions/src/python_wrappers/table.rs @@ -22,6 +22,7 @@ pub(super) const NOT_EXPORTED: &[(&str, &str)] = &[ ("bool_and", "no PySpark function"), ("bool_or", "no PySpark function"), ("count_if", "no PySpark function"), + ("random", "an alias of `rand`; no PySpark function"), ("chr", "no PySpark function; use `char`"), ("every", "no PySpark function"), ("len", "no PySpark function; use `length`"), @@ -63,19 +64,28 @@ pub(super) fn rows() -> Vec { for name in [ "cume_dist", "curdate", + "current_catalog", + "current_database", "current_date", + "current_schema", "current_timestamp", "current_user", "dense_rank", "e", + "input_file_block_length", + "input_file_block_start", + "input_file_name", "localtimestamp", + "monotonically_increasing_id", "now", "percent_rank", "pi", "rank", "row_number", "session_user", + "spark_partition_id", "user", + "version", ] { rows.push(Row::new(name, "")); } @@ -294,6 +304,18 @@ pub(super) fn rows() -> Vec { "col: ColumnOrName, upperChar: ColumnOrName = None, lowerChar: ColumnOrName = None, digitChar: ColumnOrName = None, otherChar: ColumnOrName = None", ), Row::new("ntile", "n: int"), + Row::new("rand", "seed: int = None"), + Row::new("randn", "seed: int = None"), + Row::new( + "randstr", + "length: int | Column, seed: int | Column = None", + ), + Row::new("typeof", "col: ColumnOrName"), + Row::new( + "uniform", + "min: ColumnOrName | float, max: ColumnOrName | float, seed: int | Column = None", + ), + Row::new("uuid", "seed: int | Column = None"), Row::new( "nth_value", "col: ColumnOrName, offset: int, ignoreNulls: bool = False", diff --git a/crates/krishiv-functions/src/registry/defs.rs b/crates/krishiv-functions/src/registry/defs.rs index 83c3848f4..e673d5d5d 100644 --- a/crates/krishiv-functions/src/registry/defs.rs +++ b/crates/krishiv-functions/src/registry/defs.rs @@ -848,6 +848,7 @@ pub(super) fn specs() -> Vec { out.extend(super::bitwise::specs()); out.extend(super::predicate::specs()); out.extend(super::number::specs()); + out.extend(super::misc::specs()); out.extend(super::url::specs()); out.extend(super::csv::specs()); out.extend(super::text::specs()); diff --git a/crates/krishiv-functions/src/registry/misc.rs b/crates/krishiv-functions/src/registry/misc.rs new file mode 100644 index 000000000..3c2675780 --- /dev/null +++ b/crates/krishiv-functions/src/registry/misc.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The misc scalar signatures: `version`, `current_database` +//! (`current_schema`), `current_catalog`, the `input_file_*` placeholders, +//! `monotonically_increasing_id`, `spark_partition_id`, `typeof` and the +//! seeded random functions (`rand` / `random`, `randn`, `uuid`, `uniform`, +//! `randstr`). +//! +//! Ground truth: Spark 4.2's `misc.scala` (`SparkVersion`, +//! `CurrentDatabase`, `CurrentCatalog`, `InputFileName`, `Uuid`), +//! `randomExpressions.scala` (`Rand`, `Randn`, `Uniform`, `RandStr`, +//! `NondeterministicUnaryRDG`), `MonotonicallyIncreasingID.scala`, +//! `SparkPartitionID.scala`, and the `random.sql`, +//! `current_database_catalog.sql` and `misc-functions.sql` goldens (checked +//! against a real Spark 4.2.0 run where the goldens were ambiguous). +//! +//! Arity, determinism and availability come from Spark's catalog (the +//! `FunctionDef` assembly reads them off the TSV row); only the implicit +//! casts (`Inputs`) and the result types live here. Every rule mirrors an +//! error the goldens pin: `rand`/`randn` and `randstr`/`uniform` seeds must +//! be foldable, `uuid`'s seed a plain int/long literal (Spark's +//! `expressionToSeed`), and `uniform`/`randstr` bounds the types Spark's +//! implicit casts accept. + +use krishiv_common::error::{ + DatatypeMismatchNonFoldableInput, DatatypeMismatchUnexpectedInputType, + InvalidParameterValueLong, KrishivError, SeedExpressionIsUnfoldable, +}; +use krishiv_types::DataType; + +use super::super::uniform::{UniformKind, uniform_result}; +use super::defs::{Spec, scalar}; +use crate::signature::{ArgType, CallTypes, Constant, Inputs}; + +/// `version`, `current_database` (`current_schema`), `current_catalog`, +/// `input_file_name`: a non-nullable string; `input_file_block_start` / +/// `input_file_block_length`: a non-nullable `bigint`; +/// `monotonically_increasing_id`: a non-nullable `bigint`; +/// `spark_partition_id`: a non-nullable `int`. +fn string(c: &CallTypes<'_>) -> Result { + Ok(ArgType::new(DataType::String, c.any_nullable())) +} + +fn long(c: &CallTypes<'_>) -> Result { + Ok(ArgType::new(DataType::Long, c.any_nullable())) +} + +fn int(c: &CallTypes<'_>) -> Result { + Ok(ArgType::new(DataType::Integer, c.any_nullable())) +} + +/// `typeof(x)`: a non-nullable string. The name itself is resolved by the +/// analyzer (a literal rewrite); the rule only types the call. +fn typeof_result(_: &CallTypes<'_>) -> Result { + // Spark's `typeof` is a literal of the argument's type name: never NULL, + // whatever the argument's nullability. + Ok(ArgType::new(DataType::String, false)) +} + +/// Whether `t` is an `int` or a `bigint` (what every non-`uuid` seed takes). +fn is_seed(t: Option<&DataType>) -> bool { + matches!(t, Some(DataType::Integer | DataType::Long) | None) +} + +/// The whole call as Spark quotes it in errors (`rand(id)`). +fn call_sql(name: &str, c: &CallTypes<'_>) -> String { + let args: Vec = (0..c.args().len()) + .map(|i| c.sql(i).unwrap_or_default()) + .collect(); + format!("\"{name}({})\"", args.join(", ")) +} + +/// `rand(seed?)` / `random(seed?)` / `randn(seed?)`: a non-nullable +/// `double`. The seed must be an `int`/`bigint` +/// (`DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE`, as Spark's `Rand` reports +/// it) and foldable (`SEED_EXPRESSION_IS_UNFOLDABLE`); an untyped `NULL` +/// reads as 0. +fn rand_result(name: &'static str, c: &CallTypes<'_>) -> Result { + if !c.args().is_empty() { + if !is_seed(c.data_type(0)) { + return Err(DatatypeMismatchUnexpectedInputType { + sql_expr: call_sql(name, c), + param_index: "first".to_owned(), + required_type: "(\"INT\" or \"BIGINT\")".to_owned(), + input_sql: format!("\"{}\"", c.sql(0).unwrap_or_default()), + input_type: c + .data_type(0) + .map_or_else(|| "\"VOID\"".to_owned(), |t| format!("\"{}\"", t.sql())), + } + .into()); + } + if c.constant(0).is_none() { + return Err(SeedExpressionIsUnfoldable { + seed_expr: format!("\"{}\"", c.sql(0).unwrap_or_default()), + expr_with_seed: call_sql(name, c), + } + .into()); + } + } + Ok(ArgType::new(DataType::Double, false)) +} + +fn rand(c: &CallTypes<'_>) -> Result { + rand_result("rand", c) +} + +fn randn(c: &CallTypes<'_>) -> Result { + rand_result("randn", c) +} + +/// `uuid(seed?)`: a non-nullable string. The seed must be a plain `int` / +/// `bigint` literal (Spark's `expressionToSeed`); a NULL literal draws from +/// the query seed. +fn uuid(c: &CallTypes<'_>) -> Result { + if !c.args().is_empty() { + let literal = c.is_literal(0) + && matches!( + (c.data_type(0), c.constant(0)), + (None, Some(Constant::Null)) | (Some(DataType::Integer | DataType::Long), Some(_)) + ); + if !literal { + return Err(InvalidParameterValueLong { + function_name: "`UUID`".to_owned(), + parameter: "`seed`".to_owned(), + invalid_value: format!("\"{}\"", c.sql(0).unwrap_or_default()), + } + .into()); + } + } + Ok(ArgType::new(DataType::String, false)) +} + +/// Whether `t` is numeric or an untyped `NULL` (what `uniform`'s bounds +/// take; Spark casts the `NULL` to `DOUBLE`). +fn is_uniform_bound(t: Option<&DataType>) -> bool { + t.is_none_or(|t| { + matches!( + t, + DataType::Byte + | DataType::Short + | DataType::Integer + | DataType::Long + | DataType::Float + | DataType::Double + | DataType::Decimal(_) + ) + }) +} + +/// The [`UniformKind`] of a `uniform` bound (an untyped `NULL` reads as +/// `DOUBLE`, as Spark casts it). +fn uniform_kind(t: Option<&DataType>) -> UniformKind { + match t { + None | Some(DataType::Double) => UniformKind::Double, + Some(DataType::Byte) => UniformKind::Byte, + Some(DataType::Short) => UniformKind::Short, + Some(DataType::Integer) => UniformKind::Int, + Some(DataType::Long) => UniformKind::Long, + Some(DataType::Float) => UniformKind::Float, + Some(DataType::Decimal(d)) => UniformKind::Decimal { + precision: d.precision(), + // A decimal scale is `0..=38`: the widening never fires. + scale: i8::try_from(d.scale()).unwrap_or(i8::MAX), + }, + Some(_) => UniformKind::Other, + } +} + +/// `uniform(min, max[, seed])`: Spark's `Uniform.dataType` (an untyped +/// `NULL` reads as `DOUBLE`), nullable when an argument may be. A NULL +/// bound is a NULL result; a NULL seed reads as 0. +fn uniform(c: &CallTypes<'_>) -> Result { + for i in 0..3 { + if i >= c.args().len() { + continue; + } + let bound = i < 2; + if bound && !is_uniform_bound(c.data_type(i)) || !bound && !is_seed(c.data_type(i)) { + return Err(DatatypeMismatchUnexpectedInputType { + sql_expr: call_sql("uniform", c), + param_index: ["first", "second", "third"][i].to_owned(), + required_type: if bound { + "\"NUMERIC\"".to_owned() + } else { + "(\"INT\" or \"BIGINT\")".to_owned() + }, + input_sql: format!("\"{}\"", c.sql(i).unwrap_or_default()), + input_type: c + .data_type(i) + .map_or_else(|| "\"VOID\"".to_owned(), |t| format!("\"{}\"", t.sql())), + } + .into()); + } + } + for (i, name) in [(0, "min"), (1, "max"), (2, "seed")] { + if i >= c.args().len() { + continue; + } + match c.constant(i) { + Some(Constant::Error(e)) => return Err(e.clone()), + None => { + return Err(DatatypeMismatchNonFoldableInput { + sql_expr: call_sql("uniform", c), + input_name: format!("`{name}`"), + input_type: "integer or floating-point".to_owned(), + input_expr: format!("\"{}\"", c.sql(i).unwrap_or_default()), + } + .into()); + } + Some(_) => {} + } + } + let data_type = match uniform_result(uniform_kind(c.data_type(0)), uniform_kind(c.data_type(1))) + { + UniformKind::Byte => DataType::Byte, + UniformKind::Short => DataType::Short, + UniformKind::Int => DataType::Integer, + UniformKind::Long => DataType::Long, + UniformKind::Float => DataType::Float, + UniformKind::Double | UniformKind::Other => DataType::Double, + UniformKind::Decimal { precision, scale } => { + use krishiv_types::DecimalType; + // The kind came from a valid decimal bound, so the + // reconstruction holds; `try_new` guards the impossible fall. + let scale = u8::try_from(scale).unwrap_or(u8::MAX); + DataType::Decimal(DecimalType::try_new( + u32::from(precision), + u32::from(scale), + )?) + } + }; + Ok(ArgType::new(data_type, c.any_nullable())) +} + +/// Whether `t` is a type Spark implicitly casts to `INT` (the numerics and +/// strings, or an untyped `NULL`): what `randstr`'s length takes. +fn is_length(t: Option<&DataType>) -> bool { + t.is_none_or(|t| { + matches!( + t, + DataType::Byte + | DataType::Short + | DataType::Integer + | DataType::Long + | DataType::Float + | DataType::Double + | DataType::Decimal(_) + | DataType::String + ) + }) +} + +/// `randstr(length[, seed])`: a non-nullable string. The length takes what +/// Spark casts to `INT`; the seed an `int`/`bigint`. Both must be foldable. +fn randstr(c: &CallTypes<'_>) -> Result { + if !is_length(c.data_type(0)) { + return Err(DatatypeMismatchUnexpectedInputType { + sql_expr: call_sql("randstr", c), + param_index: "first".to_owned(), + required_type: "\"INT\"".to_owned(), + input_sql: format!("\"{}\"", c.sql(0).unwrap_or_default()), + input_type: c + .data_type(0) + .map_or_else(|| "\"VOID\"".to_owned(), |t| format!("\"{}\"", t.sql())), + } + .into()); + } + if c.args().len() > 1 && !is_seed(c.data_type(1)) { + return Err(DatatypeMismatchUnexpectedInputType { + sql_expr: call_sql("randstr", c), + param_index: "second".to_owned(), + required_type: "(\"INT\" or \"BIGINT\")".to_owned(), + input_sql: format!("\"{}\"", c.sql(1).unwrap_or_default()), + input_type: c + .data_type(1) + .map_or_else(|| "\"VOID\"".to_owned(), |t| format!("\"{}\"", t.sql())), + } + .into()); + } + for (i, name) in [(0, "length"), (1, "seed")] { + if i >= c.args().len() { + continue; + } + match c.constant(i) { + Some(Constant::Error(e)) => return Err(e.clone()), + None => { + return Err(DatatypeMismatchNonFoldableInput { + sql_expr: call_sql("randstr", c), + input_name: format!("`{name}`"), + input_type: "integer".to_owned(), + input_expr: format!("\"{}\"", c.sql(i).unwrap_or_default()), + } + .into()); + } + Some(_) => {} + } + } + Ok(ArgType::new(DataType::String, false)) +} + +/// Every implemented misc function. +pub(super) fn specs() -> Vec { + let mut out = scalar( + &[ + "version", + "current_database", + "current_schema", + "current_catalog", + ], + &Inputs::Any, + string, + ); + out.extend(scalar(&["input_file_name"], &Inputs::Any, string)); + out.extend(scalar( + &["input_file_block_start", "input_file_block_length"], + &Inputs::Any, + long, + )); + out.extend(scalar(&["monotonically_increasing_id"], &Inputs::Any, long)); + out.extend(scalar(&["spark_partition_id"], &Inputs::Any, int)); + out.extend(scalar(&["typeof"], &Inputs::Any, typeof_result)); + out.extend(scalar(&["rand", "random"], &Inputs::Any, rand)); + out.extend(scalar(&["randn"], &Inputs::Any, randn)); + out.extend(scalar(&["uuid"], &Inputs::Any, uuid)); + out.extend(scalar(&["uniform"], &Inputs::Any, uniform)); + out.extend(scalar(&["randstr"], &Inputs::Any, randstr)); + out +} diff --git a/crates/krishiv-functions/src/registry/mod.rs b/crates/krishiv-functions/src/registry/mod.rs index e25958ca7..66c6952f5 100644 --- a/crates/krishiv-functions/src/registry/mod.rs +++ b/crates/krishiv-functions/src/registry/mod.rs @@ -20,6 +20,7 @@ mod csv; mod defs; mod hashing; mod math; +mod misc; mod number; mod predicate; mod r#struct; diff --git a/crates/krishiv-functions/src/registry/tests.rs b/crates/krishiv-functions/src/registry/tests.rs index d6ea09688..47c04ed3e 100644 --- a/crates/krishiv-functions/src/registry/tests.rs +++ b/crates/krishiv-functions/src/registry/tests.rs @@ -81,15 +81,32 @@ fn every_scalar_definition_binds_and_every_binding_is_defined() { fn lookups_are_case_insensitive_and_versioned() { let r = builtins(); assert!(matches!(r.lookup("UpPeR", SparkProfile::V3_5), Lookup::Found(d) if d.name == "upper")); - // `randstr` arrived in 4.0 and is not implemented. + // `randstr` arrived in 4.0 and is implemented. assert!(matches!( r.lookup("randstr", SparkProfile::V3_5), Lookup::NotFound )); assert!(matches!( r.lookup("RANDSTR", SparkProfile::V4_0), - Lookup::KnownUnsupported + Lookup::Found(_) )); + // So are `uniform` (4.0) and the misc functions. + for name in [ + "uniform", + "uuid", + "typeof", + "version", + "current_database", + "current_catalog", + "monotonically_increasing_id", + "spark_partition_id", + "input_file_name", + ] { + assert!( + matches!(r.lookup(name, SparkProfile::V4_2), Lookup::Found(_)), + "{name}" + ); + } assert!(matches!( r.lookup("no_such_fn", SparkProfile::V4_2), Lookup::NotFound @@ -132,14 +149,22 @@ fn kinds_determinism_and_execution() { assert_eq!(d.execution, Execution::Evaluator, "{name}"); assert_eq!(d.kind, FunctionKind::Scalar, "{name}"); } - for name in ["rand", "randn", "uuid", "shuffle"] { - // Not implemented here, but the optimizer's flag comes from Spark. + for name in ["rand", "randn", "uuid", "uniform", "randstr", "shuffle"] { + // The optimizer's flag comes from Spark; the implemented ones are + // nondeterministic scalar bindings (`shuffle` stays unimplemented: + // it needs an array type the engine has no representation for). assert_eq!( builtins().spark_function(name).unwrap().deterministic, Some(false), "{name}" ); } + for name in ["rand", "randn", "uuid", "uniform", "randstr"] { + let d = def(name); + assert!(!d.deterministic, "{name}"); + assert_eq!(d.execution, Execution::Scalar, "{name}"); + assert_eq!(d.kind, FunctionKind::Scalar, "{name}"); + } for name in [ "round", "pmod", "hash", "try_add", "greatest", "abs", "sha2", ] { @@ -1026,6 +1051,307 @@ fn session_and_clock_functions() { builtins().lookup("current_time", SparkProfile::V4_2), Lookup::KnownUnsupported )); + // `shuffle` needs an array type the engine has no representation for: + // it stays known-unsupported where Spark has it. + assert!(matches!( + builtins().lookup("shuffle", SparkProfile::V4_2), + Lookup::KnownUnsupported + )); +} + +/// The misc and seeded-random signatures: arities, inputs and result types. +#[test] +fn misc_signatures() { + let (s, i, l) = ( + ty(DataType::String), + ty(DataType::Integer), + ty(DataType::Long), + ); + // The session constants and `version` take no arguments and are never + // NULL. + for name in [ + "version", + "current_database", + "current_schema", + "current_catalog", + "input_file_name", + "monotonically_increasing_id", + "spark_partition_id", + "typeof", + "uuid", + "rand", + "random", + "randn", + ] { + let d = def(name); + assert_eq!(d.kind, FunctionKind::Scalar, "{name}"); + assert_eq!(d.execution, Execution::Scalar, "{name}"); + } + for name in [ + "version", + "current_database", + "current_schema", + "current_catalog", + ] { + assert_eq!(result(name, &[], &[]).unwrap(), s, "{name}"); + assert!(def(name).deterministic, "{name}"); + } + // The file placeholders and partition functions are nondeterministic. + for name in [ + "input_file_name", + "input_file_block_start", + "input_file_block_length", + "monotonically_increasing_id", + "spark_partition_id", + ] { + assert!(!def(name).deterministic, "{name}"); + } + assert_eq!(result("input_file_name", &[], &[]).unwrap(), s); + assert_eq!( + result("input_file_block_start", &[], &[]).unwrap(), + ty(DataType::Long) + ); + assert_eq!( + result("input_file_block_length", &[], &[]).unwrap(), + ty(DataType::Long) + ); + assert_eq!(result("monotonically_increasing_id", &[], &[]).unwrap(), l); + assert_eq!(result("spark_partition_id", &[], &[]).unwrap(), i); + assert_eq!(result("typeof", std::slice::from_ref(&s), &[]).unwrap(), s); + // Never NULL, even for a nullable or untyped-NULL argument. + assert_eq!(result("typeof", &[null_ty()], &[]).unwrap(), s); + assert_eq!( + result("typeof", &[ArgType::new(DataType::String, true)], &[]).unwrap(), + s + ); + assert_eq!(result("uuid", &[], &[]).unwrap(), s); + assert_eq!(result("rand", &[], &[]).unwrap(), ty(DataType::Double)); + assert_eq!( + result( + "rand", + std::slice::from_ref(&l), + &[Some(Constant::Integral(0))] + ) + .unwrap(), + ty(DataType::Double) + ); + // A NULL seed reads as 0. + assert_eq!( + result("rand", &[null_ty()], &[Some(Constant::Null)]).unwrap(), + ty(DataType::Double) + ); + // A short seed is the wrong type; a non-foldable one is refused. + let err = result( + "rand", + &[ty(DataType::Short)], + &[Some(Constant::Integral(1))], + ) + .unwrap_err(); + assert_eq!( + err.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(err.to_string().contains("(\"INT\" or \"BIGINT\")"), "{err}"); + let err = result("rand", std::slice::from_ref(&i), &[]).unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::SeedExpressionIsUnfoldable); + assert!(err.to_string().contains("must be foldable"), "{err}"); + // `uuid` wants a plain literal. + let err = result("uuid", std::slice::from_ref(&i), &[]).unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::InvalidParameterValueLong); + assert!(err.to_string().contains("`UUID`"), "{err}"); + // A non-integral literal is the wrong seed too. + let err = result( + "uuid", + &[ty(DataType::Double)], + &[Some(Constant::Fractional(1))], + ) + .unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::InvalidParameterValueLong); +} + +/// `uniform` and `randstr`: the golden types and errors. +#[test] +fn uniform_and_randstr_signatures() { + let (i, l) = (ty(DataType::Integer), ty(DataType::Long)); + let (ten, twenty, zero) = ( + Some(Constant::Integral(10)), + Some(Constant::Integral(20)), + Some(Constant::Integral(0)), + ); + // Integral bounds widen; the seed does not change the type. + assert_eq!( + result( + "uniform", + &[i.clone(), i.clone(), i.clone()], + &[ten.clone(), twenty.clone(), zero.clone()] + ) + .unwrap(), + i + ); + assert_eq!( + result( + "uniform", + &[i.clone(), l.clone(), l.clone()], + &[ten.clone(), twenty.clone(), zero.clone()] + ) + .unwrap(), + l + ); + // A NULL bound is a NULL double; a NULL seed reads as 0. + assert_eq!( + result( + "uniform", + &[null_ty(), i.clone(), i.clone()], + &[Some(Constant::Null), twenty.clone(), zero.clone()] + ) + .unwrap(), + ArgType::new(DataType::Double, true) + ); + // Narrower integrals widen; decimals meet at the wider. + assert_eq!( + result( + "uniform", + &[ty(DataType::Byte), ty(DataType::Short)], + &[Some(Constant::Integral(1)), Some(Constant::Integral(2))] + ) + .unwrap(), + ty(DataType::Short) + ); + assert_eq!( + result( + "uniform", + &[ty(dec(10, 3)), ty(dec(11, 4))], + &[Some(Constant::Fractional(1)), Some(Constant::Fractional(2))] + ) + .unwrap(), + ty(dec(11, 4)) + ); + assert_eq!( + result( + "uniform", + &[ty(DataType::Float), ty(DataType::Double)], + &[Some(Constant::Fractional(1)), Some(Constant::Fractional(2))] + ) + .unwrap(), + ty(DataType::Double) + ); + assert_eq!( + result( + "uniform", + &[i.clone(), i.clone(), null_ty()], + &[ten.clone(), twenty.clone(), Some(Constant::Null)] + ) + .unwrap(), + ArgType::new(DataType::Integer, true) + ); + // A string bound is the wrong type; a column seed is not foldable. + let s = ty(DataType::String); + let err = result( + "uniform", + &[s, i.clone(), i.clone()], + &[None, twenty.clone(), zero.clone()], + ) + .unwrap_err(); + assert_eq!( + err.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(err.to_string().contains("\"NUMERIC\""), "{err}"); + let err = result( + "uniform", + &[i.clone(), i.clone(), i.clone()], + &[ten.clone(), twenty.clone(), None], + ) + .unwrap_err(); + assert_eq!( + err.error_class(), + ErrorClass::DatatypeMismatchNonFoldableInput + ); + assert!(err.to_string().contains("`seed`"), "{err}"); + // A failing fold propagates instead of typing the call. + let failed = Constant::Error(KrishivError::from(krishiv_common::error::InternalError { + message: "boom".to_owned(), + })); + let err = result( + "uniform", + &[i.clone(), i.clone(), i.clone()], + &[ten.clone(), twenty, Some(failed.clone())], + ) + .unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::InternalError); + let err = result( + "randstr", + &[i.clone(), i.clone()], + &[ten.clone(), Some(failed)], + ) + .unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::InternalError); + // `randstr` is a string; a boolean length is the wrong type. + assert_eq!( + result("randstr", &[i.clone(), i.clone()], &[ten, zero]).unwrap(), + ty(DataType::String) + ); + let err = result( + "randstr", + &[ty(DataType::Boolean), i], + &[Some(Constant::Boolean(true)), None], + ) + .unwrap_err(); + assert_eq!( + err.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(err.to_string().contains("\"INT\""), "{err}"); +} + +/// The analyzer's `uniform` result type and the executor's bind type agree +/// on every pair of bound kinds: both layers map their own type to the +/// shared `uniform_result` rule, and the test pins the two mappers against +/// each other so one side cannot drift without the other. +#[test] +fn uniform_result_agrees_with_the_executor_bind() { + use crate::scalar::{BindOptions, bind_scalar}; + // One foldable constant per bound type, as the analyzer sees them. + let cases: Vec<(Option, Option)> = vec![ + (Some(DataType::Byte), Some(Constant::Integral(1))), + (Some(DataType::Short), Some(Constant::Integral(1))), + (Some(DataType::Integer), Some(Constant::Integral(10))), + (Some(DataType::Long), Some(Constant::Integral(20))), + (Some(DataType::Float), Some(Constant::Fractional(1))), + (Some(DataType::Double), Some(Constant::Fractional(1))), + (Some(dec(10, 3)), Some(Constant::Fractional(1))), + (Some(dec(11, 4)), Some(Constant::Fractional(1))), + (None, Some(Constant::Null)), + ]; + for (a, ca) in &cases { + for (b, cb) in &cases { + let args = [ + ArgType { + data_type: a.clone(), + nullable: a.is_none(), + }, + ArgType { + data_type: b.clone(), + nullable: b.is_none(), + }, + ]; + let reg = result("uniform", &args, &[ca.clone(), cb.clone()]).unwrap(); + let bound = bind_scalar( + "uniform", + &[ + a.clone().unwrap_or(DataType::Null).to_arrow().into(), + b.clone().unwrap_or(DataType::Null).to_arrow().into(), + ], + &BindOptions::new(), + ) + .unwrap(); + assert_eq!( + reg.data_type.clone().map(|t| t.to_arrow()), + Some(bound.return_type().clone()), + "uniform({a:?}, {b:?})" + ); + } + } } #[test] diff --git a/crates/krishiv-functions/src/scalar/misc.rs b/crates/krishiv-functions/src/scalar/misc.rs new file mode 100644 index 000000000..b31bc0e51 --- /dev/null +++ b/crates/krishiv-functions/src/scalar/misc.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Misc scalar functions: `version`, `current_database` (`current_schema`), +//! `current_catalog`, `input_file_name` (and the block functions), +//! `monotonically_increasing_id` and `spark_partition_id`. +//! +//! Ground truth: Spark 4.2's `misc.scala` (`SparkVersion`, `CurrentDatabase`, +//! `CurrentCatalog`, `InputFileName`, ...), `MonotonicallyIncreasingID.scala` +//! and `SparkPartitionID.scala`, plus the `current_database_catalog.sql` +//! golden. +//! +//! `version` reports the kernels' Spark line (`4.2.0` for the default +//! build). The database and catalog are the engine's single namespace: +//! `default` in `spark_catalog` (see `krishiv-session`'s catalog), so they +//! bind as constants, with no session state. `input_file_name` and the +//! block functions read the file being scanned; the evaluator has no file +//! context yet, so they return what Spark returns with no file being read +//! (the empty string, `-1`). +//! +//! No golden pins the `version`, `input_file_*` or `uuid` values (no such +//! value queries in any `.out` file: `string-functions.sql` only pins +//! `length(uuid())` and that distinct calls differ), so those stand on the +//! documented behavior, unverified against a golden run. +//! +//! `monotonically_increasing_id` is `(partition << 33) + row_in_partition` +//! and `spark_partition_id` is the partition index, both from the +//! executing partition (see [`super::ScalarFunction::invoke_in`]); each +//! bound function counts its partition's rows in the partitioned body. + +use std::sync::{Arc, Mutex}; + +use arrow_array::{ArrayRef, Int32Array, Int64Array, StringArray}; +use arrow_schema::DataType; +use krishiv_common::error::KrishivError; +use krishiv_kernels::SparkVersion; + +use super::args::{Inputs, internal, unsupported}; +use super::{BindArg, BindOptions, Binding, FamilyResult, bad_types, listed}; + +/// Every name this family binds. `typeof` is resolved by the analyzer +/// (a literal rewrite, like Spark's `RuntimeReplaceable`); its binding is +/// unreachable and reports that. +pub(super) const NAMES: &[&str] = &[ + "version", + "current_database", + "current_schema", + "current_catalog", + "input_file_name", + "input_file_block_start", + "input_file_block_length", + "monotonically_increasing_id", + "spark_partition_id", + "typeof", +]; + +/// The engine's single namespace and catalog (the only values +/// `current_database` / `current_catalog` can report). +const DATABASE: &str = "default"; +const CATALOG: &str = "spark_catalog"; + +/// The release `version` reports for `kernel`. +fn release(kernel: SparkVersion) -> Result<&'static str, KrishivError> { + match kernel { + SparkVersion::V4_0 => Ok("4.0.0"), + SparkVersion::V4_1 => Ok("4.1.0"), + SparkVersion::V4_2 => Ok("4.2.0"), + _ => Err(internal( + "a Spark version with no release string".to_owned(), + )), + } +} + +/// `version()`: the kernels' release line. +fn version(opts: &BindOptions) -> Result { + let version = Arc::::from(release(opts.kernel.spark_version())?); + Ok(Binding::new(DataType::Utf8, move |x: &Inputs<'_>| { + Ok(Arc::new(StringArray::from(vec![&*version; x.rows()])) as ArrayRef) + })) +} + +/// `current_database()` / `current_schema()` / `current_catalog()`. +fn catalog(name: &'static str) -> Result { + let value = Arc::::from(match name { + "current_catalog" => CATALOG, + _ => DATABASE, + }); + Ok(Binding::new(DataType::Utf8, move |x: &Inputs<'_>| { + Ok(Arc::new(StringArray::from(vec![&*value; x.rows()])) as ArrayRef) + })) +} + +/// `input_file_name()`: the empty string (no file is being read). +fn input_file_name() -> Binding { + Binding::new(DataType::Utf8, |x: &Inputs<'_>| { + Ok(Arc::new(StringArray::from(vec![""; x.rows()])) as ArrayRef) + }) +} + +/// `input_file_block_start()` / `input_file_block_length()`: -1 (no file +/// is being read). +fn input_file_block() -> Binding { + Binding::new(DataType::Int64, move |x: &Inputs<'_>| { + Ok(Arc::new(Int64Array::from(vec![-1; x.rows()])) as ArrayRef) + }) +} + +/// `spark_partition_id()`: the executing partition's index. +fn spark_partition_id() -> Binding { + Binding::partitioned(DataType::Int32, |x: &Inputs<'_>, partition| { + // Driver counts never approach `2^31` (one slot per worker thread). + debug_assert!(i32::try_from(partition).is_ok()); + Ok(Arc::new(Int32Array::from(vec![partition as i32; x.rows()])) as ArrayRef) + }) +} + +/// `monotonically_increasing_id()`: `(partition << 33) + row_in_partition`, +/// counted per partition in the partitioned body. The per-partition counts +/// are untracked by `krishiv-memory`, but bounded by the partition count +/// (bytes), like the other per-partition state. +fn monotonically_increasing_id() -> Binding { + let counts = Mutex::new(Vec::::new()); + Binding::partitioned(DataType::Int64, move |x: &Inputs<'_>, partition| { + let mut counts = counts + .lock() + .map_err(|_| internal("a row counter is poisoned".to_owned()))?; + let index = partition as usize; + if counts.len() <= index { + counts.resize(index + 1, 0); + } + let first = counts[index]; + // Java's `long` arithmetic wraps; partitions past `2^31` wrap the + // same way instead of panicking. + let base = (u64::from(partition) << 33).wrapping_add(first); + let rows = x.rows() as u64; + counts[index] = first.wrapping_add(rows); + Ok(Arc::new(Int64Array::from( + (0..rows) + .map(|i| base.wrapping_add(i) as i64) + .collect::>(), + )) as ArrayRef) + }) +} + +/// Binds the misc function `name`. +pub(super) fn bind(name: &str, args: &[BindArg], opts: &BindOptions) -> FamilyResult { + let listed = listed(NAMES, name)?; + if !args.is_empty() { + return Some((listed, Err(bad_types(listed, args)))); + } + // `typeof` never reaches execution: the analyzer rewrites it to a + // string literal, so a call that arrives here is an analyzer bug. + if listed == "typeof" { + return Some(( + listed, + Err(unsupported( + "typeof without the analyzer's literal rewrite".to_owned(), + )), + )); + } + let binding = match listed { + "version" => version(opts), + "current_database" | "current_schema" | "current_catalog" => catalog(listed), + "input_file_name" => Ok(input_file_name()), + "input_file_block_start" | "input_file_block_length" => Ok(input_file_block()), + "spark_partition_id" => Ok(spark_partition_id()), + "monotonically_increasing_id" => Ok(monotonically_increasing_id()), + _ => return Some((listed, Err(bad_types(listed, args)))), + }; + Some((listed, binding)) +} diff --git a/crates/krishiv-functions/src/scalar/mod.rs b/crates/krishiv-functions/src/scalar/mod.rs index 762ece1eb..b35f8c1b4 100644 --- a/crates/krishiv-functions/src/scalar/mod.rs +++ b/crates/krishiv-functions/src/scalar/mod.rs @@ -31,8 +31,10 @@ mod codec; mod datetime; mod hash; mod math; +mod misc; mod number; mod predicate; +mod random; mod regex; mod session; mod string; @@ -43,6 +45,7 @@ pub use regex::{extract_all_pieces, split_regex}; use std::fmt; use std::sync::Arc; +use std::sync::atomic::AtomicU64; use arrow_array::ArrayRef; use arrow_schema::DataType; @@ -121,6 +124,11 @@ pub struct BindOptions { pub decimal_precision_loss: bool, /// The session's user (`current_user()`). pub user: Arc, + /// Counts the unseeded random binds of one plan instantiation, in + /// compilation order: each draws its own query-seeded stream (Spark's + /// `ResolveRandomSeed` assigns each its own seed), while one plan + /// always draws the same streams. + pub seed_salt: Arc, } impl BindOptions { @@ -135,6 +143,7 @@ impl BindOptions { query_start_micros: 0, decimal_precision_loss: true, user: Arc::from("unknown"), + seed_salt: Arc::new(AtomicU64::new(0)), } } @@ -173,6 +182,14 @@ impl BindOptions { self } + /// With this counter of unseeded random binds (see + /// [`Self::seed_salt`]). + #[must_use] + pub fn with_seed_salt(mut self, seed_salt: Arc) -> Self { + self.seed_salt = seed_salt; + self + } + /// With this query start instant (microseconds since the epoch, UTC). #[must_use] pub fn with_query_start(mut self, micros: i64) -> Self { @@ -202,18 +219,45 @@ pub trait ScalarFunction: Send + Sync + fmt::Debug { /// The function's Spark errors (`ARITHMETIC_OVERFLOW` under ANSI, ...), /// and `INTERNAL_ERROR` for arguments that do not match the binding. fn invoke(&self, args: &[Arg], rows: usize) -> Result; + /// As [`Self::invoke`], in the executing partition `partition` (Spark's + /// partition index): a stateful function (`rand`, `uuid`, + /// `monotonically_increasing_id`, ...) owns one value stream per + /// partition, and this evaluates the next `rows` values of that + /// stream. The default runs [`Self::invoke`], for the stateless + /// functions. + /// + /// # Errors + /// As [`Self::invoke`]. + fn invoke_in( + &self, + args: &[Arg], + rows: usize, + partition: u32, + ) -> Result { + let _ = partition; + self.invoke(args, rows) + } } /// The body of a bound function: computes over [`Inputs`] that hold at /// least one column when `rows > 1`. type Body = Box) -> Result + Send + Sync>; +/// The body of a stateful bound function: computes over [`Inputs`] in the +/// executing partition `partition`, advancing that partition's stream once +/// per row it produces. A stateful body always computes every row (never +/// the one-row broadcast of [`Body`]), so the stream advances with the +/// rows even when every argument is a literal. +type PartitionedBody = + Box, u32) -> Result + Send + Sync>; + /// The one [`ScalarFunction`] implementation: a name, a type and a body. struct Bound { name: &'static str, types: Vec, return_type: DataType, body: Body, + partitioned: Option, } impl fmt::Debug for Bound { @@ -236,6 +280,23 @@ impl ScalarFunction for Bound { } fn invoke(&self, args: &[Arg], rows: usize) -> Result { + self.run(args, rows, 0) + } + + fn invoke_in( + &self, + args: &[Arg], + rows: usize, + partition: u32, + ) -> Result { + self.run(args, rows, partition) + } +} + +impl Bound { + /// Checks `args` against the binding, then computes `rows` values in + /// `partition`. + fn run(&self, args: &[Arg], rows: usize, partition: u32) -> Result { if args.len() != self.types.len() { return Err(internal(format!( "{} bound to {} arguments called with {}", @@ -251,9 +312,13 @@ impl ScalarFunction for Bound { if rows == 0 { return Ok(arrow_array::new_empty_array(&self.return_type)); } - // Every argument a literal: compute one row, then repeat it. - let all_literal = args.iter().all(|a| matches!(a, Arg::Scalar(_))); - let out = if all_literal { + let out = if let Some(run) = &self.partitioned { + // A stateful body always computes every row, in partition + // `partition`: the one-row broadcast below would repeat one + // value instead of advancing the partition's stream. + run(&Inputs::new(args, rows), partition)? + } else if args.iter().all(|a| matches!(a, Arg::Scalar(_))) { + // Every argument a literal: compute one row, then repeat it. let one: Vec = args.iter().map(Arg::as_column).collect(); let row = (self.body)(&Inputs::new(&one, 1))?; if rows == 1 { @@ -282,6 +347,7 @@ impl ScalarFunction for Bound { struct Binding { return_type: DataType, body: Body, + partitioned: Option, } impl Binding { @@ -292,6 +358,24 @@ impl Binding { Self { return_type, body: Box::new(body), + partitioned: None, + } + } + + /// A stateful binding: `body` computes every row in the executing + /// partition (see [`PartitionedBody`]). + fn partitioned( + return_type: DataType, + body: impl Fn(&Inputs<'_>, u32) -> Result + Send + Sync + 'static, + ) -> Self { + Self { + return_type, + body: Box::new(|_: &Inputs<'_>| { + Err(internal( + "a stateful function evaluated without a partition".to_owned(), + )) + }), + partitioned: Some(Box::new(body)), } } } @@ -305,7 +389,7 @@ type FamilyResult = Option<(&'static str, Result)>; type Binder = fn(&str, &[BindArg], &BindOptions) -> FamilyResult; /// The families (their names and binder), in lookup order. -const FAMILIES: [(&[&str], Binder); 14] = [ +const FAMILIES: [(&[&str], Binder); 16] = [ (string::NAMES, string::bind), (datetime::NAMES, datetime::bind), (math::NAMES, math::bind), @@ -316,6 +400,8 @@ const FAMILIES: [(&[&str], Binder); 14] = [ (bitwise::NAMES, bitwise::bind), (predicate::NAMES, predicate::bind), (number::NAMES, number::bind), + (misc::NAMES, misc::bind), + (random::NAMES, random::bind), (chars::NAMES, chars::bind), (codec::NAMES, codec::bind), (r#struct::NAMES, r#struct::bind), @@ -389,6 +475,7 @@ pub fn bind_scalar( types: args.iter().map(BindArg::data_type).collect(), return_type: binding.return_type, body: binding.body, + partitioned: binding.partitioned, })) } diff --git a/crates/krishiv-functions/src/scalar/random.rs b/crates/krishiv-functions/src/scalar/random.rs new file mode 100644 index 000000000..b2044d2a7 --- /dev/null +++ b/crates/krishiv-functions/src/scalar/random.rs @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Seeded random functions: `rand` (`random`), `randn`, `uniform`, +//! `randstr` and `uuid`. +//! +//! Ground truth: Spark 4.2's `randomExpressions.scala` (`Rand`, `Randn`, +//! `Uniform`, `RandStr`), `misc.scala` (`Uuid`) and +//! `XORShiftRandom` (`core/.../util/random/XORShiftRandom.scala`), plus the +//! `random.sql` golden (`tests/spark-golden/v4.2.0/results/random.sql.out`). +//! +//! Every function owns one value stream per partition: the stream is a pure +//! function of the seed (the folded seed argument, or the query seed for an +//! absent or NULL seed) and the partition index, advanced once per row the +//! partition produces. The analyzer only lets foldable seeds through (the +//! registry rules), so a bind reads the seed from the first row of every +//! invocation; the per-partition streams live in the partitioned body. +//! +//! `rand`/`randn` draw from Spark's `XORShiftRandom` (the MurmurHash3 seed +//! hash and the shift register below), so seeded results match Spark +//! exactly (`rand(0)` is `0.7604953758285915`). `uuid` draws its 128 bits +//! from the same generator and sets the RFC 4122 version and variant bits; +//! Spark fills those bits from a Mersenne Twister instead, so only the +//! shape (and the per-seed determinism) matches, not Spark's values. No +//! golden pins seeded `uuid` values (see `misc`). + +use std::sync::{Arc, Mutex}; + +use arrow_array::cast::AsArray; +use arrow_array::types::{Float64Type, Int32Type, Int64Type}; +use arrow_array::{ArrayRef, Float64Array, StringArray}; +use arrow_schema::DataType; +use krishiv_common::error::{KrishivError, UnableToAcquireMemory}; +use krishiv_kernels::cast::{CastMode, CastOptions, TimeZoneRules, cast}; + +use super::super::uniform::{UniformKind, uniform_result as shared_result}; +use super::args::{Inputs, internal}; +use super::{BindArg, BindOptions, Binding, FamilyResult, bad_types, listed}; + +/// Every name this family binds (`random` is Spark's alias of `rand`). +pub(super) const NAMES: &[&str] = &["rand", "random", "randn", "uniform", "randstr", "uuid"]; + +/// Spark's `XORShiftRandom`: the seed hash (Scala's `MurmurHash3`, the only +/// hashing in the stream) and the shift register. +#[derive(Debug, Clone)] +struct XorShift { + seed: u64, +} + +impl XorShift { + /// The stream for `init` (Spark's `new XORShiftRandom(init)`). + fn new(init: i64) -> Self { + Self { + seed: hash_seed(init), + } + } + + /// Spark's `next(bits)`. + #[expect( + clippy::cast_possible_truncation, + reason = "Java's `(int)` cast of the masked bits" + )] + fn next(&mut self, bits: u32) -> i32 { + let mut next = self.seed ^ self.seed.wrapping_shl(21); + next ^= next >> 35; + next ^= next.wrapping_shl(4); + self.seed = next; + (next & ((1_u64 << bits) - 1)) as i32 + } + + /// `java.util.Random.nextDouble`: 53 bits over `2^53`. + fn next_double(&mut self) -> f64 { + let high = i64::from(self.next(26)); + let low = i64::from(self.next(27)); + ((high << 27) + low) as f64 / 9_007_199_254_740_992.0 + } + + /// `java.util.Random.nextGaussian` (the polar method): `cached` is the + /// spare value (`haveNextNextGaussian` / `nextNextGaussian`). + fn next_gaussian(&mut self, cached: &mut Option) -> f64 { + if let Some(v) = cached.take() { + return v; + } + loop { + let v1 = 2.0 * self.next_double() - 1.0; + let v2 = 2.0 * self.next_double() - 1.0; + let s = v1 * v1 + v2 * v2; + if s < 1.0 && s != 0.0 { + let multiplier = (-2.0 * s.ln() / s).sqrt(); + *cached = Some(v2 * multiplier); + return v1 * multiplier; + } + } + } + + /// `java.util.Random.nextLong`. + fn next_long(&mut self) -> i64 { + (i64::from(self.next(32)) << 32) + i64::from(self.next(32)) + } +} + +/// Spark's `XORShiftRandom.hashSeed`: the seed's big-endian bytes hashed +/// with Scala's `MurmurHash3.bytesHash`. +fn hash_seed(seed: i64) -> u64 { + let bytes = seed.to_be_bytes(); + let low = bytes_hash(&bytes, 0x3c07_4a61); + let high = bytes_hash(&bytes, low); + ((i64::from(high) << 32) | (i64::from(low) & 0xffff_ffff)).cast_unsigned() +} + +/// Scala 2.13's `MurmurHash3.bytesHash` (an 8-byte input has no tail). +fn bytes_hash(data: &[u8], seed: i32) -> i32 { + let mut h = seed; + let (chunks, _) = data.as_chunks::<4>(); + for chunk in chunks { + let k = i32::from_le_bytes(*chunk); + h = murmur_mix(h, k); + } + murmur_finalize(h, data.len()) +} + +/// Scala 2.13's `MurmurHash3.mix`. +fn murmur_mix(hash: i32, data: i32) -> i32 { + murmur_mix_last(hash, data) + .rotate_left(13) + .wrapping_mul(5) + .wrapping_add(0xe654_6b64u32.cast_signed()) +} + +/// Scala 2.13's `MurmurHash3.mixLast`. +fn murmur_mix_last(hash: i32, mut data: i32) -> i32 { + data = data.wrapping_mul(0xcc9e_2d51u32.cast_signed()); + data = data.rotate_left(15); + data = data.wrapping_mul(0x1b87_3593u32.cast_signed()); + hash ^ data +} + +/// Scala 2.13's `MurmurHash3.finalizeHash`. +#[expect( + clippy::cast_possible_truncation, + reason = "the hashed seed bytes always fit" +)] +fn murmur_finalize(hash: i32, length: usize) -> i32 { + murmur_avalanche(hash ^ length as i32) +} + +/// Scala 2.13's `MurmurHash3.avalanche`. +fn murmur_avalanche(mut hash: i32) -> i32 { + hash ^= (hash.cast_unsigned() >> 16).cast_signed(); + hash = hash.wrapping_mul(0x85eb_ca6bu32.cast_signed()); + hash ^= (hash.cast_unsigned() >> 13).cast_signed(); + hash = hash.wrapping_mul(0xc2b2_ae35u32.cast_signed()); + hash ^= (hash.cast_unsigned() >> 16).cast_signed(); + hash +} + +/// SplitMix64: the per-query seed from the query start instant (fixed per +/// plan, never sampled at execution). +fn splitmix64(mut state: u64) -> u64 { + state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +/// The query seed: one value per query, from the plan-fixed query start. +fn query_seed(opts: &BindOptions) -> u64 { + opts.query_start_micros.cast_unsigned() +} + +/// An unseeded (or NULL-seeded, where Spark draws randomly) call's seed: +/// the query seed mixed with a per-occurrence salt, so two unseeded calls +/// in one query draw different streams (Spark's `ResolveRandomSeed` +/// assigns each its own random seed) while one query always draws the same +/// streams. The salt counts the unseeded binds of one plan instantiation, +/// in compilation order. +fn unseeded_seed(opts: &BindOptions) -> i64 { + use std::sync::atomic::Ordering; + let salt = opts.seed_salt.fetch_add(1, Ordering::Relaxed); + splitmix64(query_seed(opts).wrapping_add(salt)) as i64 +} + +/// One value stream: the generator and the spare Gaussian. +#[derive(Debug)] +struct Stream { + rng: XorShift, + gaussian: Option, +} + +impl Stream { + fn new(seed: i64) -> Self { + Self { + rng: XorShift::new(seed), + gaussian: None, + } + } +} + +/// The per-partition streams of one bound function, by partition index. +/// The seed is fixed per binding (the folded seed argument, or the query +/// seed for an absent or NULL seed), so the first bind records it and every +/// slot `j` draws `base + j`: a driver that binds a higher partition first +/// no longer seeds the lower slots with the wrong seed. The table is +/// untracked by `krishiv-memory`, but bounded by the partition count +/// (bytes), like the other per-partition counters. +#[derive(Debug, Default)] +struct Streams { + base: Option, + streams: Vec, +} + +impl Streams { + /// The stream of `partition`: slot `j` draws `seed + j`, seeded once. + /// (`seed` is the binding's base seed; the callers pass it unfolded.) + fn stream(&mut self, partition: u32, seed: i64) -> &mut Stream { + let base = *self.base.get_or_insert(seed); + let index = partition as usize; + if self.streams.len() <= index { + let have = self.streams.len(); + self.streams.reserve(index + 1 - have); + for slot in have..=index { + // `slot <= index`, and `index` is a `u32` widened to + // `usize`: `slot` fits a `u32` (the fallback never fires). + let p = u32::try_from(slot).unwrap_or(u32::MAX); + self.streams + .push(Stream::new(base.wrapping_add(i64::from(p)))); + } + } + &mut self.streams[index] + } +} + +/// Locks `mutex` (poison is impossible: the bodies never panic). +fn lock(mutex: &Mutex) -> Result, KrishivError> { + mutex + .lock() + .map_err(|_| internal("a random function's streams are poisoned".to_owned())) +} + +/// The folded seed argument `i` (`None` for a NULL seed). +fn seed(x: &Inputs<'_>, i: usize) -> Result, KrishivError> { + let c = x.column(i)?; + if c.data_type() == &DataType::Null || c.is_null(0) { + return Ok(None); + } + match c.data_type() { + DataType::Int32 => Ok(Some(i64::from(c.as_primitive::().value(0)))), + DataType::Int64 => Ok(Some(c.as_primitive::().value(0))), + t => Err(internal(format!("a random seed of type {t}"))), + } +} + +/// The cast to a bound of `uniform` (Spark's plain `Cast`, never `TRY_CAST`). +fn bound_cast( + c: &ArrayRef, + to: &DataType, + ansi: bool, + zone: &Arc, +) -> Result { + cast( + c.as_ref(), + to, + &CastOptions { + mode: if ansi { + CastMode::Ansi + } else { + CastMode::Legacy + }, + time_zone: zone.as_ref(), + }, + ) + .map_err(KrishivError::from) +} + +/// The folded numeric argument `i` as an `f64` (`None` for NULL): Spark's +/// `Cast` to `DOUBLE`. +fn double( + x: &Inputs<'_>, + i: usize, + ansi: bool, + zone: &Arc, +) -> Result, KrishivError> { + let c = x.column(i)?; + if c.data_type() == &DataType::Null || c.is_null(0) { + return Ok(None); + } + let d = bound_cast(&c, &DataType::Float64, ansi, zone)?; + if d.is_null(0) { + return Ok(None); + } + Ok(Some(d.as_primitive::().value(0))) +} + +/// The folded length argument `i` as an `i32` (`None` for NULL): Spark's +/// `Cast` to `INT`. +fn length( + x: &Inputs<'_>, + i: usize, + ansi: bool, + zone: &Arc, +) -> Result, KrishivError> { + let c = x.column(i)?; + if c.data_type() == &DataType::Null || c.is_null(0) { + return Ok(None); + } + let d = bound_cast(&c, &DataType::Int32, ansi, zone)?; + if d.is_null(0) { + return Ok(None); + } + Ok(Some(d.as_primitive::().value(0))) +} + +/// `rand(seed?)` / `random(seed?)`: `rng.nextDouble()` per row. +fn rand(opts: &BindOptions, args: &[BindArg]) -> Binding { + let queries = unseeded_seed(opts); + let seeded = !args.is_empty(); + let state = Mutex::new(Streams::default()); + Binding::partitioned(DataType::Float64, move |x, partition| { + let seed = if seeded { + seed(x, 0)?.unwrap_or(0) + } else { + queries + }; + let mut streams = lock(&state)?; + let s = streams.stream(partition, seed); + let out: Vec = (0..x.rows()).map(|_| s.rng.next_double()).collect(); + Ok(Arc::new(Float64Array::from(out)) as ArrayRef) + }) +} + +/// `randn(seed?)`: `rng.nextGaussian()` per row. +fn randn(opts: &BindOptions, args: &[BindArg]) -> Binding { + let queries = unseeded_seed(opts); + let seeded = !args.is_empty(); + let state = Mutex::new(Streams::default()); + Binding::partitioned(DataType::Float64, move |x, partition| { + let seed = if seeded { + seed(x, 0)?.unwrap_or(0) + } else { + queries + }; + let mut streams = lock(&state)?; + let s = streams.stream(partition, seed); + let out: Vec = (0..x.rows()) + .map(|_| s.rng.next_gaussian(&mut s.gaussian)) + .collect(); + Ok(Arc::new(Float64Array::from(out)) as ArrayRef) + }) +} + +/// `uuid(seed?)`: 128 bits per row, with the RFC 4122 version and variant +/// bits. A NULL or absent seed draws from the query seed. +fn uuid(opts: &BindOptions, args: &[BindArg]) -> Binding { + let queries = unseeded_seed(opts); + let seeded = !args.is_empty(); + let state = Mutex::new(Streams::default()); + Binding::partitioned(DataType::Utf8, move |x, partition| { + let seed = if seeded { + seed(x, 0)?.unwrap_or(queries) + } else { + queries + }; + let mut streams = lock(&state)?; + let s = streams.stream(partition, seed); + let out: Vec = (0..x.rows()) + .map(|_| { + let most = (s.rng.next_long().cast_unsigned() & 0xffff_ffff_ffff_0fff) + | 0x0000_0000_0000_4000; + let least = (s.rng.next_long().cast_unsigned() | 0x8000_0000_0000_0000) + & 0xbfff_ffff_ffff_ffff; + format!( + "{:08x}-{:04x}-{:04x}-{:04x}-{:012x}", + (most >> 32) as u32, + ((most >> 16) & 0xffff) as u16, + (most & 0xffff) as u16, + ((least >> 48) & 0xffff) as u16, + least & 0xffff_ffff_ffff, + ) + }) + .collect(); + Ok(Arc::new(StringArray::from(out)) as ArrayRef) + }) +} + +/// The [`UniformKind`] of a `uniform` bound (an untyped `NULL` reads as +/// `DOUBLE`, as Spark casts it). +fn uniform_kind(t: &DataType) -> UniformKind { + match t { + DataType::Int8 => UniformKind::Byte, + DataType::Int16 => UniformKind::Short, + DataType::Int32 => UniformKind::Int, + DataType::Int64 => UniformKind::Long, + DataType::Float32 => UniformKind::Float, + DataType::Float64 | DataType::Null => UniformKind::Double, + DataType::Decimal128(p, s) => UniformKind::Decimal { + precision: *p, + scale: *s, + }, + _ => UniformKind::Other, + } +} + +/// The Arrow result type of `uniform(min, max)` for argument types +/// `min`/`max`: Spark's `Uniform.dataType` (see `uniform_kind`). +fn uniform_result(min: &DataType, max: &DataType) -> DataType { + match shared_result(uniform_kind(min), uniform_kind(max)) { + UniformKind::Byte => DataType::Int8, + UniformKind::Short => DataType::Int16, + UniformKind::Int => DataType::Int32, + UniformKind::Long => DataType::Int64, + UniformKind::Float => DataType::Float32, + UniformKind::Double | UniformKind::Other => DataType::Float64, + UniformKind::Decimal { precision, scale } => DataType::Decimal128(precision, scale), + } +} + +/// `uniform(min, max[, seed])`: Spark's `Uniform` is a `RuntimeReplaceable` +/// over `cast(min + (max - min) * Rand(seed))`, computed here in one pass: +/// the bounds as doubles, one draw per row, then Spark's `Cast` to the +/// result type. A NULL bound is a NULL result. +fn uniform(opts: &BindOptions, args: &[BindArg]) -> Binding { + let types: Vec = args.iter().map(BindArg::data_type).collect(); + let result = uniform_result(&types[0], &types[1]); + let out_type = result.clone(); + let queries = unseeded_seed(opts); + let seeded = args.len() > 2; + let ansi = opts.ansi; + let zone = Arc::clone(&opts.time_zone); + let state = Mutex::new(Streams::default()); + Binding::partitioned(result, move |x, partition| { + let seed = if seeded { + seed(x, 2)?.unwrap_or(0) + } else { + queries + }; + let (Some(lo), Some(hi)) = (double(x, 0, ansi, &zone)?, double(x, 1, ansi, &zone)?) else { + return Ok(arrow_array::new_null_array(&out_type, x.rows())); + }; + let mut streams = lock(&state)?; + let s = streams.stream(partition, seed); + let draws: Vec = (0..x.rows()) + .map(|_| lo + (hi - lo) * s.rng.next_double()) + .collect(); + let draws = Float64Array::from(draws); + bound_cast(&(Arc::new(draws) as ArrayRef), &out_type, ansi, &zone) + }) +} + +/// `randstr(length[, seed])`: `length` characters per row, each drawn with +/// `Math.abs(rng.nextInt() % 62)` and read as a digit, a lower-case or an +/// upper-case letter (Spark's `ExpressionImplUtils.randStr`). A NULL length +/// is the empty string; a negative length is +/// `INVALID_PARAMETER_VALUE.LENGTH`. +fn randstr(opts: &BindOptions, args: &[BindArg]) -> Binding { + let queries = unseeded_seed(opts); + let seeded = args.len() > 1; + let ansi = opts.ansi; + let zone = Arc::clone(&opts.time_zone); + let state = Mutex::new(Streams::default()); + Binding::partitioned(DataType::Utf8, move |x, partition| { + let seed = if seeded { + seed(x, 1)?.unwrap_or(queries) + } else { + queries + }; + let Some(n) = length(x, 0, ansi, &zone)? else { + return Ok(Arc::new(StringArray::from(vec![""; x.rows()])) as ArrayRef); + }; + if n < 0 { + return Err(krishiv_common::error::InvalidParameterValueLength { + function_name: "`randstr`".to_owned(), + length: n.to_string(), + parameter: "`length`".to_owned(), + } + .into()); + } + // The output must fit a `StringArray`'s `i32` offsets: refuse before + // drawing or allocating, as the other string builders do. + let total = usize::try_from(n) + .ok() + .and_then(|n| n.checked_mul(x.rows())) + .filter(|t| i32::try_from(*t).is_ok()); + let Some(_) = total else { + return Err(UnableToAcquireMemory { + requested_bytes: n.to_string(), + received_bytes: "0".to_owned(), + } + .into()); + }; + let mut streams = lock(&state)?; + let s = streams.stream(partition, seed); + let out: Vec = (0..x.rows()) + .map(|_| { + (0..n) + .map(|_| { + // Java's `Math.abs(rng.nextInt() % 62)`: `%` keeps + // the dividend's sign, then `abs`. + let v = (s.rng.next(32) % 62).abs(); + // `v` is `abs(nextInt() % 62)`, so in `0..62`. + let v = u8::try_from(v).unwrap_or(0); + if v < 10 { + (b'0' + v) as char + } else if v < 36 { + (b'a' + v - 10) as char + } else { + (b'A' + v - 36) as char + } + }) + .collect() + }) + .collect(); + Ok(Arc::new(StringArray::from(out)) as ArrayRef) + }) +} + +/// Whether `t` is an `int` or `bigint` seed (an untyped `NULL` reads as +/// one): what the analyzer lets through for every seed but `uuid`'s. +fn is_seed(t: &DataType) -> bool { + matches!(t, DataType::Int32 | DataType::Int64 | DataType::Null) +} + +/// Binds the random function `name`. +pub(super) fn bind(name: &str, args: &[BindArg], opts: &BindOptions) -> FamilyResult { + let listed = listed(NAMES, name)?; + let types: Vec = args.iter().map(BindArg::data_type).collect(); + let binding = match (listed, types.as_slice()) { + ("rand" | "random", []) => rand(opts, args), + ("rand" | "random" | "randn" | "uuid", [t]) if is_seed(t) => match listed { + "randn" => randn(opts, args), + "uuid" => uuid(opts, args), + _ => rand(opts, args), + }, + ("randn", []) => randn(opts, args), + ("uuid", []) => uuid(opts, args), + ("uniform", [_, _] | [_, _, _]) => uniform(opts, args), + ("randstr", [_] | [_, _]) => randstr(opts, args), + _ => return Some((listed, Err(bad_types(listed, args)))), + }; + Some((listed, Ok(binding))) +} diff --git a/crates/krishiv-functions/src/scalar/tests/misc.rs b/crates/krishiv-functions/src/scalar/tests/misc.rs new file mode 100644 index 000000000..86250458f --- /dev/null +++ b/crates/krishiv-functions/src/scalar/tests/misc.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The misc bindings: the session constants, the file placeholders and the +//! partition functions. + +use std::sync::Arc; + +use arrow_array::Array; +use arrow_array::cast::AsArray; +use arrow_array::types::{Int32Type, Int64Type}; +use arrow_schema::DataType; +use krishiv_common::error::ErrorClass; + +use super::{BindOptions, bind, bind_scalar, class, strs}; + +fn strings(f: &Arc, rows: usize, partition: u32) -> Vec { + let out = f.invoke_in(&[], rows, partition).unwrap(); + out.as_string::() + .iter() + .map(|s| s.unwrap().to_owned()) + .collect() +} + +#[test] +fn version_reports_the_kernels_release_line() { + let f = bind("version", &[]); + assert_eq!(f.return_type(), &DataType::Utf8); + assert_eq!(strings(&f, 3, 0), vec!["4.2.0"; 3]); + // The line follows the kernels' Spark version. + let opts = BindOptions::new().with_kernel( + krishiv_kernels::KernelCtx::new().with_spark_version(krishiv_kernels::SparkVersion::V4_0), + ); + let f = super::bind_scalar("version", &[], &opts).unwrap(); + assert_eq!(strings(&f, 1, 0), vec!["4.0.0"]); + // `typeof` never reaches execution: the analyzer rewrites it first. + assert_eq!( + class( + &super::bind_scalar("typeof", &[DataType::Int32.into()], &BindOptions::new()) + .unwrap_err() + ), + ErrorClass::KrishivUnsupportedFeature + ); +} + +#[test] +fn current_database_and_catalog() { + assert_eq!( + strings(&bind("current_database", &[]), 2, 0), + vec!["default"; 2] + ); + assert_eq!(strings(&bind("current_schema", &[]), 1, 0), vec!["default"]); + assert_eq!( + strings(&bind("current_catalog", &[]), 1, 0), + vec!["spark_catalog"] + ); +} + +#[test] +fn input_file_functions_read_no_file() { + assert_eq!(strings(&bind("input_file_name", &[]), 2, 0), vec![""; 2]); + for name in ["input_file_block_start", "input_file_block_length"] { + let f = bind(name, &[]); + assert_eq!(f.return_type(), &DataType::Int64); + let out = f.invoke_in(&[], 3, 0).unwrap(); + assert_eq!( + out.as_primitive::().values(), + &[-1, -1, -1], + "{name}" + ); + } +} + +#[test] +fn spark_partition_id_is_the_partition() { + let f = bind("spark_partition_id", &[]); + assert_eq!(f.return_type(), &DataType::Int32); + for partition in [0, 1, 7] { + let out = f.invoke_in(&[], 3, partition).unwrap(); + assert_eq!( + out.as_primitive::().values(), + &[partition as i32; 3], + "partition {partition}" + ); + } + // Without a partition the single partition reads 0. + let out = f.invoke(&[], 2).unwrap(); + assert_eq!(out.as_primitive::().values(), &[0, 0]); +} + +#[test] +fn monotonically_increasing_id_counts_per_partition() { + let f = bind("monotonically_increasing_id", &[]); + assert_eq!(f.return_type(), &DataType::Int64); + let ids = |rows: usize, partition: u32| { + f.invoke_in(&[], rows, partition) + .unwrap() + .as_primitive::() + .values() + .to_vec() + }; + assert_eq!(ids(3, 0), vec![0, 1, 2]); + // The next batch continues the partition's count. + assert_eq!(ids(2, 0), vec![3, 4]); + // Another partition counts from its own base. + let base = 2_i64 << 33; + assert_eq!(ids(2, 2), vec![base, base + 1]); + assert_eq!(ids(1, 0), vec![5]); +} + +#[test] +fn misc_functions_take_no_arguments() { + for name in [ + "version", + "current_database", + "current_schema", + "current_catalog", + "input_file_name", + "input_file_block_start", + "input_file_block_length", + "spark_partition_id", + "monotonically_increasing_id", + ] { + assert_eq!(bind(name, &[]).name(), name); + let arg = strs(&[Some("x")]); + let err = + bind_scalar(name, &[arg.data_type().clone().into()], &BindOptions::new()).unwrap_err(); + assert_eq!(class(&err), ErrorClass::KrishivUnsupportedFeature, "{name}"); + // The error names the bad signature, not a missing function. + assert!(err.to_string().contains(name), "{name}: {err}"); + } +} diff --git a/crates/krishiv-functions/src/scalar/tests/mod.rs b/crates/krishiv-functions/src/scalar/tests/mod.rs index 2b7e5f18a..9a06369fc 100644 --- a/crates/krishiv-functions/src/scalar/tests/mod.rs +++ b/crates/krishiv-functions/src/scalar/tests/mod.rs @@ -581,6 +581,7 @@ fn results_are_checked_against_the_binding() { types: vec![], return_type: DataType::Int32, body: Box::new(|_| Ok(Arc::new(StringArray::from(vec!["x"])) as ArrayRef)), + partitioned: None, }; let err = wrong.invoke(&[], 1).unwrap_err(); assert_eq!(class(&err), ErrorClass::InternalError); @@ -590,8 +591,10 @@ fn results_are_checked_against_the_binding() { mod bitwise; mod calendar; mod families; +mod misc; mod number; mod predicate; +mod random; mod r#struct; mod text; mod url; diff --git a/crates/krishiv-functions/src/scalar/tests/random.rs b/crates/krishiv-functions/src/scalar/tests/random.rs new file mode 100644 index 000000000..7d379fb40 --- /dev/null +++ b/crates/krishiv-functions/src/scalar/tests/random.rs @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The seeded random bindings: Spark's streams, golden values and the +//! per-partition contract. + +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::types::{Float64Type, Int32Type}; +use arrow_array::{Array, ArrayRef, Int32Array, Int64Array}; +use arrow_schema::DataType; + +use super::{Arg, BindArg, BindOptions, ints, lit, strs}; + +fn seed_i32(v: i32) -> ArrayRef { + Arc::new(Int32Array::from(vec![v])) +} + +fn seed_i64(v: i64) -> ArrayRef { + Arc::new(Int64Array::from(vec![v])) +} + +fn null_i32() -> ArrayRef { + Arc::new(Int32Array::from(vec![None as Option])) +} + +/// The `f64` values of `a`. +fn f64s(a: &ArrayRef) -> Vec> { + a.as_primitive::().iter().collect() +} + +/// `f` over `rows` rows in `partition`, with a literal seed `s`. +fn seeded(name: &str, s: &ArrayRef, rows: usize, partition: u32) -> ArrayRef { + let bound = vec![BindArg::Literal(Arc::clone(s))]; + let f = super::bind_scalar(name, &bound, &BindOptions::new()).unwrap(); + assert_eq!(f.return_type(), &DataType::Float64); + f.invoke_in(&[Arg::Scalar(Arc::clone(s))], rows, partition) + .unwrap() +} + +#[test] +fn rand_zero_matches_spark() { + // `SELECT rand(0)` in `random.sql`. + let s = seed_i32(0); + let got = seeded("rand", &s, 1, 0); + assert_eq!(f64s(&got), vec![Some(0.7604953758285915)]); + // The alias binds the same stream. + let got = seeded("random", &s, 1, 0); + assert_eq!(f64s(&got), vec![Some(0.7604953758285915)]); +} + +#[test] +fn rand_advances_one_draw_per_row() { + // Oracle: `SELECT rand(0) FROM range(0, 3, 1, 1)` on a Spark 4.2.0 + // server: the rows draw consecutively from one stream. + let s = seed_i32(0); + let got = seeded("rand", &s, 3, 0); + assert_eq!( + f64s(&got), + vec![ + Some(0.7604953758285915), + Some(0.5234194256885571), + Some(0.0953472826424725) + ] + ); +} + +#[test] +fn randn_zero_matches_spark() { + // `SELECT randn(0)` in `random.sql`. + let s = seed_i64(0); + let got = seeded("randn", &s, 1, 0); + assert_eq!(f64s(&got), vec![Some(1.6034991609278433)]); +} + +#[test] +fn null_seeds_read_as_zero() { + // `rand(NULL)` and `rand(cast(NULL AS INT))` are `rand(0)`. + let s = null_i32(); + let got = seeded("rand", &s, 1, 0); + assert_eq!(f64s(&got), vec![Some(0.7604953758285915)]); + let got = seeded("randn", &s, 1, 0); + assert_eq!(f64s(&got), vec![Some(1.6034991609278433)]); +} + +#[test] +fn long_seeds_match_int_seeds() { + let i = seed_i32(0); + let l = seed_i64(0); + assert_eq!( + f64s(&seeded("rand", &i, 2, 0)), + f64s(&seeded("rand", &l, 2, 0)) + ); + assert_eq!( + f64s(&seeded("randn", &i, 2, 0)), + f64s(&seeded("randn", &l, 2, 0)) + ); +} + +#[test] +fn partitions_own_their_streams() { + // The same query in another partition draws another stream; the + // partition seeds the generator (`seed + partitionIndex`, Spark's + // `initializeInternal`). Oracle: `SELECT spark_partition_id(), rand(0) + // FROM range(0, 6, 1, 2)` on a Spark 4.2.0 server gives partition 0 + // 0.7604953758285915, 0.5234194256885571, 0.0953472826424725 and + // partition 1 0.6363787615254752, 0.5993846534021868, 0.134842710012538. + let s = seed_i32(0); + let p0 = f64s(&seeded("rand", &s, 3, 0)); + let p1 = f64s(&seeded("rand", &s, 3, 1)); + assert_eq!( + p1, + vec![ + Some(0.6363787615254752), + Some(0.5993846534021868), + Some(0.134842710012538) + ] + ); + assert_eq!( + p0, + vec![ + Some(0.7604953758285915), + Some(0.5234194256885571), + Some(0.0953472826424725) + ] + ); + // One partition continues its stream across batches: the second batch + // starts where the first stopped (the third draw of the stream). + let f = super::bind_scalar( + "rand", + &[BindArg::Literal(Arc::clone(&s))], + &BindOptions::new(), + ) + .unwrap(); + let lit = Arg::Scalar(Arc::clone(&s)); + let a = f.invoke_in(std::slice::from_ref(&lit), 2, 0).unwrap(); + let b = f.invoke_in(std::slice::from_ref(&lit), 2, 0).unwrap(); + assert_eq!( + f64s(&a), + vec![Some(0.7604953758285915), Some(0.5234194256885571)] + ); + assert_eq!(f64s(&b)[0], Some(0.0953472826424725)); +} + +#[test] +fn a_late_partition_does_not_reuse_an_early_partitions_seed() { + // Regression: a driver that bound a higher partition first seeded the + // lower slots with its own seed. Each slot draws `base + slot`, so + // partition 0 still draws Spark's first draw of seed 0 here. + let s = seed_i32(0); + let bound = vec![BindArg::Literal(Arc::clone(&s))]; + let f = super::bind_scalar("rand", &bound, &BindOptions::new()).unwrap(); + let a = lit(&s); + let p1 = f64s(&f.invoke_in(std::slice::from_ref(&a), 1, 1).unwrap()); + let p0 = f64s(&f.invoke_in(std::slice::from_ref(&a), 1, 0).unwrap()); + assert_eq!(p1, vec![Some(0.6363787615254752)]); + assert_eq!(p0, vec![Some(0.7604953758285915)]); +} + +#[test] +fn uniform_matches_the_golden_values() { + // `SELECT uniform(0, 10, 0)` is 7, `(10, 20, 0)` is 17. + let bound = |lo: &ArrayRef, hi: &ArrayRef, sd: &ArrayRef| { + let bound = vec![ + BindArg::Literal(Arc::clone(lo)), + BindArg::Literal(Arc::clone(hi)), + BindArg::Literal(Arc::clone(sd)), + ]; + super::bind_scalar("uniform", &bound, &BindOptions::new()).unwrap() + }; + let (zero, ten, twenty) = (seed_i32(0), seed_i32(10), seed_i32(20)); + let f = bound(&zero, &ten, &zero); + assert_eq!(f.return_type(), &DataType::Int32); + let out = f + .invoke_in(&[lit(&zero), lit(&ten), lit(&zero)], 1, 0) + .unwrap(); + assert_eq!(out.as_primitive::().values(), &[7]); + let f = bound(&ten, &twenty, &zero); + let out = f + .invoke_in(&[lit(&ten), lit(&twenty), lit(&zero)], 1, 0) + .unwrap(); + assert_eq!(out.as_primitive::().values(), &[17]); + // Partition 1 draws its own stream (`16`, then `15`). + let out = f + .invoke_in(&[lit(&ten), lit(&twenty), lit(&zero)], 2, 1) + .unwrap(); + assert_eq!(out.as_primitive::().values(), &[16, 15]); +} + +#[test] +fn randstr_matches_the_golden_strings() { + // `randstr(10, 0)` is `ceV0PXaR2I`, `randstr(3, 0)` is `ceV`. + let len = seed_i32(10); + let sd = seed_i32(0); + let bound = vec![ + BindArg::Literal(Arc::clone(&len)), + BindArg::Literal(Arc::clone(&sd)), + ]; + let f = super::bind_scalar("randstr", &bound, &BindOptions::new()).unwrap(); + assert_eq!(f.return_type(), &DataType::Utf8); + let out = f.invoke_in(&[lit(&len), lit(&sd)], 1, 0).unwrap(); + let texts: Vec> = out + .as_string::() + .iter() + .map(|s| s.map(str::to_owned)) + .collect(); + assert_eq!(texts, vec![Some("ceV0PXaR2I".to_owned())]); + // A fresh bind draws the same stream again. + let len3 = seed_i32(3); + let bound3 = vec![ + BindArg::Literal(Arc::clone(&len3)), + BindArg::Literal(Arc::clone(&sd)), + ]; + let g = super::bind_scalar("randstr", &bound3, &BindOptions::new()).unwrap(); + let out = g.invoke_in(&[lit(&len3), lit(&sd)], 1, 0).unwrap(); + let texts: Vec> = out + .as_string::() + .iter() + .map(|s| s.map(str::to_owned)) + .collect(); + assert_eq!(texts, vec![Some("ceV".to_owned())]); +} + +#[test] +fn randstr_rows_draw_consecutively() { + // One partition draws one stream: rows are consecutive slices of it + // (as a real Spark run with one partition shows). + let len = seed_i32(10); + let sd = seed_i32(0); + let bound = vec![ + BindArg::Literal(Arc::clone(&len)), + BindArg::Literal(Arc::clone(&sd)), + ]; + let f = super::bind_scalar("randstr", &bound, &BindOptions::new()).unwrap(); + let out = f.invoke_in(&[lit(&len), lit(&sd)], 3, 0).unwrap(); + let texts: Vec> = out + .as_string::() + .iter() + .map(|s| s.map(str::to_owned)) + .collect(); + assert_eq!( + texts, + vec![ + Some("ceV0PXaR2I".to_owned()), + Some("lBhHi56d0u".to_owned()), + Some("CtuwSOTLiL".to_owned()) + ] + ); + // The golden's second partition draws its own stream: with partitions + // `[0]`, `[1, 2]` the rows are `ceV0PXaR2I`, `fYxVfArnv7`, + // `iSIv0VT2XL`. + let out = f.invoke_in(&[lit(&len), lit(&sd)], 2, 1).unwrap(); + let texts: Vec> = out + .as_string::() + .iter() + .map(|s| s.map(str::to_owned)) + .collect(); + assert_eq!( + texts, + vec![Some("fYxVfArnv7".to_owned()), Some("iSIv0VT2XL".to_owned())] + ); +} + +#[test] +fn randstr_null_length_is_empty_and_negative_fails() { + use krishiv_common::error::ErrorClass; + let sd = seed_i32(0); + // A NULL length is the empty string. + let bound = vec![ + BindArg::Literal(null_i32()), + BindArg::Literal(Arc::clone(&sd)), + ]; + let f = super::bind_scalar("randstr", &bound, &BindOptions::new()).unwrap(); + let out = f.invoke_in(&[lit(&null_i32()), lit(&sd)], 2, 0).unwrap(); + assert_eq!(out.len(), 2); + assert!(out.as_string::().iter().all(|s| s == Some("")),); + // A negative length is `INVALID_PARAMETER_VALUE.LENGTH`. + let neg = seed_i32(-1); + let bound = vec![ + BindArg::Literal(Arc::clone(&neg)), + BindArg::Literal(Arc::clone(&sd)), + ]; + let f = super::bind_scalar("randstr", &bound, &BindOptions::new()).unwrap(); + let err = f.invoke_in(&[lit(&neg), lit(&sd)], 1, 0).unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::InvalidParameterValueLength); + assert!(err.to_string().contains("`randstr`"), "{err}"); + assert!(err.to_string().contains("-1"), "{err}"); +} + +#[test] +fn randstr_refuses_an_output_past_the_string_offset_limit() { + use krishiv_common::error::ErrorClass; + // 1e9 characters over 3 rows exceed `i32` offsets: an error, not a panic. + let (big, sd) = (seed_i32(1_000_000_000), seed_i32(0)); + let bound = vec![ + BindArg::Literal(Arc::clone(&big)), + BindArg::Literal(Arc::clone(&sd)), + ]; + let f = super::bind_scalar("randstr", &bound, &BindOptions::new()).unwrap(); + let err = f.invoke_in(&[lit(&big), lit(&sd)], 3, 0).unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::UnableToAcquireMemory); + // The binding is still usable afterwards (nothing was poisoned). + let small = seed_i32(3); + let bound = vec![ + BindArg::Literal(Arc::clone(&small)), + BindArg::Literal(Arc::clone(&sd)), + ]; + let f = super::bind_scalar("randstr", &bound, &BindOptions::new()).unwrap(); + let out = f.invoke_in(&[lit(&small), lit(&sd)], 2, 0).unwrap(); + assert_eq!(out.as_string::().value(0), "ceV"); +} + +#[test] +fn uuids_have_shape_and_partitioned_streams() { + let sd = seed_i32(42); + let bound = vec![BindArg::Literal(Arc::clone(&sd))]; + let f = super::bind_scalar("uuid", &bound, &BindOptions::new()).unwrap(); + assert_eq!(f.return_type(), &DataType::Utf8); + let out = f.invoke_in(&[lit(&sd)], 2, 0).unwrap(); + let texts: Vec = out + .as_string::() + .iter() + .map(|s| s.unwrap().to_owned()) + .collect(); + assert_eq!(texts.len(), 2); + assert_ne!(texts[0], texts[1]); + for u in &texts { + assert_eq!(u.len(), 36, "{u}"); + assert_eq!(&u[8..9], "-"); + assert_eq!(&u[14..15], "4"); + assert!(matches!(&u[19..20], "8" | "9" | "a" | "b"), "{u}"); + } + // Another partition draws another stream. + let p1 = f.invoke_in(&[lit(&sd)], 2, 1).unwrap(); + let other: Vec = p1 + .as_string::() + .iter() + .map(|s| s.unwrap().to_owned()) + .collect(); + assert_ne!(texts, other); + // The unseeded form binds with no arguments. + let g = super::bind_scalar("uuid", &[], &BindOptions::new()).unwrap(); + let out = g.invoke_in(&[], 1, 0).unwrap(); + assert_eq!(out.len(), 1); +} + +#[test] +fn unseeded_draws_come_from_the_query_seed() { + // Two binds of `rand()` under one query seed agree; another query + // seed draws another stream. (Separate `BindOptions` hold separate + // salts; one query's binds draw different streams, below.) + let f = + super::bind_scalar("rand", &[], &BindOptions::new().with_query_start(1_000_000)).unwrap(); + let g = + super::bind_scalar("rand", &[], &BindOptions::new().with_query_start(1_000_000)).unwrap(); + assert_eq!( + f64s(&f.invoke_in(&[], 2, 0).unwrap()), + f64s(&g.invoke_in(&[], 2, 0).unwrap()) + ); + let other = BindOptions::new().with_query_start(2_000_000); + let h = super::bind_scalar("rand", &[], &other).unwrap(); + assert_ne!( + f64s(&f.invoke_in(&[], 2, 0).unwrap()), + f64s(&h.invoke_in(&[], 2, 0).unwrap()) + ); +} + +#[test] +fn unseeded_calls_in_one_query_draw_different_streams() { + // Two unseeded binds sharing one query's salt draw different streams + // (Spark's `ResolveRandomSeed` assigns each its own seed), and the + // same holds for `uuid` and `uniform` without a seed. + let opts = BindOptions::new().with_query_start(1_000_000); + let f = super::bind_scalar("rand", &[], &opts).unwrap(); + let g = super::bind_scalar("rand", &[], &opts).unwrap(); + assert_ne!( + f64s(&f.invoke_in(&[], 2, 0).unwrap()), + f64s(&g.invoke_in(&[], 2, 0).unwrap()) + ); + let u = super::bind_scalar("uuid", &[], &opts).unwrap(); + let v = super::bind_scalar("uuid", &[], &opts).unwrap(); + let texts = |f: &Arc| { + f.invoke_in(&[], 1, 0) + .unwrap() + .as_string::() + .iter() + .map(|s| s.unwrap().to_owned()) + .collect::>() + }; + assert_ne!(texts(&u), texts(&v)); +} + +#[test] +fn wrong_argument_types_do_not_bind() { + use krishiv_common::error::ErrorClass; + // A seed that is not an `int`/`bigint` is not bound (the analyzer + // reports `DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE` first): the error + // names the bad signature, not a missing function. + for t in [DataType::Float64, DataType::Utf8] { + let a = arrow_array::new_empty_array(&t); + let err = super::bind_scalar("rand", &[t.into()], &BindOptions::new()).unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::KrishivUnsupportedFeature); + assert!(err.to_string().contains("rand("), "{err}"); + let _ = a; + } + // `uniform` takes two or three arguments. + let err = super::bind_scalar("uniform", &[], &BindOptions::new()).unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::KrishivUnsupportedFeature); + // `randstr` takes one or two. + let ty = || ints(&[Some(1)]).data_type().clone(); + let err = super::bind_scalar( + "randstr", + &[ty().into(), ty().into(), ty().into()], + &BindOptions::new(), + ) + .unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::KrishivUnsupportedFeature); + // A string length casts like Spark's implicit cast; a bad one fails + // the cast at runtime. + let len = || { + super::bind_scalar( + "randstr", + &[BindArg::Literal(strs(&[Some("10")]))], + &BindOptions::new(), + ) + .unwrap() + }; + let out = len() + .invoke_in(&[Arg::Scalar(strs(&[Some("10")]))], 1, 0) + .unwrap(); + assert_eq!(out.as_string::().value(0).len(), 10); + let err = len() + .invoke_in(&[Arg::Scalar(strs(&[Some("abc")]))], 1, 0) + .unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::CastInvalidInput); +} diff --git a/crates/krishiv-functions/src/uniform.rs b/crates/krishiv-functions/src/uniform.rs new file mode 100644 index 000000000..8e1dbc0ca --- /dev/null +++ b/crates/krishiv-functions/src/uniform.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The `uniform(min, max)` result-type rule, shared by the analyzer rule +//! (`registry::misc`, over [`krishiv_types::DataType`]) and the executor +//! binding (`scalar::random`, over Arrow types): Spark's `Uniform.dataType`. +//! +//! Both layers map their own type to a [`UniformKind`]; [`uniform_result`] +//! then picks the result kind, so the rank order, the `double`/`float` +//! priority and the decimal rule live once. The layers only map kinds back +//! to their own type. `uniform_result_agrees_with_the_executor_bind` (in +//! `registry::tests`) pins every pair of kinds end to end across both +//! layers, so a mapper cannot drift from the rule. + +/// The numeric kinds `uniform`'s result rule distinguishes: the four +/// integral widths in Spark's `UpCastRule` order, the two floating widths, a +/// decimal width, and everything else (strings, booleans, timestamps, ...), +/// which reads as `DOUBLE` — as does an untyped `NULL` bound, which Spark +/// casts to `DOUBLE`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum UniformKind { + Byte, + Short, + Int, + Long, + Float, + Double, + Decimal { precision: u8, scale: i8 }, + Other, +} + +/// Whether `kind` is one of the four integral widths. +fn is_integral(kind: UniformKind) -> bool { + matches!( + kind, + UniformKind::Byte | UniformKind::Short | UniformKind::Int | UniformKind::Long + ) +} + +/// The result kind of `uniform(min, max)` for bound kinds `min`/`max`. +pub(crate) fn uniform_result(min: UniformKind, max: UniformKind) -> UniformKind { + let rank = |kind: UniformKind| match kind { + UniformKind::Byte => 0, + UniformKind::Short => 1, + UniformKind::Int => 2, + _ => 3, + }; + match (min, max) { + (a, b) if is_integral(a) && is_integral(b) => { + if rank(a) >= rank(b) { + a + } else { + b + } + } + (UniformKind::Double, _) | (_, UniformKind::Double) => UniformKind::Double, + (UniformKind::Float, _) | (_, UniformKind::Float) => UniformKind::Float, + ( + UniformKind::Decimal { + precision: p1, + scale: s1, + }, + UniformKind::Decimal { + precision: p2, + scale: s2, + }, + ) => { + if is_wider_decimal((p1, s1), (p2, s2)) { + min + } else { + max + } + } + (d @ UniformKind::Decimal { .. }, _) | (_, d @ UniformKind::Decimal { .. }) => d, + _ => UniformKind::Double, + } +} + +/// Whether the decimal `(p1, s1)` is wider than `(p2, s2)` (Spark's +/// `DecimalType.isWiderThan`: every part at least as wide). +pub(crate) fn is_wider_decimal(left: (u8, i8), right: (u8, i8)) -> bool { + let (p1, s1) = (i32::from(left.0), i32::from(left.1)); + let (p2, s2) = (i32::from(right.0), i32::from(right.1)); + p1 - s1 >= p2 - s2 && s1 >= s2 && p1 >= p2 +} diff --git a/crates/krishiv-plan/src/analyzer/exprs.rs b/crates/krishiv-plan/src/analyzer/exprs.rs index 146023ff3..e7f39a289 100644 --- a/crates/krishiv-plan/src/analyzer/exprs.rs +++ b/crates/krishiv-plan/src/analyzer/exprs.rs @@ -1340,6 +1340,9 @@ impl Resolver<'_> { } let data_type = ty.data_type.clone().unwrap_or(DataType::Null); call_kids.extend(kids.get(n).cloned()); + // `typeof(x)` is Spark's `RuntimeReplaceable`; its type + // name is known once the call is typed. + let is_typeof = entry.id.name() == "typeof"; let call = ResolvedFunction { id: entry.id, kind: entry.kind, @@ -1352,6 +1355,21 @@ impl Resolver<'_> { }; let expr = rebuilt(ExprNode::Resolved(ResolvedExpr::Function(call))); let shown = self.call_shown(lookup_name, &expr, kids, &call_kids); + // `typeof(x)` is Spark's `RuntimeReplaceable`: the analyzer + // replaces it with the type name literal, keeping the call + // as the shown form (the executor gets no type logic). + let (expr, shown) = if is_typeof { + let name = call_kids + .first() + .and_then(|k| k.ty.data_type.clone()) + .map_or_else(|| "void".to_owned(), |t| t.catalog_string()); + ( + rebuilt(ExprNode::Literal(Literal::String(name))), + Some(expr), + ) + } else { + (expr, shown) + }; let mut bound = Bound { agg: if entry.kind == FunctionKind::Aggregate && !window_fn { Some(e.origin) @@ -3787,6 +3805,16 @@ fn call( let d = if distinct { "DISTINCT " } else { "" }; format!("to_char({d}{})", k.join(", ")) } + // `current_database` and `current_schema` are one expression + // (Spark's `CurrentDatabase`, shown as `current_schema`); + // `current_catalog` shows without parentheses. + _ if name.eq_ignore_ascii_case("current_database") => { + let d = if distinct { "DISTINCT " } else { "" }; + format!("current_schema({d}{})", k.join(", ")) + } + _ if name.eq_ignore_ascii_case("current_catalog") && k.is_empty() && !distinct => { + "current_catalog".to_owned() + } // Single-argument math (`sec`, `csc`, `cot`, `exp`, ...) and // two-argument math (`power`, `atan2`, `hypot`) print upper case, // as Spark's `UnaryMathExpression` / `BinaryMathExpression` do; diff --git a/crates/krishiv-plan/src/tests/analyzer.rs b/crates/krishiv-plan/src/tests/analyzer.rs index 04f705f26..e7817755f 100644 --- a/crates/krishiv-plan/src/tests/analyzer.rs +++ b/crates/krishiv-plan/src/tests/analyzer.rs @@ -1564,7 +1564,8 @@ fn unsupported_inputs_fail_explicitly() { old.profile = SparkProfile::V3_5; let randstr = LogicalPlan::table(["t"]).project(vec![Expr::function("randstr", vec![int(3)])]); assert_eq!(class(&run(randstr.clone(), &old)), "UNRESOLVED_ROUTINE"); - assert_eq!(class(&run(randstr, &ctx)), "KRISHIV_UNSUPPORTED_FEATURE"); + // `randstr` is implemented: it resolves on current versions. + assert_eq!(class(&run(randstr, &ctx)), "OK"); // The argument count is checked first, with Spark's `expectedNum` text. let upper = LogicalPlan::table(["t"]).project(vec![Expr::function("upper", vec![int(1), int(2)])]); @@ -1775,7 +1776,7 @@ fn function_entries_print_their_signature() { let uuid = ["uuid".to_owned()]; assert!(matches!( b.lookup(&uuid, SparkProfile::V4_2), - FunctionLookup::KnownUnsupported + FunctionLookup::Found(_) )); assert_eq!(b.version(), 0); } diff --git a/crates/krishiv-session/tests/misc_functions.rs b/crates/krishiv-session/tests/misc_functions.rs new file mode 100644 index 000000000..470e98d7a --- /dev/null +++ b/crates/krishiv-session/tests/misc_functions.rs @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The misc-function goldens end to end: every `misc-functions.sql`, +//! `random.sql` and `current_database_catalog.sql` statement that uses only +//! the implemented functions matches Spark 4.2.0 exactly (schema, rows, +//! error class and parameters), except the multi-row `VALUES` statements: +//! the engine runs those tiny scans in one partition while Spark CI splits +//! them over two, so the row multisets differ. The single-partition draws +//! are asserted here; their decomposition into per-partition streams is +//! pinned at the binding level (`scalar/tests/random.rs`: partition 0's +//! first `uniform(10, 20, 0)` draw is 17, partition 1's next two are 16 and +//! 15 — the golden multiset — and `randstr`'s partition-1 rows match the +//! golden's rows 2-3 exactly). The nondeterministic functions are otherwise +//! deterministic per partition. +#![cfg(test)] + +use arrow_array::cast::AsArray; +use arrow_array::{Array, RecordBatch}; +use arrow_schema::DataType as ArrowType; +use arrow_schema::{Field, Schema}; +use krishiv_common::error::{ErrorClass, KrishivError}; +use krishiv_session::Session; +use std::sync::Arc; + +fn block_on(f: impl Future) -> T { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(f) +} + +/// The schema names and every row of `sql`, values as strings (`~` for +/// NULL), rows in output order. +fn rows(s: &Session, sql: &str) -> (Vec, Vec>) { + let df = block_on(s.sql(sql)).unwrap(); + let schema = block_on(df.schema()).unwrap(); + let names: Vec = schema.fields().iter().map(|f| f.name().clone()).collect(); + let batches = block_on(df.collect()).unwrap(); + let mut out = Vec::new(); + for b in &batches { + for i in 0..b.num_rows() { + let mut row = Vec::new(); + for c in b.columns() { + if c.is_null(i) { + row.push("~".to_owned()); + continue; + } + row.push(match c.data_type() { + ArrowType::Utf8View => c.as_string_view().value(i).to_owned(), + ArrowType::Utf8 => c.as_string::().value(i).to_owned(), + ArrowType::Int32 => c + .as_primitive::() + .value(i) + .to_string(), + ArrowType::Int64 => c + .as_primitive::() + .value(i) + .to_string(), + ArrowType::Float32 => c + .as_primitive::() + .value(i) + .to_string(), + ArrowType::Float64 => c + .as_primitive::() + .value(i) + .to_string(), + ArrowType::Boolean => c.as_boolean().value(i).to_string(), + ArrowType::Decimal128(_, s) => { + let v = c + .as_primitive::() + .value(i); + render_decimal(v, *s) + } + t => panic!("{sql}: unexpected type {t}"), + }); + } + out.push(row); + } + } + (names, out) +} + +fn session() -> Session { + Session::new().unwrap() +} + +/// Renders a decimal `(unscaled, scale)` as Spark does. +fn render_decimal(unscaled: i128, scale: i8) -> String { + let neg = unscaled < 0; + let digits = unscaled.unsigned_abs().to_string(); + let s = scale.cast_unsigned() as usize; + let text = if s == 0 { + digits + } else if digits.len() > s { + let (i, f) = digits.split_at(digits.len() - s); + format!("{i}.{f}") + } else { + format!("0.{:0>width$}", digits, width = s) + }; + format!("{}{text}", if neg { "-" } else { "" }) +} + +/// A session with a three-row table (`SELECT ... FROM vals` stands in for +/// the goldens' `FROM VALUES (0), (1), (2)`, which the engine does not +/// support yet). +fn values_session() -> Session { + use arrow_array::Int32Array; + let s = Session::new().unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new( + "col", + ArrowType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![0, 1, 2]))], + ) + .unwrap(); + s.register_batches("vals", schema, vec![batch]).unwrap(); + s +} + +fn error(s: &Session, sql: &str) -> KrishivError { + block_on(async { s.sql(sql).await?.collect().await }).unwrap_err() +} + +#[test] +fn typeof_names_the_analyzers_type() { + let s = session(); + // `select typeof(null)` in `misc-functions.sql`. + assert_eq!( + rows(&s, "select typeof(null)"), + ( + vec!["typeof(NULL)".to_owned()], + vec![vec!["void".to_owned()]] + ) + ); + assert_eq!( + rows(&s, "select typeof(true)"), + ( + vec!["typeof(true)".to_owned()], + vec![vec!["boolean".to_owned()]] + ) + ); + assert_eq!( + rows(&s, "select typeof(1Y), typeof(1S), typeof(1), typeof(1L)"), + ( + vec![ + "typeof(1)".to_owned(), + "typeof(1)".to_owned(), + "typeof(1)".to_owned(), + "typeof(1)".to_owned(), + ], + vec![vec![ + "tinyint".to_owned(), + "smallint".to_owned(), + "int".to_owned(), + "bigint".to_owned(), + ]] + ) + ); + assert_eq!( + rows( + &s, + "select typeof(cast(1.0 as float)), typeof(1.0D), typeof(1.2)" + ), + ( + vec![ + "typeof(CAST(1.0 AS FLOAT))".to_owned(), + "typeof(1.0)".to_owned(), + "typeof(1.2)".to_owned(), + ], + vec![vec![ + "float".to_owned(), + "double".to_owned(), + "decimal(2,1)".to_owned(), + ]] + ) + ); + assert_eq!( + rows(&s, "select typeof(x'ABCD'), typeof('SPARK')"), + ( + vec!["typeof(X'ABCD')".to_owned(), "typeof(SPARK)".to_owned()], + vec![vec!["binary".to_owned(), "string".to_owned()]] + ) + ); + assert_eq!( + rows( + &s, + "select typeof(date '1986-05-23'), typeof(timestamp '1986-05-23')" + ), + ( + vec![ + "typeof(DATE '1986-05-23')".to_owned(), + "typeof(TIMESTAMP '1986-05-23 00:00:00')".to_owned(), + ], + vec![vec!["date".to_owned(), "timestamp".to_owned()]] + ) + ); +} + +#[test] +fn typeof_needs_no_execution() { + // The rewrite is a literal: the argument is never evaluated, so a + // failing argument still names its type (with the analyzer's casts, as + // Spark's `toPrettySQL` shows them). + let s = session(); + assert_eq!( + rows(&s, "select typeof(1 / 0)").0, + vec!["typeof((CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)))".to_owned()] + ); + assert_eq!( + rows(&s, "select typeof(1 / 0)").1, + vec![vec!["double".to_owned()]] + ); +} + +#[test] +fn current_database_and_catalog() { + // `current_database_catalog.sql`, exactly. + let s = session(); + assert_eq!( + rows( + &s, + "select current_database(), current_schema(), current_catalog()" + ), + ( + vec![ + "current_schema()".to_owned(), + "current_schema()".to_owned(), + "current_catalog".to_owned(), + ], + vec![vec![ + "default".to_owned(), + "default".to_owned(), + "spark_catalog".to_owned(), + ]] + ) + ); + assert_eq!( + rows(&s, "select version()").1, + vec![vec!["4.2.0".to_owned()]] + ); +} + +/// Seeded draws match Spark exactly; the multi-row `VALUES` cases assert +/// the engine's single-partition stream (Spark CI splits those rows over +/// two partitions — see the module docs). +#[test] +fn rand_and_randn_match_spark_single_partition() { + let s = session(); + // `random.sql`, exactly. + assert_eq!( + rows(&s, "SELECT rand(0)").1, + vec![vec!["0.7604953758285915".to_owned()]] + ); + assert_eq!( + rows(&s, "SELECT rand(cast(3 / 7 AS int))").1, + vec![vec!["0.7604953758285915".to_owned()]] + ); + assert_eq!( + rows(&s, "SELECT rand(NULL)").1, + vec![vec!["0.7604953758285915".to_owned()]] + ); + assert_eq!( + rows(&s, "SELECT rand(cast(NULL AS int))").1, + vec![vec!["0.7604953758285915".to_owned()]] + ); + assert_eq!( + rows(&s, "SELECT randn(0L)").1, + vec![vec!["1.6034991609278433".to_owned()]] + ); + assert_eq!( + rows(&s, "SELECT randn(cast(3 / 7 AS long))").1, + vec![vec!["1.6034991609278433".to_owned()]] + ); + assert_eq!( + rows(&s, "SELECT randn(NULL)").1, + vec![vec!["1.6034991609278433".to_owned()]] + ); + assert_eq!( + rows(&s, "SELECT randn(cast(NULL AS long))").1, + vec![vec!["1.6034991609278433".to_owned()]] + ); + // The seed types Spark rejects. + let e = error(&s, "SELECT rand(1.0)"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(e.to_string().contains("(\"INT\" or \"BIGINT\")"), "{e}"); + let e = error(&s, "SELECT rand('1')"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(e.to_string().contains("\"rand(1)\""), "{e}"); + // A non-foldable seed is refused. + let v = values_session(); + let e = error(&v, "SELECT rand(col) FROM vals"); + assert_eq!(e.error_class(), ErrorClass::SeedExpressionIsUnfoldable); + let e = error(&s, "SELECT rand(id) FROM (SELECT 1 AS id)"); + assert_eq!(e.error_class(), ErrorClass::SeedExpressionIsUnfoldable); + // Too many seeds is an arity error. + let e = error(&s, "SELECT rand(1, 2)"); + assert_eq!(e.error_class(), ErrorClass::WrongNumArgsWithoutSuggestion); +} + +/// Seeded values match Spark exactly; the multi-row `VALUES` case asserts +/// the engine's single-partition draws (Spark CI splits those rows over +/// two partitions — see the module docs). +#[test] +fn uniform_matches_spark_single_partition() { + let s = session(); + let value = |sql: &str| rows(&s, sql).1.into_iter().flatten().collect::>(); + assert_eq!(value("SELECT uniform(0, 1, 0) AS result"), ["0"]); + assert_eq!(value("SELECT uniform(0, 10, 0) AS result"), ["7"]); + assert_eq!(value("SELECT uniform(0L, 10L, 0) AS result"), ["7"]); + assert_eq!(value("SELECT uniform(10, 20, 0) AS result"), ["17"]); + assert_eq!( + value("SELECT uniform(10.0F, 20.0F, 0) AS result"), + ["17.604954"] + ); + assert_eq!( + value( + "SELECT uniform(cast(10 as decimal(10, 3)), cast(20 as decimal(10, 3)), 0) AS result" + ), + ["17.605"] + ); + assert_eq!( + value( + "SELECT uniform(cast(10 as decimal(10, 3)), cast(20 as decimal(11, 4)), 0) AS result" + ), + ["17.6050"] + ); + assert_eq!( + value("SELECT uniform(10.0D, 20.0D, CAST(3 / 7 AS LONG)) AS result"), + ["17.604953758285916"] + ); + assert_eq!(value("SELECT uniform(-10L, 10L, 0) AS result"), ["5"]); + assert_eq!(value("SELECT uniform(-20L, -10L, 0) AS result"), ["-12"]); + assert_eq!(value("SELECT uniform(-20L, -10L, -10) AS result"), ["-17"]); + // NULL bounds are NULL results; a NULL seed reads as 0. + assert_eq!(value("SELECT uniform(NULL, 1, 0) AS result"), ["~"]); + assert_eq!( + value("SELECT uniform(cast(NULL AS int), 1, 0) AS result"), + ["~"] + ); + assert_eq!(value("SELECT uniform(0, NULL, 0) AS result"), ["~"]); + assert_eq!(value("SELECT uniform(0, 1, NULL) AS result"), ["0"]); + assert_eq!( + value("SELECT uniform(0, 1, cast(NULL as int)) AS result"), + ["0"] + ); + // The seed and bound checks Spark reports. + let v = values_session(); + let e = error(&v, "SELECT uniform(10, 20, col) AS result FROM vals"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchNonFoldableInput + ); + assert!(e.to_string().contains("`seed`"), "{e}"); + let e = error(&s, "SELECT uniform(10) AS result"); + assert_eq!(e.error_class(), ErrorClass::WrongNumArgsWithoutSuggestion); + let e = error(&s, "SELECT uniform(10, 20, 30, 40) AS result"); + assert_eq!(e.error_class(), ErrorClass::WrongNumArgsWithoutSuggestion); + assert!(e.to_string().contains("[2, 3]"), "{e}"); + let e = error(&s, "SELECT uniform(10.0F, 20.0F, 0.0F) AS result"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(e.to_string().contains("third"), "{e}"); + let e = error(&s, "SELECT uniform('abc', 10, 0) AS result"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(e.to_string().contains("\"NUMERIC\""), "{e}"); + let e = error(&s, "SELECT uniform(0, 'def', 0) AS result"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(e.to_string().contains("\"NUMERIC\""), "{e}"); + assert!(e.to_string().contains("second"), "{e}"); + // Several partitions draw several streams: the engine runs the tiny + // scan in one partition, so the rows are one stream's first draws + // (Spark CI splits the rows over two partitions and sorts). + let v = values_session(); + let mut got = rows(&v, "SELECT uniform(10, 20, 0) AS result FROM vals") + .1 + .into_iter() + .flatten() + .collect::>(); + got.sort(); + assert_eq!(got, ["10", "15", "17"]); +} + +/// Seeded strings match Spark exactly; the multi-row `VALUES` case asserts +/// the engine's single-partition draws (Spark CI splits those rows over +/// two partitions — see the module docs). +#[test] +fn randstr_matches_spark_single_partition() { + let s = session(); + let value = |sql: &str| rows(&s, sql).1.into_iter().flatten().collect::>(); + assert_eq!(value("SELECT randstr(1, 0) AS result"), ["c"]); + assert_eq!(value("SELECT randstr(5, 0) AS result"), ["ceV0P"]); + assert_eq!(value("SELECT randstr(10, 0) AS result"), ["ceV0PXaR2I"]); + assert_eq!(value("SELECT randstr(10S, 0) AS result"), ["ceV0PXaR2I"]); + assert_eq!( + value("SELECT randstr(CAST(10 AS TINYINT), 0) AS result"), + ["ceV0PXaR2I"] + ); + assert_eq!( + value("SELECT randstr(CAST(10 AS BIGINT), 0) AS result"), + ["ceV0PXaR2I"] + ); + assert_eq!(value("SELECT randstr(1.0F, 0) AS result"), ["c"]); + assert_eq!(value("SELECT randstr(1.0D, 0) AS result"), ["c"]); + assert_eq!( + value("SELECT randstr(cast(1 AS DECIMAL(10, 2)), 0) AS result"), + ["c"] + ); + assert_eq!(value("SELECT randstr(1, -1) AS result"), ["S"]); + assert_eq!(value("SELECT randstr(NULL, 0) AS result"), [""]); + assert_eq!(value("SELECT randstr(0, NULL) AS result"), [""]); + let v = values_session(); + let e = error(&v, "SELECT randstr(col, 0) AS result FROM vals"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchNonFoldableInput + ); + let e = error(&s, "SELECT randstr(10, 0, 1) AS result"); + assert_eq!(e.error_class(), ErrorClass::WrongNumArgsWithoutSuggestion); + let e = error(&s, "SELECT randstr(-1, 0) AS result"); + assert_eq!(e.error_class(), ErrorClass::InvalidParameterValueLength); + // A string seed is neither an `int` nor a `bigint`. + let e = error(&s, "SELECT randstr(10, 'a') AS result"); + assert_eq!( + e.error_class(), + ErrorClass::DatatypeMismatchUnexpectedInputType + ); + assert!(e.to_string().contains("second"), "{e}"); + assert!(e.to_string().contains("(\"INT\" or \"BIGINT\")"), "{e}"); + // The engine's tiny scan is one partition: consecutive draws of one + // stream (Spark CI splits the rows over two partitions and sorts). + let v = values_session(); + assert_eq!( + rows(&v, "SELECT randstr(10, 0) AS result FROM vals").1, + [ + vec!["ceV0PXaR2I".to_owned()], + vec!["lBhHi56d0u".to_owned()], + vec!["CtuwSOTLiL".to_owned()] + ] + ); +} + +#[test] +fn nondeterministic_queries_are_deterministic_per_partition() { + let s = session(); + let value = |sql: &str| rows(&s, sql).1.into_iter().flatten().collect::>(); + // The same query twice draws the same stream (one partition). + let once = value("SELECT rand(0), randn(0), uuid(42), monotonically_increasing_id()"); + let twice = value("SELECT rand(0), randn(0), uuid(42), monotonically_increasing_id()"); + assert_eq!(once, twice); + // An unseeded call is not NULL. + assert_eq!(value("SELECT randstr(10) IS NOT NULL AS result"), ["true"]); + assert_eq!( + value("SELECT uniform(10, 20.0F) IS NOT NULL AS result"), + ["true"] + ); + // `uuid()` draws from the query seed: two queries differ. + let a = value("SELECT uuid()"); + let b = value("SELECT uuid()"); + assert_ne!(a, b); + // Two unseeded calls in one query draw different streams. + let both = rows(&s, "SELECT rand() AS a, rand() AS b").1; + assert_ne!(both[0][0], both[0][1]); + // Partition functions in one partition. + assert_eq!(value("SELECT spark_partition_id()"), ["0"]); + assert_eq!(value("SELECT monotonically_increasing_id()"), ["0"]); + assert_eq!(value("SELECT input_file_name()"), [""]); + assert_eq!(value("SELECT input_file_block_start()"), ["-1"]); + assert_eq!(value("SELECT input_file_block_length()"), ["-1"]); + // UNION ALL numbers its children's partitions one after another, as + // Spark's `UnionRDD` does (Spark 4.2.0, local[1]: 0, 1, 2 and + // 0, 8589934592). + assert_eq!( + value( + "SELECT spark_partition_id() AS p FROM (SELECT 1 AS x UNION ALL \ + SELECT 2 AS x UNION ALL SELECT 3 AS x) ORDER BY p" + ), + ["0", "1", "2"] + ); + assert_eq!( + value( + "SELECT monotonically_increasing_id() AS m FROM (SELECT 1 AS x \ + UNION ALL SELECT 2 AS x) ORDER BY m" + ), + ["0", "8589934592"] + ); + // A stateful predicate sees the same partition stream. + let v = values_session(); + assert_eq!( + rows(&v, "SELECT col FROM vals WHERE rand(0) > 0.5").1, + [vec!["0".to_owned()], vec!["1".to_owned()]] + ); +} + +#[test] +fn misc_rejects_bad_calls() { + let s = session(); + // `uuid` wants a plain literal seed. + let e = error(&s, "SELECT uuid(1 + 1)"); + assert_eq!(e.error_class(), ErrorClass::InvalidParameterValueLong); + // `typeof` takes exactly one argument. + let e = error(&s, "SELECT typeof()"); + assert_eq!(e.error_class(), ErrorClass::WrongNumArgsWithoutSuggestion); + for sql in [ + "SELECT version(1)", + "SELECT current_database(1)", + "SELECT monotonically_increasing_id(1)", + "SELECT input_file_name('x')", + "SELECT spark_partition_id(1)", + ] { + let e = error(&s, sql); + assert_eq!( + e.error_class(), + ErrorClass::WrongNumArgsWithoutSuggestion, + "{sql}" + ); + } +} diff --git a/crates/krishiv-session/tests/session_variables.rs b/crates/krishiv-session/tests/session_variables.rs index 3ee3c9c97..749dcfd80 100644 --- a/crates/krishiv-session/tests/session_variables.rs +++ b/crates/krishiv-session/tests/session_variables.rs @@ -11,7 +11,7 @@ //! TABLE`, persistent views, `INTERVAL` values, `LATERAL` subqueries or //! struct, map and array values), and the comment names it. Statements that //! fail deterministically without the feature assert the engine's exact -//! error instead (the `RAND` declarations, the struct variable). The second +//! error instead (the struct variable). The second //! golden file, `session-variable-precedence.sql`, needs `DECLARE all` (a //! reserved word here), `GROUP BY ALL` / `ORDER BY ALL` and `CREATE TABLE` //! throughout and is skipped as a whole; precedence without them is covered @@ -392,16 +392,12 @@ const CASES: &[(&str, Expect)] = &[ rows: &[&["1", "1"]], }, ), - // `RAND` is unsupported, so the declaration fails and `var1` keeps its - // `INT` value of 1: the `SELECT` below asserts the retained value, and - // asserts nothing about the golden `DOUBLE`. Pinning the failure here - // means `RAND` support must update this case. + // `RAND` is supported, so the declaration replaces `var1` with the + // `DOUBLE` value of `1 + RAND(5)`: the `SELECT` below asserts it is at + // least 1, as the golden does. ( "DECLARE OR REPLACE VARIABLE var1 DOUBLE DEFAULT 1 + RAND(5)", - Err { - class: ErrorClass::KrishivUnsupportedFeature, - params: &[("feature", "function RAND")], - }, + E, ), ( "SELECT true AS Expected, var1 >= 1 AS result", @@ -616,26 +612,20 @@ const CASES: &[(&str, Expect)] = &[ ("DECLARE VARIABLE var1, var2, var3 INT", E), ( "DECLARE VARIABLE var4, var5, var6 INT DEFAULT CAST(RAND(0) * 10 AS INT)", - Err { - class: ErrorClass::KrishivUnsupportedFeature, - params: &[("feature", "function RAND")], - }, + E, ), ("DECLARE VARIABLE var7, var8, var9 DEFAULT 5", E), ( "SELECT var4 = var5, var4 = var6, var5 = var6", - Err { - class: ErrorClass::UnresolvedColumnWithoutSuggestion, - params: &[("objectName", "`var4`")], + R { + schema: "struct<(var4 = var5):boolean,(var4 = var6):boolean,(var5 = var6):boolean>", + rows: &[&["true", "true", "true"]], }, ), ("DECLARE OR REPLACE VARIABLE var1, var2, var3 DOUBLE", E), ( "DECLARE OR REPLACE VARIABLE var4, var5, var6 DOUBLE DEFAULT RAND(0)", - Err { - class: ErrorClass::KrishivUnsupportedFeature, - params: &[("feature", "function RAND")], - }, + E, ), ( "DECLARE OR REPLACE VARIABLE var7, var8, var9 DEFAULT 1.5", @@ -643,35 +633,17 @@ const CASES: &[(&str, Expect)] = &[ ), ( "SELECT var4 = var5, var4 = var6, var5 = var6", - Err { - class: ErrorClass::UnresolvedColumnWithoutSuggestion, - params: &[("objectName", "`var4`")], + R { + schema: "struct<(var4 = var5):boolean,(var4 = var6):boolean,(var5 = var6):boolean>", + rows: &[&["true", "true", "true"]], }, ), ("DROP TEMPORARY VARIABLE var1", E), ("DROP TEMPORARY VARIABLE var2", E), ("DROP TEMPORARY VARIABLE var3", E), - ( - "DROP TEMPORARY VARIABLE var4", - Err { - class: ErrorClass::VariableNotFound, - params: &[("variableName", "`system`.`session`.`var4`")], - }, - ), - ( - "DROP TEMPORARY VARIABLE var5", - Err { - class: ErrorClass::VariableNotFound, - params: &[("variableName", "`system`.`session`.`var5`")], - }, - ), - ( - "DROP TEMPORARY VARIABLE var6", - Err { - class: ErrorClass::VariableNotFound, - params: &[("variableName", "`system`.`session`.`var6`")], - }, - ), + ("DROP TEMPORARY VARIABLE var4", E), + ("DROP TEMPORARY VARIABLE var5", E), + ("DROP TEMPORARY VARIABLE var6", E), ("DROP TEMPORARY VARIABLE var7", E), ("DROP TEMPORARY VARIABLE var8", E), ("DROP TEMPORARY VARIABLE var9", E), diff --git a/crates/krishiv-session/tests/string_function_goldens.rs b/crates/krishiv-session/tests/string_function_goldens.rs index 517be460e..1430957bc 100644 --- a/crates/krishiv-session/tests/string_function_goldens.rs +++ b/crates/krishiv-session/tests/string_function_goldens.rs @@ -300,7 +300,7 @@ fn check(file: &str, passed: usize, skipped: usize) { #[test] fn string_functions_match_spark() { - check("string-functions.sql.out", 212, 31); + check("string-functions.sql.out", 213, 30); } #[test] diff --git a/python/krishiv/sql/_generated_functions.py b/python/krishiv/sql/_generated_functions.py index 24a5956fd..7233ceb30 100644 --- a/python/krishiv/sql/_generated_functions.py +++ b/python/krishiv/sql/_generated_functions.py @@ -66,7 +66,10 @@ "csc", "cume_dist", "curdate", + "current_catalog", + "current_database", "current_date", + "current_schema", "current_timestamp", "current_user", "date_add", @@ -106,6 +109,9 @@ "hypot", "ifnull", "initcap", + "input_file_block_length", + "input_file_block_start", + "input_file_name", "instr", "is_valid_utf8", "isnan", @@ -139,6 +145,7 @@ "mean", "min", "minute", + "monotonically_increasing_id", "month", "monthname", "months_between", @@ -166,6 +173,9 @@ "quote", "radians", "raise_error", + "rand", + "randn", + "randstr", "rank", "regexp", "regexp_count", @@ -201,6 +211,7 @@ "sinh", "skewness", "soundex", + "spark_partition_id", "split_part", "sqrt", "startswith", @@ -235,18 +246,22 @@ "try_to_number", "try_url_decode", "try_validate_utf8", + "typeof", "ucase", "unbase64", "unhex", + "uniform", "unix_date", "upper", "url_decode", "url_encode", "user", + "uuid", "validate_utf8", "var_pop", "var_samp", "variance", + "version", "weekday", "weekofyear", "xxhash64", @@ -461,11 +476,26 @@ def curdate() -> Column: return _call("curdate") +def current_catalog() -> Column: + """Calls the registry function ``current_catalog``.""" + return _call("current_catalog") + + +def current_database() -> Column: + """Calls the registry function ``current_database``.""" + return _call("current_database") + + def current_date() -> Column: """Calls the registry function ``current_date``.""" return _call("current_date") +def current_schema() -> Column: + """Calls the registry function ``current_schema``.""" + return _call("current_schema") + + def current_timestamp() -> Column: """Calls the registry function ``current_timestamp``.""" return _call("current_timestamp") @@ -662,6 +692,21 @@ def initcap(col: ColumnOrName) -> Column: return _call("initcap", _named(col, "col")) +def input_file_block_length() -> Column: + """Calls the registry function ``input_file_block_length``.""" + return _call("input_file_block_length") + + +def input_file_block_start() -> Column: + """Calls the registry function ``input_file_block_start``.""" + return _call("input_file_block_start") + + +def input_file_name() -> Column: + """Calls the registry function ``input_file_name``.""" + return _call("input_file_name") + + def instr(str: ColumnOrName, substr: str | Column) -> Column: """Calls the registry function ``instr``.""" return _call("instr", _named(str, "str"), _lit(substr, "substr")) @@ -841,6 +886,11 @@ def minute(col: ColumnOrName) -> Column: return _call("minute", _named(col, "col")) +def monotonically_increasing_id() -> Column: + """Calls the registry function ``monotonically_increasing_id``.""" + return _call("monotonically_increasing_id") + + def month(col: ColumnOrName) -> Column: """Calls the registry function ``month``.""" return _call("month", _named(col, "col")) @@ -978,6 +1028,21 @@ def raise_error(errMsg: Column | str) -> Column: return _call("raise_error", _lit(errMsg, "errMsg")) +def rand(seed: int | None = None) -> Column: + """Calls the registry function ``rand``.""" + return _call("rand", *_opt(_lit, seed, "seed")) + + +def randn(seed: int | None = None) -> Column: + """Calls the registry function ``randn``.""" + return _call("randn", *_opt(_lit, seed, "seed")) + + +def randstr(length: int | Column, seed: int | Column | None = None) -> Column: + """Calls the registry function ``randstr``.""" + return _call("randstr", _lit(length, "length"), *_opt(_lit, seed, "seed")) + + def rank() -> Column: """Calls the registry function ``rank``.""" return _call("rank") @@ -1157,6 +1222,11 @@ def soundex(col: ColumnOrName) -> Column: return _call("soundex", _named(col, "col")) +def spark_partition_id() -> Column: + """Calls the registry function ``spark_partition_id``.""" + return _call("spark_partition_id") + + def split_part(src: ColumnOrName, delimiter: ColumnOrName, partNum: ColumnOrName) -> Column: """Calls the registry function ``split_part``.""" return _call("split_part", _named(src, "src"), _named(delimiter, "delimiter"), _named(partNum, "partNum")) @@ -1329,6 +1399,11 @@ def try_validate_utf8(str: ColumnOrName) -> Column: return _call("try_validate_utf8", _named(str, "str")) +def typeof(col: ColumnOrName) -> Column: + """Calls the registry function ``typeof``.""" + return _call("typeof", _named(col, "col")) + + def ucase(str: ColumnOrName) -> Column: """Calls the registry function ``ucase``.""" return _call("ucase", _named(str, "str")) @@ -1344,6 +1419,11 @@ def unhex(col: ColumnOrName) -> Column: return _call("unhex", _named(col, "col")) +def uniform(min: ColumnOrName | float, max: ColumnOrName | float, seed: int | Column | None = None) -> Column: + """Calls the registry function ``uniform``.""" + return _call("uniform", _num_or_col(min, "min"), _num_or_col(max, "max"), *_opt(_lit, seed, "seed")) + + def unix_date(col: ColumnOrName) -> Column: """Calls the registry function ``unix_date``.""" return _call("unix_date", _named(col, "col")) @@ -1369,6 +1449,11 @@ def user() -> Column: return _call("user") +def uuid(seed: int | Column | None = None) -> Column: + """Calls the registry function ``uuid``.""" + return _call("uuid", *_opt(_lit, seed, "seed")) + + def validate_utf8(str: ColumnOrName) -> Column: """Calls the registry function ``validate_utf8``.""" return _call("validate_utf8", _named(str, "str")) @@ -1389,6 +1474,11 @@ def variance(col: ColumnOrName) -> Column: return _call("variance", _named(col, "col")) +def version() -> Column: + """Calls the registry function ``version``.""" + return _call("version") + + def weekday(col: ColumnOrName) -> Column: """Calls the registry function ``weekday``.""" return _call("weekday", _named(col, "col")) diff --git a/python/krishiv/sql/_generated_functions.pyi b/python/krishiv/sql/_generated_functions.pyi index 53e0316c2..9db616f2a 100644 --- a/python/krishiv/sql/_generated_functions.pyi +++ b/python/krishiv/sql/_generated_functions.pyi @@ -47,7 +47,10 @@ __all__ = [ "csc", "cume_dist", "curdate", + "current_catalog", + "current_database", "current_date", + "current_schema", "current_timestamp", "current_user", "date_add", @@ -87,6 +90,9 @@ __all__ = [ "hypot", "ifnull", "initcap", + "input_file_block_length", + "input_file_block_start", + "input_file_name", "instr", "is_valid_utf8", "isnan", @@ -120,6 +126,7 @@ __all__ = [ "mean", "min", "minute", + "monotonically_increasing_id", "month", "monthname", "months_between", @@ -147,6 +154,9 @@ __all__ = [ "quote", "radians", "raise_error", + "rand", + "randn", + "randstr", "rank", "regexp", "regexp_count", @@ -182,6 +192,7 @@ __all__ = [ "sinh", "skewness", "soundex", + "spark_partition_id", "split_part", "sqrt", "startswith", @@ -216,18 +227,22 @@ __all__ = [ "try_to_number", "try_url_decode", "try_validate_utf8", + "typeof", "ucase", "unbase64", "unhex", + "uniform", "unix_date", "upper", "url_decode", "url_encode", "user", + "uuid", "validate_utf8", "var_pop", "var_samp", "variance", + "version", "weekday", "weekofyear", "xxhash64", @@ -276,7 +291,10 @@ def crc32(col: ColumnOrName) -> Column: ... def csc(col: ColumnOrName) -> Column: ... def cume_dist() -> Column: ... def curdate() -> Column: ... +def current_catalog() -> Column: ... +def current_database() -> Column: ... def current_date() -> Column: ... +def current_schema() -> Column: ... def current_timestamp() -> Column: ... def current_user() -> Column: ... def date_add(start: ColumnOrName, days: ColumnOrName | int) -> Column: ... @@ -316,6 +334,9 @@ def hour(col: ColumnOrName) -> Column: ... def hypot(col1: ColumnOrName | float, col2: ColumnOrName | float) -> Column: ... def ifnull(col1: ColumnOrName, col2: ColumnOrName) -> Column: ... def initcap(col: ColumnOrName) -> Column: ... +def input_file_block_length() -> Column: ... +def input_file_block_start() -> Column: ... +def input_file_name() -> Column: ... def instr(str: ColumnOrName, substr: str | Column) -> Column: ... def is_valid_utf8(str: ColumnOrName) -> Column: ... def isnan(col: ColumnOrName) -> Column: ... @@ -355,6 +376,7 @@ def md5(col: ColumnOrName) -> Column: ... def mean(col: ColumnOrName) -> Column: ... def min(col: ColumnOrName) -> Column: ... def minute(col: ColumnOrName) -> Column: ... +def monotonically_increasing_id() -> Column: ... def month(col: ColumnOrName) -> Column: ... def monthname(col: ColumnOrName) -> Column: ... def months_between(date1: ColumnOrName, date2: ColumnOrName, roundOff: bool = True) -> Column: ... @@ -384,6 +406,9 @@ def quarter(col: ColumnOrName) -> Column: ... def quote(col: ColumnOrName) -> Column: ... def radians(col: ColumnOrName) -> Column: ... def raise_error(errMsg: Column | str) -> Column: ... +def rand(seed: int | None = None) -> Column: ... +def randn(seed: int | None = None) -> Column: ... +def randstr(length: int | Column, seed: int | Column | None = None) -> Column: ... def rank() -> Column: ... def regexp(str: ColumnOrName, regexp: ColumnOrName) -> Column: ... def regexp_count(str: ColumnOrName, regexp: ColumnOrName) -> Column: ... @@ -419,6 +444,7 @@ def sin(col: ColumnOrName) -> Column: ... def sinh(col: ColumnOrName) -> Column: ... def skewness(col: ColumnOrName) -> Column: ... def soundex(col: ColumnOrName) -> Column: ... +def spark_partition_id() -> Column: ... def split_part(src: ColumnOrName, delimiter: ColumnOrName, partNum: ColumnOrName) -> Column: ... def sqrt(col: ColumnOrName) -> Column: ... def startswith(str: ColumnOrName, prefix: ColumnOrName) -> Column: ... @@ -453,18 +479,22 @@ def try_to_binary(col: ColumnOrName, format: ColumnOrName | None = None) -> Colu def try_to_number(col: ColumnOrName, format: ColumnOrName) -> Column: ... def try_url_decode(str: ColumnOrName) -> Column: ... def try_validate_utf8(str: ColumnOrName) -> Column: ... +def typeof(col: ColumnOrName) -> Column: ... def ucase(str: ColumnOrName) -> Column: ... def unbase64(col: ColumnOrName) -> Column: ... def unhex(col: ColumnOrName) -> Column: ... +def uniform(min: ColumnOrName | float, max: ColumnOrName | float, seed: int | Column | None = None) -> Column: ... def unix_date(col: ColumnOrName) -> Column: ... def upper(col: ColumnOrName) -> Column: ... def url_decode(str: ColumnOrName) -> Column: ... def url_encode(str: ColumnOrName) -> Column: ... def user() -> Column: ... +def uuid(seed: int | Column | None = None) -> Column: ... def validate_utf8(str: ColumnOrName) -> Column: ... def var_pop(col: ColumnOrName) -> Column: ... def var_samp(col: ColumnOrName) -> Column: ... def variance(col: ColumnOrName) -> Column: ... +def version() -> Column: ... def weekday(col: ColumnOrName) -> Column: ... def weekofyear(col: ColumnOrName) -> Column: ... def xxhash64(*cols: ColumnOrName) -> Column: ... diff --git a/python/tests/_pyspark_signatures.py b/python/tests/_pyspark_signatures.py index 6aa6ee29e..eae19fa61 100644 --- a/python/tests/_pyspark_signatures.py +++ b/python/tests/_pyspark_signatures.py @@ -58,7 +58,10 @@ "csc": "col", "cume_dist": "", "curdate": "", + "current_catalog": "", + "current_database": "", "current_date": "", + "current_schema": "", "current_timestamp": "", "current_user": "", "date_add": "start, days", @@ -100,6 +103,9 @@ "hypot": "col1, col2", "ifnull": "col1, col2", "initcap": "col", + "input_file_block_length": "", + "input_file_block_start": "", + "input_file_name": "", "instr": "str, substr", "is_valid_utf8": "str", "isnan": "col", @@ -135,6 +141,7 @@ "mean": "col", "min": "col", "minute": "col", + "monotonically_increasing_id": "", "month": "col", "monthname": "col", "months_between": "date1, date2, roundOff=True", @@ -162,6 +169,9 @@ "quote": "col", "radians": "col", "raise_error": "errMsg", + "rand": "seed=None", + "randn": "seed=None", + "randstr": "length, seed=None", "rank": "", "regexp": "str, regexp", "regexp_count": "str, regexp", @@ -182,6 +192,7 @@ "sec": "col", "second": "col", "session_user": "", + "spark_partition_id": "", "sha": "col", "sha1": "col", "sha2": "col, numBits", @@ -231,17 +242,21 @@ "try_to_number": "col, format", "try_url_decode": "str", "try_validate_utf8": "str", + "typeof": "col", "ucase": "str", "unbase64": "col", "unhex": "col", + "uniform": "min, max, seed=None", "unix_date": "col", "upper": "col", "url_decode": "str", "url_encode": "str", "user": "", + "uuid": "seed=None", "var_pop": "col", "var_samp": "col", "variance": "col", + "version": "", "weekday": "col", "weekofyear": "col", "validate_utf8": "str", diff --git a/tests/fixtures/analysis/registry-matrix/rand.json b/tests/fixtures/analysis/registry-matrix/rand.json new file mode 100644 index 000000000..c92cc4430 --- /dev/null +++ b/tests/fixtures/analysis/registry-matrix/rand.json @@ -0,0 +1,38 @@ +{ + "entries": [ + { + "id": "rand#114bc082913c/ansi_on", + "reason": "SELECT rand(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "rand#114bc082913c/ansi_off", + "reason": "SELECT rand(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "rand#16b39c8effd3/ansi_on", + "reason": "SELECT rand(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "rand#16b39c8effd3/ansi_off", + "reason": "SELECT rand(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "rand#2883ca45911e/ansi_on", + "reason": "SELECT rand(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "rand#2883ca45911e/ansi_off", + "reason": "SELECT rand(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "rand#a4f856765524/ansi_on", + "reason": "SELECT rand(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "rand#a4f856765524/ansi_off", + "reason": "SELECT rand(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + } + ], + "format_version": 1, + "function": "rand" +} diff --git a/tests/fixtures/analysis/registry-matrix/randn.json b/tests/fixtures/analysis/registry-matrix/randn.json new file mode 100644 index 000000000..d01eca515 --- /dev/null +++ b/tests/fixtures/analysis/registry-matrix/randn.json @@ -0,0 +1,38 @@ +{ + "entries": [ + { + "id": "randn#8df4dfcb761d/ansi_on", + "reason": "SELECT randn(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randn#8df4dfcb761d/ansi_off", + "reason": "SELECT randn(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randn#905f1cb314d6/ansi_on", + "reason": "SELECT randn(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randn#905f1cb314d6/ansi_off", + "reason": "SELECT randn(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randn#9df340e4c4ed/ansi_on", + "reason": "SELECT randn(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randn#9df340e4c4ed/ansi_off", + "reason": "SELECT randn(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randn#db9474536e6f/ansi_on", + "reason": "SELECT randn(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randn#db9474536e6f/ansi_off", + "reason": "SELECT randn(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + } + ], + "format_version": 1, + "function": "randn" +} diff --git a/tests/fixtures/analysis/registry-matrix/random.json b/tests/fixtures/analysis/registry-matrix/random.json new file mode 100644 index 000000000..83f5dfa94 --- /dev/null +++ b/tests/fixtures/analysis/registry-matrix/random.json @@ -0,0 +1,38 @@ +{ + "entries": [ + { + "id": "random#6a872774b235/ansi_on", + "reason": "SELECT random(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "random#6a872774b235/ansi_off", + "reason": "SELECT random(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "random#73bb8aca50b7/ansi_on", + "reason": "SELECT random(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "random#73bb8aca50b7/ansi_off", + "reason": "SELECT random(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "random#aafd575271fd/ansi_on", + "reason": "SELECT random(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "random#aafd575271fd/ansi_off", + "reason": "SELECT random(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "random#b218d468723d/ansi_on", + "reason": "SELECT random(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "random#b218d468723d/ansi_off", + "reason": "SELECT random(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + } + ], + "format_version": 1, + "function": "random" +} diff --git a/tests/fixtures/analysis/registry-matrix/randstr.json b/tests/fixtures/analysis/registry-matrix/randstr.json new file mode 100644 index 000000000..c38e1ca9d --- /dev/null +++ b/tests/fixtures/analysis/registry-matrix/randstr.json @@ -0,0 +1,70 @@ +{ + "entries": [ + { + "id": "randstr#175c23e922dc/ansi_on", + "reason": "SELECT randstr(3, INTERVAL '1-2' YEAR TO MONTH) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#175c23e922dc/ansi_off", + "reason": "SELECT randstr(3, INTERVAL '1-2' YEAR TO MONTH) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#55df7b2e1334/ansi_on", + "reason": "SELECT randstr(3, INTERVAL '1 02:03:04' DAY TO SECOND) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#55df7b2e1334/ansi_off", + "reason": "SELECT randstr(3, INTERVAL '1 02:03:04' DAY TO SECOND) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#72a2793557e6/ansi_on", + "reason": "SELECT randstr(array(1, 2), 0) AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#72a2793557e6/ansi_off", + "reason": "SELECT randstr(array(1, 2), 0) AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#78a2b0d4ee1a/ansi_on", + "reason": "SELECT randstr(3, array(1, 2)) AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#78a2b0d4ee1a/ansi_off", + "reason": "SELECT randstr(3, array(1, 2)) AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#9e4c25337d0d/ansi_on", + "reason": "SELECT randstr(map('a', 1), 0) AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#9e4c25337d0d/ansi_off", + "reason": "SELECT randstr(map('a', 1), 0) AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#a0e222978047/ansi_on", + "reason": "SELECT randstr(INTERVAL '1 02:03:04' DAY TO SECOND, 0) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#a0e222978047/ansi_off", + "reason": "SELECT randstr(INTERVAL '1 02:03:04' DAY TO SECOND, 0) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#d06ac69e5e3a/ansi_on", + "reason": "SELECT randstr(3, map('a', 1)) AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#d06ac69e5e3a/ansi_off", + "reason": "SELECT randstr(3, map('a', 1)) AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#e2c6f88c4702/ansi_on", + "reason": "SELECT randstr(INTERVAL '1-2' YEAR TO MONTH, 0) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "randstr#e2c6f88c4702/ansi_off", + "reason": "SELECT randstr(INTERVAL '1-2' YEAR TO MONTH, 0) AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + } + ], + "format_version": 1, + "function": "randstr" +} diff --git a/tests/fixtures/analysis/registry-matrix/typeof.json b/tests/fixtures/analysis/registry-matrix/typeof.json new file mode 100644 index 000000000..57307746d --- /dev/null +++ b/tests/fixtures/analysis/registry-matrix/typeof.json @@ -0,0 +1,62 @@ +{ + "entries": [ + { + "id": "typeof#737efe3cca2b/ansi_on", + "reason": "SELECT typeof(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map" + }, + { + "id": "typeof#737efe3cca2b/ansi_off", + "reason": "SELECT typeof(map('a', 1)) | KRISHIV_UNSUPPORTED_FEATURE: function map" + }, + { + "id": "typeof#85119c3e6111/ansi_on", + "reason": "SELECT typeof(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals" + }, + { + "id": "typeof#85119c3e6111/ansi_off", + "reason": "SELECT typeof(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals" + }, + { + "id": "typeof#86964662cdab/ansi_on", + "reason": "SELECT typeof(array(1)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#86964662cdab/ansi_off", + "reason": "SELECT typeof(array(1)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#8b64929c858b/ansi_on", + "reason": "SELECT typeof(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#8b64929c858b/ansi_off", + "reason": "SELECT typeof(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#94be9d2b4025/ansi_on", + "reason": "SELECT typeof(CAST(array() AS array)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#94be9d2b4025/ansi_off", + "reason": "SELECT typeof(CAST(array() AS array)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#bea7c4b2a1ca/ansi_on", + "reason": "SELECT typeof(CAST(array(NULL) AS array)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#bea7c4b2a1ca/ansi_off", + "reason": "SELECT typeof(CAST(array(NULL) AS array)) | KRISHIV_UNSUPPORTED_FEATURE: function array" + }, + { + "id": "typeof#ef1847dcc457/ansi_on", + "reason": "SELECT typeof(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals" + }, + { + "id": "typeof#ef1847dcc457/ansi_off", + "reason": "SELECT typeof(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals" + } + ], + "format_version": 1, + "function": "typeof" +} diff --git a/tests/fixtures/analysis/registry-matrix/uniform.json b/tests/fixtures/analysis/registry-matrix/uniform.json new file mode 100644 index 000000000..05efb925d --- /dev/null +++ b/tests/fixtures/analysis/registry-matrix/uniform.json @@ -0,0 +1,166 @@ +{ + "entries": [ + { + "id": "uniform#183cfc5f631a/ansi_on", + "reason": "SELECT uniform(NULL, 20, 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#183cfc5f631a/ansi_off", + "reason": "SELECT uniform(NULL, 20, 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#1e99715e4fd3/ansi_on", + "reason": "SELECT uniform(10, INTERVAL '1 02:03:04' DAY TO SECOND, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#1e99715e4fd3/ansi_off", + "reason": "SELECT uniform(10, INTERVAL '1 02:03:04' DAY TO SECOND, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#30c2e2b398e9/ansi_on", + "reason": "SELECT uniform(array(1, 2), 20, 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#30c2e2b398e9/ansi_off", + "reason": "SELECT uniform(array(1, 2), 20, 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#310b01a5b4ae/ansi_on", + "reason": "SELECT uniform(10, INTERVAL '1-2' YEAR TO MONTH, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#310b01a5b4ae/ansi_off", + "reason": "SELECT uniform(10, INTERVAL '1-2' YEAR TO MONTH, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#3fd714d3fbcf/ansi_on", + "reason": "SELECT uniform(10, map('a', 1), 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#3fd714d3fbcf/ansi_off", + "reason": "SELECT uniform(10, map('a', 1), 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#454025458e8a/ansi_on", + "reason": "SELECT uniform(10, 20, NULL) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#454025458e8a/ansi_off", + "reason": "SELECT uniform(10, 20, NULL) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#5656d353ced6/ansi_on", + "reason": "SELECT uniform(INTERVAL '1-2' YEAR TO MONTH, 20, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#5656d353ced6/ansi_off", + "reason": "SELECT uniform(INTERVAL '1-2' YEAR TO MONTH, 20, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#5ff5470f6623/ansi_on", + "reason": "SELECT uniform(10, 20, array(1, 2)) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#5ff5470f6623/ansi_off", + "reason": "SELECT uniform(10, 20, array(1, 2)) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#681f70cb51f1/ansi_on", + "reason": "SELECT uniform(map('a', 1), 20, 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#681f70cb51f1/ansi_off", + "reason": "SELECT uniform(map('a', 1), 20, 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#6afd046a5774/ansi_on", + "reason": "SELECT uniform(10, 20, INTERVAL '1-2' YEAR TO MONTH) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#6afd046a5774/ansi_off", + "reason": "SELECT uniform(10, 20, INTERVAL '1-2' YEAR TO MONTH) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#96fbf5c2c8f2/ansi_on", + "reason": "SELECT uniform(10, CAST(1.5 AS DECIMAL(38,18)), 0) > 0 AS result | boolean not null where Spark has boolean" + }, + { + "id": "uniform#96fbf5c2c8f2/ansi_off", + "reason": "SELECT uniform(10, CAST(1.5 AS DECIMAL(38,18)), 0) > 0 AS result | boolean not null where Spark has boolean" + }, + { + "id": "uniform#a51977da5bbb/ansi_on", + "reason": "SELECT uniform(10, 20, CAST(NULL AS int)) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#a51977da5bbb/ansi_off", + "reason": "SELECT uniform(10, 20, CAST(NULL AS int)) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#a9c82a62607d/ansi_on", + "reason": "SELECT uniform(10, CAST(NULL AS int), 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#a9c82a62607d/ansi_off", + "reason": "SELECT uniform(10, CAST(NULL AS int), 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#ab6b99e0333b/ansi_on", + "reason": "SELECT uniform(10, NULL, 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#ab6b99e0333b/ansi_off", + "reason": "SELECT uniform(10, NULL, 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#ac2a6eb487eb/ansi_on", + "reason": "SELECT uniform(10, array(1, 2), 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#ac2a6eb487eb/ansi_off", + "reason": "SELECT uniform(10, array(1, 2), 0) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#d54a4a610339/ansi_on", + "reason": "SELECT uniform(10, 20, map('a', 1)) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#d54a4a610339/ansi_off", + "reason": "SELECT uniform(10, 20, map('a', 1)) > 0 AS result | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#d942efa11da8/ansi_on", + "reason": "SELECT uniform(CAST(1.5 AS DECIMAL(38,18)), 20, 0) > 0 AS result | boolean not null where Spark has boolean" + }, + { + "id": "uniform#d942efa11da8/ansi_off", + "reason": "SELECT uniform(CAST(1.5 AS DECIMAL(38,18)), 20, 0) > 0 AS result | boolean not null where Spark has boolean" + }, + { + "id": "uniform#e6040ed43075/ansi_on", + "reason": "SELECT uniform(INTERVAL '1 02:03:04' DAY TO SECOND, 20, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#e6040ed43075/ansi_off", + "reason": "SELECT uniform(INTERVAL '1 02:03:04' DAY TO SECOND, 20, 0) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#f476aa44bc4b/ansi_on", + "reason": "SELECT uniform(CAST(NULL AS int), 20, 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#f476aa44bc4b/ansi_off", + "reason": "SELECT uniform(CAST(NULL AS int), 20, 0) > 0 AS result | boolean where Spark has boolean not null" + }, + { + "id": "uniform#fa2e75b8ab98/ansi_on", + "reason": "SELECT uniform(10, 20, INTERVAL '1 02:03:04' DAY TO SECOND) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "uniform#fa2e75b8ab98/ansi_off", + "reason": "SELECT uniform(10, 20, INTERVAL '1 02:03:04' DAY TO SECOND) > 0 AS result | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + } + ], + "format_version": 1, + "function": "uniform" +} diff --git a/tests/spark-golden/known-failures-cases/cte.json b/tests/spark-golden/known-failures-cases/cte.json index 7066cd644..4a84cce33 100644 --- a/tests/spark-golden/known-failures-cases/cte.json +++ b/tests/spark-golden/known-failures-cases/cte.json @@ -13,9 +13,6 @@ 47, 48 ], - "fn-misc": [ - 22 - ], "query-features": [ 17 ], diff --git a/tests/spark-golden/known-failures-cases/datetime-legacy.json b/tests/spark-golden/known-failures-cases/datetime-legacy.json index fd71080b3..be61ea048 100644 --- a/tests/spark-golden/known-failures-cases/datetime-legacy.json +++ b/tests/spark-golden/known-failures-cases/datetime-legacy.json @@ -166,7 +166,6 @@ "depends-on-command": [ 68, 69, - 163, 165, 166, 211, diff --git a/tests/spark-golden/known-failures-cases/group-by-ordinal.json b/tests/spark-golden/known-failures-cases/group-by-ordinal.json deleted file mode 100644 index e72e86e62..000000000 --- a/tests/spark-golden/known-failures-cases/group-by-ordinal.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cases": { - "depends-on-command": [ - 13 - ] - }, - "file": "group-by-ordinal", - "format_version": 1 -} diff --git a/tests/spark-golden/known-failures-cases/group-by.json b/tests/spark-golden/known-failures-cases/group-by.json index 4655b3f0e..b8c3d280e 100644 --- a/tests/spark-golden/known-failures-cases/group-by.json +++ b/tests/spark-golden/known-failures-cases/group-by.json @@ -4,7 +4,6 @@ 9, 10, 15, - 61, 96, 97, 98, diff --git a/tests/spark-golden/known-failures-cases/misc-functions.json b/tests/spark-golden/known-failures-cases/misc-functions.json index 894713271..4d1d1cf00 100644 --- a/tests/spark-golden/known-failures-cases/misc-functions.json +++ b/tests/spark-golden/known-failures-cases/misc-functions.json @@ -4,11 +4,6 @@ 4 ], "fn-misc": [ - 0, - 1, - 2, - 3, - 5, 6, 7, 9, diff --git a/tests/spark-golden/known-failures-cases/nonansi/string-functions.json b/tests/spark-golden/known-failures-cases/nonansi/string-functions.json index acaf9e556..b33c316fe 100644 --- a/tests/spark-golden/known-failures-cases/nonansi/string-functions.json +++ b/tests/spark-golden/known-failures-cases/nonansi/string-functions.json @@ -1,8 +1,5 @@ { "cases": { - "fn-misc": [ - 5 - ], "fn-regex": [ 13, 14, diff --git a/tests/spark-golden/known-failures-cases/nonansi/timestamp.json b/tests/spark-golden/known-failures-cases/nonansi/timestamp.json index 585ca2a96..c3c78b768 100644 --- a/tests/spark-golden/known-failures-cases/nonansi/timestamp.json +++ b/tests/spark-golden/known-failures-cases/nonansi/timestamp.json @@ -115,7 +115,6 @@ 138 ], "depends-on-command": [ - 61, 63, 64, 109, diff --git a/tests/spark-golden/known-failures-cases/order-by.json b/tests/spark-golden/known-failures-cases/order-by.json index 85effa546..68720aee3 100644 --- a/tests/spark-golden/known-failures-cases/order-by.json +++ b/tests/spark-golden/known-failures-cases/order-by.json @@ -13,9 +13,6 @@ 21, 27 ], - "query-features": [ - 12 - ], "variant-type": [ 14 ] diff --git a/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence-legacy.json b/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence-legacy.json index 627be2b28..bbb3f78e3 100644 --- a/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence-legacy.json +++ b/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence-legacy.json @@ -3,10 +3,6 @@ "depends-on-command": [ 13 ], - "fn-misc": [ - 9, - 10 - ], "sql-commands": [ 6, 8 diff --git a/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence.json b/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence.json index c7a664118..fc0125935 100644 --- a/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence.json +++ b/tests/spark-golden/known-failures-cases/parameterless-function-name-precedence.json @@ -3,10 +3,6 @@ "depends-on-command": [ 13 ], - "fn-misc": [ - 9, - 10 - ], "sql-commands": [ 6, 8 diff --git a/tests/spark-golden/known-failures-cases/postgreSQL/case.json b/tests/spark-golden/known-failures-cases/postgreSQL/case.json index f820c93a2..c0fd0a057 100644 --- a/tests/spark-golden/known-failures-cases/postgreSQL/case.json +++ b/tests/spark-golden/known-failures-cases/postgreSQL/case.json @@ -13,9 +13,6 @@ 30, 31 ], - "fn-string": [ - 17 - ], "sql-commands": [ 0, 1, diff --git a/tests/spark-golden/known-failures-cases/postgreSQL/limit.json b/tests/spark-golden/known-failures-cases/postgreSQL/limit.json index 8c8e6b80e..3888e8869 100644 --- a/tests/spark-golden/known-failures-cases/postgreSQL/limit.json +++ b/tests/spark-golden/known-failures-cases/postgreSQL/limit.json @@ -3,9 +3,6 @@ "depends-on-command": [ 9, 10 - ], - "fn-string": [ - 12 ] }, "file": "postgreSQL/limit", diff --git a/tests/spark-golden/known-failures-cases/predicate-functions.json b/tests/spark-golden/known-failures-cases/predicate-functions.json index 9a3540b72..e8e545cd7 100644 --- a/tests/spark-golden/known-failures-cases/predicate-functions.json +++ b/tests/spark-golden/known-failures-cases/predicate-functions.json @@ -16,10 +16,6 @@ 57, 80, 86 - ], - "fn-string": [ - 81, - 87 ] }, "file": "predicate-functions", diff --git a/tests/spark-golden/known-failures-cases/random.json b/tests/spark-golden/known-failures-cases/random.json index 77ce2531c..e304bac15 100644 --- a/tests/spark-golden/known-failures-cases/random.json +++ b/tests/spark-golden/known-failures-cases/random.json @@ -1,82 +1,8 @@ { "cases": { - "fn-string": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 43, - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 58, - 59, - 61, - 62, - 63, - 64, - 65, - 66, - 67, - 70, - 71 - ], "query-features": [ 25, - 41, - 42, - 60, - 68, - 69, - 72, - 73 + 60 ] }, "file": "random", diff --git a/tests/spark-golden/known-failures-cases/sql-session-variables.json b/tests/spark-golden/known-failures-cases/sql-session-variables.json index 3fe5e6aad..348877652 100644 --- a/tests/spark-golden/known-failures-cases/sql-session-variables.json +++ b/tests/spark-golden/known-failures-cases/sql-session-variables.json @@ -6,11 +6,7 @@ 15, 60, 63, - 83, - 85, - 87, 89, - 93, 144, 148, 197, @@ -27,15 +23,8 @@ "sql-commands": [ 62, 64, - 80, 88, - 92, 135, - 142, - 146, - 152, - 153, - 154, 196, 202, 205, diff --git a/tests/spark-golden/known-failures-cases/string-functions.json b/tests/spark-golden/known-failures-cases/string-functions.json index 83b4c0f36..73ece90f4 100644 --- a/tests/spark-golden/known-failures-cases/string-functions.json +++ b/tests/spark-golden/known-failures-cases/string-functions.json @@ -1,8 +1,5 @@ { "cases": { - "fn-misc": [ - 5 - ], "fn-regex": [ 13, 14, diff --git a/tests/spark-golden/known-failures-cases/timestamp.json b/tests/spark-golden/known-failures-cases/timestamp.json index ce78a9a38..dbe1e4164 100644 --- a/tests/spark-golden/known-failures-cases/timestamp.json +++ b/tests/spark-golden/known-failures-cases/timestamp.json @@ -115,7 +115,6 @@ 138 ], "depends-on-command": [ - 61, 63, 64, 109, diff --git a/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp-ansi.json b/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp-ansi.json index 18c668c57..29f8e44f0 100644 --- a/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp-ansi.json +++ b/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp-ansi.json @@ -114,7 +114,6 @@ 138 ], "depends-on-command": [ - 61, 63, 64, 109, diff --git a/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp.json b/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp.json index b3e2cfa9a..5da956bf9 100644 --- a/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp.json +++ b/tests/spark-golden/known-failures-cases/timestampNTZ/timestamp.json @@ -114,7 +114,6 @@ 138 ], "depends-on-command": [ - 61, 63, 64, 109,