diff --git a/crates/krishiv-exec/src/aggregate/accumulator.rs b/crates/krishiv-exec/src/aggregate/accumulator.rs index 958b2ac25..332792e87 100644 --- a/crates/krishiv-exec/src/aggregate/accumulator.rs +++ b/crates/krishiv-exec/src/aggregate/accumulator.rs @@ -112,6 +112,92 @@ impl Values { Self::Bytes(x) => v(x), } } + + /// Whether row `i` of `src` orders after (`max`) or before group `g`'s + /// value, strictly: a tie keeps the value held. + fn beats(&self, g: usize, src: &Col<'_>, i: usize, max: bool) -> Result { + Ok(match (self, src) { + (Self::Int(v), Col::Int(c)) => c[i] != v[g] && (c[i] > v[g]) == max, + (Self::Float(v), Col::Float(c)) => { + if max { + float_lt(v[g], c[i]) + } else { + float_lt(c[i], v[g]) + } + } + (Self::Bytes(v), Col::Bytes(c)) => { + let cur: &[u8] = &v[g]; + if max { c[i] > cur } else { c[i] < cur } + } + _ => { + return Err(internal( + "an ordering key of another kind than its state".to_owned(), + )); + } + }) + } + + /// Stores row `i` of `src` as group `g`'s value, keeping `heap` (the + /// bytes of `Bytes` values) current. + fn put(&mut self, g: usize, src: &Col<'_>, i: usize, heap: &mut usize) -> Result<()> { + match (self, src) { + (Self::Int(v), Col::Int(c)) => v[g] = c[i], + (Self::Float(v), Col::Float(c)) => v[g] = c[i], + (Self::Bytes(v), Col::Bytes(c)) => { + *heap = heap.saturating_sub(v[g].len()) + c[i].len(); + v[g] = c[i].into(); + } + _ => { + return Err(internal( + "a value of another kind than its state".to_owned(), + )); + } + } + Ok(()) + } + + /// Clears group `g`'s value (a NULL win), keeping `heap` current. + fn clear(&mut self, g: usize, heap: &mut usize) { + match self { + Self::Int(v) => v[g] = 0, + Self::Float(v) => v[g] = 0.0, + Self::Bytes(v) => { + *heap = heap.saturating_sub(v[g].len()); + v[g] = Box::default(); + } + } + } + + /// Copies group `from`'s value to group `to` (a running frame's + /// snapshot). Byte values are never snapshotted: the copies would not + /// be accounted for. + fn copy_slot(&mut self, from: usize, to: usize) -> Result<()> { + match self { + Self::Int(v) => v[to] = v[from], + Self::Float(v) => v[to] = v[from], + Self::Bytes(_) => return Err(internal("a snapshot of byte values".to_owned())), + } + Ok(()) + } +} + +/// A column read as one physical kind (the kinds of [`Values`]). +enum Col<'a> { + Int(Cow<'a, [i128]>), + Float(Cow<'a, [f64]>), + Bytes(Vec<&'a [u8]>), +} + +impl<'a> Col<'a> { + fn of(a: &'a ArrayRef) -> Result { + Ok(match a.data_type() { + DataType::Float32 | DataType::Float64 => Self::Float(floats(a)?), + DataType::Utf8 | DataType::Utf8View | DataType::Binary | DataType::BinaryView => { + Self::Bytes(byte_values(a)?) + } + _ => Self::Int(int_like(a)?), + }) + } } #[derive(Debug)] @@ -155,6 +241,17 @@ enum Kind { null: Vec, heap: usize, }, + /// `max_by` / `min_by`: `set` = a row with a non-NULL key was taken + /// (`keys` holds its key), `null` = its value is NULL. `heap` counts + /// the bytes of both `Bytes` vectors. + By { + max: bool, + values: Values, + keys: Values, + set: Vec, + null: Vec, + heap: usize, + }, } /// One aggregate's state for every group of a table. @@ -794,7 +891,25 @@ impl Accumulator { set: Vec::new(), heap: 0, }, - (f @ (F::First { ignore_nulls } | F::Last { ignore_nulls }), t) => Kind::Pick { + (F::MaxBy | F::MinBy, _) => { + let (v, k) = expr + .by_types() + .ok_or_else(|| internal("max_by without value and key types".to_owned()))?; + Kind::By { + max: expr.function == F::MaxBy, + values: Values::empty(v), + keys: Values::empty(k), + set: Vec::new(), + null: Vec::new(), + heap: 0, + } + } + ( + f @ (F::First { ignore_nulls } + | F::Last { ignore_nulls } + | F::AnyValue { ignore_nulls }), + t, + ) => Kind::Pick { last: matches!(f, F::Last { .. }), ignore_nulls, values: Values::empty(t), @@ -845,13 +960,20 @@ impl Accumulator { Kind::Pick { values, set, null, .. } => vec![values.vec(), v(set), v(null)], + Kind::By { + values, + keys, + set, + null, + .. + } => vec![values.vec(), keys.vec(), v(set), v(null)], } } /// Bytes held. pub(crate) fn size(&self) -> usize { let heap = match &self.kind { - Kind::MinMax { heap, .. } | Kind::Pick { heap, .. } => *heap, + Kind::MinMax { heap, .. } | Kind::Pick { heap, .. } | Kind::By { heap, .. } => *heap, _ => 0, }; heap + self.vecs().iter().map(|(_, c, e)| c * e).sum::() @@ -859,8 +981,14 @@ impl Accumulator { /// [`size`](Self::size) after [`resize`](Self::resize) to `new` more /// groups and an update with `input` (string values it may keep). - /// `arg` is the value column the update reads (raw or state). - pub(crate) fn projected(&self, new: usize, arg: Option<&ArrayRef>) -> usize { + /// `arg` is the value column the update reads (raw or state); `key` the + /// ordering key column of `max_by`/`min_by`. + pub(crate) fn projected( + &self, + new: usize, + arg: Option<&ArrayRef>, + key: Option<&ArrayRef>, + ) -> usize { let vecs: usize = self .vecs() .iter() @@ -885,6 +1013,15 @@ impl Accumulator { heap, .. } => heap + arg.map_or(0, value_bytes), + Kind::By { + values, keys, heap, .. + } => { + let bytes = |v: &Values, a: Option<&ArrayRef>| match v { + Values::Bytes(_) => a.map_or(0, value_bytes), + _ => 0, + }; + heap + bytes(values, arg) + bytes(keys, key) + } Kind::MinMax { heap, .. } | Kind::Pick { heap, .. } => *heap, _ => 0, }; @@ -931,6 +1068,18 @@ impl Accumulator { null.resize(n, false); values.resize(n); } + Kind::By { + values, + keys, + set, + null, + .. + } => { + set.resize(n, false); + null.resize(n, false); + values.resize(n); + keys.resize(n); + } } } @@ -974,6 +1123,18 @@ impl Accumulator { null.reserve(new); values.reserve(new); } + Kind::By { + values, + keys, + set, + null, + .. + } => { + set.reserve(new); + null.reserve(new); + values.reserve(new); + keys.reserve(new); + } } } @@ -1051,7 +1212,13 @@ impl Accumulator { args.len() ))); } - if (state || matches!(self.kind, Kind::MinMax { .. } | Kind::Pick { .. })) && !is_unit(w) { + if (state + || matches!( + self.kind, + Kind::MinMax { .. } | Kind::Pick { .. } | Kind::By { .. } + )) + && !is_unit(w) + { return Err(unsupported(format!( "{:?} over rows with weights other than 1 (needs incremental view maintenance)", self.expr.function @@ -1099,6 +1266,18 @@ impl Accumulator { self.expr.function, self.expr.arg_type ))); } + if run.is_some() + && matches!( + &self.kind, + Kind::By { values, keys, .. } + if matches!(values, Values::Bytes(_)) || matches!(keys, Values::Bytes(_)) + ) + { + return Err(unsupported(format!( + "{:?} over strings or binaries in a running window frame", + self.expr.function + ))); + } match (&mut self.kind, state) { (Kind::Count(c), false) => { for (i, &g) in groups.iter().enumerate() { @@ -1305,6 +1484,8 @@ impl Accumulator { null[g] = !valid(i); if valid(i) { vals[g] = v[i]; + } else { + vals[g] = 0; } } snapshot!(i, vals, set, null); @@ -1319,6 +1500,8 @@ impl Accumulator { null[g] = !valid(i); if valid(i) { vals[g] = v[i]; + } else { + vals[g] = 0.0; } } snapshot!(i, vals, set, null); @@ -1334,12 +1517,60 @@ impl Accumulator { if valid(i) { *heap = heap.saturating_sub(vals[g].len()) + v[i].len(); vals[g] = v[i].into(); + } else { + *heap = heap.saturating_sub(vals[g].len()); + vals[g] = Box::default(); } } } } } } + ( + Kind::By { + max, + values, + keys, + set, + null, + heap, + }, + _, + ) => { + let &[xa, ya] = args else { + return Err(internal("max_by without a value and a key".to_owned())); + }; + if xa.len() != groups.len() || ya.len() != groups.len() { + return Err(internal( + "max_by columns of other lengths than the group ids".to_owned(), + )); + } + let (x, y) = (Col::of(xa)?, Col::of(ya)?); + let key_valid = |i: usize| nulls[1].as_ref().is_none_or(|n| n.is_valid(i)); + for (i, &g) in groups.iter().enumerate() { + let g = g as usize; + if key_valid(i) && (!set[g] || keys.beats(g, &y, i, *max)?) { + keys.put(g, &y, i, heap)?; + if valid(i) { + values.put(g, &x, i, heap)?; + } else { + values.clear(g, heap); + } + set[g] = true; + null[g] = !valid(i); + } + if let Some(r) = run { + let to = r.snapshots[i]; + if to != Running::NONE { + let (to, from) = (to as usize, r.parts[i] as usize); + values.copy_slot(from, to)?; + keys.copy_slot(from, to)?; + set[to] = set[from]; + null[to] = null[from]; + } + } + } + } } if let Kind::SumDec { sum, .. } | Kind::AvgDec { sum, .. } = &self.kind { self.max_sum = refresh_max(self.max_sum, sum, groups); @@ -1359,7 +1590,8 @@ impl Accumulator { Kind::SumDec { .. } | Kind::AvgFloat { .. } | Kind::AvgDec { .. } - | Kind::Pick { .. } => 2, + | Kind::Pick { .. } + | Kind::By { .. } => 2, } } @@ -1409,16 +1641,32 @@ impl Accumulator { )), ], Kind::MinMax { values, set, .. } => { - vec![self.value_array(values, |i| set[i], ids)?] + vec![self.value_array(values, &self.result, |i| set[i], ids)?] } Kind::Pick { values, set, null, .. } => vec![ - self.value_array(values, |i| set[i] && !null[i], ids)?, + self.value_array(values, &self.result, |i| set[i] && !null[i], ids)?, Arc::new(BooleanArray::from_iter( ids.iter().map(|i| Some(set[pick(i)])), )), ], + Kind::By { + values, + keys, + set, + null, + .. + } => { + let (_, key_type) = self + .expr + .by_types() + .ok_or_else(|| internal("max_by without value and key types".to_owned()))?; + vec![ + self.value_array(values, &self.result, |i| set[i] && !null[i], ids)?, + self.value_array(keys, key_type, |i| set[i], ids)?, + ] + } }) } @@ -1432,17 +1680,17 @@ impl Accumulator { } } - /// `values` of groups `ids` as the result type; `on(group)` says - /// whether the group has a value (else NULL). + /// `values` of groups `ids` as type `t`; `on(group)` says whether the + /// group has a value (else NULL). fn value_array( &self, values: &Values, + t: &DataType, on: impl Fn(usize) -> bool, ids: &[u32], ) -> Result { let on = |i: &u32| on(*i as usize); let at = |i: &u32| *i as usize; - let t = &self.result; Ok(match values { Values::Float(v) => { let it = ids.iter().map(|i| on(i).then(|| v[at(i)])); @@ -1548,6 +1796,10 @@ impl Accumulator { } self.decimal(out.into_iter(), &self.result)? } + // Only the value: the ordering key is state, not a result. + Kind::By { + values, set, null, .. + } => self.value_array(values, &self.result, |i| set[i] && !null[i], ids)?, Kind::Count(_) | Kind::SumInt { .. } | Kind::SumFloat { .. } diff --git a/crates/krishiv-exec/src/aggregate/accumulator/differential.rs b/crates/krishiv-exec/src/aggregate/accumulator/differential.rs index 5f9ecfb9f..0a4265153 100644 --- a/crates/krishiv-exec/src/aggregate/accumulator/differential.rs +++ b/crates/krishiv-exec/src/aggregate/accumulator/differential.rs @@ -330,3 +330,137 @@ fn min_max_on_decimal64() { assert_eq!(v.as_primitive::().value(0), want); } } + +/// Keys of the `max_by`/`min_by` merge test: NaN (the largest), the +/// infinities, and `-0.0 = 0.0` (a tie). +const KEYS: [f64; 7] = [ + f64::NAN, + f64::NEG_INFINITY, + -1.0, + -0.0, + 0.0, + 1.0, + f64::INFINITY, +]; + +/// `(group, value, key)`: a NULL value is legal, a NULL key skips the row. +type ByRow = (u32, Option, Option); + +/// The value and key columns of `rows`, as strings (`strings`) or as +/// `bigint`/`double` (keys by their index in strings, so ties stay ties). +fn by_columns(rows: &[ByRow], strings: (bool, bool)) -> Vec { + let x: ArrayRef = if strings.0 { + Arc::new(StringViewArray::from_iter( + rows.iter().map(|r| r.1.map(|v| format!("v{v:03}"))), + )) + } else { + Arc::new(Int64Array::from_iter(rows.iter().map(|r| r.1))) + }; + let y: ArrayRef = if strings.1 { + Arc::new(StringViewArray::from_iter( + rows.iter().map(|r| r.2.map(|k| format!("k{k}"))), + )) + } else { + Arc::new(Float64Array::from_iter( + rows.iter().map(|r| r.2.map(|k| KEYS[usize::from(k)])), + )) + }; + vec![x, y] +} + +fn by_expr(f: AggregateFunction, strings: (bool, bool)) -> AggregateExpr { + let x = if strings.0 { + DataType::Utf8View + } else { + DataType::Int64 + }; + let y = if strings.1 { + DataType::Utf8View + } else { + DataType::Float64 + }; + match f { + AggregateFunction::MaxBy | AggregateFunction::MinBy => { + AggregateExpr::by(f, (0, 1), (x, y), "b") + } + _ => AggregateExpr::new(f, 0, x, "a"), + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: 256, + failure_persistence: None, + ..ProptestConfig::default() + })] + + /// `max_by`, `min_by` and `any_value`: the input cut anywhere into + /// consecutive parts, each part's partial states merged in order (also + /// through a second level), give the single pass's states and result, + /// ties and NULLs included. + #[test] + fn by_merge_differential( + rows in proptest::collection::vec( + (0..4_u32, proptest::option::of(0..40_i64), proptest::option::of(0..7_u8)), + 0..60, + ), + cuts in proptest::collection::vec(0..61_usize, 0..4), + strings in (any::(), any::()), + ) { + let groups = 4_u32; + let ids: Vec = (0..groups).collect(); + let mut cuts: Vec = cuts.into_iter().map(|c| c.min(rows.len())).collect(); + cuts.push(0); + cuts.push(rows.len()); + cuts.sort_unstable(); + let fold = |range: &[ByRow]| { + (range.iter().map(|r| r.0).collect::>(), by_columns(range, strings)) + }; + for f in [ + AggregateFunction::MaxBy, + AggregateFunction::MinBy, + AggregateFunction::AnyValue { ignore_nulls: false }, + AggregateFunction::AnyValue { ignore_nulls: true }, + ] { + let expr = by_expr(f, strings); + let new = || { + let mut a = Accumulator::new(&expr, true).unwrap(); + a.resize(groups as usize); + a + }; + let mut single = new(); + let (g, cols) = fold(&rows); + single.update(&g, &cols, &Weights::Const(1)).unwrap(); + let partials: Vec> = cuts + .windows(2) + .map(|w| { + let mut a = new(); + let (g, cols) = fold(&rows[w[0]..w[1]]); + a.update(&g, &cols, &Weights::Const(1)).unwrap(); + a.state(&ids).unwrap() + }) + .collect(); + let mut flat = new(); + for st in &partials { + flat.merge(&ids, st).unwrap(); + } + // Two levels: the halves merged apart, then together. + let half = partials.len() / 2; + let mut top = new(); + for part in [&partials[..half], &partials[half..]] { + let mut mid = new(); + for st in part { + mid.merge(&ids, st).unwrap(); + } + top.merge(&ids, &mid.state(&ids).unwrap()).unwrap(); + } + let want = (single.state(&ids).unwrap(), single.evaluate(&ids).unwrap()); + for merged in [&flat, &top] { + prop_assert_eq!( + &(merged.state(&ids).unwrap(), merged.evaluate(&ids).unwrap()), + &want + ); + } + } + } +} diff --git a/crates/krishiv-exec/src/aggregate/mod.rs b/crates/krishiv-exec/src/aggregate/mod.rs index ffc87e155..869539766 100644 --- a/crates/krishiv-exec/src/aggregate/mod.rs +++ b/crates/krishiv-exec/src/aggregate/mod.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 -//! Hash aggregation (ADR 0011): `count`, `sum`, `avg`, `min`, `max` with -//! Spark 4.2 result types, in two phases, with spill and `DISTINCT`. +//! Hash aggregation (ADR 0011): `count`, `sum`, `avg`, `min`, `max`, +//! `first`, `last`, `any_value`, `max_by`, `min_by` with Spark 4.2 result +//! types, in two phases, with spill and `DISTINCT`. `count_if` and the +//! boolean aggregates are `count` and `min`/`max` over a rewritten argument +//! (the physical planner). //! //! # Operators //! - [`HashAggregate`] is a pipeline breaker. Each driver aggregates into @@ -49,9 +52,10 @@ //! # Weights //! Rows carry Z-set weights ([`Weights`](krishiv_types::batch::Weights)): //! `count` adds the weight, `sum` adds `value × weight`, `avg` divides the -//! weighted sum by the weighted count. `min`/`max` need retractions to be -//! maintained incrementally (P4 IVM), so they and `DISTINCT` reject any weight -//! other than 1 with `KRISHIV_UNSUPPORTED_FEATURE`. Output rows have weight 1; +//! weighted sum by the weighted count. `min`/`max`, `first`, `last`, +//! `any_value`, `max_by` and `min_by` need retractions to be maintained +//! incrementally (P4 IVM), so they and `DISTINCT` reject any weight other +//! than 1 with `KRISHIV_UNSUPPORTED_FEATURE`. Output rows have weight 1; //! a group exists once it received a row. mod accumulator; @@ -66,7 +70,7 @@ mod tests; use std::sync::Arc; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef}; use krishiv_common::error::{ ArithmeticOverflow, KrishivError, KrishivUnsupportedFeature, NumericValueOutOfRangeWithSuggestion, @@ -112,6 +116,23 @@ pub enum AggregateFunction { /// `last(x, true)` / `IGNORE NULLS`: the last non-NULL value. ignore_nulls: bool, }, + /// `any_value(x)`: some row's value of the group; here the first row's, + /// as `first` (a legal choice); NULL for no value. The input type. + /// In-order merges keep the first row in input order; a `DISTINCT` + /// first phase groups in hash order. + AnyValue { + /// `any_value(x, true)` / `IGNORE NULLS`: the first non-NULL value. + ignore_nulls: bool, + }, + /// `max_by(x, y)`: `x` of the row with the largest non-NULL `y` (Spark + /// ordering), the first such row on a tie for in-order merges; rows + /// with a NULL `y` are ignored and a NULL `x` is a result. NULL for no + /// row. The type of `x`. Built with [`AggregateExpr::by`]. A + /// `DISTINCT` first phase groups in hash order, so a tie there keeps + /// the first row in hash order, not in input order. + MaxBy, + /// `min_by(x, y)`, as `max_by` for the smallest `y` (same tie caveat). + MinBy, } /// Where an aggregate's arguments come from. @@ -142,7 +163,9 @@ pub struct AggregateExpr { /// Where the arguments come from. pub input: AggInput, /// The type of the raw argument (`count` ignores it). Fixed by the - /// planner; state columns derive from it. + /// planner; state columns derive from it. For `max_by`/`min_by`, a + /// struct of the value's type then the ordering key's (see + /// [`AggregateExpr::by`]). pub arg_type: DataType, /// The output column name. pub name: String, @@ -165,6 +188,27 @@ impl AggregateExpr { } } + /// `max_by(x, y)` / `min_by(x, y)` of columns `value` and `key`, of + /// types `value_type` and `key_type`. Both must be types `min`/`max` + /// take. + pub fn by( + function: AggregateFunction, + (value, key): (usize, usize), + (value_type, key_type): (DataType, DataType), + name: impl Into, + ) -> Self { + Self { + function, + args: vec![value, key], + input: AggInput::Raw, + arg_type: DataType::Struct(Fields::from(vec![ + Field::new("value", value_type, true), + Field::new("key", key_type, true), + ])), + name: name.into(), + } + } + /// `count(*)`. pub fn count_star(name: impl Into) -> Self { Self { @@ -207,11 +251,24 @@ impl AggregateExpr { (F::Avg, DataType::Decimal64(p, s) | DataType::Decimal128(p, s)) if *s >= 0 => { decimal_on(t, (*p + 4).min(38), s.saturating_add(4).min(38)) } - (F::Min | F::Max | F::First { .. } | F::Last { .. }, t) + (F::Min | F::Max | F::First { .. } | F::Last { .. } | F::AnyValue { .. }, t) if accumulator::min_max_supported(t) => { t.clone() } + (F::MaxBy | F::MinBy, _) => match self.by_types() { + Some((v, k)) + if accumulator::min_max_supported(v) && accumulator::min_max_supported(k) => + { + v.clone() + } + _ => { + return Err(unsupported(format!( + "{:?} of {}", + self.function, self.arg_type + ))); + } + }, (f, t) => return Err(unsupported(format!("{f:?} of {t}"))), }) } @@ -237,13 +294,30 @@ impl AggregateExpr { ], (F::Min | F::Max, _) => vec![Field::new(n("value"), out, true)], // Whether a row was taken: a NULL value is a value. - (F::First { .. } | F::Last { .. }, _) => vec![ + (F::First { .. } | F::Last { .. } | F::AnyValue { .. }, _) => vec![ Field::new(n("value"), out, true), Field::new(n("valueSet"), DataType::Boolean, false), ], + // The key is NULL until a row with a key was taken: it says + // whether the value (which may be NULL) counts. + (F::MaxBy | F::MinBy, _) => { + let key = self.by_types().map_or(DataType::Null, |(_, k)| k.clone()); + vec![ + Field::new(n("value"), out, true), + Field::new(n("key"), key, true), + ] + } }) } + /// The value and ordering key types of a `max_by`/`min_by`. + fn by_types(&self) -> Option<(&DataType, &DataType)> { + match &self.arg_type { + DataType::Struct(f) if f.len() == 2 => Some((f[0].data_type(), f[1].data_type())), + _ => None, + } + } + /// The type `avg` sums in: `double`, or `sum`'s decimal type. fn avg_sum_type(&self) -> DataType { match self.arg_type { diff --git a/crates/krishiv-exec/src/aggregate/rowtable.rs b/crates/krishiv-exec/src/aggregate/rowtable.rs index 26999852e..3ac57e97f 100644 --- a/crates/krishiv-exec/src/aggregate/rowtable.rs +++ b/crates/krishiv-exec/src/aggregate/rowtable.rs @@ -719,7 +719,7 @@ impl RowTable { .map(|(part, n)| { part.accs .iter() - .map(|a| a.projected(n, None)) + .map(|a| a.projected(n, None, None)) .sum::() }) .sum(); @@ -728,12 +728,16 @@ impl RowTable { .iter() .zip(&self.core.ranges) .map(|(a, r)| { - let at = match fold { - Fold::States { offset } => r.start.checked_sub(offset), - Fold::Raw => a.args().first().copied(), + // The value column, then (`max_by`/`min_by`) the key. + let at = |n: usize| { + let i = match fold { + Fold::States { offset } => r.start.checked_sub(offset).map(|i| i + n), + Fold::Raw => a.args().get(n).copied(), + }; + i.and_then(|i| cols.get(i)) }; - let col = at.and_then(|i| cols.get(i)); - a.projected(0, col).saturating_sub(a.projected(0, None)) + a.projected(0, at(0), at(1)) + .saturating_sub(a.projected(0, None, None)) }) .sum() }); diff --git a/crates/krishiv-exec/src/aggregate/table.rs b/crates/krishiv-exec/src/aggregate/table.rs index 72491087d..5f3d44c59 100644 --- a/crates/krishiv-exec/src/aggregate/table.rs +++ b/crates/krishiv-exec/src/aggregate/table.rs @@ -245,12 +245,16 @@ impl AggTable { .iter() .zip(&self.core.ranges) .map(|(a, r)| { - let at = if p.states { - Some(r.start) - } else { - a.args().first().copied() + // The value column, then (`max_by`/`min_by`) the key. + let at = |n: usize| { + let i = if p.states { + Some(r.start + n) + } else { + a.args().get(n).copied() + }; + i.and_then(|i| p.cols.get(i)) }; - a.projected(new, at.and_then(|i| p.cols.get(i))) + a.projected(new, at(0), at(1)) }) .sum::() } diff --git a/crates/krishiv-exec/src/aggregate/tests.rs b/crates/krishiv-exec/src/aggregate/tests.rs index 6ee02d4cd..00fb1ed42 100644 --- a/crates/krishiv-exec/src/aggregate/tests.rs +++ b/crates/krishiv-exec/src/aggregate/tests.rs @@ -1461,6 +1461,8 @@ async fn first_and_last_pick_rows_in_input_order() { ); } +mod simple; + #[test] fn first_and_last_types_and_states() { use AggregateFunction::{First, Last}; diff --git a/crates/krishiv-exec/src/aggregate/tests/simple.rs b/crates/krishiv-exec/src/aggregate/tests/simple.rs new file mode 100644 index 000000000..e08ef1c4a --- /dev/null +++ b/crates/krishiv-exec/src/aggregate/tests/simple.rs @@ -0,0 +1,738 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `any_value`, `max_by` and `min_by`: semantics (Spark 4.2: the first row +//! on a tie, NULL keys ignored, a NULL value a legal result, NaN largest, +//! `-0.0 = 0.0`), merges of partial states, the row-layout tables, spills, +//! memory accounting and the types they take. Every test that runs a plan +//! through `exec` also runs it with the row-layout tables. + +use arrow_array::types::TimestampMicrosecondType; +use arrow_array::{ + BooleanArray, Decimal128Array, Float32Array, Float64Array, Int16Array, PrimitiveArray, +}; + +use super::*; +use AggregateFunction::{AnyValue, MaxBy, MinBy}; + +fn strs(v: &[Option<&str>]) -> ArrayRef { + Arc::new(StringViewArray::from(v.to_vec())) +} + +fn f64s(v: &[Option]) -> ArrayRef { + Arc::new(Float64Array::from(v.to_vec())) +} + +/// `by(x, y)` over columns 1 and 2 of `s`. +fn by(f: AggregateFunction, s: &SchemaRef) -> AggregateExpr { + AggregateExpr::by( + f, + (1, 2), + ( + s.field(1).data_type().clone(), + s.field(2).data_type().clone(), + ), + format!("{f:?}"), + ) +} + +/// Groups of `(x, y)` rows: `1` has a tie at the largest key across the +/// two batches (`c` first) and a NULL key; `2` has only NULL keys; `3` +/// holds a NULL value with the largest key; `4` a NULL value and key; `5` +/// one row. +/// +/// `tie: false` makes `d`'s key 6, so no group has a tie (the order a +/// `DISTINCT` plan merges its groups in is not the input's). +fn by_input(tie: bool) -> (SchemaRef, Vec>) { + let s = schema(vec![ + ("k", DataType::Int64), + ("x", DataType::Utf8View), + ("y", DataType::Int64), + ]); + let k = i64s(&[1, 1, 1, 1, 2, 2, 3, 3, 4, 5].map(Some)); + let x = strs(&[ + Some("a"), + Some("b"), + Some("c"), + Some("d"), + Some("p"), + Some("q"), + None, + Some("z"), + None, + Some("m"), + ]); + let y = i64s(&[ + Some(3), + None, + Some(7), + Some(if tie { 7 } else { 6 }), + None, + None, + Some(9), + Some(1), + None, + Some(5), + ]); + // The tie `c`/`d` straddles the batches. + let cut = |a: &ArrayRef| (a.slice(0, 3), a.slice(3, 7)); + let ((k1, k2), (x1, x2), (y1, y2)) = (cut(&k), cut(&x), cut(&y)); + (s, vec![vec![k1, x1, y1], vec![k2, x2, y2]]) +} + +const BY_MAX: [&str; 5] = ["1|c", "2|NULL", "3|NULL", "4|NULL", "5|m"]; +const BY_MIN: [&str; 5] = ["1|a", "2|NULL", "3|z", "4|NULL", "5|m"]; + +fn by_aggs(s: &SchemaRef) -> Vec { + vec![by(MaxBy, s), by(MinBy, s)] +} + +/// `1|max|min` rows from the two single-aggregate expectations. +fn both(max: &[&str], min: &[&str]) -> Vec { + max.iter() + .zip(min) + .map(|(a, b)| format!("{a}|{}", b.split_once('|').map_or("", |p| p.1))) + .collect() +} + +/// Spark 4.2: `SELECT k, max_by(x, y), min_by(x, y) FROM t GROUP BY k`. +/// The first row wins a tie (also across batches); NULL keys are ignored; a +/// NULL value is a result; no non-NULL key gives NULL. +#[tokio::test] +async fn max_by_and_min_by_pick_the_extreme_key() { + let (s, b) = by_input(true); + let single = node(scan(&s, b.clone()), spec(&s, vec![0], by_aggs(&s))); + let expected = both(&BY_MAX, &BY_MIN); + assert_eq!(run_one(single).await, expected); + // Global: the largest key is 9, whose value is NULL (a NULL value is a + // legal result); the smallest is 1, of `z`. + let global = node(scan(&s, b.clone()), spec(&s, vec![], by_aggs(&s))); + assert_eq!(run_one(global).await, ["NULL|z"]); + // Over no rows, and over rows whose keys are all NULL: NULL. + let none = node( + scan(&s, vec![vec![i64s(&[]), strs(&[]), i64s(&[])]]), + spec(&s, vec![], by_aggs(&s)), + ); + assert_eq!(run_one(none).await, ["NULL|NULL"]); + let null_keys = node( + scan( + &s, + vec![vec![ + i64s(&[Some(1), Some(1)]), + strs(&[Some("a"), Some("b")]), + i64s(&[None, None]), + ]], + ), + spec(&s, vec![], by_aggs(&s)), + ); + assert_eq!(run_one(null_keys).await, ["NULL|NULL"]); +} + +/// Partial states from any split of the input merge to the single-phase +/// result: through the partial phase (batches in order) and as `DISTINCT`'s +/// first phase. +#[tokio::test] +async fn max_by_and_min_by_merge_partial_states() { + let (s, b) = by_input(true); + let expected = both(&BY_MAX, &BY_MIN); + let mut first = spec(&s, vec![0], by_aggs(&s)); + first.output = AggregateOutput::Partial; + // The partial states: value then key, per aggregate. + let names: Vec = first + .output_schema() + .unwrap() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect(); + assert_eq!( + names, + ["k", "MaxBy#value", "MaxBy#key", "MinBy#value", "MinBy#key"] + ); + let second = second_phase(&first, None); + let partial = PartialAggregate::new(first, BypassPolicy::default(), Some(7)).unwrap(); + let plan = node(PartialNode::new(scan(&s, b.clone()), partial), second); + assert_eq!(run_one(plan).await, expected); + // Bypass: one state row per input row, merged by the final phase. + let mut first = spec(&s, vec![0], by_aggs(&s)); + first.output = AggregateOutput::Partial; + let second = second_phase(&first, None); + let always = BypassPolicy { + min_rows: 1, + max_group_percent: 0, + ..BypassPolicy::default() + }; + let partial = PartialAggregate::new(first, always, Some(7)).unwrap(); + let plan = node(PartialNode::new(scan(&s, b.clone()), partial), second); + assert_eq!(run_one(plan).await, expected); + // `count(DISTINCT x)` next to them: the other aggregates' states are + // merged by the second phase (without a tie, as that order is the + // hash order of the first phase's groups). + let (s, b) = by_input(false); + let mut aggs = by_aggs(&s); + aggs.push(agg(Count, 1, &s)); + let sp = spec(&s, vec![0], aggs); + let plan = distinct_plan(scan(&s, b), sp, &[false, false, true]).unwrap(); + let want = [ + "1|c|a|4", + "2|NULL|NULL|2", + "3|NULL|z|1", + "4|NULL|NULL|0", + "5|m|m|1", + ]; + assert_eq!(run_one(plan).await, want); +} + +/// Spark's ordering of `double` keys: NaN is the largest (and equals +/// itself), `-0.0 = 0.0`, infinities order as usual; the first row wins ties. +#[tokio::test] +async fn float_keys_follow_spark_ordering() { + let s = schema(vec![ + ("k", DataType::Int64), + ("x", DataType::Int64), + ("y", DataType::Float64), + ]); + let nan = f64::NAN; + let k = i64s(&[1, 1, 2, 2, 3, 3, 4, 4, 4].map(Some)); + let x = i64s(&[1, 2, 1, 2, 1, 2, 1, 2, 3].map(Some)); + let y = f64s(&[ + Some(0.0), + Some(-0.0), + Some(nan), + Some(f64::INFINITY), + Some(nan), + Some(nan), + Some(f64::NEG_INFINITY), + None, + Some(f64::NEG_INFINITY), + ]); + let plan = node( + scan(&s, vec![vec![k, x, y]]), + spec(&s, vec![0], by_aggs(&s)), + ); + // k | max_by | min_by: a `0.0`/`-0.0` tie keeps the first row; NaN beats + // infinity; all-NaN keys tie. + assert_eq!(run_one(plan).await, ["1|1|1", "2|1|2", "3|1|1", "4|1|1"]); +} + +/// `any_value` is the first row's value, or with `ignoreNulls` the first +/// non-NULL one; through the partial phase as well. +#[tokio::test] +async fn any_value_is_the_first_value() { + let (s, b) = picks_input(); + let aggs = vec![ + agg( + AnyValue { + ignore_nulls: false, + }, + 1, + &s, + ), + agg(AnyValue { ignore_nulls: true }, 1, &s), + agg( + AnyValue { + ignore_nulls: false, + }, + 2, + &s, + ), + agg(AnyValue { ignore_nulls: true }, 2, &s), + ]; + let expected = ["1|NULL|5|a|a", "2|NULL|NULL|NULL|NULL", "3|9|9|z|z"]; + let single = node(scan(&s, b.clone()), spec(&s, vec![0], aggs.clone())); + assert_eq!(run_one(single).await, expected); + let mut first = spec(&s, vec![0], aggs.clone()); + first.output = AggregateOutput::Partial; + let second = second_phase(&first, None); + let partial = PartialAggregate::new(first, BypassPolicy::default(), Some(7)).unwrap(); + let plan = node(PartialNode::new(scan(&s, b), partial), second); + assert_eq!(run_one(plan).await, expected); + // No rows: NULL. + let empty = |t: DataType| arrow_array::new_empty_array(&t); + let none = node( + scan( + &s, + vec![vec![ + empty(DataType::Int64), + empty(DataType::Int64), + empty(DataType::Utf8View), + ]], + ), + spec(&s, vec![], aggs), + ); + assert_eq!(run_one(none).await, ["NULL|NULL|NULL|NULL"]); +} + +/// A column of Spark type `t` holding `v` (`i64` stands for the value in +/// every type's domain; strings are zero-padded so they order as numbers). +fn typed(t: &DataType, v: &[Option]) -> ArrayRef { + let on = |f: fn(i64) -> i128| v.iter().map(|x| x.map(f)).collect::>(); + match t { + DataType::Boolean => Arc::new(BooleanArray::from( + v.iter().map(|x| x.map(|x| x % 2 != 0)).collect::>(), + )), + DataType::Int8 => Arc::new(Int8Array::from( + v.iter() + .map(|x| x.map(|x| i8::try_from(x).unwrap())) + .collect::>(), + )), + DataType::Int16 => Arc::new(Int16Array::from( + v.iter() + .map(|x| x.map(|x| i16::try_from(x).unwrap())) + .collect::>(), + )), + DataType::Int32 => Arc::new(Int32Array::from( + v.iter() + .map(|x| x.map(|x| i32::try_from(x).unwrap())) + .collect::>(), + )), + DataType::Int64 => i64s(v), + DataType::Float32 => Arc::new(Float32Array::from( + v.iter().map(|x| x.map(|x| x as f32)).collect::>(), + )), + DataType::Float64 => f64s(&v.iter().map(|x| x.map(|x| x as f64)).collect::>()), + DataType::Date32 => Arc::new(Date32Array::from( + v.iter() + .map(|x| x.map(|x| i32::try_from(x).unwrap())) + .collect::>(), + )), + DataType::Timestamp(..) => { + Arc::new(PrimitiveArray::::from(v.to_vec())) + } + DataType::Decimal128(p, s) => Arc::new( + Decimal128Array::from(on(|x| i128::from(x) * 100)) + .with_precision_and_scale(*p, *s) + .unwrap(), + ), + DataType::Utf8View => Arc::new(StringViewArray::from( + v.iter() + .map(|x| x.map(|x| format!("{x:05}"))) + .collect::>(), + )), + t => panic!("no test column of {t}"), + } +} + +/// The types `max_by`/`min_by` take as value and as key. +fn by_types() -> Vec { + vec![ + DataType::Boolean, + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Int64, + DataType::Float32, + DataType::Float64, + DataType::Decimal128(10, 2), + DataType::Date32, + DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, None), + DataType::Utf8View, + ] +} + +/// What `max_by`/`min_by` of rows `(g, x, y)` give per group, by a plain +/// loop: the first row with a non-NULL key and the extreme key. +fn by_oracle( + rows: &[(i64, Option, Option)], + max: bool, + key: impl Fn(i64) -> i64, +) -> BTreeMap> { + let mut best: BTreeMap)>> = BTreeMap::new(); + for &(g, x, y) in rows { + let slot = best.entry(g).or_insert(None); + let Some(y) = y else { continue }; + let better = match slot { + None => true, + Some((cur, _)) => (key(y) > key(*cur)) == max && key(y) != key(*cur), + }; + if better { + *slot = Some((y, x)); + } + } + best.into_iter() + .map(|(g, v)| (g, v.and_then(|(_, x)| x))) + .collect() +} + +/// Every value type as `x` and as `y`, grouped, through both phases and +/// both table layouts, against a plain loop. +#[tokio::test] +async fn every_value_and_key_type() { + // Distinct non-negative keys per group (except for booleans), NULLs in + // both columns, a NULL key on the largest value. + let rows: Vec<(i64, Option, Option)> = vec![ + (1, Some(10), Some(5)), + (1, Some(11), Some(9)), + (1, None, None), + (1, Some(12), Some(2)), + (2, None, Some(7)), + (2, Some(21), Some(3)), + (3, Some(30), None), + (4, Some(40), Some(1)), + ]; + let g: Vec> = rows.iter().map(|r| Some(r.0)).collect(); + let xs: Vec> = rows.iter().map(|r| r.1).collect(); + let ys: Vec> = rows.iter().map(|r| r.2).collect(); + for tx in by_types() { + for ty in by_types() { + let s = schema(vec![ + ("k", DataType::Int64), + ("x", tx.clone()), + ("y", ty.clone()), + ]); + let batch = vec![i64s(&g), typed(&tx, &xs), typed(&ty, &ys)]; + let key = |y: i64| if ty == DataType::Boolean { y % 2 } else { y }; + for (f, max) in [(MaxBy, true), (MinBy, false)] { + let want = by_oracle(&rows, max, key); + let render = |v: Option| cell(&typed(&tx, &[v]), 0); + let expected: Vec = { + let mut r: Vec = want + .iter() + .map(|(g, v)| format!("{g}|{}", render(*v))) + .collect(); + r.sort(); + r + }; + let sp = spec(&s, vec![0], vec![by(f, &s)]); + let single = node(scan(&s, vec![batch.clone()]), sp.clone()); + assert_eq!(run_one(single).await, expected, "{f:?} of {tx} by {ty}"); + // Two batches through the partial phase. + let halves: Vec> = vec![ + batch.iter().map(|c| c.slice(0, 3)).collect(), + batch.iter().map(|c| c.slice(3, 5)).collect(), + ]; + let mut first = sp.clone(); + first.output = AggregateOutput::Partial; + let second = second_phase(&first, None); + let partial = + PartialAggregate::new(first, BypassPolicy::default(), Some(7)).unwrap(); + let plan = node(PartialNode::new(scan(&s, halves), partial), second); + assert_eq!( + run_one(plan).await, + expected, + "{f:?} of {tx} by {ty}, two phases" + ); + } + } + } +} + +/// Group `k` holds `(x, y)`; the keys' order is not the values' order, so a +/// wrong pick shows. +fn spill_input(n: i64, keys: i64) -> (SchemaRef, Vec>) { + let s = schema(vec![ + ("k", DataType::Int64), + ("x", DataType::Utf8View), + ("y", DataType::Utf8View), + ]); + let batches = (0..n) + .step_by(1000) + .map(|start| { + let r = start..(start + 1000).min(n); + let k: Vec<_> = r.clone().map(|i| Some(i % keys)).collect(); + let x: Vec<_> = r.clone().map(|i| Some(format!("value-{i:08}"))).collect(); + let y: Vec<_> = r + .map(|i| (i % 11 != 0).then(|| format!("key-{:08}", (i * 7919) % 100_003))) + .collect(); + vec![ + i64s(&k), + Arc::new(StringViewArray::from(x)) as ArrayRef, + Arc::new(StringViewArray::from(y)) as ArrayRef, + ] + }) + .collect(); + (s, batches) +} + +/// `max_by`/`min_by` of strings by strings under a tiny budget: the runs +/// spill, merge partition by partition and give the in-memory result, which +/// is the plain loop's. +#[tokio::test] +async fn by_aggregates_spill_and_merge() { + let (s, b) = spill_input(60_000, 40_000); + let aggs = vec![by(MaxBy, &s), by(MinBy, &s)]; + let expected = run_rows(node(scan(&s, b.clone()), spec(&s, vec![0], aggs.clone()))).await; + // The same answer by a plain loop (ties keep the first row). + let mut best: BTreeMap; 2]> = BTreeMap::new(); + for i in 0..60_000_i64 { + if i % 11 == 0 { + continue; + } + let (x, y) = ( + format!("value-{i:08}"), + format!("key-{:08}", (i * 7919) % 100_003), + ); + let slot = best.entry(i % 40_000).or_default(); + if slot[0].as_ref().is_none_or(|(_, cur)| y > *cur) { + slot[0] = Some((x.clone(), y.clone())); + } + if slot[1].as_ref().is_none_or(|(_, cur)| y < *cur) { + slot[1] = Some((x, y)); + } + } + let mut plain: Vec = (0..40_000_i64) + .map(|k| { + let v = best.get(&k); + let pick = |n: usize| { + v.and_then(|v| v[n].as_ref()) + .map_or("NULL".to_owned(), |(x, _)| x.clone()) + }; + format!("{k}|{}|{}", pick(0), pick(1)) + }) + .collect(); + plain.sort(); + assert_eq!(expected, plain); + let (_m, scope) = spill_dir(); + let mut sp = spec(&s, vec![0], aggs); + sp.spill = Some(scope.clone()); + let o = Opts { + limit: 3 << 20, + ..Opts::default() + }; + let (out, metrics) = exec_each(|| node(scan(&s, b.clone()), sp.clone()), o).await; + assert_eq!(out, expected); + for m in &metrics { + assert!(extra(m, "HashAggregate", "spill_runs") > 0, "{m:?}"); + assert_eq!(extra(m, "HashAggregate", "groups"), 40_000); + } + // Spilled partial states merged in a second phase. + let mut first = spec(&s, vec![0], vec![by(MaxBy, &s), by(MinBy, &s)]); + first.output = AggregateOutput::Partial; + let second = second_phase(&first, Some(scope)); + let partial = PartialAggregate::new(first, BypassPolicy::default(), Some(7)).unwrap(); + let plan = node(PartialNode::new(scan(&s, b), partial), second); + assert_eq!(rows(&exec(plan, o).await.unwrap().0), expected); +} + +/// A table that outgrows the pool without a spill directory fails the query +/// with a memory error and releases everything. +#[tokio::test] +async fn by_aggregates_without_spill_fail_cleanly() { + let (s, b) = spill_input(60_000, 60_000); + let plan = node(scan(&s, b), spec(&s, vec![0], vec![by(MaxBy, &s)])); + let o = Opts { + limit: 1 << 20, + ..Opts::default() + }; + let e = exec(plan, o).await.err().map(|e| e.error_class()); + assert_eq!(e, Some(ErrorClass::UnableToAcquireMemory)); +} + +#[test] +fn types_states_and_errors() { + let s = schema(vec![ + ("k", DataType::Int64), + ("x", DataType::Decimal128(12, 2)), + ("y", DataType::Date32), + ]); + let a = by(MaxBy, &s); + assert_eq!(a.result_type().unwrap(), DataType::Decimal128(12, 2)); + let fields = a.state_fields().unwrap(); + let types: Vec<_> = fields.iter().map(|f| f.data_type().clone()).collect(); + assert_eq!(types, [DataType::Decimal128(12, 2), DataType::Date32]); + assert!(fields.iter().all(Field::is_nullable)); + assert_eq!(a.merging(4).args, [4, 5]); + assert_eq!(a.args, [1, 2]); + let any = AggregateExpr::new(AnyValue { ignore_nulls: true }, 1, DataType::Utf8View, "a"); + assert_eq!(any.result_type().unwrap(), DataType::Utf8View); + assert_eq!(any.state_fields().unwrap().len(), 2); + let unsupported = ErrorClass::KrishivUnsupportedFeature; + // A type `min`/`max` do not take, as value or as key; a plain argument + // type (no key) for `max_by`. + let bad = |x: DataType, y: DataType| { + AggregateExpr::by(MinBy, (0, 1), (x, y), "b") + .result_type() + .map_err(|e| e.error_class()) + }; + assert_eq!(bad(DataType::Null, DataType::Int64), Err(unsupported)); + assert_eq!(bad(DataType::Int64, DataType::Null), Err(unsupported)); + // Spark reports these as `DATATYPE_MISMATCH`; the engine has no + // ordering for them at all, so they are unsupported here. + let interval = DataType::Interval(arrow_schema::IntervalUnit::YearMonth); + let array = DataType::List(Arc::new(Field::new("item", DataType::Int32, true))); + assert_eq!(bad(interval.clone(), DataType::Int64), Err(unsupported)); + assert_eq!(bad(DataType::Int64, interval), Err(unsupported)); + assert_eq!(bad(array.clone(), DataType::Int64), Err(unsupported)); + assert_eq!(bad(DataType::Int64, array), Err(unsupported)); + assert_eq!( + AggregateExpr::new(MaxBy, 0, DataType::Int64, "b") + .result_type() + .map_err(|e| e.error_class()), + Err(unsupported) + ); + assert_eq!( + AggregateExpr::new( + AnyValue { + ignore_nulls: false + }, + 0, + DataType::Null, + "b" + ) + .result_type() + .map_err(|e| e.error_class()), + Err(unsupported) + ); +} + +/// Weights other than 1 need incremental view maintenance, as for `min`. +#[tokio::test] +async fn weights_are_rejected() { + let (s, b) = by_input(true); + let aggs = vec![ + by(MaxBy, &s), + agg( + AnyValue { + ignore_nulls: false, + }, + 1, + &s, + ), + ]; + let weighted: Vec = b + .into_iter() + .map(|cols| { + let data = RecordBatch::try_new(Arc::clone(&s), cols).unwrap(); + Batch::with_weights(data, Weights::Const(2)).unwrap() + }) + .collect(); + let plan: Arc = Arc::new(Scan(Batches::new(Arc::clone(&s), weighted))); + let e = exec(node(plan, spec(&s, vec![0], aggs)), Opts::default()) + .await + .err() + .map(|e| e.error_class()); + assert_eq!(e, Some(ErrorClass::KrishivUnsupportedFeature)); +} + +/// A running window frame snapshots state per row; byte values would be +/// copied per row without being accounted, so they are refused, as is a +/// partial state. Other types snapshot. +#[test] +fn running_frames_snapshot_fixed_width_states_only() { + let int = |a: &AggregateExpr| Accumulator::new(a, true).unwrap(); + let mk = |tx: DataType, ty: DataType| { + let expr = AggregateExpr::by(MaxBy, (0, 1), (tx, ty), "b"); + int(&expr) + }; + // Rows 0..3 of one partition (group 0); each row's state goes to its + // own group 1..=3. + let parts = [0u32; 3]; + let snaps = [1u32, 2, 3]; + let run = Running { + parts: &parts, + snapshots: &snaps, + }; + let mut a = mk(DataType::Int64, DataType::Int64); + a.resize(4); + let x = i64s(&[Some(10), None, Some(30)]); + let y = i64s(&[Some(1), Some(5), Some(5)]); + a.update_running(&[0, 0, 0], &[x, y], &run).unwrap(); + // After each row: 10; then NULL (key 5 beats 1); then NULL again (a tie + // keeps the first). + let out = a.evaluate(&[1, 2, 3]).unwrap(); + let out = out.as_primitive::(); + assert_eq!( + (0..3) + .map(|i| out.is_valid(i).then(|| out.value(i))) + .collect::>(), + [Some(10), None, None] + ); + for (tx, ty) in [ + (DataType::Utf8View, DataType::Int64), + (DataType::Int64, DataType::Utf8View), + ] { + let mut a = mk(tx.clone(), ty.clone()); + a.resize(4); + let cols = [typed(&tx, &[Some(1); 3]), typed(&ty, &[Some(1); 3])]; + let e = a.update_running(&[0, 0, 0], &cols, &run).unwrap_err(); + assert_eq!(e.error_class(), ErrorClass::KrishivUnsupportedFeature); + // The whole-partition fold takes them. + let mut a = mk(tx.clone(), ty.clone()); + a.resize(1); + a.update(&[0, 0, 0], &cols, &Weights::Const(1)).unwrap(); + } + // Columns of other lengths than the group ids are an internal error, + // not a panic. + let mut a = mk(DataType::Int64, DataType::Int64); + a.resize(1); + let e = a + .update( + &[0, 0], + &[i64s(&[Some(1)]), i64s(&[Some(1), Some(2)])], + &Weights::Const(1), + ) + .unwrap_err(); + assert_eq!(e.error_class(), ErrorClass::InternalError); +} + +/// The projected size bounds the size after the update, for byte values +/// and keys (what the reservation is made from). +#[test] +fn projected_size_bounds_the_size_after_an_update() { + for (tx, ty) in [ + (DataType::Utf8View, DataType::Int64), + (DataType::Int64, DataType::Utf8View), + (DataType::Utf8View, DataType::Utf8View), + (DataType::Int64, DataType::Int64), + ] { + let expr = AggregateExpr::by(MaxBy, (0, 1), (tx.clone(), ty.clone()), "b"); + let mut a = Accumulator::new(&expr, true).unwrap(); + let v: Vec> = (0..200).map(Some).collect(); + let (x, y) = (typed(&tx, &v), typed(&ty, &v)); + let projected = a.projected(200, Some(&x), Some(&y)); + a.reserve(200); + a.resize(200); + let groups: Vec = (0..200).collect(); + a.update(&groups, &[x, y], &Weights::Const(1)).unwrap(); + assert!( + a.size() <= projected, + "{tx} by {ty}: {} > {projected}", + a.size() + ); + // Replacing every value by a longer one stays within the bound too. + let longer: Vec> = (1000..1200).map(Some).collect(); + let (x, y) = (typed(&tx, &longer), typed(&ty, &longer)); + let projected = a.projected(0, Some(&x), Some(&y)); + a.update(&groups, &[x, y], &Weights::Const(1)).unwrap(); + assert!( + a.size() <= projected, + "{tx} by {ty}: {} > {projected}", + a.size() + ); + } +} + +/// A column of another physical kind than the state's (a plan that typed +/// its argument wrongly) is an internal error, not a panic or a wrong +/// answer. +#[test] +fn columns_of_the_wrong_kind_are_internal_errors() { + let expr = AggregateExpr::by(MaxBy, (0, 1), (DataType::Int64, DataType::Int64), "b"); + let mut a = Accumulator::new(&expr, true).unwrap(); + a.resize(1); + let ints = i64s(&[Some(1)]); + let doubles = f64s(&[Some(1.0)]); + let internal = |r: crate::Result<()>| r.unwrap_err().error_class(); + // The first row is stored: a key of another kind cannot be. + let r = a.update( + &[0], + &[Arc::clone(&ints), Arc::clone(&doubles)], + &Weights::Const(1), + ); + assert_eq!(internal(r), ErrorClass::InternalError); + a.update( + &[0], + &[Arc::clone(&ints), Arc::clone(&ints)], + &Weights::Const(1), + ) + .unwrap(); + // A later row is compared with the key held first. + let r = a.update( + &[0], + &[Arc::clone(&ints), Arc::clone(&doubles)], + &Weights::Const(1), + ); + assert_eq!(internal(r), ErrorClass::InternalError); + // A value of another kind, on a row that wins (key 2 beats key 1). + let larger = i64s(&[Some(2)]); + let r = a.update(&[0], &[doubles, larger], &Weights::Const(1)); + assert_eq!(internal(r), ErrorClass::InternalError); +} diff --git a/crates/krishiv-exec/src/planner/aggregate.rs b/crates/krishiv-exec/src/planner/aggregate.rs index e5a7d4ae4..81ba3c3a5 100644 --- a/crates/krishiv-exec/src/planner/aggregate.rs +++ b/crates/krishiv-exec/src/planner/aggregate.rs @@ -10,6 +10,8 @@ use std::collections::HashMap; use std::sync::Arc; +use krishiv_functions::registry::{Lookup, builtins}; +use krishiv_functions::signature::SparkProfile; use krishiv_plan::{ AttrId, BoundColumn, Expr, ExprNode, FunctionId, FunctionKind, Grouping, ResolvedExpr, ResolvedFunction, @@ -48,75 +50,24 @@ impl Planner<'_> { // Step 1: keys, then each call's arguments. let mut wanted: Vec> = keys.clone(); let mut arg_at = Vec::with_capacity(calls.len()); + let mut lowered = Vec::with_capacity(calls.len()); for c in &calls { - let args: Vec<&Arc> = - c.f.args - .iter() - .take(if picks(c.f.id.name()).is_some() { - 1 - } else { - usize::MAX - }) - .filter(|a| !non_null_literal(a) || c.f.id.name() != "count") - .collect(); - arg_at.push(wanted.len()..wanted.len() + args.len()); - wanted.extend(args.into_iter().cloned()); + let l = lower(&c.f) + .ok_or_else(|| unsupported(&format!("the aggregate function {}", c.f.id.name())))?; + arg_at.push(wanted.len()..wanted.len() + l.args.len()); + wanted.extend(l.args.iter().cloned()); + lowered.push(l.function); } let (input, columns) = self.with_columns(input, &wanted)?; let k = keys.len(); let mut aggregates = Vec::with_capacity(calls.len()); let mut distinct = Vec::with_capacity(calls.len()); - for (c, at) in calls.iter().zip(&arg_at) { - let function = match c.f.id.name() { - "count" => AggregateFunction::Count, - "sum" => AggregateFunction::Sum, - "avg" => AggregateFunction::Avg, - "min" => AggregateFunction::Min, - "max" => AggregateFunction::Max, - // `first(x, ignoreNulls)`: the analyzer checked the flag is a - // boolean literal. - name if picks(name).is_some() => { - let ignore_nulls = matches!( - c.f.args.get(1).map(|a| &a.node), - Some(ExprNode::Literal(krishiv_plan::Literal::Boolean(true))) - ); - if picks(name) == Some(true) { - AggregateFunction::Last { ignore_nulls } - } else { - AggregateFunction::First { ignore_nulls } - } - } - other => { - return Err(unsupported(&format!("the aggregate function {other}"))); - } - }; + for ((c, at), function) in calls.iter().zip(&arg_at).zip(lowered) { if c.f.filter.is_some() { return Err(unsupported("aggregate FILTER (WHERE ...) clauses")); } let args: Vec = columns[at.clone()].to_vec(); - if function != AggregateFunction::Count && args.len() != 1 { - return Err(internal("an aggregate of several arguments")); - } - // Spark's STRING and BINARY are views here whatever carrier a - // registered table uses; the accumulators read any carrier. - let arg_type = args.first().map_or(arrow_schema::DataType::Null, |a| { - match input.schema.field(*a).data_type() { - arrow_schema::DataType::Utf8 | arrow_schema::DataType::LargeUtf8 => { - arrow_schema::DataType::Utf8View - } - arrow_schema::DataType::Binary | arrow_schema::DataType::LargeBinary => { - arrow_schema::DataType::BinaryView - } - t => t.clone(), - } - }); - let agg = AggregateExpr { - function, - args, - input: crate::aggregate::AggInput::Raw, - arg_type, - name: c.key.clone(), - }; + let agg = aggregate_expr(function, args, &input.schema, c.key.clone())?; let want = c.f.data_type.to_arrow(); if agg.result_type()? != want { return Err(internal(&format!( @@ -264,16 +215,127 @@ fn replace( }) } -/// `Some(true)` for `last`/`last_value`, `Some(false)` for `first`/ -/// `first_value`: the aggregates that pick a row's value. -pub(super) fn picks(name: &str) -> Option { - match name { - "first" | "first_value" => Some(false), - "last" | "last_value" => Some(true), - _ => None, +/// An aggregate call as the executor runs it: the function and the +/// argument expressions it reads (`first(x, flag)` reads `x`; `count(1)` +/// reads none). The one place that maps registry functions to executor +/// ones, for aggregates and window aggregates alike. Some aggregates are +/// defined by another, whose accumulator they reuse (the output keeps their +/// own name): `count_if(c)` is `count(nullif(c, false))`, `bool_and` and +/// `every` are `min`, `bool_or`, `some` and `any` are `max` (the registry +/// types their argument as `boolean`). +pub(super) struct Lowered { + pub(super) function: AggregateFunction, + pub(super) args: Vec>, +} + +/// `None` for a function the executor does not run as an aggregate. +pub(super) fn lower(f: &ResolvedFunction) -> Option { + // `first(x, ignoreNulls)`: the analyzer checked the flag is a boolean + // literal. + let ignore_nulls = matches!( + f.args.get(1).map(|a| &a.node), + Some(ExprNode::Literal(krishiv_plan::Literal::Boolean(true))) + ); + let value: Vec> = f.args.iter().take(1).cloned().collect(); + let (function, args) = match f.id.name() { + "count" => ( + AggregateFunction::Count, + f.args + .iter() + .filter(|a| !non_null_literal(a)) + .cloned() + .collect(), + ), + "count_if" => ( + AggregateFunction::Count, + vec![null_if_false(f.args.first()?)?], + ), + "sum" => (AggregateFunction::Sum, f.args.clone()), + "avg" | "mean" => (AggregateFunction::Avg, f.args.clone()), + "min" | "bool_and" | "every" => (AggregateFunction::Min, f.args.clone()), + "max" | "bool_or" | "some" | "any" => (AggregateFunction::Max, f.args.clone()), + "first" | "first_value" => (AggregateFunction::First { ignore_nulls }, value), + "last" | "last_value" => (AggregateFunction::Last { ignore_nulls }, value), + "any_value" => (AggregateFunction::AnyValue { ignore_nulls }, value), + "max_by" if f.args.len() == 2 => (AggregateFunction::MaxBy, f.args.clone()), + "min_by" if f.args.len() == 2 => (AggregateFunction::MinBy, f.args.clone()), + _ => return None, + }; + Some(Lowered { function, args }) +} + +/// `nullif(c, false)`: `c` where it is true, else NULL, so `count` of it +/// counts the rows where `c` is true. +fn null_if_false(c: &Arc) -> Option> { + let profile = SparkProfile::ALL.into_iter().max()?; + let Lookup::Found(def) = builtins().lookup("nullif", profile) else { + return None; + }; + Some(Arc::new(Expr::new(ExprNode::Resolved( + ResolvedExpr::Function(ResolvedFunction { + id: FunctionId::Registry { + name: def.name.to_owned(), + id: def.id, + }, + kind: FunctionKind::Scalar, + args: vec![ + Arc::clone(c), + Arc::new(Expr::literal(krishiv_plan::Literal::Boolean(false))), + ], + distinct: false, + filter: None, + data_type: krishiv_types::DataType::Boolean, + nullable: true, + deterministic: true, + }), + )))) +} + +/// The executor's aggregate `function` over columns `args` of `schema`. +/// Spark's STRING and BINARY are views here whatever carrier a registered +/// table uses; the accumulators read any carrier. +pub(super) fn aggregate_expr( + function: AggregateFunction, + args: Vec, + schema: &arrow_schema::Schema, + name: String, +) -> Result { + let typed = |a: &usize| match schema.field(*a).data_type() { + arrow_schema::DataType::Utf8 | arrow_schema::DataType::LargeUtf8 => { + arrow_schema::DataType::Utf8View + } + arrow_schema::DataType::Binary | arrow_schema::DataType::LargeBinary => { + arrow_schema::DataType::BinaryView + } + t => t.clone(), + }; + if matches!( + function, + AggregateFunction::MaxBy | AggregateFunction::MinBy + ) { + let [value, key] = args[..] else { + return Err(internal("max_by without a value and a key")); + }; + return Ok(AggregateExpr::by( + function, + (value, key), + (typed(&value), typed(&key)), + name, + )); } + if function != AggregateFunction::Count && args.len() != 1 { + return Err(internal("an aggregate of several arguments")); + } + let arg_type = args.first().map_or(arrow_schema::DataType::Null, typed); + Ok(AggregateExpr { + function, + args, + input: crate::aggregate::AggInput::Raw, + arg_type, + name, + }) } -pub(super) fn non_null_literal(e: &Expr) -> bool { +fn non_null_literal(e: &Expr) -> bool { matches!(&e.node, ExprNode::Literal(l) if !matches!(l, krishiv_plan::Literal::Null(_))) } diff --git a/crates/krishiv-exec/src/planner/window.rs b/crates/krishiv-exec/src/planner/window.rs index 15a0479aa..92504cc3e 100644 --- a/crates/krishiv-exec/src/planner/window.rs +++ b/crates/krishiv-exec/src/planner/window.rs @@ -11,10 +11,9 @@ use krishiv_plan::{ SortDirection, SortOrder, WindowSpecRef, }; -use super::aggregate::{non_null_literal, picks}; +use super::aggregate::{aggregate_expr, lower}; use super::build::{Planned, Planner, internal, list, orders, unsupported}; use super::properties::Distribution; -use crate::aggregate::{AggInput, AggregateExpr, AggregateFunction}; use crate::error::Result; use crate::operator::Source; use crate::operators::SortKey; @@ -66,9 +65,9 @@ impl Planner<'_> { for c in &calls { let args = arguments(c.f); let start = wanted.len(); - wanted.extend(args.iter().map(|a| Arc::clone(a))); - wanted.extend(c.f.filter.iter().cloned()); arg_at.push((start, args.len())); + wanted.extend(args); + wanted.extend(c.f.filter.iter().cloned()); } let (input, columns) = self.with_columns(input, &wanted)?; let np = partition_by.len(); @@ -290,53 +289,20 @@ impl Planner<'_> { other => return Err(unsupported(&format!("the window function {other}"))), }); } - let function = match name { - "count" => AggregateFunction::Count, - "sum" => AggregateFunction::Sum, - "avg" | "mean" => AggregateFunction::Avg, - "min" => AggregateFunction::Min, - "max" => AggregateFunction::Max, - name if picks(name).is_some() => { - let ignore_nulls = matches!( - c.f.args.get(1).map(|a| &a.node), - Some(ExprNode::Literal(krishiv_plan::Literal::Boolean(true))) - ); - if picks(name) == Some(true) { - AggregateFunction::Last { ignore_nulls } - } else { - AggregateFunction::First { ignore_nulls } - } - } - other => return Err(unsupported(&format!("the window aggregate {other}"))), - }; + let function = lower(c.f) + .ok_or_else(|| unsupported(&format!("the window aggregate {name}")))? + .function; // The analyzer rejects `count(DISTINCT x) OVER (...)` as Spark does; // a plan that still holds one must not be computed as `count(x)`. if c.f.distinct { return Err(unsupported("DISTINCT window aggregates")); } - if function != AggregateFunction::Count && args.len() != 1 { - return Err(internal("a window aggregate of several arguments")); - } - // Spark's STRING and BINARY are views here whatever carrier the - // input uses; the accumulators read any carrier. - let arg_type = args.first().map_or(arrow_schema::DataType::Null, |a| { - match input.schema.field(*a).data_type() { - arrow_schema::DataType::Utf8 | arrow_schema::DataType::LargeUtf8 => { - arrow_schema::DataType::Utf8View - } - arrow_schema::DataType::Binary | arrow_schema::DataType::LargeBinary => { - arrow_schema::DataType::BinaryView - } - t => t.clone(), - } - }); - Ok(WindowFunction::Aggregate(AggregateExpr { + Ok(WindowFunction::Aggregate(aggregate_expr( function, - args: args.to_vec(), - input: AggInput::Raw, - arg_type, - name: c.output.name.clone(), - })) + args.to_vec(), + &input.schema, + c.output.name.clone(), + )?)) } } @@ -383,27 +349,22 @@ fn call(e: &Expr) -> Result> { Ok(Call { f, frame, output }) } -/// The argument expressions an aggregate reads: `first(x, flag)` reads -/// `x`; `count(1)` (a non-NULL literal) reads none. Window functions read -/// none (their arguments are constants). -fn arguments(f: &krishiv_plan::ResolvedFunction) -> Vec<&Arc> { +/// The argument expressions an aggregate reads (see [`lower`]). Window +/// functions read none (their arguments are constants). +fn arguments(f: &krishiv_plan::ResolvedFunction) -> Vec> { if f.kind == FunctionKind::Window { // `lag`/`lead` read their input and default, `nth_value` its input. return match f.id.name() { "lag" | "lead" => [f.args.first(), f.args.get(2)] .into_iter() .flatten() + .cloned() .collect(), - "nth_value" => f.args.first().into_iter().collect(), + "nth_value" => f.args.first().into_iter().cloned().collect(), _ => Vec::new(), }; } - let name = f.id.name(); - f.args - .iter() - .take(if picks(name).is_some() { 1 } else { usize::MAX }) - .filter(|a| !non_null_literal(a) || name != "count") - .collect() + lower(f).map(|l| l.args).unwrap_or_default() } /// Whether argument `i` of `f` is the literal `true` (an `IGNORE NULLS` diff --git a/crates/krishiv-exec/src/window/eval.rs b/crates/krishiv-exec/src/window/eval.rs index 5eeb708d4..09629a6ac 100644 --- a/crates/krishiv-exec/src/window/eval.rs +++ b/crates/krishiv-exec/src/window/eval.rs @@ -582,13 +582,14 @@ fn is_bytes(t: &DataType) -> bool { } /// Whether `a` picks one row's string or binary value (`min`, `max`, -/// `first`, `last`): evaluated by [`byte_values`], without copying values. +/// `first`, `last`, `any_value`): evaluated by [`byte_values`], without +/// copying values. pub(super) fn picks_bytes(a: &AggregateExpr) -> bool { use AggregateFunction as F; is_bytes(&a.arg_type) && matches!( a.function, - F::Min | F::Max | F::First { .. } | F::Last { .. } + F::Min | F::Max | F::First { .. } | F::Last { .. } | F::AnyValue { .. } ) } @@ -629,9 +630,9 @@ fn segment(shape: Shape, u: &Unit<'_>, row: usize) -> (usize, usize) { } } -/// `min`/`max`/`first`/`last` of string or binary values over a frame: -/// the row each frame picks, then the picked values gathered as views of -/// the input's buffers (no value is copied; Spark compares by bytes). +/// `min`/`max`/`first`/`last`/`any_value` of string or binary values over a +/// frame: the row each frame picks, then the picked values gathered as views +/// of the input's buffers (no value is copied; Spark compares by bytes). fn byte_values( a: &AggregateExpr, f: &WindowFunctionPlan, @@ -687,8 +688,8 @@ fn byte_values( } /// Whether a row (`valid`, its bytes `value`) replaces the value a -/// `min`/`max`/`first`/`last` of strings or binaries picked so far -/// (`picked`: `None` before any row, `Some(None)` for a picked NULL); +/// `min`/`max`/`first`/`last`/`any_value` of strings or binaries picked so +/// far (`picked`: `None` before any row, `Some(None)` for a picked NULL); /// Spark compares by bytes. pub(super) fn replaces<'v>( function: AggregateFunction, @@ -710,7 +711,9 @@ pub(super) fn replaces<'v>( } None => true, }, - F::First { ignore_nulls } => picked.is_none() && (valid || !ignore_nulls), + F::First { ignore_nulls } | F::AnyValue { ignore_nulls } => { + picked.is_none() && (valid || !ignore_nulls) + } F::Last { ignore_nulls } => valid || !ignore_nulls, other => return Err(internal(format!("{other:?} of strings by row"))), }) diff --git a/crates/krishiv-exec/src/window/tests.rs b/crates/krishiv-exec/src/window/tests.rs index f4ecd441b..b6693d683 100644 --- a/crates/krishiv-exec/src/window/tests.rs +++ b/crates/krishiv-exec/src/window/tests.rs @@ -477,6 +477,11 @@ fn reference( let s = vals.iter().fold(0.0, |s, v| s + *v as f64); format!("{:?}", s / to_f(vals.len())) } + AggregateFunction::AnyValue { .. } + | AggregateFunction::MaxBy + | AggregateFunction::MinBy => { + unreachable!("no window test aggregates with this function") + } } } } diff --git a/crates/krishiv-functions/CLAUDE.md b/crates/krishiv-functions/CLAUDE.md index eed4cb74c..e0b4326dd 100644 --- a/crates/krishiv-functions/CLAUDE.md +++ b/crates/krishiv-functions/CLAUDE.md @@ -12,7 +12,7 @@ Spark-compatible function library: scalar, aggregate, window and generator funct `ResultType`, `Arity`); krishiv-plan re-exports them. - `registry` (P2-T153): `builtins()` → `Registry::lookup(name, profile)` = `Found(&FunctionDef)` / `KnownUnsupported` / `NotFound`. A `FunctionDef` has kind, arity, inputs, result rule, `deterministic`, `since` and `Execution` (`Scalar` = - `bind_scalar`, `Aggregate(Aggregate)` for krishiv-exec (count, sum, avg, min, max, first, last; `first(x, ignoreNulls)` takes the flag from a boolean-literal second argument), `Evaluator` = the conditionals, evaluated lazily by the exec evaluator). Signatures live in + `bind_scalar`, `Aggregate(Aggregate)` for krishiv-exec (count, sum, avg, min, max, first, last, any_value, max_by, min_by; `count_if` and the boolean aggregates name `count`/`min`/`max`, the planner rewrites their argument; `first(x, ignoreNulls)` and `any_value` take the flag from a boolean-literal second argument; the three-argument `max_by` is rejected as unsupported), `Evaluator` = the conditionals, evaluated lazily by the exec evaluator). Signatures live in `registry/defs.rs` (strings, datetime, aggregates), `registry/math.rs` (math and `try_*`/`mod` arithmetic, typed as the operators through `Inputs::Operator`/`TryOperator`), `registry/hashing.rs` and `registry/conditional.rs` (`Evaluator` entries typed through `Inputs::Leading`, `Common`, `CommonWithZero`, `Comparable`); arity, determinism, kind and availability come from `vendored/spark-functions.tsv`, generated by diff --git a/crates/krishiv-functions/src/python_wrappers/table.rs b/crates/krishiv-functions/src/python_wrappers/table.rs index 0dad2e791..99e9e4417 100644 --- a/crates/krishiv-functions/src/python_wrappers/table.rs +++ b/crates/krishiv-functions/src/python_wrappers/table.rs @@ -17,6 +17,15 @@ pub(super) const NOT_EXPORTED: &[(&str, &str)] = &[ (">>>", "operator"), ("if", "no PySpark function"), ("mod", "no PySpark function"), + ("any", "no PySpark function"), + ("any_value", "no PySpark function"), + ("bool_and", "no PySpark function"), + ("bool_or", "no PySpark function"), + ("count_if", "no PySpark function"), + ("every", "no PySpark function"), + ("max_by", "no PySpark function"), + ("min_by", "no PySpark function"), + ("some", "no PySpark function"), ( "mean", "an alias of `avg`; the exported `mean` is a row that sends `avg`", diff --git a/crates/krishiv-functions/src/registry/defs.rs b/crates/krishiv-functions/src/registry/defs.rs index de824f8a3..d4ed63b05 100644 --- a/crates/krishiv-functions/src/registry/defs.rs +++ b/crates/krishiv-functions/src/registry/defs.rs @@ -6,7 +6,8 @@ use krishiv_common::error::{ DatatypeMismatchNonFoldableInput, DatatypeMismatchUnexpectedInputType, DatatypeMismatchValueOutOfRange, InvalidExtractField, InvalidParameterValueBoolean, - KrishivError, LegacyErrorTemp1210, NonFoldableArgument, SecondFunctionArgumentNotInteger, + KrishivError, KrishivUnsupportedFeature, LegacyErrorTemp1210, NonFoldableArgument, + SecondFunctionArgumentNotInteger, }; use krishiv_kernels::datetime::DatePart; use krishiv_types::{DataType, DecimalType}; @@ -457,8 +458,32 @@ fn last(c: &CallTypes<'_>) -> Result { pick(c, "last") } +fn any_value(c: &CallTypes<'_>) -> Result { + pick(c, "any_value") +} + +/// `bool_and`, `bool_or` and their aliases: `boolean`, NULL for no non-NULL +/// input. +fn nullable_boolean(_: &CallTypes<'_>) -> Result { + Ok(ArgType::new(DataType::Boolean, true)) +} + +/// `max_by(x, y)`, `min_by(x, y)`: the type of `x`. The three-argument +/// form (`max_by(x, y, k)`) returns an array of up to `k` values, which +/// this engine has no type for yet. +fn by(c: &CallTypes<'_>) -> Result { + if c.args().len() > 2 { + return Err(KrishivUnsupportedFeature { + feature: "max_by and min_by with a third argument (they return an array)".to_owned(), + } + .into()); + } + min_max(c) +} + fn aggregates() -> Vec { let numeric = || Inputs::Expects(vec![InputType::Numeric { intervals: true }]); + let boolean_arg = || Inputs::Expects(vec![exact(DataType::Boolean)]); vec![ aggregate("count", Inputs::Any, count, Aggregate::Count), aggregate("sum", numeric(), sum, Aggregate::Sum), @@ -470,6 +495,16 @@ fn aggregates() -> Vec { aggregate("first_value", Inputs::Any, first, Aggregate::First), aggregate("last", Inputs::Any, last, Aggregate::Last), aggregate("last_value", Inputs::Any, last, Aggregate::Last), + aggregate("any_value", Inputs::Any, any_value, Aggregate::AnyValue), + // Spark runs these as `count`, `min` and `max` of a rewritten argument. + aggregate("count_if", boolean_arg(), count, Aggregate::Count), + aggregate("bool_and", boolean_arg(), nullable_boolean, Aggregate::Min), + aggregate("every", boolean_arg(), nullable_boolean, Aggregate::Min), + aggregate("bool_or", boolean_arg(), nullable_boolean, Aggregate::Max), + aggregate("some", boolean_arg(), nullable_boolean, Aggregate::Max), + aggregate("any", boolean_arg(), nullable_boolean, Aggregate::Max), + aggregate("max_by", Inputs::Any, by, Aggregate::MaxBy), + aggregate("min_by", Inputs::Any, by, Aggregate::MinBy), ] } diff --git a/crates/krishiv-functions/src/registry/mod.rs b/crates/krishiv-functions/src/registry/mod.rs index c6af46e3e..da1b6b7a9 100644 --- a/crates/krishiv-functions/src/registry/mod.rs +++ b/crates/krishiv-functions/src/registry/mod.rs @@ -66,6 +66,12 @@ pub enum Aggregate { First, /// `last` / `last_value`; `ignoreNulls` is the second argument. Last, + /// `any_value`; `ignoreNulls` is the second argument. + AnyValue, + /// `max_by(x, y)`. + MaxBy, + /// `min_by(x, y)`. + MinBy, } /// A built-in function. diff --git a/crates/krishiv-functions/src/registry/tests.rs b/crates/krishiv-functions/src/registry/tests.rs index 6388993b5..b212e859b 100644 --- a/crates/krishiv-functions/src/registry/tests.rs +++ b/crates/krishiv-functions/src/registry/tests.rs @@ -688,6 +688,87 @@ fn first_and_last_take_a_boolean_literal_flag() { assert_eq!(r.data_type, None); } +/// `count_if` and the boolean aggregates run as `count`, `min` and `max` +/// (Spark rewrites them so); `max_by(x, y, k)` returns an array, which +/// Krishiv has no type for, so only the two-argument form types. +#[test] +fn simple_aggregates() { + for (name, agg) in [ + ("count_if", Aggregate::Count), + ("bool_and", Aggregate::Min), + ("every", Aggregate::Min), + ("bool_or", Aggregate::Max), + ("some", Aggregate::Max), + ("any", Aggregate::Max), + ("any_value", Aggregate::AnyValue), + ("max_by", Aggregate::MaxBy), + ("min_by", Aggregate::MinBy), + ] { + let d = def(name); + assert_eq!(d.kind, FunctionKind::Aggregate, "{name}"); + assert_eq!(d.execution, Execution::Aggregate(agg), "{name}"); + assert!(d.deterministic, "{name}"); + } + let boolean = Some(vec![InputType::Exact(DataType::Boolean)]); + let expects = |name: &str| def(name).inputs.expected(&CallTypes::new(&[], &[])); + let bool_result = ArgType::new(DataType::Boolean, true); + for name in ["count_if", "bool_and", "every", "bool_or", "some", "any"] { + assert_eq!(expects(name), boolean, "{name}"); + assert!(def(name).arity.accepts(1) && !def(name).arity.accepts(2)); + } + let x = ty(DataType::Boolean); + // `count_if` is `bigint` and never NULL; the others are nullable + // `boolean`s whatever their argument. + assert_eq!( + result("count_if", std::slice::from_ref(&x), &[]).unwrap(), + ArgType::new(DataType::Long, false) + ); + for name in ["bool_and", "every", "bool_or", "some", "any"] { + assert_eq!( + result(name, std::slice::from_ref(&x), &[]).unwrap(), + bool_result + ); + } + // `max_by(x, y)` has the type of `x`, nullable. + let (v, k) = (ty(DataType::String), ty(DataType::Integer)); + for name in ["max_by", "min_by"] { + let d = def(name); + assert!(d.arity.accepts(2) && d.arity.accepts(3) && !d.arity.accepts(1)); + assert_eq!( + result(name, &[v.clone(), k.clone()], &[]).unwrap(), + ArgType::new(DataType::String, true) + ); + let err = result(name, &[v.clone(), k.clone(), ty(DataType::Integer)], &[]).unwrap_err(); + assert_eq!(err.error_class(), ErrorClass::KrishivUnsupportedFeature); + } + // `any_value(x [, ignoreNulls])` takes the flag as `first` does. + let nullable = ArgType::new(DataType::Integer, true); + assert_eq!( + result("any_value", &[ty(DataType::Integer)], &[]).unwrap(), + nullable + ); + let err = result( + "any_value", + &[ty(DataType::Integer), ty(DataType::Boolean)], + &[None, None], + ) + .unwrap_err(); + assert_eq!( + err.to_string(), + "The second argument in any_value should be a boolean literal." + ); + // The aggregates that return arrays stay known-unsupported. + for name in ["collect_list", "collect_set", "array_agg"] { + assert!( + matches!( + builtins().lookup(name, SparkProfile::V4_2), + Lookup::KnownUnsupported + ), + "{name}" + ); + } +} + #[test] fn session_and_clock_functions() { for name in ["current_user", "user", "session_user"] { diff --git a/crates/krishiv-plan-codec/src/encode.rs b/crates/krishiv-plan-codec/src/encode.rs index 63a3b6e86..19c0579f1 100644 --- a/crates/krishiv-plan-codec/src/encode.rs +++ b/crates/krishiv-plan-codec/src/encode.rs @@ -313,7 +313,11 @@ impl Encoder { Ok(pb::AggregateNode { input: self.node_index(input)?, group_by: indexes(&plan.group_by), - aggregates: plan.aggregates.iter().map(aggregate_expr).collect(), + aggregates: plan + .aggregates + .iter() + .map(aggregate_expr) + .collect::>()?, distinct: plan.distinct.clone(), output: match plan.output { AggregateOutput::Final => pb::AggregateOutput::Final, @@ -611,7 +615,7 @@ fn window_function(f: &WindowFunctionPlan) -> Result { WindowFunction::PercentRank => Function::PercentRank(pb::Empty {}), WindowFunction::CumeDist => Function::CumeDist(pb::Empty {}), WindowFunction::Ntile(buckets) => Function::Ntile(*buckets), - WindowFunction::Aggregate(a) => Function::Aggregate(aggregate_expr(a)), + WindowFunction::Aggregate(a) => Function::Aggregate(aggregate_expr(a)?), WindowFunction::Offset { input, offset, @@ -672,7 +676,7 @@ fn frame_bound(b: FrameBound) -> Result { } } -fn aggregate_expr(a: &AggregateExpr) -> pb::AggregateExpr { +fn aggregate_expr(a: &AggregateExpr) -> Result { let (function, ignore_nulls) = match a.function { AggregateFunction::Count => (pb::AggregateFunction::Count, false), AggregateFunction::Sum => (pb::AggregateFunction::Sum, false), @@ -681,8 +685,16 @@ fn aggregate_expr(a: &AggregateExpr) -> pb::AggregateExpr { AggregateFunction::Max => (pb::AggregateFunction::Max, false), AggregateFunction::First { ignore_nulls } => (pb::AggregateFunction::First, ignore_nulls), AggregateFunction::Last { ignore_nulls } => (pb::AggregateFunction::Last, ignore_nulls), + // The format has no ids for these yet. + AggregateFunction::AnyValue { .. } + | AggregateFunction::MaxBy + | AggregateFunction::MinBy => { + return Err(CodecError::Unsupported( + "an aggregate function the format does not know".to_owned(), + )); + } }; - pb::AggregateExpr { + Ok(pb::AggregateExpr { function: function as i32, ignore_nulls, args: indexes(&a.args), @@ -692,7 +704,7 @@ fn aggregate_expr(a: &AggregateExpr) -> pb::AggregateExpr { } as i32, arg_type: type_bytes(&a.arg_type), name: a.name.clone(), - } + }) } fn read_source(source: &ReadSource) -> Result { diff --git a/crates/krishiv-plan-codec/src/features.rs b/crates/krishiv-plan-codec/src/features.rs index 6ecb77ba4..9869c9f9b 100644 --- a/crates/krishiv-plan-codec/src/features.rs +++ b/crates/krishiv-plan-codec/src/features.rs @@ -238,7 +238,10 @@ pub fn feature_of(construct: Construct<'_>) -> Option { | AggregateFunction::Min | AggregateFunction::Max | AggregateFunction::First { .. } - | AggregateFunction::Last { .. } => None, + | AggregateFunction::Last { .. } + | AggregateFunction::AnyValue { .. } + | AggregateFunction::MaxBy + | AggregateFunction::MinBy => None, }, } } diff --git a/crates/krishiv-plan-codec/tests/roundtrip.rs b/crates/krishiv-plan-codec/tests/roundtrip.rs index 3870bd5c9..11acb46db 100644 --- a/crates/krishiv-plan-codec/tests/roundtrip.rs +++ b/crates/krishiv-plan-codec/tests/roundtrip.rs @@ -781,3 +781,70 @@ fn an_exchange_without_a_partitioning_kind_is_corrupt() { e.partitioning = Some(pb::Partitioning { kind: None }); corrupt(&m, "partitioning kind is missing"); } + +/// `any_value`, `max_by` and `min_by` have no ids in the format yet: a plan +/// holding one is refused with the typed error (not encoded as another +/// function), in an aggregate node and as a window function. +#[test] +fn aggregates_the_format_does_not_know_are_refused() { + use krishiv_exec::aggregate::{AggregateExpr, AggregateFunction, AggregateOutput}; + use krishiv_exec::physical::AggregatePlan; + use krishiv_exec::window::{WindowFrame, WindowFunction, WindowFunctionPlan, WindowPlan}; + let exprs = [ + AggregateExpr::new( + AggregateFunction::AnyValue { ignore_nulls: true }, + 0, + arrow_schema::DataType::Int32, + "a", + ), + AggregateExpr::by( + AggregateFunction::MaxBy, + (0, 1), + ( + arrow_schema::DataType::Utf8View, + arrow_schema::DataType::Int64, + ), + "b", + ), + AggregateExpr::by( + AggregateFunction::MinBy, + (0, 1), + ( + arrow_schema::DataType::Int32, + arrow_schema::DataType::Float64, + ), + "c", + ), + ]; + for e in exprs { + let refused = |plan: &PhysicalPlan| { + let err = encode(plan).unwrap_err(); + assert!( + matches!(&err, CodecError::Unsupported(m) if m.contains("aggregate function")), + "{err:?}" + ); + }; + refused(&plan_of(PhysNode::Aggregate { + input: Box::new(inventory::scan()), + aggregate: AggregatePlan { + group_by: vec![0], + aggregates: vec![e.clone()], + distinct: vec![false], + output: AggregateOutput::Final, + }, + })); + refused(&plan_of(PhysNode::Window { + input: Box::new(inventory::scan()), + window: WindowPlan { + partition_by: Vec::new(), + order_by: Vec::new(), + functions: vec![WindowFunctionPlan { + function: WindowFunction::Aggregate(e), + frame: WindowFrame::whole(krishiv_exec::window::FrameUnits::Rows), + filter: None, + name: "w".to_owned(), + }], + }, + })); + } +} diff --git a/crates/krishiv-plan/src/analyzer/exprs.rs b/crates/krishiv-plan/src/analyzer/exprs.rs index 09edc9281..742566351 100644 --- a/crates/krishiv-plan/src/analyzer/exprs.rs +++ b/crates/krishiv-plan/src/analyzer/exprs.rs @@ -2496,7 +2496,7 @@ fn call( // Spark's name of `first(x, ignoreNulls)` leaves the flag out. _ if k.len() == 2 && matches!(k[1], "true" | "false") - && ["first", "last", "first_value", "last_value"] + && ["first", "last", "first_value", "last_value", "any_value"] .iter() .any(|n| name.eq_ignore_ascii_case(n)) => { diff --git a/crates/krishiv-session/tests/aggregate_goldens.rs b/crates/krishiv-session/tests/aggregate_goldens.rs new file mode 100644 index 000000000..9086954df --- /dev/null +++ b/crates/krishiv-session/tests/aggregate_goldens.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Spark 4.2 `SQLQueryTestSuite` goldens for the simple aggregates +//! (`every`, `some`, `any`, `bool_and`, `bool_or`, `any_value`): each claimed +//! statement of the vendored `group-by` and `window` result files runs +//! through a session over the tables the file creates, and its schema, rows +//! (in order when the statement orders them) or error class, SQLSTATE and +//! message parameters must be Spark's. +//! +//! Claimed: the `group-by.sql` statements on `test_agg` (the boolean +//! aggregates grouped, global, with `HAVING`, in subqueries, in windows and +//! with wrong argument types, but not `every("true")`, which needs a string +//! to boolean cast) and the `window.sql` statement of +//! `first_value`/`any_value`/`last_value` over `()`. +//! +//! Not claimed (they need other functions or syntax): the other `group-by` +//! statements, which test `count`, `sum`, maps and arrays, and the +//! `first`/`last`/`any_value` ones over `t1` (they use `GROUP BY ALL` and a +//! correlated `EXISTS`); +//! the `window.sql` statements that also use `nth_value`, `lead`, `lag`, +//! `stddev`, `collect_list` and the like; the `try_aggregates.sql` ones +//! (`try_sum`, `try_avg`); every `max-min-by-k.sql` statement (all use the +//! three-argument form, which returns an array). +#![forbid(unsafe_code)] + +use std::path::Path; +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::types::{Int32Type, Int64Type}; +use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::executor::block_on; +use krishiv_common::error::KrishivError; +use krishiv_session::Session; +use serde_json::Value; + +type R = Result>; + +/// One golden statement: the query and its expected schema and output. +struct Case { + query: String, + schema: String, + output: String, +} + +fn load(file: &str) -> R> { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/spark-golden/v4.2.0/results") + .join(file); + let text = std::fs::read_to_string(path)?; + let blocks: Vec<&str> = text.split("-- !query\n").skip(1).collect(); + let cases: Vec = blocks + .iter() + .filter_map(|block| { + let (query, rest) = block.split_once("\n-- !query schema\n")?; + let (schema, output) = rest.split_once("\n-- !query output\n")?; + Some(Case { + query: query.trim().to_owned(), + schema: schema.trim().to_owned(), + output: output.trim_end().to_owned(), + }) + }) + .collect(); + if cases.len() != blocks.len() { + return Err(format!("dropped statements of {file}").into()); + } + Ok(cases) +} + +/// The tuples of a `VALUES` list as text fields (top-level commas only). +fn tuples(values: &str) -> Vec> { + let mut rows = Vec::new(); + let (mut depth, mut quote) = (0usize, None::); + let mut field = String::new(); + let mut row: Vec = Vec::new(); + for ch in values.chars() { + match (quote, ch) { + (Some(q), c) => { + field.push(c); + if c == q { + quote = None; + } + } + (None, '"' | '\'') => { + field.push(ch); + quote = Some(ch); + } + (None, '(') => { + depth += 1; + if depth > 1 { + field.push(ch); + } + } + (None, ')') => { + depth -= 1; + if depth == 0 { + row.push(field.trim().to_owned()); + field.clear(); + rows.push(std::mem::take(&mut row)); + } else { + field.push(ch); + } + } + (None, ',') if depth == 1 => { + row.push(field.trim().to_owned()); + field.clear(); + } + (None, c) if depth >= 1 => field.push(c), + _ => {} + } + } + rows +} + +/// Registers the view `name` the setup statement `create` makes, with the +/// columns `keep` (indices of its column list) typed from their values. +fn register(s: &Session, create: &str, name: &str, keep: &[usize]) -> R { + let from = create.find("FROM VALUES").ok_or("no VALUES")? + "FROM VALUES".len(); + let as_at = create.rfind(&format!("AS {name}(")).ok_or("no alias")?; + let columns: Vec<&str> = create[as_at..] + .split_once('(') + .and_then(|(_, c)| c.split_once(')')) + .ok_or("no column list")? + .0 + .split(',') + .map(str::trim) + .collect(); + let rows = tuples(&create[from..as_at]); + let mut fields = Vec::new(); + let mut arrays: Vec = Vec::new(); + for &c in keep { + let col: Vec> = rows + .iter() + .map(|r| r.get(c).map(String::as_str).filter(|v| *v != "null")) + .collect(); + let nullable = col.iter().any(Option::is_none); + let name = columns.get(c).ok_or("column out of range")?; + let (t, a): (DataType, ArrayRef) = + if col.iter().flatten().all(|v| *v == "true" || *v == "false") { + let a = BooleanArray::from_iter(col.iter().map(|v| v.map(|v| v == "true"))); + (DataType::Boolean, Arc::new(a)) + } else if col.iter().flatten().all(|v| v.parse::().is_ok()) { + let a = Int32Array::from_iter(col.iter().map(|v| v.and_then(|v| v.parse().ok()))); + (DataType::Int32, Arc::new(a)) + } else { + let a = StringArray::from_iter( + col.iter().map(|v| v.map(|v| v.trim_matches(['"', '\'']))), + ); + (DataType::Utf8, Arc::new(a)) + }; + fields.push(Field::new(*name, t, nullable)); + arrays.push(a); + } + let schema = Arc::new(Schema::new(fields)); + let batch = RecordBatch::try_new(Arc::clone(&schema), arrays)?; + s.register_batches(name, schema, vec![batch])?; + Ok(()) +} + +fn spark_type(t: &DataType) -> String { + match t { + DataType::Int32 => "int".into(), + DataType::Int64 => "bigint".into(), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => "string".into(), + DataType::Boolean => "boolean".into(), + other => format!("{other}"), + } +} + +fn cell(c: &dyn Array, row: usize) -> String { + if c.is_null(row) { + return "NULL".into(); + } + match c.data_type() { + DataType::Int32 => c.as_primitive::().value(row).to_string(), + DataType::Int64 => c.as_primitive::().value(row).to_string(), + DataType::Boolean => c.as_boolean().value(row).to_string(), + DataType::Utf8 => c.as_string::().value(row).to_owned(), + DataType::Utf8View => c.as_string_view().value(row).to_owned(), + other => format!("<{other}>"), + } +} + +fn render(schema: &Schema, batches: &[RecordBatch]) -> (String, Vec) { + let columns: Vec = schema + .fields() + .iter() + .map(|f| format!("{}:{}", f.name(), spark_type(f.data_type()))) + .collect(); + let rows = batches + .iter() + .flat_map(|b| { + (0..b.num_rows()).map(|r| { + b.columns() + .iter() + .map(|c| cell(c.as_ref(), r)) + .collect::>() + .join("\t") + }) + }) + .collect(); + (format!("struct<{}>", columns.join(",")), rows) +} + +/// Whether `query` has an `ORDER BY` at parenthesis depth 0, outside +/// string literals, quoted identifiers and comments. An `ORDER BY` inside +/// `OVER (...)` orders only the window frame, as in Spark, and does not +/// fix the order of the query's rows: the engine runs window partitions +/// in parallel across hash buckets, so rows come back in bucket order. +/// (Same approximation as `krishiv_testkit::golden::parse::text_sorted`.) +fn outer_order_by(query: &str) -> bool { + let mut top = String::with_capacity(query.len()); + let mut depth = 0_usize; + let mut chars = query.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\'' | '"' | '`' => { + while let Some(n) = chars.next() { + if n == '\\' { + chars.next(); + } else if n == c { + break; + } + } + top.push(' '); + } + '-' if chars.peek() == Some(&'-') => { + for n in chars.by_ref() { + if n == '\n' { + break; + } + } + top.push(' '); + } + '/' if chars.peek() == Some(&'*') => { + let mut prev = ' '; + for n in chars.by_ref() { + if prev == '*' && n == '/' { + break; + } + prev = n; + } + top.push(' '); + } + '(' => { + depth += 1; + top.push(' '); + } + ')' => { + depth = depth.saturating_sub(1); + top.push(' '); + } + _ if depth == 0 => top.push(c.to_ascii_lowercase()), + _ => {} + } + } + let mut words = top + .split(|c: char| !c.is_ascii_alphanumeric() && c != '_') + .filter(|w| !w.is_empty()) + .peekable(); + while let Some(w) = words.next() { + if w == "order" && words.peek() == Some(&"by") { + return true; + } + } + false +} + +/// The JSON body of a golden error: class, SQLSTATE, message parameters. +fn expected_error(output: &str) -> R<(String, String, Value)> { + let body = output.split_once('\n').ok_or("an error without a body")?.1; + let json: Value = serde_json::Deserializer::from_str(body) + .into_iter::() + .next() + .ok_or("no error JSON")??; + let text = |k: &str| json[k].as_str().unwrap_or_default().to_owned(); + Ok(( + text("errorClass"), + text("sqlState"), + json["messageParameters"].clone(), + )) +} + +/// Runs `case` over `s`; the disagreement with Spark, if any. +fn run(s: &Session, case: &Case) -> R> { + let result = block_on(async { + let df = s.sql(&case.query).await?; + let schema = df.schema().await?; + Ok::<_, KrishivError>((schema, df.collect().await?)) + }); + if case.schema == "struct<>" { + let (class, state, params) = expected_error(&case.output)?; + let Err(e) = result else { + return Ok(Some(format!("{}\n expected error {class}", case.query))); + }; + let got: serde_json::Map = e + .message_parameters() + .iter() + .map(|(k, v)| ((*k).to_owned(), Value::String(v.clone()))) + .collect(); + let same = e.error_class().name() == class + && e.sql_state() == Some(state.as_str()) + && Value::Object(got.clone()) == params; + return Ok((!same).then(|| { + format!( + "{}\n error {} {:?} {got:?}\n want {class} {state} {params}", + case.query, + e.error_class().name(), + e.sql_state() + ) + })); + } + let (schema, batches) = match result { + Ok(r) => r, + Err(e) => return Ok(Some(format!("{}\n error {e}", case.query))), + }; + let (schema, mut rows) = render(&schema, &batches); + let mut want: Vec = case.output.lines().map(str::to_owned).collect(); + // Only an outer `ORDER BY` fixes the order of Spark's rows; an + // `ORDER BY` inside `OVER (...)` orders only the window frame. + if !outer_order_by(&case.query) { + rows.sort(); + want.sort(); + } + Ok((schema != case.schema || rows != want).then(|| { + format!( + "{}\n schema {schema} want {}\n rows {rows:?}\n want {want:?}", + case.query, case.schema + ) + })) +} + +/// The setup statement creating view `name`. +fn setup<'a>(cases: &'a [Case], name: &str) -> R<&'a str> { + cases + .iter() + .find(|c| c.query.contains(&format!("VIEW {name} AS"))) + .map(|c| c.query.as_str()) + .ok_or_else(|| format!("no setup of {name}").into()) +} + +fn check(bad: &[String], claimed: usize, want: usize) -> R { + if claimed != want { + return Err(format!("{claimed} statements claimed, expected {want}").into()); + } + if bad.is_empty() { + Ok(()) + } else { + Err(format!("{} mismatches:\n{}", bad.len(), bad.join("\n")).into()) + } +} + +/// The boolean aggregates of `group-by.sql`: every statement that calls one. +#[test] +fn boolean_aggregates_match_spark() -> R { + let cases = load("group-by.sql.out")?; + let create = setup(&cases, "test_agg")?; + let names = ["every(", "some(", "any(", "bool_and(", "bool_or("]; + let mut bad = Vec::new(); + let mut claimed = 0; + for case in cases.iter().filter(|c| { + let q = c.query.to_ascii_lowercase(); + c.query != create && names.iter().any(|n| q.contains(n)) && !c.query.contains("\"true\"") + }) { + claimed += 1; + let s = Session::new()?; + register(&s, create, "test_agg", &[0, 1])?; + bad.extend(run(&s, case)?); + } + check(&bad, claimed, 18) +} + +/// `first_value`, `any_value` and `last_value` over an empty window +/// definition (`WINDOW w AS ()`), with and without `ignoreNulls`. +#[test] +fn any_value_over_the_whole_partition_matches_spark() -> R { + let cases = load("window.sql.out")?; + let create = setup(&cases, "testData")?; + let mut bad = Vec::new(); + let mut claimed = 0; + for case in cases + .iter() + .filter(|c| c.query.contains("any_value(false) OVER w AS any_value")) + { + claimed += 1; + let s = Session::new()?; + // Only `val` and `cate` are read: columns 0 and 5. + register(&s, create, "testData", &[0, 5])?; + bad.extend(run(&s, case)?); + } + check(&bad, claimed, 1) +} diff --git a/crates/krishiv-session/tests/aggregate_simple.rs b/crates/krishiv-session/tests/aggregate_simple.rs new file mode 100644 index 000000000..81253c818 --- /dev/null +++ b/crates/krishiv-session/tests/aggregate_simple.rs @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `count_if`, `any_value`, `max_by` and `min_by` through SQL: the planner's +//! rewrite of `count_if` to `count`, the typing errors, grouped, global and +//! window forms. Expected values follow Spark 4.2's definitions (the +//! boolean aggregates, `every`/`some`/`any`/`bool_and`/`bool_or`, are +//! checked against its goldens in `aggregate_goldens.rs`). +#![forbid(unsafe_code)] + +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::types::{Float64Type, Int32Type, Int64Type}; +use arrow_array::{ + Array, ArrayRef, BooleanArray, Float64Array, Int32Array, RecordBatch, StringArray, +}; +use arrow_schema::{DataType, Field, Schema}; +use futures::executor::block_on; +use krishiv_common::error::KrishivError; +use krishiv_session::Session; + +type R = Result>; + +/// Seven rows `(id, k, v, x, y, d)`: `k` groups them (1: ids 1–4, 2: ids +/// 5–6, 3: id 7); `y` and `d` hold NULLs, ties (`0.0 = -0.0`, `y = 7`), +/// NaN and infinity. +fn session() -> R { + let s = Session::new()?; + let nan = f64::NAN; + let columns: Vec<(&str, DataType, ArrayRef)> = vec![ + ( + "id", + DataType::Int32, + Arc::new(Int32Array::from((1..=7).collect::>())), + ), + ( + "k", + DataType::Int32, + Arc::new(Int32Array::from(vec![1, 1, 1, 1, 2, 2, 3])), + ), + ( + "v", + DataType::Boolean, + Arc::new(BooleanArray::from(vec![ + Some(true), + Some(false), + None, + Some(true), + None, + None, + Some(false), + ])), + ), + ( + "x", + DataType::Utf8, + Arc::new(StringArray::from(vec![ + Some("a"), + Some("b"), + Some("c"), + Some("d"), + None, + Some("p"), + Some("q"), + ])), + ), + ( + "y", + DataType::Int32, + Arc::new(Int32Array::from(vec![ + Some(3), + None, + Some(7), + Some(7), + None, + Some(2), + Some(9), + ])), + ), + ( + "d", + DataType::Float64, + Arc::new(Float64Array::from(vec![ + Some(0.0), + Some(-0.0), + Some(nan), + Some(f64::INFINITY), + None, + Some(nan), + Some(nan), + ])), + ), + ]; + let schema = Arc::new(Schema::new( + columns + .iter() + .map(|(n, t, _)| Field::new(*n, t.clone(), true)) + .collect::>(), + )); + let arrays = columns.into_iter().map(|c| c.2).collect(); + let batch = RecordBatch::try_new(Arc::clone(&schema), arrays)?; + s.register_batches("t", schema, vec![batch])?; + Ok(s) +} + +fn cell(c: &dyn Array, row: usize) -> String { + if c.is_null(row) { + return "NULL".into(); + } + match c.data_type() { + DataType::Int32 => c.as_primitive::().value(row).to_string(), + DataType::Int64 => c.as_primitive::().value(row).to_string(), + DataType::Boolean => c.as_boolean().value(row).to_string(), + DataType::Float64 => format!("{:?}", c.as_primitive::().value(row)), + DataType::Utf8 => c.as_string::().value(row).to_owned(), + DataType::Utf8View => c.as_string_view().value(row).to_owned(), + other => format!("<{other}>"), + } +} + +/// The column names and types, and the rows (`a|b|c`) of `sql`, in order. +fn query(s: &Session, sql: &str) -> Result<(Vec, Vec), KrishivError> { + block_on(async { + let df = s.sql(sql).await?; + let schema = df.schema().await?; + let batches = df.collect().await?; + let fields = schema + .fields() + .iter() + .map(|f| format!("{}:{}", f.name(), f.data_type())) + .collect(); + let rows = batches + .iter() + .flat_map(|b| { + (0..b.num_rows()).map(|r| { + let cells: Vec = + b.columns().iter().map(|c| cell(c.as_ref(), r)).collect(); + cells.join("|") + }) + }) + .collect(); + Ok((fields, rows)) + }) +} + +fn rows(s: &Session, sql: &str) -> R> { + Ok(query(s, sql)?.1) +} + +/// The error class and message parameters of `sql`, which must fail. +fn error(s: &Session, sql: &str) -> R<(String, Vec<(String, String)>)> { + match query(s, sql) { + Ok(_) => Err(format!("{sql} did not fail").into()), + Err(e) => Ok(( + e.error_class().name().to_owned(), + e.message_parameters() + .iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect(), + )), + } +} + +fn param<'a>(params: &'a [(String, String)], key: &str) -> Option<&'a str> { + params + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) +} + +/// `count_if(c)` counts the rows where `c` is true: `bigint`, never NULL, +/// 0 for no rows and for NULL conditions; `DISTINCT` counts at most one. +#[test] +fn count_if_counts_true_rows() -> R { + let s = session()?; + let (names, got) = query( + &s, + "SELECT k, count_if(v), count_if(v IS NULL), count_if(k = 1) \ + FROM t GROUP BY k ORDER BY k", + )?; + assert_eq!( + names, + [ + "k:Int32", + "count_if(v):Int64", + "count_if((v IS NULL)):Int64", + "count_if((k = 1)):Int64" + ] + ); + assert_eq!(got, ["1|2|1|4", "2|0|2|0", "3|0|0|0"]); + // Global: one row, 0 without rows or without a true one. + assert_eq!(rows(&s, "SELECT count_if(v) FROM t")?, ["2"]); + assert_eq!(rows(&s, "SELECT count_if(v) FROM t WHERE 1 = 0")?, ["0"]); + assert_eq!(rows(&s, "SELECT count_if(v) FROM t WHERE k = 2")?, ["0"]); + assert_eq!( + rows(&s, "SELECT count_if(true), count_if(false) FROM t")?, + ["7|0"] + ); + // A grouped aggregate over no rows has no group. + assert!(rows(&s, "SELECT k, count_if(v) FROM t WHERE 1 = 0 GROUP BY k")?.is_empty()); + assert_eq!( + rows( + &s, + "SELECT k, count_if(DISTINCT v) FROM t GROUP BY k ORDER BY k" + )?, + ["1|1", "2|0", "3|0"] + ); + assert_eq!( + rows( + &s, + "SELECT k FROM t GROUP BY k HAVING count_if(v) > 1 ORDER BY k" + )?, + ["1"] + ); + // Not nullable: the result of the aggregate is `count`'s. + assert_eq!( + rows(&s, "SELECT count_if(v) + 1 FROM t WHERE k = 3")?, + ["1"] + ); + Ok(()) +} + +#[test] +fn count_if_in_windows() -> R { + let s = session()?; + assert_eq!( + rows( + &s, + "SELECT id, count_if(v) OVER (PARTITION BY k ORDER BY id) FROM t ORDER BY id" + )?, + ["1|1", "2|1", "3|1", "4|2", "5|0", "6|0", "7|0"] + ); + assert_eq!( + rows( + &s, + "SELECT id, count_if(v) OVER (PARTITION BY k) FROM t ORDER BY id" + )?, + ["1|2", "2|2", "3|2", "4|2", "5|0", "6|0", "7|0"] + ); + Ok(()) +} + +/// The argument must be a `boolean` (a string is cast, a number is an +/// error), as `DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE` with the registry's +/// required type. +#[test] +fn count_if_checks_its_argument_type() -> R { + let s = session()?; + for (sql, input_sql, input_type) in [ + ("SELECT count_if(1)", "\"1\"", "\"INT\""), + ("SELECT count_if(y) FROM t", "\"y\"", "\"INT\""), + ("SELECT count_if(d) FROM t", "\"d\"", "\"DOUBLE\""), + ] { + let (class, params) = error(&s, sql)?; + assert_eq!(class, "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", "{sql}"); + assert_eq!(param(¶ms, "requiredType"), Some("\"BOOLEAN\"")); + assert_eq!(param(¶ms, "paramIndex"), Some("first")); + assert_eq!(param(¶ms, "inputSql"), Some(input_sql)); + assert_eq!(param(¶ms, "inputType"), Some(input_type)); + assert!(param(¶ms, "sqlExpr").is_some_and(|e| e.contains("count_if("))); + } + let (class, _) = error(&s, "SELECT count_if(v, v) FROM t")?; + assert_eq!(class, "WRONG_NUM_ARGS.WITHOUT_SUGGESTION"); + // The boolean aggregates take the same check. + let (class, params) = error(&s, "SELECT bool_and(y) FROM t")?; + assert_eq!(class, "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE"); + assert_eq!(param(¶ms, "requiredType"), Some("\"BOOLEAN\"")); + Ok(()) +} + +/// Spark 4.2: the value of the row with the extreme non-NULL key, the first +/// such row on a tie; NULL keys are skipped; NaN is the largest `double`, +/// `-0.0 = 0.0`. +#[test] +fn max_by_and_min_by() -> R { + let s = session()?; + let (names, got) = query( + &s, + "SELECT k, max_by(x, y), min_by(x, y) FROM t GROUP BY k ORDER BY k", + )?; + assert_eq!( + names, + ["k:Int32", "max_by(x, y):Utf8View", "min_by(x, y):Utf8View"] + ); + // Group 1: keys 3, NULL, 7, 7: the largest is the first 7 (`c`). + // Group 2: the NULL key of the first row is skipped. + assert_eq!(got, ["1|c|a", "2|p|p", "3|q|q"]); + assert_eq!( + rows(&s, "SELECT max_by(x, y), min_by(x, y) FROM t")?, + ["q|p"] + ); + // Doubles: NaN beats infinity; `0.0` and `-0.0` tie, the first wins. + assert_eq!( + rows( + &s, + "SELECT k, max_by(y, d), min_by(y, d) FROM t GROUP BY k ORDER BY k" + )?, + ["1|7|3", "2|2|2", "3|9|9"] + ); + // Expressions as arguments; a NULL value is a result. + assert_eq!(rows(&s, "SELECT max_by(y, id) FROM t WHERE k = 1")?, ["7"]); + assert_eq!( + rows( + &s, + "SELECT max_by(y, id), min_by(y, id) FROM t WHERE k = 1 AND id < 3" + )?, + ["NULL|3"] + ); + // No rows, or only NULL keys: NULL. + assert_eq!( + rows(&s, "SELECT max_by(x, y), min_by(x, y) FROM t WHERE 1 = 0")?, + ["NULL|NULL"] + ); + assert_eq!( + rows(&s, "SELECT max_by(x, y), min_by(x, y) FROM t WHERE id = 5")?, + ["NULL|NULL"] + ); + assert!(rows(&s, "SELECT k, max_by(x, y) FROM t WHERE 1 = 0 GROUP BY k")?.is_empty()); + Ok(()) +} + +#[test] +fn max_by_and_min_by_in_windows() -> R { + let s = session()?; + assert_eq!( + rows( + &s, + "SELECT id, max_by(x, y) OVER (PARTITION BY k), \ + min_by(x, y) OVER (PARTITION BY k) FROM t ORDER BY id" + )?, + [ + "1|c|a", "2|c|a", "3|c|a", "4|c|a", "5|p|p", "6|p|p", "7|q|q" + ] + ); + // A running frame: the first row wins ties (`-0.0` after `0.0`, `inf` + // after NaN), a NULL key leaves the state as it was. + assert_eq!( + rows( + &s, + "SELECT id, max_by(y, d) OVER (PARTITION BY k ORDER BY id) FROM t ORDER BY id" + )?, + ["1|3", "2|3", "3|7", "4|7", "5|NULL", "6|2", "7|9"] + ); + // Strings would be copied for every row of a running frame. + let (class, _) = error( + &s, + "SELECT max_by(x, y) OVER (PARTITION BY k ORDER BY id) FROM t", + )?; + assert_eq!(class, "KRISHIV_UNSUPPORTED_FEATURE"); + Ok(()) +} + +/// `max_by(x, y, k)` returns an array: not supported, and the wrong number +/// of arguments is Spark's error. +#[test] +fn max_by_with_a_limit_is_unsupported() -> R { + let s = session()?; + for f in ["max_by", "min_by"] { + let (class, params) = error(&s, &format!("SELECT {f}(x, y, 2) FROM t"))?; + assert_eq!(class, "KRISHIV_UNSUPPORTED_FEATURE"); + assert!(param(¶ms, "feature").is_some_and(|m| m.contains("array"))); + let (class, _) = error(&s, &format!("SELECT {f}(x) FROM t"))?; + assert_eq!(class, "WRONG_NUM_ARGS.WITHOUT_SUGGESTION"); + } + // The other aggregates that return arrays stay unsupported. + for f in ["collect_list", "collect_set"] { + let (class, _) = error(&s, &format!("SELECT {f}(x) FROM t"))?; + assert_eq!(class, "KRISHIV_UNSUPPORTED_FEATURE", "{f}"); + } + Ok(()) +} + +/// `any_value(x)` is the first row's value, `any_value(x, true)` the first +/// non-NULL one; the flag must be a boolean literal and is not part of the +/// column's name (Spark: `any_value(col)`). +#[test] +fn any_value_returns_the_first_value() -> R { + let s = session()?; + let (names, got) = query( + &s, + "SELECT k, any_value(x), any_value(x, true), any_value(x, false) \ + FROM t GROUP BY k ORDER BY k", + )?; + assert_eq!( + names, + [ + "k:Int32", + "any_value(x):Utf8View", + "any_value(x):Utf8View", + "any_value(x):Utf8View" + ] + ); + assert_eq!(got, ["1|a|a|a", "2|NULL|p|NULL", "3|q|q|q"]); + assert_eq!( + rows( + &s, + "SELECT any_value(x), any_value(y, true) FROM t WHERE 1 = 0" + )?, + ["NULL|NULL"] + ); + assert_eq!( + rows( + &s, + "SELECT id, any_value(x, true) OVER (PARTITION BY k ORDER BY id) FROM t ORDER BY id" + )?, + ["1|a", "2|a", "3|a", "4|a", "5|NULL", "6|p", "7|q"] + ); + assert_eq!( + rows( + &s, + "SELECT id, any_value(x) IGNORE NULLS OVER (PARTITION BY k ORDER BY id) \ + FROM t ORDER BY id" + )?, + ["1|a", "2|a", "3|a", "4|a", "5|NULL", "6|p", "7|q"] + ); + let (class, params) = error(&s, "SELECT any_value(x, 1 = 1) FROM t")?; + assert_eq!(class, "_LEGACY_ERROR_TEMP_1210"); + assert_eq!(param(¶ms, "funcName"), Some("any_value")); + Ok(()) +} + +/// The planner maps every registry name of an aggregate, aliases included: +/// `mean` is `avg`, grouped and in a window (one mapping serves both). +#[test] +fn aliases_run_as_their_aggregate() -> R { + let s = session()?; + assert_eq!( + rows( + &s, + "SELECT k, mean(y) FROM t WHERE k > 1 GROUP BY k ORDER BY k" + )?, + ["2|2.0", "3|9.0"] + ); + assert_eq!( + rows( + &s, + "SELECT id, mean(y) OVER (PARTITION BY k) FROM t WHERE k > 1 ORDER BY id" + )?, + ["5|2.0", "6|2.0", "7|9.0"] + ); + Ok(()) +} + +/// `DISTINCT` on the new aggregates deduplicates the `(x, y)` pairs first +/// (so it agrees with the plain form where there is no tie); `FILTER` is +/// not supported. +#[test] +fn distinct_and_filter_with_new_aggregates() -> R { + let s = session()?; + // No tie in `k = 2` or `k = 3`, so the hash order of the `DISTINCT` + // first phase cannot change the winner. + assert_eq!( + rows( + &s, + "SELECT k, max_by(DISTINCT x, y), min_by(DISTINCT x, y) \ + FROM t WHERE k > 1 GROUP BY k ORDER BY k" + )?, + ["2|p|p", "3|q|q"] + ); + assert_eq!( + rows( + &s, + "SELECT max_by(DISTINCT x, y), min_by(DISTINCT x, y) \ + FROM t WHERE k = 2" + )?, + ["p|p"] + ); + assert_eq!( + rows(&s, "SELECT any_value(DISTINCT x) FROM t WHERE k = 3")?, + ["q"] + ); + for sql in [ + "SELECT max_by(x, y) FILTER (WHERE v) FROM t", + "SELECT min_by(x, y) FILTER (WHERE v) FROM t", + "SELECT any_value(x) FILTER (WHERE v) FROM t", + ] { + let (class, params) = error(&s, sql)?; + assert_eq!(class, "KRISHIV_UNSUPPORTED_FEATURE", "{sql}"); + assert!( + param(¶ms, "feature").is_some_and(|m| m.contains("aggregate FILTER")), + "{sql}: {params:?}" + ); + } + Ok(()) +} diff --git a/crates/krishiv-sql/src/lower.rs b/crates/krishiv-sql/src/lower.rs index e0428fb6c..2da2be0ef 100644 --- a/crates/krishiv-sql/src/lower.rs +++ b/crates/krishiv-sql/src/lower.rs @@ -1361,6 +1361,7 @@ fn takes_null_treatment(name: &str) -> bool { "last", "first_value", "last_value", + "any_value", "lag", "lead", "nth_value", diff --git a/crates/krishiv-sql/tests/lower.rs b/crates/krishiv-sql/tests/lower.rs index c955f0c7a..f4657d2c3 100644 --- a/crates/krishiv-sql/tests/lower.rs +++ b/crates/krishiv-sql/tests/lower.rs @@ -128,6 +128,8 @@ fn null_treatment_sets_the_ignore_nulls_argument() { ("first(a, false) IGNORE NULLS", "first(a, true)"), ("last(a, true) RESPECT NULLS", "last(a, false)"), ("first(a, b) IGNORE NULLS", "first(a, b)"), + ("any_value(a) IGNORE NULLS", "any_value(a, true)"), + ("any_value(a, true) RESPECT NULLS", "any_value(a, false)"), ]; for (call, shown_call) in cases { let plan = shown(&format!("SELECT {call} FROM t")); diff --git a/tests/fixtures/analysis/registry-matrix-manifest.json b/tests/fixtures/analysis/registry-matrix-manifest.json index db2c44814..166dbc051 100644 --- a/tests/fixtures/analysis/registry-matrix-manifest.json +++ b/tests/fixtures/analysis/registry-matrix-manifest.json @@ -1,5 +1,5 @@ { - "cases": 12566, + "cases": 12784, "entries": [ { "id": "c1a16bec609/ansi_off", @@ -505,6 +505,46 @@ "id": "acosh#de074fcfa8d3/ansi_on", "reason": "SELECT acosh(INTERVAL '1 02:03:04' DAY TO SECOND) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" }, + { + "id": "any_value#04ba50b0a41d/ansi_off", + "reason": "SELECT any_value(col, INTERVAL '1 02:03:04' DAY TO SECOND) FROM VALUES (NULL), (5), (20) AS tab(col) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#04ba50b0a41d/ansi_on", + "reason": "SELECT any_value(col, INTERVAL '1 02:03:04' DAY TO SECOND) FROM VALUES (NULL), (5), (20) AS tab(col) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#6188e23671f4/ansi_off", + "reason": "SELECT any_value(col, map('a', 1)) FROM VALUES (NULL), (5), (20) AS tab(col) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#6188e23671f4/ansi_on", + "reason": "SELECT any_value(col, map('a', 1)) FROM VALUES (NULL), (5), (20) AS tab(col) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#94942fcbfe3e/ansi_off", + "reason": "SELECT any_value(col, array(1, 2)) FROM VALUES (NULL), (5), (20) AS tab(col) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#94942fcbfe3e/ansi_on", + "reason": "SELECT any_value(col, array(1, 2)) FROM VALUES (NULL), (5), (20) AS tab(col) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#a4824560fdf2/ansi_off", + "reason": "SELECT any_value(col, named_struct('a', 1, 'b', 'x')) FROM VALUES (NULL), (5), (20) AS tab(col) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#a4824560fdf2/ansi_on", + "reason": "SELECT any_value(col, named_struct('a', 1, 'b', 'x')) FROM VALUES (NULL), (5), (20) AS tab(col) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#ba624a02fb3a/ansi_off", + "reason": "SELECT any_value(col, INTERVAL '1-2' YEAR TO MONTH) FROM VALUES (NULL), (5), (20) AS tab(col) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has _LEGACY_ERROR_TEMP_1210" + }, + { + "id": "any_value#ba624a02fb3a/ansi_on", + "reason": "SELECT any_value(col, INTERVAL '1-2' YEAR TO MONTH) FROM VALUES (NULL), (5), (20) AS tab(col) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has _LEGACY_ERROR_TEMP_1210" + }, { "id": "asin#31dfa83e35e0/ansi_off", "reason": "SELECT asin(named_struct('a', 1, 'b', 'x')) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" @@ -5609,6 +5649,206 @@ "id": "ltrim#bf0dac2666b9/ansi_on", "reason": "SELECT ltrim(named_struct('a', 1, 'b', 'x')) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" }, + { + "id": "max_by#05512ce12f17/ansi_off", + "reason": "SELECT max_by(x, y, 1.5D) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#05512ce12f17/ansi_on", + "reason": "SELECT max_by(x, y, 1.5D) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#0e50ca5bcb18/ansi_off", + "reason": "SELECT max_by(x, y, 1L) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#0e50ca5bcb18/ansi_on", + "reason": "SELECT max_by(x, y, 1L) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#127d584c1fa6/ansi_off", + "reason": "SELECT max_by(x, y, CAST(1.5 AS DECIMAL(38,18))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#127d584c1fa6/ansi_on", + "reason": "SELECT max_by(x, y, CAST(1.5 AS DECIMAL(38,18))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#14dbc2514ae8/ansi_off", + "reason": "SELECT max_by(x, y, NULL) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#14dbc2514ae8/ansi_on", + "reason": "SELECT max_by(x, y, NULL) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#1b47195e7fd6/ansi_off", + "reason": "SELECT max_by(x, y, '1') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#1b47195e7fd6/ansi_on", + "reason": "SELECT max_by(x, y, '1') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#3463d4380cb1/ansi_off", + "reason": "SELECT max_by(x, y, INTERVAL '1-2' YEAR TO MONTH) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#3463d4380cb1/ansi_on", + "reason": "SELECT max_by(x, y, INTERVAL '1-2' YEAR TO MONTH) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#3910f72903d7/ansi_off", + "reason": "SELECT max_by(x, y, CAST(1 AS SMALLINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#3910f72903d7/ansi_on", + "reason": "SELECT max_by(x, y, CAST(1 AS SMALLINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#3f96e614e136/ansi_off", + "reason": "SELECT max_by(x, y, CAST(1.5 AS FLOAT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#3f96e614e136/ansi_on", + "reason": "SELECT max_by(x, y, CAST(1.5 AS FLOAT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#44dca1a0228a/ansi_off", + "reason": "SELECT max_by(x, y, CAST(1.5 AS DECIMAL(10,2))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#44dca1a0228a/ansi_on", + "reason": "SELECT max_by(x, y, CAST(1.5 AS DECIMAL(10,2))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#4e7242e71dee/ansi_off", + "reason": "SELECT max_by(x, y, 'abc') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#4e7242e71dee/ansi_on", + "reason": "SELECT max_by(x, y, 'abc') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#6d9fa4bae78f/ansi_off", + "reason": "SELECT max_by(x, y, CAST(NULL AS int)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#6d9fa4bae78f/ansi_on", + "reason": "SELECT max_by(x, y, CAST(NULL AS int)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#7f4914ee361a/ansi_off", + "reason": "SELECT max_by(x, y, 1) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#7f4914ee361a/ansi_on", + "reason": "SELECT max_by(x, y, 1) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#86fc166ee8e0/ansi_off", + "reason": "SELECT max_by(x, y, X'01') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#86fc166ee8e0/ansi_on", + "reason": "SELECT max_by(x, y, X'01') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#9207124931b4/ansi_off", + "reason": "SELECT max_by(x, y, CAST(2147483647 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#9207124931b4/ansi_on", + "reason": "SELECT max_by(x, y, CAST(2147483647 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#95f7fc49eca2/ansi_off", + "reason": "SELECT max_by(x, y, CAST(-2147483648 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#95f7fc49eca2/ansi_on", + "reason": "SELECT max_by(x, y, CAST(-2147483648 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "max_by#99ec05fac0e5/ansi_off", + "reason": "SELECT max_by(x, y, TIMESTAMP'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#99ec05fac0e5/ansi_on", + "reason": "SELECT max_by(x, y, TIMESTAMP'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#9df2dec34beb/ansi_off", + "reason": "SELECT max_by(x, y, CAST(1 AS TINYINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#9df2dec34beb/ansi_on", + "reason": "SELECT max_by(x, y, CAST(1 AS TINYINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#bc01ee66b357/ansi_off", + "reason": "SELECT max_by(x, y, named_struct('a', 1, 'b', 'x')) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#bc01ee66b357/ansi_on", + "reason": "SELECT max_by(x, y, named_struct('a', 1, 'b', 'x')) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#c45683d91f90/ansi_off", + "reason": "SELECT max_by(x, y, TIMESTAMP_NTZ'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#c45683d91f90/ansi_on", + "reason": "SELECT max_by(x, y, TIMESTAMP_NTZ'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#c55f1e8b3e18/ansi_off", + "reason": "SELECT max_by(x, y, map('a', 1)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#c55f1e8b3e18/ansi_on", + "reason": "SELECT max_by(x, y, map('a', 1)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#d67b46ca8103/ansi_off", + "reason": "SELECT max_by(x, y, DATE'2024-02-29') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#d67b46ca8103/ansi_on", + "reason": "SELECT max_by(x, y, DATE'2024-02-29') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#e2de8c6a8128/ansi_off", + "reason": "SELECT max_by(x, y, INTERVAL '1 02:03:04' DAY TO SECOND) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#e2de8c6a8128/ansi_on", + "reason": "SELECT max_by(x, y, INTERVAL '1 02:03:04' DAY TO SECOND) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#ea6e2bbfc9e7/ansi_off", + "reason": "SELECT max_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#ea6e2bbfc9e7/ansi_on", + "reason": "SELECT max_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "max_by#f0872159688c/ansi_off", + "reason": "SELECT max_by(x, y, array(1, 2)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#f0872159688c/ansi_on", + "reason": "SELECT max_by(x, y, array(1, 2)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#ff115f23cf8f/ansi_off", + "reason": "SELECT max_by(x, y, true) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "max_by#ff115f23cf8f/ansi_on", + "reason": "SELECT max_by(x, y, true) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, { "id": "md5#494fc1423cbf/ansi_off", "reason": "SELECT md5(array(1, 2)) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" @@ -5649,6 +5889,206 @@ "id": "md5#fd708175c7a6/ansi_on", "reason": "SELECT md5(INTERVAL '1-2' YEAR TO MONTH) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" }, + { + "id": "min_by#110479e28bf2/ansi_off", + "reason": "SELECT min_by(x, y, map('a', 1)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#110479e28bf2/ansi_on", + "reason": "SELECT min_by(x, y, map('a', 1)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function map where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#1db3db810285/ansi_off", + "reason": "SELECT min_by(x, y, CAST(NULL AS int)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#1db3db810285/ansi_on", + "reason": "SELECT min_by(x, y, CAST(NULL AS int)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#1e60613841d9/ansi_off", + "reason": "SELECT min_by(x, y, CAST(1 AS TINYINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#1e60613841d9/ansi_on", + "reason": "SELECT min_by(x, y, CAST(1 AS TINYINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#24ee45e7bda4/ansi_off", + "reason": "SELECT min_by(x, y, CAST(1.5 AS DECIMAL(10,2))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#24ee45e7bda4/ansi_on", + "reason": "SELECT min_by(x, y, CAST(1.5 AS DECIMAL(10,2))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#25cff7efd50b/ansi_off", + "reason": "SELECT min_by(x, y, INTERVAL '1-2' YEAR TO MONTH) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#25cff7efd50b/ansi_on", + "reason": "SELECT min_by(x, y, INTERVAL '1-2' YEAR TO MONTH) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#337188cc2002/ansi_off", + "reason": "SELECT min_by(x, y, CAST(1.5 AS FLOAT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#337188cc2002/ansi_on", + "reason": "SELECT min_by(x, y, CAST(1.5 AS FLOAT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#38f797b62e71/ansi_off", + "reason": "SELECT min_by(x, y, TIMESTAMP'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#38f797b62e71/ansi_on", + "reason": "SELECT min_by(x, y, TIMESTAMP'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#446282845e4c/ansi_off", + "reason": "SELECT min_by(x, y, 1) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#446282845e4c/ansi_on", + "reason": "SELECT min_by(x, y, 1) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#5863f1218462/ansi_off", + "reason": "SELECT min_by(x, y, 1L) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#5863f1218462/ansi_on", + "reason": "SELECT min_by(x, y, 1L) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#5f34f8cd49c2/ansi_off", + "reason": "SELECT min_by(x, y, TIMESTAMP_NTZ'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#5f34f8cd49c2/ansi_on", + "reason": "SELECT min_by(x, y, TIMESTAMP_NTZ'2024-02-29 12:34:56.123456') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#676312ae136e/ansi_off", + "reason": "SELECT min_by(x, y, INTERVAL '1 02:03:04' DAY TO SECOND) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#676312ae136e/ansi_on", + "reason": "SELECT min_by(x, y, INTERVAL '1 02:03:04' DAY TO SECOND) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#80d0ca30807a/ansi_off", + "reason": "SELECT min_by(x, y, array(1, 2)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#80d0ca30807a/ansi_on", + "reason": "SELECT min_by(x, y, array(1, 2)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function array where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#897e17cdfb93/ansi_off", + "reason": "SELECT min_by(x, y, 1.5D) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#897e17cdfb93/ansi_on", + "reason": "SELECT min_by(x, y, 1.5D) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#8f53d124656c/ansi_off", + "reason": "SELECT min_by(x, y, CAST(1 AS SMALLINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#8f53d124656c/ansi_on", + "reason": "SELECT min_by(x, y, CAST(1 AS SMALLINT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#9f369d142638/ansi_off", + "reason": "SELECT min_by(x, y, X'01') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#9f369d142638/ansi_on", + "reason": "SELECT min_by(x, y, X'01') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#9fbbe5ee2d5b/ansi_off", + "reason": "SELECT min_by(x, y, CAST(1.5 AS DECIMAL(38,18))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#9fbbe5ee2d5b/ansi_on", + "reason": "SELECT min_by(x, y, CAST(1.5 AS DECIMAL(38,18))) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#a4dd19d714cf/ansi_off", + "reason": "SELECT min_by(x, y, true) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#a4dd19d714cf/ansi_on", + "reason": "SELECT min_by(x, y, true) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#b0ca70d894fe/ansi_off", + "reason": "SELECT min_by(x, y, '1') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#b0ca70d894fe/ansi_on", + "reason": "SELECT min_by(x, y, '1') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#c283a1906ffa/ansi_off", + "reason": "SELECT min_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#c283a1906ffa/ansi_on", + "reason": "SELECT min_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#c5734adff05f/ansi_off", + "reason": "SELECT min_by(x, y, NULL) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#c5734adff05f/ansi_on", + "reason": "SELECT min_by(x, y, NULL) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#dca05b6fd08d/ansi_off", + "reason": "SELECT min_by(x, y, named_struct('a', 1, 'b', 'x')) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#dca05b6fd08d/ansi_on", + "reason": "SELECT min_by(x, y, named_struct('a', 1, 'b', 'x')) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: function named_struct where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#e72b3df6398f/ansi_off", + "reason": "SELECT min_by(x, y, DATE'2024-02-29') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#e72b3df6398f/ansi_on", + "reason": "SELECT min_by(x, y, DATE'2024-02-29') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE" + }, + { + "id": "min_by#ecbbd72e0aa5/ansi_off", + "reason": "SELECT min_by(x, y, CAST(-2147483648 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#ecbbd72e0aa5/ansi_on", + "reason": "SELECT min_by(x, y, CAST(-2147483648 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#f239b696d6ea/ansi_off", + "reason": "SELECT min_by(x, y, 'abc') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#f239b696d6ea/ansi_on", + "reason": "SELECT min_by(x, y, 'abc') FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array)" + }, + { + "id": "min_by#f4262cb06090/ansi_off", + "reason": "SELECT min_by(x, y, CAST(2147483647 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, + { + "id": "min_by#f4262cb06090/ansi_on", + "reason": "SELECT min_by(x, y, CAST(2147483647 AS INT)) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y) | KRISHIV_UNSUPPORTED_FEATURE: max_by and min_by with a third argument (they return an array) where Spark has DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE" + }, { "id": "mod#05114d19d1a3/ansi_off", "reason": "SELECT MOD(INTERVAL '1 02:03:04' DAY TO SECOND, 1.8) | parse: KRISHIV_UNSUPPORTED_FEATURE: INTERVAL literals where Spark has DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES" @@ -11584,6 +12024,16 @@ "function": "acosh", "pass": 40 }, + { + "cases": 10, + "function": "any", + "pass": 10 + }, + { + "cases": 54, + "function": "any_value", + "pass": 44 + }, { "cases": 52, "function": "asin", @@ -11634,6 +12084,16 @@ "function": "bit_get", "pass": 80 }, + { + "cases": 10, + "function": "bool_and", + "pass": 10 + }, + { + "cases": 10, + "function": "bool_or", + "pass": 10 + }, { "cases": 106, "function": "bround", @@ -11704,6 +12164,11 @@ "function": "count", "pass": 12 }, + { + "cases": 10, + "function": "count_if", + "pass": 10 + }, { "cases": 54, "function": "crc32", @@ -11804,6 +12269,11 @@ "function": "equal_null", "pass": 86 }, + { + "cases": 10, + "function": "every", + "pass": 10 + }, { "cases": 50, "function": "exp", @@ -11974,6 +12444,11 @@ "function": "max", "pass": 8 }, + { + "cases": 52, + "function": "max_by", + "pass": 2 + }, { "cases": 54, "function": "md5", @@ -11989,6 +12464,11 @@ "function": "min", "pass": 8 }, + { + "cases": 52, + "function": "min_by", + "pass": 2 + }, { "cases": 100, "function": "mod", @@ -12214,6 +12694,11 @@ "function": "sinh", "pass": 40 }, + { + "cases": 10, + "function": "some", + "pass": 10 + }, { "cases": 50, "function": "sqrt", @@ -12345,5 +12830,5 @@ "pass": 40 } ], - "pass": 9681 + "pass": 9789 } diff --git a/tests/spark-golden/known-failures.json b/tests/spark-golden/known-failures.json index 1f6b65e02..e3a903817 100644 --- a/tests/spark-golden/known-failures.json +++ b/tests/spark-golden/known-failures.json @@ -489,7 +489,7 @@ "extract-value-resolution-edge-cases": {"depends-on-command":[1,2],"nested-interval-types":[5,6,7],"query-features":[3],"sql-commands":[0,4]}, "generators": {"depends-on-command":[1,2,3,10,11,12,19,20,21,28,29,30,39,40,46,47],"fn-json":[51,52,53,54,55],"generators":[4,5,7,8,13,14,16,17,22,23,25,26,31,32,34,35,41,42,48,49,56,57,58,59,60],"nested-interval-types":[6,9,15,18,24,27,33,36,37,38,43,44,45,50]}, "generators-resolution-edge-cases": {"fn-array":[24,25,26,28],"generators":[0,1,2,3,4,5,10,11,12,13,27,29,32,33,34,35,44,45,46,47,48,49],"query-features":[6,7,8,9,14,15,16,17,18,19,20,21,22,23,36,37,38,39,40,41,42,43,50,51,52,53],"window":[30,31]}, - "group-by": {"depends-on-command":[9,10,14,15,34,35,36,37,38,39,40,41,48,49,50,51,52,61,96,97,98,99,100,101],"fn-aggregate":[42,43,44,45,46,47],"query-features":[16,24,25,26,27,29,30,31,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,82,83,84,85,86,87,88,89,90,91,92,93,94,95],"sql-commands":[1,102]}, + "group-by": {"depends-on-command":[9,10,14,15,61,96,97,98,99,100,101],"fn-aggregate":[45],"query-features":[16,24,25,26,27,29,30,31,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,82,83,84,85,86,87,88,89,90,91,92,93,94,95],"sql-commands":[1,102]}, "group-by-alias": {"depends-on-command":[8,16,17],"query-features":[28,35]}, "group-by-all": {"depends-on-command":[1,2,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,25,26,28],"query-features":[27]}, "group-by-all-duckdb": {"depends-on-command":[1,2,3,4,5,6],"query-features":[7,8]}, @@ -498,7 +498,7 @@ "group-by-ordinal": {"depends-on-command":[13]}, "grouping_set": {"depends-on-command":[5]}, "grouping_set_grand_total_disabled": {"query-features":[0,1]}, - "having": {"depends-on-command":[3,19,21,22,28,29,30,31,32],"query-features":[6,7,8,9,24,25,33,34,35,36,41,44,45,46,47,49]}, + "having": {"depends-on-command":[3,19,21,22,28,29,30,31,32],"query-features":[6,7,8,9,24,25,33,34,35,36,41,44,45,46,49]}, "having-and-order-by-recursive-type-name-resolution": {"query-features":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44]}, "higher-order-functions": {"depends-on-command":[3,4,5,6,7,9,11,12,13,14,16,17,18,20,22,26,27,28,29,30,31],"hof":[1,2,23,24,34,35,36,38,39,40,42,43,44,46,47,48,50,51,52,54,55,56,58,59,60,62,63,64,66,67,68,70,71,72,74,75,76,78,79,80,82],"nested-interval-types":[8,10,15,19,21,37,41,45,49,53,57,61,65,69,73,77,81],"query-features":[32,33],"sql-commands":[0,25]}, "hll": {"depends-on-command":[8,9,10,11,12,13,14,15,16,17],"fn-aggregate":[31,32,33],"query-features":[18,19,20,21,22,23,24,25,26,27,28,29,30],"sql-commands":[0,1,2,3,4,5,6,7,34,35,36]}, @@ -573,7 +573,7 @@ "pipe-operators": {"depends-on-command":[25,26,27,28,29,30,31,32,33,34,37,38,39,41,42,43,44,45,47,48,49,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,88,89,92,93,94,95,96,99,100,101,103,105,106,107,110,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,184,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,205,206,207,208,209,210,211,212,214,215,216,217,218,219,220,233,234,235,238,241,242,244,245,246,247,248,250,251,252,254,256,257,258,259,260,261,262,263,264,265,266,267,268,269,277,281,282,283,284,285,287,288,290,296,300,301,302,303,304,305,307,308,310,311,312,313],"parser-pipe-collate":[35,40,46,50,90,91,97,98,102,104,166,204,221,222,223,224,225,226,227,228,229,230,231,232,236,237,239,240,243,255,270,271,272,273,274,275,276,278,279,280,286,306,309],"sql-commands":[0,1,2,3,4,5,6,7,8,12,19,24,314,315,316]}, "pivot": {"depends-on-command":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33],"sql-commands":[2]}, "postgreSQL/aggregates_part1": {"depends-on-command":[45,46,47,48],"fn-aggregate":[9,10,11,12,13,14,15,16,17,18,35,36,37,38,39,40,41,42,43],"query-features":[19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34],"subqueries":[54,55]}, - "postgreSQL/aggregates_part2": {"depends-on-command":[2,3,4,5,6,7,8,12,13,14,15],"generators":[27]}, + "postgreSQL/aggregates_part2": {"depends-on-command":[2,3,4,5,6,7,8,13,14,15],"generators":[27]}, "postgreSQL/aggregates_part3": {"query-features":[1,2],"subqueries":[3]}, "postgreSQL/aggregates_part4": {"fn-aggregate":[0,1,2,3]}, "postgreSQL/boolean": {"cast-matrix":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39],"depends-on-command":[44,45,46,47,49,56,57,58,59,60,61,62,63,64,65,66,67,68,73,76,77,78,79,80,81,82,83,84,85,86,87],"sql-commands":[40,41,42,43,48,50,51,52,53,54,55,69,70,71,72,74,75,88,89,90,91]}, @@ -714,7 +714,7 @@ "udaf/udaf-group-by-ordinal": {"depends-on-command":[1,2,3,4,5,6,7,8,9,10,13,14,15,16,17,18,19,20,21,22,23,24,25,26,28],"test-udfs":[11,12]}, "udaf/udaf-grouping-set": {"depends-on-command":[1,2,3,4,5,6,7,8,11,12,13,14]}, "udf/postgreSQL/udf-aggregates_part1": {"depends-on-command":[44,45,46,47],"test-udfs":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,49,50,51,52,53,54,55]}, - "udf/postgreSQL/udf-aggregates_part2": {"depends-on-command":[2,3,4,8,9,10,11],"test-udfs":[12,13,14,15,16,17,18,19,20,21,22,23,24]}, + "udf/postgreSQL/udf-aggregates_part2": {"depends-on-command":[2,3,4,9,10,11],"test-udfs":[12,13,14,15,16,17,18,19,20,21,22,23,24]}, "udf/postgreSQL/udf-aggregates_part3": {"test-udfs":[0,1]}, "udf/postgreSQL/udf-aggregates_part4": {"test-udfs":[0,1,2,3]}, "udf/postgreSQL/udf-case": {"depends-on-command":[21,22,23,24,25,26,27,28,29,30],"sql-commands":[0,1,2,3,4,5,6,7,8,9,10,11,32,33],"test-udfs":[12,13,14,15,16,17,18,19,20,31]}, @@ -726,7 +726,7 @@ "udf/udf-except": {"depends-on-command":[2,3,4,5,6,7,8]}, "udf/udf-except-all": {"depends-on-command":[4,5,6,7,8,9,10,11,12,13,14,15,17,18,19,20,21,22]}, "udf/udf-group-analytics": {"depends-on-command":[1,2,3,4,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28]}, - "udf/udf-group-by": {"depends-on-command":[1,2,3,4,5,6,7,8,9,10,11,12,13,15,16,17,19,21,22,23,24,31,32,33,34,35,36,37,38,43,44,45,46,47,48,49,50,51],"test-udfs":[14,25,26,27,28,29,39,40,41,42]}, + "udf/udf-group-by": {"depends-on-command":[1,2,3,4,5,6,7,8,9,10,11,12,13,15,16,17,19,21,22,23,24,31,32,33,34,35,36,37,38,44,45,46,47,48,49,50,51],"test-udfs":[14,25,26,27,28,29,39,40,41,42]}, "udf/udf-having": {"depends-on-command":[1,2,3],"test-udfs":[4]}, "udf/udf-inline-table": {"test-udfs":[0,1,2,3,4,5,6,7,8,9,10,12,15,16]}, "udf/udf-inner-join": {"depends-on-command":[6],"sql-commands":[4,5]}, @@ -758,7 +758,7 @@ "view-schema-evolution": {"depends-on-command":[4,9,14,20,24,30,35,40,46,50,54,64,70,76,83,86,96],"sql-commands":[0,1,2,3,5,6,7,8,10,11,12,13,15,16,17,18,19,21,22,23,25,26,27,28,29,31,32,33,34,36,37,38,39,41,42,43,44,45,47,48,49,51,52,53,55,56,57,58,59,60,61,62,63,65,66,67,68,69,71,72,73,74,75,77,78,79,80,81,82,84,85,87,88,89,90,91,92,93,94,95,97,99]}, "view-schema-type-evolution": {"depends-on-command":[4,9,14,20,24,28,36,42,51,54],"sql-commands":[0,1,2,3,5,6,7,8,10,11,12,13,15,16,17,18,19,21,22,23,25,26,27,29,30,31,32,33,34,35,37,38,39,40,41,43,44,45,46,47,48,49,50,52,53,55,57]}, "view-with-default-collation": {"depends-on-command":[2,5,11,12,18,19,25,26,29,30,31,32,37,38,39,40,41,42,46,47,49,56,57,58,61,64,70,71,77,78,84,85,88,89,90,91,96,97,98,99,100,101,105,106,108,115,116,117],"sql-commands":[1,4,7,8,9,10,14,15,16,17,21,22,23,24,28,33,34,35,36,44,45,48,51,52,53,54,55,60,63,66,67,68,69,73,74,75,76,80,81,82,83,87,92,93,94,95,103,104,107,110,111,112,113,114,119]}, - "window": {"depends-on-command":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,26,27,28,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,49,50,51,52,53,54,55,56,57,62,63,64,65,66,67,68,69,70],"sql-commands":[0]}, + "window": {"depends-on-command":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,26,27,28,30,31,32,33,34,35,36,37,41,42,43,46,52,53,54,57,62,63,64,65,66,67,68,69,70],"sql-commands":[0]}, "window-with-aggregates": {"query-features":[2,4,5,6,7,8,10,11,12]}, "xml-functions": {"depends-on-command":[30],"fn-url-xml-csv":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,15,16,17,18,19,20,21,22,23,24,25,26,27,28,32,33,34,35,36,37,38,39,50,51,52,53,54,55],"time-geo-types":[40,41,42,43,44,45,46,47,48,49]} }